diff --git a/CHANGELOG.md b/CHANGELOG.md index 629f51bb..330026af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [0.9.4] 2020 +### Improvements +- Arch + - faster caching data process during initialization + +### Fixes +- not caching data of repository packages with unknown repository names + ## [0.9.3] 2020-05-12 ### Features - new **restore** action to restore all bauh settings and cache through the 'custom actions' button (**+**). It is equivalent to the command `bauh --reset`. diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index ff4c6119..cddf956b 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -484,11 +484,11 @@ class ArchManager(SoftwareManager): pkgs.append(pkg) - def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult: + def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None, names: Iterable[str] = None) -> SearchResult: self.aur_client.clean_caches() arch_config = read_config() - installed = pacman.map_installed() + installed = pacman.map_installed(names=names) aur_pkgs, repo_pkgs = None, None diff --git a/bauh/gems/arch/disk.py b/bauh/gems/arch/disk.py index 0680b006..f0b1e9e3 100644 --- a/bauh/gems/arch/disk.py +++ b/bauh/gems/arch/disk.py @@ -14,11 +14,10 @@ RE_CLEAN_NAME = re.compile(r'[+*?%]') def write(pkg: ArchPackage): data = pkg.get_data_to_cache() - if data: - Path(pkg.get_disk_cache_path()).mkdir(parents=True, exist_ok=True) + Path(pkg.get_disk_cache_path()).mkdir(parents=True, exist_ok=True) - with open(pkg.get_disk_data_path(), 'w+') as f: - f.write(json.dumps(data)) + with open(pkg.get_disk_data_path(), 'w+') as f: + f.write(json.dumps(data)) def fill_icon_path(pkg: ArchPackage, icon_paths: List[str], only_exact_match: bool): diff --git a/bauh/gems/arch/pacman.py b/bauh/gems/arch/pacman.py index 1e5476fb..7d11e4d3 100644 --- a/bauh/gems/arch/pacman.py +++ b/bauh/gems/arch/pacman.py @@ -92,12 +92,12 @@ def _fill_ignored(res: dict): res['pkgs'] = list_ignored_packages() -def map_installed() -> dict: # returns a dict with with package names as keys and versions as values +def map_installed(names: Iterable[str] = None) -> dict: # returns a dict with with package names as keys and versions as values ignored = {} thread_ignored = Thread(target=_fill_ignored, args=(ignored,), daemon=True) thread_ignored.start() - allinfo = run_cmd('pacman -Qi') + allinfo = run_cmd('pacman -Qi{}'.format(' ' + ' '.join(names) if names else '')) pkgs = {'signed': {}, 'not_signed': {}} current_pkg = {} @@ -689,10 +689,6 @@ def map_updates_data(pkgs: Iterable[str], files: bool = False) -> dict: return res -def list_installed_names() -> Set[str]: - return {p for p in run_cmd('pacman -Qq').split('\n') if p} - - def upgrade_several(pkgnames: Iterable[str], root_password: str, overwrite_conflicting_files: bool = False) -> SimpleProcess: cmd = ['pacman', '-S', *pkgnames, '--noconfirm'] @@ -1019,3 +1015,9 @@ def list_unnecessary_deps(pkgs: Iterable[str], all_provided: Dict[str, Set[str]] unnecessary.add(dep) return unnecessary.difference(pkgs) + + +def list_installed_names() -> Set[str]: + output = run_cmd('pacman -Qq', print_error=False) + return {name.strip() for name in output.split('\n') if name} if output else set() + diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index 31a5abcf..5bbc4e28 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -1,3 +1,4 @@ +import glob import logging import os import re @@ -13,7 +14,7 @@ from bauh.api.abstract.handler import TaskManager from bauh.commons.html import bold from bauh.commons.system import run_cmd, new_root_subprocess, ProcessHandler from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_FILE, CONFIG_DIR, BUILD_DIR, \ - AUR_INDEX_FILE, get_icon_path, database, mirrors + AUR_INDEX_FILE, get_icon_path, database, mirrors, ARCH_CACHE_PATH from bauh.gems.arch.aur import URL_INDEX from bauh.view.util.translation import I18n @@ -75,6 +76,8 @@ class ArchDiskCacheUpdater(Thread): self.aur = bool(arch_config['aur']) self.controller = controller self.internet_available = internet_available + self.installed_hash_path = '{}/installed.sha1'.format(ARCH_CACHE_PATH) + self.installed_cache_dir = '{}/installed'.format(ARCH_CACHE_PATH) def update_prepared(self, pkgname: str, add: bool = True): if add: @@ -97,10 +100,31 @@ class ArchDiskCacheUpdater(Thread): ti = time.time() self.task_man.register_task(self.task_id, self.i18n['arch.task.disk_cache'], get_icon_path()) + self.task_man.update_progress(self.task_id, 1, '') + + self.logger.info("Checking already cached package data") + + cache_dirs = [fpath for fpath in glob.glob('{}/*'.format(self.installed_cache_dir)) if os.path.isdir(fpath)] + + not_cached_names = None + + if cache_dirs: # if there are cache data + installed_names = pacman.list_installed_names() + cached_pkgs = {cache_dir.split('/')[-1] for cache_dir in cache_dirs} + + not_cached_names = installed_names.difference(cached_pkgs) + if not not_cached_names: + self.task_man.update_progress(self.task_id, 100, '') + self.task_man.finish_task(self.task_id) + tf = time.time() + time_msg = '{0:.2f} seconds'.format(tf - ti) + self.logger.info('Finished: no package data to cache ({})'.format(time_msg)) + return + self.logger.info('Pre-caching installed Arch packages data to disk') installed = self.controller.read_installed(disk_loader=None, internet_available=self.internet_available, - only_apps=False, pkg_types=None, limit=-1).installed + only_apps=False, pkg_types=None, limit=-1, names=not_cached_names).installed self.task_man.update_progress(self.task_id, 0, self.i18n['arch.task.disk_cache.reading']) @@ -111,13 +135,14 @@ class ArchDiskCacheUpdater(Thread): self.progress = self.to_index * 2 self.update_prepared(None, add=False) - saved += disk.save_several(pkgs, when_prepared=self.update_prepared, after_written=self.update_indexed) + # overwrite == True because the verification already happened + saved += disk.save_several(pkgs, when_prepared=self.update_prepared, after_written=self.update_indexed, overwrite=True) self.task_man.update_progress(self.task_id, 100, None) self.task_man.finish_task(self.task_id) tf = time.time() - time_msg = 'Took {0:.2f} seconds'.format(tf - ti) - self.logger.info('Pre-cached data of {} Arch packages to the disk. {}'.format(saved, time_msg)) + time_msg = '{0:.2f} seconds'.format(tf - ti) + self.logger.info('Finished: pre-cached data of {} Arch packages to the disk ({})'.format(saved, time_msg)) class ArchCompilationOptimizer(Thread):