[improvement][arch] faster caching data process during initialization

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.9.4] 2020
+### Improvements
+- Arch
+    - faster caching data process during initialization
+
+### Fixes
+- not caching data of repository packages with unknown repository names
+
 ## [0.9.3] 2020-05-12
 ### Features
 - new **restore** action to restore all bauh settings and cache through the 'custom actions' button (**+**). It is equivalent to the command `bauh --reset`.

bauh/gems/arch/controller.py

@@ -484,11 +484,11 @@ class ArchManager(SoftwareManager):
 
             pkgs.append(pkg)
 
-    def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult:
+    def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None, names: Iterable[str] = None) -> SearchResult:
         self.aur_client.clean_caches()
         arch_config = read_config()
 
-        installed = pacman.map_installed()
+        installed = pacman.map_installed(names=names)
 
         aur_pkgs, repo_pkgs = None, None
 

bauh/gems/arch/disk.py

@@ -14,11 +14,10 @@ RE_CLEAN_NAME = re.compile(r'[+*?%]')
 def write(pkg: ArchPackage):
     data = pkg.get_data_to_cache()
 
-    if data:
+    Path(pkg.get_disk_cache_path()).mkdir(parents=True, exist_ok=True)
-        Path(pkg.get_disk_cache_path()).mkdir(parents=True, exist_ok=True)
 
-        with open(pkg.get_disk_data_path(), 'w+') as f:
+    with open(pkg.get_disk_data_path(), 'w+') as f:
-            f.write(json.dumps(data))
+        f.write(json.dumps(data))
 
 
 def fill_icon_path(pkg: ArchPackage, icon_paths: List[str], only_exact_match: bool):

bauh/gems/arch/pacman.py

@@ -92,12 +92,12 @@ def _fill_ignored(res: dict):
     res['pkgs'] = list_ignored_packages()
 
 
-def map_installed() -> dict:  # returns a dict with with package names as keys and versions as values
+def map_installed(names: Iterable[str] = None) -> dict:  # returns a dict with with package names as keys and versions as values
     ignored = {}
     thread_ignored = Thread(target=_fill_ignored, args=(ignored,), daemon=True)
     thread_ignored.start()
 
-    allinfo = run_cmd('pacman -Qi')
+    allinfo = run_cmd('pacman -Qi{}'.format(' ' + ' '.join(names) if names else ''))
 
     pkgs = {'signed': {}, 'not_signed': {}}
     current_pkg = {}
@@ -689,10 +689,6 @@ def map_updates_data(pkgs: Iterable[str], files: bool = False) -> dict:
         return res
 
 
-def list_installed_names() -> Set[str]:
-    return {p for p in run_cmd('pacman -Qq').split('\n') if p}
-
-
 def upgrade_several(pkgnames: Iterable[str], root_password: str, overwrite_conflicting_files: bool = False) -> SimpleProcess:
     cmd = ['pacman', '-S', *pkgnames, '--noconfirm']
 
@@ -1019,3 +1015,9 @@ def list_unnecessary_deps(pkgs: Iterable[str], all_provided: Dict[str, Set[str]]
                         unnecessary.add(dep)
 
     return unnecessary.difference(pkgs)
+
+
+def list_installed_names() -> Set[str]:
+    output = run_cmd('pacman -Qq', print_error=False)
+    return {name.strip() for name in output.split('\n') if name} if output else set()
+

bauh/gems/arch/worker.py

@@ -1,3 +1,4 @@
+import glob
 import logging
 import os
 import re
@@ -13,7 +14,7 @@ from bauh.api.abstract.handler import TaskManager
 from bauh.commons.html import bold
 from bauh.commons.system import run_cmd, new_root_subprocess, ProcessHandler
 from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_FILE, CONFIG_DIR, BUILD_DIR, \
-    AUR_INDEX_FILE, get_icon_path, database, mirrors
+    AUR_INDEX_FILE, get_icon_path, database, mirrors, ARCH_CACHE_PATH
 from bauh.gems.arch.aur import URL_INDEX
 from bauh.view.util.translation import I18n
 
@@ -75,6 +76,8 @@ class ArchDiskCacheUpdater(Thread):
         self.aur = bool(arch_config['aur'])
         self.controller = controller
         self.internet_available = internet_available
+        self.installed_hash_path = '{}/installed.sha1'.format(ARCH_CACHE_PATH)
+        self.installed_cache_dir = '{}/installed'.format(ARCH_CACHE_PATH)
 
     def update_prepared(self, pkgname: str, add: bool = True):
         if add:
@@ -97,10 +100,31 @@ class ArchDiskCacheUpdater(Thread):
         ti = time.time()
         self.task_man.register_task(self.task_id, self.i18n['arch.task.disk_cache'], get_icon_path())
 
+        self.task_man.update_progress(self.task_id, 1, '')
+
+        self.logger.info("Checking already cached package data")
+
+        cache_dirs = [fpath for fpath in glob.glob('{}/*'.format(self.installed_cache_dir)) if os.path.isdir(fpath)]
+
+        not_cached_names = None
+
+        if cache_dirs:  # if there are cache data
+            installed_names = pacman.list_installed_names()
+            cached_pkgs = {cache_dir.split('/')[-1] for cache_dir in cache_dirs}
+
+            not_cached_names = installed_names.difference(cached_pkgs)
+            if not not_cached_names:
+                self.task_man.update_progress(self.task_id, 100, '')
+                self.task_man.finish_task(self.task_id)
+                tf = time.time()
+                time_msg = '{0:.2f} seconds'.format(tf - ti)
+                self.logger.info('Finished: no package data to cache ({})'.format(time_msg))
+                return
+
         self.logger.info('Pre-caching installed Arch packages data to disk')
 
         installed = self.controller.read_installed(disk_loader=None, internet_available=self.internet_available,
-                                                   only_apps=False, pkg_types=None, limit=-1).installed
+                                                   only_apps=False, pkg_types=None, limit=-1, names=not_cached_names).installed
 
         self.task_man.update_progress(self.task_id, 0, self.i18n['arch.task.disk_cache.reading'])
 
@@ -111,13 +135,14 @@ class ArchDiskCacheUpdater(Thread):
         self.progress = self.to_index * 2
         self.update_prepared(None, add=False)
 
-        saved += disk.save_several(pkgs, when_prepared=self.update_prepared, after_written=self.update_indexed)
+        # overwrite == True because the verification already happened
+        saved += disk.save_several(pkgs, when_prepared=self.update_prepared, after_written=self.update_indexed, overwrite=True)
         self.task_man.update_progress(self.task_id, 100, None)
         self.task_man.finish_task(self.task_id)
 
         tf = time.time()
-        time_msg = 'Took {0:.2f} seconds'.format(tf - ti)
+        time_msg = '{0:.2f} seconds'.format(tf - ti)
-        self.logger.info('Pre-cached data of {} Arch packages to the disk. {}'.format(saved, time_msg))
+        self.logger.info('Finished: pre-cached data of {} Arch packages to the disk ({})'.format(saved, time_msg))
 
 
 class ArchCompilationOptimizer(Thread):