[snap] caching categories to the disk | not locking the application boot while categories are being retrieved

CHANGELOG.md

@@ -17,6 +17,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - showing a "user-friendly" popup when there are integrity issues with the source-files of a building package
     - not waiting for the categories file to be retrieved from the cloud during application boot ( reduces boot time )
     - caching cloud to categories to the disk so they can be used in scenarios when it is not possible to retrieve them ( e.g: internet is off )
+- Snap:
+    - not waiting for the categories file to be retrieved from the cloud during application boot ( reduces boot time )
+    - caching cloud to categories to the disk so they can be used in scenarios when it is not possible to retrieve them ( e.g: internet is off )
 - minor thread improvements
     
 ### Fixes

bauh/gems/arch/controller.py

@@ -52,10 +52,10 @@ class ArchManager(SoftwareManager):
         self.enabled = True
         self.arch_distro = context.distro == 'arch'
         self.categories_mapper = AURCategoriesMapper(context.http_client, context.logger, self, self.context.disk_cache)
-        self.categories_map = {}
+        self.categories = {}
 
     def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader):
-        app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories_map)
+        app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories)
         app.downgrade_enabled = downgrade_enabled
 
         if app.installed:
@@ -117,7 +117,7 @@ class ArchManager(SoftwareManager):
 
                 if pkgsinfo:
                     for pkgdata in pkgsinfo:
-                        pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories_map)
+                        pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories)
                         pkg.downgrade_enabled = downgrade_enabled
 
                         if disk_loader:
@@ -136,7 +136,7 @@ class ArchManager(SoftwareManager):
                               latest_version=data.get('version'), description=data.get('description'),
                               installed=True, mirror='aur')
 
-            pkg.categories = self.categories_map.get(pkg.name)
+            pkg.categories = self.categories.get(pkg.name)
             pkg.downgrade_enabled = downgrade_enabled
 
             if disk_loader:
@@ -731,7 +731,7 @@ class ArchManager(SoftwareManager):
             self.categories_mapper.join()
             for pkg in api_res:
                 if pkg.get('Name') in sug_names:
-                    res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}, self.categories_map), suggestions.ALL.get(pkg['Name'])))
+                    res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}, self.categories), suggestions.ALL.get(pkg['Name'])))
 
         return res
 

bauh/gems/arch/worker.py

@@ -198,8 +198,7 @@ class AURCategoriesMapper(Thread):
                     self.logger.info('Loaded categories for {} AUR packages'.format(len(categories_map)))
 
                     if self.disk_cache and categories_map:
-                        t = Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True)
+                        Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start()
-                        t.start()
 
                     return categories_map
                 except:
@@ -218,5 +217,6 @@ class AURCategoriesMapper(Thread):
 
         if categories:
             self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__))
-            self.manager.categories_map = categories
+            self.manager.categories = categories
-            self.logger.info('Finished')
+
+        self.logger.info('Finished')

bauh/gems/snap/__init__.py

@@ -1,2 +1,6 @@
 import os
+
+from bauh.api.constants import CACHE_PATH
+
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+SNAP_CACHE_PATH = CACHE_PATH + '/snap'

bauh/gems/snap/controller.py

@@ -19,6 +19,7 @@ from bauh.gems.snap.worker import SnapAsyncDataLoader, CategoriesDownloader
 
 RE_AVAILABLE_CHANNELS = re.compile(re.compile(r'(\w+)\s+(snap install.+)'))
 
+
 class SnapManager(SoftwareManager):
 
     def __init__(self, context: ApplicationContext):
@@ -31,7 +32,7 @@ class SnapManager(SoftwareManager):
         self.logger = context.logger
         self.ubuntu_distro = context.distro == 'ubuntu'
         self.categories = {}
-        self.categories_downloader = CategoriesDownloader(self.http_client, self.logger)
+        self.categories_downloader = CategoriesDownloader(self.http_client, self.logger, self, context.disk_cache)
 
     def map_json(self, app_json: dict, installed: bool,  disk_loader: DiskCacheLoader, internet: bool = True) -> SnapApplication:
         app = SnapApplication(publisher=app_json.get('publisher'),
@@ -91,6 +92,7 @@ class SnapManager(SoftwareManager):
 
     def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult:
         if snap.is_snapd_running():
+            self.categories_downloader.join()
             installed = [self.map_json(app_json, installed=True, disk_loader=disk_loader, internet=internet_available) for app_json in snap.read_installed(self.ubuntu_distro)]
             return SearchResult(installed, None, len(installed))
         else:
@@ -173,10 +175,7 @@ class SnapManager(SoftwareManager):
         return ProcessHandler(watcher).handle(SystemProcess(subproc=snap.refresh_and_stream(pkg.name, root_password)))
 
     def prepare(self):
-        categories = self.categories_downloader.get_categories()
+        self.categories_downloader.start()
-
-        if categories:
-            self.categories = categories
 
     def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
         pass
@@ -207,6 +206,8 @@ class SnapManager(SoftwareManager):
             sugs.sort(key=lambda t: t[1].value, reverse=True)
 
             threads = []
+            self.categories_downloader.join()
+
             for sug in sugs:
 
                 if limit <= 0 or len(res) < limit:

bauh/gems/snap/worker.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import traceback
+from pathlib import Path
 from threading import Thread
 from typing import Dict, List
 
@@ -11,7 +12,7 @@ from bauh.api.abstract.context import ApplicationContext
 from bauh.api.abstract.controller import SoftwareManager
 from bauh.api.abstract.model import PackageStatus
 from bauh.api.http import HttpClient
-from bauh.gems.snap import snap
+from bauh.gems.snap import snap, SNAP_CACHE_PATH
 from bauh.gems.snap.constants import SNAP_API_URL
 from bauh.gems.snap.model import SnapApplication
 
@@ -76,13 +77,51 @@ class SnapAsyncDataLoader(Thread):
                 self.manager.cache_to_disk(pkg=self.app, icon_bytes=None, only_icon=False)
 
 
-class CategoriesDownloader:
+class CategoriesDownloader(Thread):
 
     URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/categories.txt'
+    CATEGORIES_FILE_PATH = SNAP_CACHE_PATH + '/categories.txt'
 
-    def __init__(self, http_client: HttpClient, logger: logging.Logger):
+    def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool):
+        super(CategoriesDownloader, self).__init__(daemon=True)
         self.http_client = http_client
         self.logger = logger
+        self.manager = manager
+        self.disk_cache = disk_cache
+
+    def _read_categories_from_disk(self) -> dict:
+        if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH):
+            self.logger.info("Reading cached categories from the disk")
+
+            with open(self.CATEGORIES_FILE_PATH) as f:
+                categories = f.read()
+
+            return self._map_categories(categories)
+
+        return {}
+
+    def _map_categories(self, categories: str) -> dict:
+        categories_map = {}
+        for l in categories.split('\n'):
+            if l:
+                data = l.split('=')
+                categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
+
+        return categories_map
+
+    def _cache_categories_to_disk(self, categories: str):
+        self.logger.info('Caching Snap categories to the disk')
+
+        try:
+            Path(SNAP_CACHE_PATH).mkdir(parents=True, exist_ok=True)
+
+            with open(self.CATEGORIES_FILE_PATH, 'w+') as f:
+                f.write(categories)
+
+            self.logger.info("Snap categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
+        except:
+            self.logger.error("Could not cache Snap categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
+            traceback.print_exc()
 
     def get_categories(self) -> Dict[str, List[str]]:
         self.logger.info('Downloading Snap category definitions from {}'.format(self.URL_CATEGORIES_FILE))
@@ -92,20 +131,29 @@ class CategoriesDownloader:
 
             if res:
                 try:
-                    categories_map = {}
+                    categories = self._map_categories(res.text)
-                    for l in res.text.split('\n'):
+                    self.logger.info('Loaded categories for {} Snap applications'.format(len(categories)))
-                        if l:
-                            data = l.split('=')
-                            categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
 
-                    self.logger.info('Loaded categories for {} Snap applications'.format(len(categories_map)))
+                    if self.disk_cache and categories:
-                    return categories_map
+                        Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start()
+
+                    return categories
                 except:
                     self.logger.error("Could not parse categories definitions")
                     traceback.print_exc()
-                    return {}
             else:
                 self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE))
+
         except requests.exceptions.ConnectionError:
             self.logger.warning('The internet connection seems to be off.')
-            return {}
+
+        return self._read_categories_from_disk()
+
+    def run(self):
+        categories = self.get_categories()
+
+        if categories:
+            self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__))
+            self.manager.categories = categories
+
+        self.logger.info('Finished')