[aur] caching categories to the disk | not locking the application boot while the AUR categories are being retrieved

CHANGELOG.md

@@ -15,6 +15,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - showing an error popup when **AppImageLauncher** messes up with an application installation
 - AUR:
     - showing a "user-friendly" popup when there are integrity issues with the source-files of a building package
+    - not waiting for the categories file to be retrieved from the cloud during application boot ( reduces boot time )
+    - caching cloud to categories to the disk so they can be used in scenarios when it is not possible to retrieve them ( e.g: internet is off )
+- minor thread improvements
     
 ### Fixes
 - application not initializing when there is no internet connection
@@ -24,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - not respecting **ignorepkg** settings in **pacman.conf**
     - not able to handle **missing dependencies with symbols** ( e.g: libpng++ )
     - not able to work with **.xpm** icons
+    - not mapping categories to the search results
 
 ## [0.7.0] 2019-10-18
 ### Features

bauh/gems/arch/__init__.py

@@ -1,4 +1,8 @@
 import os
+
+from bauh.api.constants import CACHE_PATH
+
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 BUILD_DIR = '/tmp/bauh/aur'
+ARCH_CACHE_PATH = CACHE_PATH + '/arch'
 

bauh/gems/arch/controller.py

@@ -22,7 +22,7 @@ from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, c
 from bauh.gems.arch.aur import AURClient
 from bauh.gems.arch.mapper import ArchDataMapper
 from bauh.gems.arch.model import ArchPackage
-from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, CategoriesDownloader
+from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, AURCategoriesMapper
 
 URL_GIT = 'https://aur.archlinux.org/{}.git'
 URL_PKG_DOWNLOAD = 'https://aur.archlinux.org/cgit/aur.git/snapshot/{}.tar.gz'
@@ -51,11 +51,11 @@ class ArchManager(SoftwareManager):
         self.logger = context.logger
         self.enabled = True
         self.arch_distro = context.distro == 'arch'
-        self.categories_downloader = CategoriesDownloader(context.http_client, context.logger)
+        self.categories_mapper = AURCategoriesMapper(context.http_client, context.logger, self, self.context.disk_cache)
         self.categories_map = {}
 
     def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader):
-        app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'])
+        app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories_map)
         app.downgrade_enabled = downgrade_enabled
 
         if app.installed:
@@ -66,7 +66,7 @@ class ArchManager(SoftwareManager):
         else:
             res.new.append(app)
 
-        Thread(target=self.mapper.fill_package_build, args=(app,)).start()
+        Thread(target=self.mapper.fill_package_build, args=(app,), daemon=True).start()
 
     def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1) -> SearchResult:
         self.comp_optimizer.join()
@@ -75,7 +75,7 @@ class ArchManager(SoftwareManager):
         res = SearchResult([], [], 0)
 
         installed = {}
-        read_installed = Thread(target=lambda: installed.update(pacman.list_and_map_installed()))
+        read_installed = Thread(target=lambda: installed.update(pacman.list_and_map_installed()), daemon=True)
         read_installed.start()
 
         api_res = self.aur_client.search(words)
@@ -117,9 +117,8 @@ class ArchManager(SoftwareManager):
 
                 if pkgsinfo:
                     for pkgdata in pkgsinfo:
-                        pkg = self.mapper.map_api_data(pkgdata, not_signed)
+                        pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories_map)
                         pkg.downgrade_enabled = downgrade_enabled
-                        pkg.categories = self.categories_map.get(pkg.name)
 
                         if disk_loader:
                             disk_loader.fill(pkg)
@@ -161,6 +160,7 @@ class ArchManager(SoftwareManager):
         apps = []
         if installed and installed['not_signed']:
             self.dcache_updater.join()
+            self.categories_mapper.join()
 
             self._fill_aur_pkgs(installed['not_signed'], apps, disk_loader, internet_available)
 
@@ -684,7 +684,7 @@ class ArchManager(SoftwareManager):
             return False
 
     def cache_to_disk(self, pkg: ArchPackage, icon_bytes: bytes, only_icon: bool):
-       pass
+        pass
 
     def requires_root(self, action: str, pkg: ArchPackage):
         return action != 'search'
@@ -693,10 +693,7 @@ class ArchManager(SoftwareManager):
         self.dcache_updater.start()
         self.comp_optimizer.start()
         self.aur_index_updater.start()
-        categories = self.categories_downloader.get_categories()
+        self.categories_mapper.start()
-
-        if categories:
-            self.categories_map = categories
 
     def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
         installed = self.read_installed(disk_loader=None, internet_available=internet_available).installed
@@ -731,9 +728,10 @@ class ArchManager(SoftwareManager):
         api_res = self.aur_client.get_info(sug_names)
 
         if api_res:
+            self.categories_mapper.join()
             for pkg in api_res:
                 if pkg.get('Name') in sug_names:
-                    res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}), suggestions.ALL.get(pkg['Name'])))
+                    res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}, self.categories_map), suggestions.ALL.get(pkg['Name'])))
 
         return res
 

bauh/gems/arch/mapper.py

@@ -110,14 +110,18 @@ class ArchDataMapper:
         if res and res.status_code == 200 and res.text:
             pkg.pkgbuild = res.text
 
-    def map_api_data(self, apidata: dict, installed: dict) -> ArchPackage:
+    def map_api_data(self, apidata: dict, installed: dict, categories: dict) -> ArchPackage:
         data = installed.get(apidata.get('Name'))
         app = ArchPackage(name=apidata.get('Name'), installed=bool(data), mirror='aur')
         app.status = PackageStatus.LOADING_DATA
 
+        if categories:
+            app.categories = categories.get(app.name)
+
         if data:
             app.version = data.get('version')
             app.description = data.get('description')
 
         self.fill_api_data(app, apidata, fill_version=not data)
+
         return app

bauh/gems/arch/model.py

@@ -2,10 +2,8 @@ import datetime
 from typing import List
 
 from bauh.api.abstract.model import SoftwarePackage
-from bauh.api.constants import CACHE_PATH
 from bauh.commons import resource
-
+from bauh.gems.arch import ROOT_DIR, ARCH_CACHE_PATH
-from bauh.gems.arch import ROOT_DIR
 
 CACHED_ATTRS = {'command', 'icon_path', 'mirror', 'maintainer', 'desktop_entry'}
 
@@ -35,7 +33,7 @@ class ArchPackage(SoftwarePackage):
 
     @staticmethod
     def disk_cache_path(pkgname: str, mirror: str):
-        return CACHE_PATH + '/arch/installed/' + ('aur' if mirror == 'aur' else 'mirror') + '/' + pkgname
+        return ARCH_CACHE_PATH + '/installed/' + ('aur' if mirror == 'aur' else 'mirror') + '/' + pkgname
 
     def get_pkg_build_url(self):
         if self.package_base:

bauh/gems/arch/worker.py

@@ -5,6 +5,7 @@ import time
 import traceback
 from math import ceil
 from multiprocessing import Process
+from pathlib import Path
 from threading import Thread
 from typing import Dict, List
 
@@ -12,9 +13,9 @@ import requests
 
 from bauh.api.abstract.context import ApplicationContext
 from bauh.api.abstract.controller import SoftwareManager
-from bauh.api.constants import HOME_PATH
+from bauh.api.constants import HOME_PATH, CACHE_PATH
 from bauh.api.http import HttpClient
-from bauh.gems.arch import pacman, disk
+from bauh.gems.arch import pacman, disk, ARCH_CACHE_PATH
 
 URL_INDEX = 'https://aur.archlinux.org/packages.gz'
 URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg={}'
@@ -139,13 +140,51 @@ class ArchCompilationOptimizer(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else P
                 self.logger.warning("A custom 'makepkg.conf' is already defined at '{}'".format(HOME_PATH))
 
 
-class CategoriesDownloader:
+class AURCategoriesMapper(Thread):
 
     URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt'
+    CATEGORIES_CACHE_DIR = ARCH_CACHE_PATH + '/categories'
+    CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt'
 
-    def __init__(self, http_client: HttpClient, logger: logging.Logger):
+    def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool):
+        super(AURCategoriesMapper, self).__init__(daemon=True)
         self.http_client = http_client
         self.logger = logger
+        self.manager = manager
+        self.disk_cache = disk_cache
+
+    def _read_categories_from_disk(self) -> dict:
+        if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH):
+            self.logger.info("Reading cached categories from the disk")
+
+            with open(self.CATEGORIES_FILE_PATH) as f:
+                categories = f.read()
+
+            return self._map_categories(categories)
+
+        return {}
+
+    def _map_categories(self, categories: str) -> dict:
+        categories_map = {}
+        for l in categories.split('\n'):
+            if l:
+                data = l.split('=')
+                categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
+        return categories_map
+
+    def _cache_categories_to_disk(self, categories: str):
+        self.logger.info('Caching AUR categories to the disk')
+
+        try:
+            Path(self.CATEGORIES_CACHE_DIR).mkdir(parents=True, exist_ok=True)
+
+            with open(self.CATEGORIES_FILE_PATH, 'w+') as f:
+                f.write(categories)
+
+            self.logger.info("AUR categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
+        except:
+            self.logger.error("Could not cache AUR categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
+            traceback.print_exc()
 
     def get_categories(self) -> Dict[str, List[str]]:
         self.logger.info('Downloading AUR category definitions from {}'.format(self.URL_CATEGORIES_FILE))
@@ -155,21 +194,29 @@ class CategoriesDownloader:
 
             if res:
                 try:
-                    categories_map = {}
+                    categories_map = self._map_categories(res.text)
-                    for l in res.text.split('\n'):
-                        if l:
-                            data = l.split('=')
-                            categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
-
                     self.logger.info('Loaded categories for {} AUR packages'.format(len(categories_map)))
+
+                    if self.disk_cache and categories_map:
+                        t = Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True)
+                        t.start()
+
                     return categories_map
                 except:
                     self.logger.error("Could not parse AUR categories definitions")
                     traceback.print_exc()
-                    return {}
             else:
                 self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE))
-                return {}
+
         except requests.exceptions.ConnectionError:
             self.logger.warning('The internet connection seems to be off.')
-            return {}
+
+        return self._read_categories_from_disk()
+
+    def run(self):
+        categories = self.get_categories()
+
+        if categories:
+            self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__))
+            self.manager.categories_map = categories
+            self.logger.info('Finished')

bauh/view/qt/screenshots.py

@@ -64,7 +64,7 @@ class ScreenshotsDialog(QDialog):
         self.img_idx = 0
 
         for idx, s in enumerate(self.screenshots):
-            t = Thread(target=self._download_img, args=(idx, s))
+            t = Thread(target=self._download_img, args=(idx, s), daemon=True)
             t.start()
 
         self._load_img()