diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c32ccc7..259e6710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - showing a "user-friendly" popup when there are integrity issues with the source-files of a building package - not waiting for the categories file to be retrieved from the cloud during application boot ( reduces boot time ) - caching cloud to categories to the disk so they can be used in scenarios when it is not possible to retrieve them ( e.g: internet is off ) +- Snap: + - not waiting for the categories file to be retrieved from the cloud during application boot ( reduces boot time ) + - caching cloud to categories to the disk so they can be used in scenarios when it is not possible to retrieve them ( e.g: internet is off ) - minor thread improvements ### Fixes diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index f4fd9a94..37a645bd 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -52,10 +52,10 @@ class ArchManager(SoftwareManager): self.enabled = True self.arch_distro = context.distro == 'arch' self.categories_mapper = AURCategoriesMapper(context.http_client, context.logger, self, self.context.disk_cache) - self.categories_map = {} + self.categories = {} def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader): - app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories_map) + app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories) app.downgrade_enabled = downgrade_enabled if app.installed: @@ -117,7 +117,7 @@ class ArchManager(SoftwareManager): if pkgsinfo: for pkgdata in pkgsinfo: - pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories_map) + pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories) pkg.downgrade_enabled = downgrade_enabled if disk_loader: @@ -136,7 +136,7 @@ class ArchManager(SoftwareManager): latest_version=data.get('version'), description=data.get('description'), installed=True, mirror='aur') - pkg.categories = self.categories_map.get(pkg.name) + pkg.categories = self.categories.get(pkg.name) pkg.downgrade_enabled = downgrade_enabled if disk_loader: @@ -731,7 +731,7 @@ class ArchManager(SoftwareManager): self.categories_mapper.join() for pkg in api_res: if pkg.get('Name') in sug_names: - res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}, self.categories_map), suggestions.ALL.get(pkg['Name']))) + res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}, self.categories), suggestions.ALL.get(pkg['Name']))) return res diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index b5fa9c05..1a359d87 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -198,8 +198,7 @@ class AURCategoriesMapper(Thread): self.logger.info('Loaded categories for {} AUR packages'.format(len(categories_map))) if self.disk_cache and categories_map: - t = Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True) - t.start() + Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start() return categories_map except: @@ -218,5 +217,6 @@ class AURCategoriesMapper(Thread): if categories: self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__)) - self.manager.categories_map = categories - self.logger.info('Finished') + self.manager.categories = categories + + self.logger.info('Finished') diff --git a/bauh/gems/snap/__init__.py b/bauh/gems/snap/__init__.py index 8824c2d4..6e7f3aaf 100644 --- a/bauh/gems/snap/__init__.py +++ b/bauh/gems/snap/__init__.py @@ -1,2 +1,6 @@ import os + +from bauh.api.constants import CACHE_PATH + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +SNAP_CACHE_PATH = CACHE_PATH + '/snap' diff --git a/bauh/gems/snap/controller.py b/bauh/gems/snap/controller.py index 0ab35e23..006f3cdb 100644 --- a/bauh/gems/snap/controller.py +++ b/bauh/gems/snap/controller.py @@ -19,6 +19,7 @@ from bauh.gems.snap.worker import SnapAsyncDataLoader, CategoriesDownloader RE_AVAILABLE_CHANNELS = re.compile(re.compile(r'(\w+)\s+(snap install.+)')) + class SnapManager(SoftwareManager): def __init__(self, context: ApplicationContext): @@ -31,7 +32,7 @@ class SnapManager(SoftwareManager): self.logger = context.logger self.ubuntu_distro = context.distro == 'ubuntu' self.categories = {} - self.categories_downloader = CategoriesDownloader(self.http_client, self.logger) + self.categories_downloader = CategoriesDownloader(self.http_client, self.logger, self, context.disk_cache) def map_json(self, app_json: dict, installed: bool, disk_loader: DiskCacheLoader, internet: bool = True) -> SnapApplication: app = SnapApplication(publisher=app_json.get('publisher'), @@ -91,6 +92,7 @@ class SnapManager(SoftwareManager): def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult: if snap.is_snapd_running(): + self.categories_downloader.join() installed = [self.map_json(app_json, installed=True, disk_loader=disk_loader, internet=internet_available) for app_json in snap.read_installed(self.ubuntu_distro)] return SearchResult(installed, None, len(installed)) else: @@ -173,10 +175,7 @@ class SnapManager(SoftwareManager): return ProcessHandler(watcher).handle(SystemProcess(subproc=snap.refresh_and_stream(pkg.name, root_password))) def prepare(self): - categories = self.categories_downloader.get_categories() - - if categories: - self.categories = categories + self.categories_downloader.start() def list_updates(self, internet_available: bool) -> List[PackageUpdate]: pass @@ -207,6 +206,8 @@ class SnapManager(SoftwareManager): sugs.sort(key=lambda t: t[1].value, reverse=True) threads = [] + self.categories_downloader.join() + for sug in sugs: if limit <= 0 or len(res) < limit: diff --git a/bauh/gems/snap/worker.py b/bauh/gems/snap/worker.py index 54bd1271..17a609d0 100644 --- a/bauh/gems/snap/worker.py +++ b/bauh/gems/snap/worker.py @@ -1,6 +1,7 @@ import logging import os import traceback +from pathlib import Path from threading import Thread from typing import Dict, List @@ -11,7 +12,7 @@ from bauh.api.abstract.context import ApplicationContext from bauh.api.abstract.controller import SoftwareManager from bauh.api.abstract.model import PackageStatus from bauh.api.http import HttpClient -from bauh.gems.snap import snap +from bauh.gems.snap import snap, SNAP_CACHE_PATH from bauh.gems.snap.constants import SNAP_API_URL from bauh.gems.snap.model import SnapApplication @@ -76,13 +77,51 @@ class SnapAsyncDataLoader(Thread): self.manager.cache_to_disk(pkg=self.app, icon_bytes=None, only_icon=False) -class CategoriesDownloader: +class CategoriesDownloader(Thread): URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/categories.txt' + CATEGORIES_FILE_PATH = SNAP_CACHE_PATH + '/categories.txt' - def __init__(self, http_client: HttpClient, logger: logging.Logger): + def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool): + super(CategoriesDownloader, self).__init__(daemon=True) self.http_client = http_client self.logger = logger + self.manager = manager + self.disk_cache = disk_cache + + def _read_categories_from_disk(self) -> dict: + if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH): + self.logger.info("Reading cached categories from the disk") + + with open(self.CATEGORIES_FILE_PATH) as f: + categories = f.read() + + return self._map_categories(categories) + + return {} + + def _map_categories(self, categories: str) -> dict: + categories_map = {} + for l in categories.split('\n'): + if l: + data = l.split('=') + categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c] + + return categories_map + + def _cache_categories_to_disk(self, categories: str): + self.logger.info('Caching Snap categories to the disk') + + try: + Path(SNAP_CACHE_PATH).mkdir(parents=True, exist_ok=True) + + with open(self.CATEGORIES_FILE_PATH, 'w+') as f: + f.write(categories) + + self.logger.info("Snap categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH)) + except: + self.logger.error("Could not cache Snap categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH)) + traceback.print_exc() def get_categories(self) -> Dict[str, List[str]]: self.logger.info('Downloading Snap category definitions from {}'.format(self.URL_CATEGORIES_FILE)) @@ -92,20 +131,29 @@ class CategoriesDownloader: if res: try: - categories_map = {} - for l in res.text.split('\n'): - if l: - data = l.split('=') - categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c] + categories = self._map_categories(res.text) + self.logger.info('Loaded categories for {} Snap applications'.format(len(categories))) - self.logger.info('Loaded categories for {} Snap applications'.format(len(categories_map))) - return categories_map + if self.disk_cache and categories: + Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start() + + return categories except: self.logger.error("Could not parse categories definitions") traceback.print_exc() - return {} else: self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE)) + except requests.exceptions.ConnectionError: self.logger.warning('The internet connection seems to be off.') - return {} + + return self._read_categories_from_disk() + + def run(self): + categories = self.get_categories() + + if categories: + self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__)) + self.manager.categories = categories + + self.logger.info('Finished')