diff --git a/CHANGELOG.md b/CHANGELOG.md index 0631d5ab..3c32ccc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - showing an error popup when **AppImageLauncher** messes up with an application installation - AUR: - showing a "user-friendly" popup when there are integrity issues with the source-files of a building package + - not waiting for the categories file to be retrieved from the cloud during application boot ( reduces boot time ) + - caching cloud to categories to the disk so they can be used in scenarios when it is not possible to retrieve them ( e.g: internet is off ) +- minor thread improvements ### Fixes - application not initializing when there is no internet connection @@ -24,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - not respecting **ignorepkg** settings in **pacman.conf** - not able to handle **missing dependencies with symbols** ( e.g: libpng++ ) - not able to work with **.xpm** icons + - not mapping categories to the search results ## [0.7.0] 2019-10-18 ### Features diff --git a/bauh/gems/arch/__init__.py b/bauh/gems/arch/__init__.py index 3b1cfda6..87c42aa7 100644 --- a/bauh/gems/arch/__init__.py +++ b/bauh/gems/arch/__init__.py @@ -1,4 +1,8 @@ import os + +from bauh.api.constants import CACHE_PATH + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) BUILD_DIR = '/tmp/bauh/aur' +ARCH_CACHE_PATH = CACHE_PATH + '/arch' diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 358e1a59..f4fd9a94 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -22,7 +22,7 @@ from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, c from bauh.gems.arch.aur import AURClient from bauh.gems.arch.mapper import ArchDataMapper from bauh.gems.arch.model import ArchPackage -from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, CategoriesDownloader +from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, AURCategoriesMapper URL_GIT = 'https://aur.archlinux.org/{}.git' URL_PKG_DOWNLOAD = 'https://aur.archlinux.org/cgit/aur.git/snapshot/{}.tar.gz' @@ -51,11 +51,11 @@ class ArchManager(SoftwareManager): self.logger = context.logger self.enabled = True self.arch_distro = context.distro == 'arch' - self.categories_downloader = CategoriesDownloader(context.http_client, context.logger) + self.categories_mapper = AURCategoriesMapper(context.http_client, context.logger, self, self.context.disk_cache) self.categories_map = {} def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader): - app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed']) + app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories_map) app.downgrade_enabled = downgrade_enabled if app.installed: @@ -66,7 +66,7 @@ class ArchManager(SoftwareManager): else: res.new.append(app) - Thread(target=self.mapper.fill_package_build, args=(app,)).start() + Thread(target=self.mapper.fill_package_build, args=(app,), daemon=True).start() def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1) -> SearchResult: self.comp_optimizer.join() @@ -75,7 +75,7 @@ class ArchManager(SoftwareManager): res = SearchResult([], [], 0) installed = {} - read_installed = Thread(target=lambda: installed.update(pacman.list_and_map_installed())) + read_installed = Thread(target=lambda: installed.update(pacman.list_and_map_installed()), daemon=True) read_installed.start() api_res = self.aur_client.search(words) @@ -117,9 +117,8 @@ class ArchManager(SoftwareManager): if pkgsinfo: for pkgdata in pkgsinfo: - pkg = self.mapper.map_api_data(pkgdata, not_signed) + pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories_map) pkg.downgrade_enabled = downgrade_enabled - pkg.categories = self.categories_map.get(pkg.name) if disk_loader: disk_loader.fill(pkg) @@ -161,6 +160,7 @@ class ArchManager(SoftwareManager): apps = [] if installed and installed['not_signed']: self.dcache_updater.join() + self.categories_mapper.join() self._fill_aur_pkgs(installed['not_signed'], apps, disk_loader, internet_available) @@ -684,7 +684,7 @@ class ArchManager(SoftwareManager): return False def cache_to_disk(self, pkg: ArchPackage, icon_bytes: bytes, only_icon: bool): - pass + pass def requires_root(self, action: str, pkg: ArchPackage): return action != 'search' @@ -693,10 +693,7 @@ class ArchManager(SoftwareManager): self.dcache_updater.start() self.comp_optimizer.start() self.aur_index_updater.start() - categories = self.categories_downloader.get_categories() - - if categories: - self.categories_map = categories + self.categories_mapper.start() def list_updates(self, internet_available: bool) -> List[PackageUpdate]: installed = self.read_installed(disk_loader=None, internet_available=internet_available).installed @@ -731,9 +728,10 @@ class ArchManager(SoftwareManager): api_res = self.aur_client.get_info(sug_names) if api_res: + self.categories_mapper.join() for pkg in api_res: if pkg.get('Name') in sug_names: - res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}), suggestions.ALL.get(pkg['Name']))) + res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}, self.categories_map), suggestions.ALL.get(pkg['Name']))) return res diff --git a/bauh/gems/arch/mapper.py b/bauh/gems/arch/mapper.py index d058d3c3..723c01ee 100644 --- a/bauh/gems/arch/mapper.py +++ b/bauh/gems/arch/mapper.py @@ -110,14 +110,18 @@ class ArchDataMapper: if res and res.status_code == 200 and res.text: pkg.pkgbuild = res.text - def map_api_data(self, apidata: dict, installed: dict) -> ArchPackage: + def map_api_data(self, apidata: dict, installed: dict, categories: dict) -> ArchPackage: data = installed.get(apidata.get('Name')) app = ArchPackage(name=apidata.get('Name'), installed=bool(data), mirror='aur') app.status = PackageStatus.LOADING_DATA + if categories: + app.categories = categories.get(app.name) + if data: app.version = data.get('version') app.description = data.get('description') self.fill_api_data(app, apidata, fill_version=not data) + return app diff --git a/bauh/gems/arch/model.py b/bauh/gems/arch/model.py index 9003f983..9da563eb 100644 --- a/bauh/gems/arch/model.py +++ b/bauh/gems/arch/model.py @@ -2,10 +2,8 @@ import datetime from typing import List from bauh.api.abstract.model import SoftwarePackage -from bauh.api.constants import CACHE_PATH from bauh.commons import resource - -from bauh.gems.arch import ROOT_DIR +from bauh.gems.arch import ROOT_DIR, ARCH_CACHE_PATH CACHED_ATTRS = {'command', 'icon_path', 'mirror', 'maintainer', 'desktop_entry'} @@ -35,7 +33,7 @@ class ArchPackage(SoftwarePackage): @staticmethod def disk_cache_path(pkgname: str, mirror: str): - return CACHE_PATH + '/arch/installed/' + ('aur' if mirror == 'aur' else 'mirror') + '/' + pkgname + return ARCH_CACHE_PATH + '/installed/' + ('aur' if mirror == 'aur' else 'mirror') + '/' + pkgname def get_pkg_build_url(self): if self.package_base: diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index a0f69558..b5fa9c05 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -5,6 +5,7 @@ import time import traceback from math import ceil from multiprocessing import Process +from pathlib import Path from threading import Thread from typing import Dict, List @@ -12,9 +13,9 @@ import requests from bauh.api.abstract.context import ApplicationContext from bauh.api.abstract.controller import SoftwareManager -from bauh.api.constants import HOME_PATH +from bauh.api.constants import HOME_PATH, CACHE_PATH from bauh.api.http import HttpClient -from bauh.gems.arch import pacman, disk +from bauh.gems.arch import pacman, disk, ARCH_CACHE_PATH URL_INDEX = 'https://aur.archlinux.org/packages.gz' URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg={}' @@ -139,13 +140,51 @@ class ArchCompilationOptimizer(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else P self.logger.warning("A custom 'makepkg.conf' is already defined at '{}'".format(HOME_PATH)) -class CategoriesDownloader: +class AURCategoriesMapper(Thread): URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt' + CATEGORIES_CACHE_DIR = ARCH_CACHE_PATH + '/categories' + CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt' - def __init__(self, http_client: HttpClient, logger: logging.Logger): + def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool): + super(AURCategoriesMapper, self).__init__(daemon=True) self.http_client = http_client self.logger = logger + self.manager = manager + self.disk_cache = disk_cache + + def _read_categories_from_disk(self) -> dict: + if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH): + self.logger.info("Reading cached categories from the disk") + + with open(self.CATEGORIES_FILE_PATH) as f: + categories = f.read() + + return self._map_categories(categories) + + return {} + + def _map_categories(self, categories: str) -> dict: + categories_map = {} + for l in categories.split('\n'): + if l: + data = l.split('=') + categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c] + return categories_map + + def _cache_categories_to_disk(self, categories: str): + self.logger.info('Caching AUR categories to the disk') + + try: + Path(self.CATEGORIES_CACHE_DIR).mkdir(parents=True, exist_ok=True) + + with open(self.CATEGORIES_FILE_PATH, 'w+') as f: + f.write(categories) + + self.logger.info("AUR categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH)) + except: + self.logger.error("Could not cache AUR categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH)) + traceback.print_exc() def get_categories(self) -> Dict[str, List[str]]: self.logger.info('Downloading AUR category definitions from {}'.format(self.URL_CATEGORIES_FILE)) @@ -155,21 +194,29 @@ class CategoriesDownloader: if res: try: - categories_map = {} - for l in res.text.split('\n'): - if l: - data = l.split('=') - categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c] - + categories_map = self._map_categories(res.text) self.logger.info('Loaded categories for {} AUR packages'.format(len(categories_map))) + + if self.disk_cache and categories_map: + t = Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True) + t.start() + return categories_map except: self.logger.error("Could not parse AUR categories definitions") traceback.print_exc() - return {} else: self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE)) - return {} + except requests.exceptions.ConnectionError: self.logger.warning('The internet connection seems to be off.') - return {} + + return self._read_categories_from_disk() + + def run(self): + categories = self.get_categories() + + if categories: + self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__)) + self.manager.categories_map = categories + self.logger.info('Finished') diff --git a/bauh/view/qt/screenshots.py b/bauh/view/qt/screenshots.py index 8f8dea43..395e89b0 100644 --- a/bauh/view/qt/screenshots.py +++ b/bauh/view/qt/screenshots.py @@ -64,7 +64,7 @@ class ScreenshotsDialog(QDialog): self.img_idx = 0 for idx, s in enumerate(self.screenshots): - t = Thread(target=self._download_img, args=(idx, s)) + t = Thread(target=self._download_img, args=(idx, s), daemon=True) t.start() self._load_img()