From fbd98d8b3167dfbb78c25373be19515c11ae067a Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Fri, 20 May 2022 17:17:54 -0300 Subject: [PATCH] [arch] improvement: suggestions available for repository packages --- CHANGELOG.md | 3 + README.md | 1 + bauh/api/abstract/model.py | 8 ++ bauh/gems/arch/config.py | 3 +- bauh/gems/arch/controller.py | 153 +++++++++++++++++++-- bauh/gems/arch/pacman.py | 87 ++++++++---- bauh/gems/arch/resources/locale/ca | 2 + bauh/gems/arch/resources/locale/de | 2 + bauh/gems/arch/resources/locale/en | 2 + bauh/gems/arch/resources/locale/es | 2 + bauh/gems/arch/resources/locale/fr | 2 + bauh/gems/arch/resources/locale/it | 2 + bauh/gems/arch/resources/locale/pt | 2 + bauh/gems/arch/resources/locale/ru | 2 + bauh/gems/arch/resources/locale/tr | 2 + bauh/gems/arch/suggestions.py | 211 +++++++++++++++++++++++++++++ 16 files changed, 446 insertions(+), 38 deletions(-) create mode 100644 bauh/gems/arch/suggestions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f53b609..01569306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - new parameter `--suggestions`: forces loading software suggestions after the initialization process [#260](https://github.com/vinifmor/bauh/issues/260) ### Improvements +- Arch + - suggestions available for repository packages (and the associated caching expiration property `suggestions_exp`. Default: 24 hours) + - General - preventing command injection through the search mechanism [#266](https://github.com/vinifmor/bauh/issues/266) - code refactoring diff --git a/README.md b/README.md index c07c4a51..5da3ba51 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,7 @@ suggest_optdep_uninstall: false # if the optional dependencies associated with categories_exp: 24 # It defines the expiration time (in HOURS) of the packages categories mapping file stored in disc. Use 0 so that it is always updated during initialization. aur_rebuild_detector: true # it checks if packages built with old library versions require to be rebuilt. If a package needs to be rebuilt, it will be marked for update ('rebuild-detector' must be installed). Default: true. prefer_repository_provider: true # when there is just one repository provider for a given a dependency and several from AUR, it will be automatically picked. +suggestions_exp: 24 # it defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. ``` ##### Debian packages diff --git a/bauh/api/abstract/model.py b/bauh/api/abstract/model.py index 8e63e257..0c9f1d7e 100644 --- a/bauh/api/abstract/model.py +++ b/bauh/api/abstract/model.py @@ -299,6 +299,14 @@ class SuggestionPriority(Enum): HIGH = 2 TOP = 3 + def __gt__(self, other): + if isinstance(other, SuggestionPriority): + return self.value > other.value + + def __lt__(self, other): + if isinstance(other, SuggestionPriority): + return self.value < other.value + class PackageSuggestion: diff --git a/bauh/gems/arch/config.py b/bauh/gems/arch/config.py index b17800e3..615499b5 100644 --- a/bauh/gems/arch/config.py +++ b/bauh/gems/arch/config.py @@ -40,4 +40,5 @@ class ArchConfigManager(YAMLConfigManager): 'categories_exp': 24, 'aur_rebuild_detector': False, "aur_rebuild_detector_no_bin": True, - "prefer_repository_provider": True} + "prefer_repository_provider": True, + 'suggestions_exp': 24} diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index b175cae4..757aa766 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -21,7 +21,7 @@ from bauh.api.abstract.controller import SearchResult, SoftwareManager, Applicat from bauh.api.abstract.disk import DiskCacheLoader from bauh.api.abstract.handler import ProcessWatcher, TaskManager from bauh.api.abstract.model import PackageUpdate, PackageHistory, SoftwarePackage, PackageStatus, \ - CustomSoftwareAction + CustomSoftwareAction, PackageSuggestion from bauh.api.abstract.view import MessageType, FormComponent, InputOption, SingleSelectComponent, SelectViewType, \ ViewComponent, PanelComponent, MultipleSelectComponent, TextInputComponent, TextInputType, \ FileChooserComponent, TextComponent @@ -49,6 +49,7 @@ from bauh.gems.arch.model import ArchPackage from bauh.gems.arch.output import TransactionStatusHandler from bauh.gems.arch.pacman import RE_DEP_OPERATORS from bauh.gems.arch.proc_util import write_as_user +from bauh.gems.arch.suggestions import RepositorySuggestionsDownloader from bauh.gems.arch.updates import UpdatesSummarizer from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, RefreshMirrors, \ SyncDatabases @@ -215,6 +216,7 @@ class ArchManager(SoftwareManager, SettingsController): self.re_file_conflict = re.compile(r'[\w\d\-_.]+:') self.disk_cache_updater = disk_cache_updater self.pkgbuilder_user: Optional[str] = f'{__app_name__}-aur' if context.root_user else None + self._suggestions_downloader: Optional[RepositorySuggestionsDownloader] = None def refresh_mirrors(self, root_password: Optional[str], watcher: ProcessWatcher) -> bool: handler = ProcessHandler(watcher) @@ -583,7 +585,7 @@ class ArchManager(SoftwareManager, SettingsController): rebuild_ignored = Thread(target=self.__fill_ignored_by_rebuild_detector, args=(rebuild_output, ), daemon=True) rebuild_ignored.start() - installed = pacman.map_installed(names=names) + installed = pacman.map_packages(names=names) aur_pkgs, repo_pkgs, aur_index = None, None, None @@ -630,7 +632,7 @@ class ArchManager(SoftwareManager, SettingsController): t.join() if pkgs: - ignored = self._list_ignored_updates() + ignored = self._fill_ignored_updates(set()) if ignored: for p in pkgs: @@ -1484,7 +1486,7 @@ class ArchManager(SoftwareManager, SettingsController): return uninstalled def _map_installed_data_for_removal(self, names: Iterable[str]) -> Optional[Dict[str, Dict[str, str]]]: - data = pacman.map_installed(names) + data = pacman.map_packages(names) if data: remapped_data = {} @@ -2796,6 +2798,10 @@ class ArchManager(SoftwareManager, SettingsController): self.index_aur = AURIndexUpdater(context=self.context, taskman=task_manager, create_config=create_config) # must always execute to properly determine the installed packages (even that AUR is disabled) self.index_aur.start() + self.suggestions_downloader.create_config = create_config + self.suggestions_downloader.register_task(task_manager) + self.suggestions_downloader.start() + refresh_mirrors = RefreshMirrors(taskman=task_manager, i18n=self.i18n, root_password=root_password, logger=self.logger, create_config=create_config) refresh_mirrors.start() @@ -2938,6 +2944,12 @@ class ArchManager(SoftwareManager, SettingsController): only_int=True, capitalize_label=False, value=arch_config['categories_exp'] if isinstance(arch_config['categories_exp'], int) else ''), + TextInputComponent(id_='arch_sugs_exp', + label=self.i18n['arch.config.suggestions_exp'], + tooltip=self.i18n['arch.config.suggestions_exp.tip'], + only_int=True, + capitalize_label=False, + value=arch_config['suggestions_exp'] if isinstance(arch_config['suggestions_exp'], int) else '') ] return SettingsView(self, PanelComponent([FormComponent(fields, spaces=False)], id_="repo"), icon_path=get_repo_icon_path()) @@ -3045,6 +3057,7 @@ class ArchManager(SoftwareManager, SettingsController): arch_config['suggest_unneeded_uninstall'] = sug_unneeded_uni arch_config['categories_exp'] = form.get_component('arch_cats_exp', TextInputComponent).get_int_value() + arch_config['suggestions_exp'] = form.get_component('arch_sugs_exp', TextInputComponent).get_int_value() def _fill_aur_settings(self, arch_config: dict, form: FormComponent): arch_config['optimize'] = form.get_component('opts', SingleSelectComponent).get_selected() @@ -3316,8 +3329,7 @@ class ArchManager(SoftwareManager, SettingsController): return True - def _list_ignored_updates(self) -> Set[str]: - ignored = set() + def _fill_ignored_updates(self, output: Set[str]) -> Set[str]: if os.path.exists(UPDATES_IGNORED_FILE): with open(UPDATES_IGNORED_FILE) as f: ignored_lines = f.readlines() @@ -3327,9 +3339,9 @@ class ArchManager(SoftwareManager, SettingsController): line_clean = line.strip() if line_clean: - ignored.add(line_clean) + output.add(line_clean) - return ignored + return output def _write_ignored(self, names: Set[str]): Path(ARCH_CONFIG_DIR).mkdir(parents=True, exist_ok=True) @@ -3344,7 +3356,7 @@ class ArchManager(SoftwareManager, SettingsController): f.write('') def ignore_update(self, pkg: ArchPackage): - ignored = self._list_ignored_updates() + ignored = self._fill_ignored_updates(set()) if pkg.name not in ignored: ignored.add(pkg.name) @@ -3353,7 +3365,7 @@ class ArchManager(SoftwareManager, SettingsController): pkg.update_ignored = True def _revert_ignored_updates(self, pkgs: Iterable[str]): - ignored = self._list_ignored_updates() + ignored = self._fill_ignored_updates(set()) for p in pkgs: if p in ignored: @@ -3691,3 +3703,124 @@ class ArchManager(SoftwareManager, SettingsController): return added return True + + def _fill_available_packages(self, output: Dict[str, Set[str]]): + output.update(pacman.map_available_packages()) + + def _fill_suggestions(self, output: Dict[str, int]): + self.suggestions_downloader.register_task(None) + suggestions = self.suggestions_downloader.read(self.configman.read_config()) + + if suggestions: + output.update(suggestions) + + def list_suggestions(self, limit: int, filter_installed: bool) -> Optional[List[PackageSuggestion]]: + if limit == 0: + return [] + + arch_config = self.configman.get_config() + + if not arch_config['repositories']: + return + + name_priority = dict() + + fill_suggestions = Thread(target=self._fill_suggestions, args=(name_priority,)) + fill_suggestions.start() + + available_packages = dict() + fill_available = Thread(target=self._fill_available_packages, args=(available_packages,)) + fill_available.start() + + ignored_pkgs = set() + fill_ignored = Thread(target=pacman.fill_ignored_packages, args=(ignored_pkgs,)) + fill_ignored.start() + + fill_suggestions.join() + + if not name_priority: + self.logger.info("No Arch package suggestions found") + return [] + + self.logger.info(f"Found {len(name_priority)} named Arch package suggestions") + + if fill_available: + fill_available.join() + + if not available_packages: + self.logger.error("No available Arch package found. It will not be possible to return suggestions") + return [] + + fill_ignored.join() + + available_suggestions = dict() + + for n in name_priority: + if n not in ignored_pkgs: + data = available_packages.get(n) + + if data and (not filter_installed or not data['i']): + available_suggestions[n] = data + + if not available_suggestions: + self.logger.info("No Arch package suggestion to return") + return [] + + if filter_installed: + ignored_updates = set() + thread_fill_ignored_updates = Thread(target=self._fill_ignored_updates, args=(ignored_updates,)) + thread_fill_ignored_updates.start() + else: + ignored_updates, thread_fill_ignored_updates = None, None + + # sorting by priority + suggestion_by_priority = tuple(pair[1] for pair in sorted(((name_priority[n], n) for n in available_suggestions), + reverse=True)) + + if 0 < limit < len(available_suggestions): + suggestion_by_priority = suggestion_by_priority[0:limit] + + self.logger.info(f'Available Arch package suggestions: {len(suggestion_by_priority)}') + + if thread_fill_ignored_updates: + thread_fill_ignored_updates.join() + + full_data = pacman.map_packages(names=suggestion_by_priority, remote=filter_installed, not_signed=False, + skip_ignored=True) + + if full_data and full_data.get('signed'): + full_data = full_data['signed'] + + suggestions = [] + for name in suggestion_by_priority: + pkg_data = available_suggestions[name] + pkg_full_data = full_data.get(name) + description = None + + if pkg_full_data: + description = pkg_full_data.get('description') + + pkg_updates_ignored = pkg_data['i'] and ignored_updates and name in ignored_updates + + suggestions.append(PackageSuggestion(package=ArchPackage(name=name, + version=pkg_data['v'], + latest_version=pkg_data['v'], + repository=pkg_data['r'], + installed=pkg_data['i'], + description=description, + categories=self.categories.get(name), + i18n=self.i18n, + maintainer=pkg_data['r'], + update_ignored=pkg_updates_ignored), + priority=name_priority[name])) + + return suggestions + + @property + def suggestions_downloader(self) -> RepositorySuggestionsDownloader: + if not self._suggestions_downloader: + self._suggestions_downloader = RepositorySuggestionsDownloader(logger=self.logger, + http_client=self.http_client, + i18n=self.i18n) + + return self._suggestions_downloader diff --git a/bauh/gems/arch/pacman.py b/bauh/gems/arch/pacman.py index c2f73c23..75531d22 100644 --- a/bauh/gems/arch/pacman.py +++ b/bauh/gems/arch/pacman.py @@ -2,9 +2,10 @@ import logging import os import re import shutil +import traceback from io import StringIO from threading import Thread -from typing import List, Set, Tuple, Dict, Iterable, Optional +from typing import List, Set, Tuple, Dict, Iterable, Optional, Any from colorama import Fore @@ -17,7 +18,7 @@ RE_DEPS = re.compile(r'[\w\-_]+:[\s\w_\-.]+\s+\[\w+]') RE_OPTDEPS = re.compile(r'[\w._\-]+\s*:') RE_DEP_NOTFOUND = re.compile(r'error:.+\'(.+)\'') RE_DEP_OPERATORS = re.compile(r'[<>=]') -RE_INSTALLED_FIELDS = re.compile(r'(Name|Description|Version|Install Date|Validated By)\s*:\s*(.+)') +RE_REPOSITORY_FIELDS = re.compile(r'(Repository|Name|Description|Version|Install Date|Validated By)\s*:\s*(.+)') RE_INSTALLED_SIZE = re.compile(r'Installed Size\s*:\s*([0-9,.]+)\s(\w+)\n?', re.IGNORECASE) RE_DOWNLOAD_SIZE = re.compile(r'Download Size\s*:\s*([0-9,.]+)\s(\w+)\n?', re.IGNORECASE) RE_UPDATE_REQUIRED_FIELDS = re.compile(r'(\bProvides\b|\bInstalled Size\b|\bConflicts With\b)\s*:\s(.+)\n') @@ -92,23 +93,31 @@ def check_installed(pkg: str) -> bool: return bool(res) -def _fill_ignored(res: dict): - res['pkgs'] = list_ignored_packages() +def fill_ignored_packages(output: Set[str]): + output.update(list_ignored_packages()) -def map_installed(names: Optional[Iterable[str]] = None) -> Dict[str, Dict[str, str]]: - ignored = {} - thread_ignored = Thread(target=_fill_ignored, args=(ignored,), daemon=True) - thread_ignored.start() +def map_packages(names: Optional[Iterable[str]] = None, remote: bool = False, signed: bool = True, + not_signed: bool = True, skip_ignored: bool = False) -> Dict[str, Dict[str, Dict[str, str]]]: + if not signed and not not_signed: + return {} - allinfo = run_cmd('pacman -Qi{}'.format(' ' + ' '.join(names) if names else ''), print_error=False) + ignored, thread_ignored = None, None + if not skip_ignored: + ignored = set() + thread_ignored = Thread(target=fill_ignored_packages, args=(ignored,), daemon=True) + thread_ignored.start() + + allinfo = run_cmd(f"pacman -{'S' if remote else 'Q'}i {' '.join(names) if names else ''}", print_error=False) pkgs = {'signed': {}, 'not_signed': {}} current_pkg = {} if allinfo: - for idx, field_tuple in enumerate(RE_INSTALLED_FIELDS.findall(allinfo)): - if field_tuple[0].startswith('N'): + for idx, field_tuple in enumerate(RE_REPOSITORY_FIELDS.findall(allinfo)): + if field_tuple[0].startswith('R'): + current_pkg['repository'] = field_tuple[1].strip() + elif field_tuple[0].startswith('N'): current_pkg['name'] = field_tuple[1].strip() elif field_tuple[0].startswith('Ve'): current_pkg['version'] = field_tuple[1].strip() @@ -117,25 +126,25 @@ def map_installed(names: Optional[Iterable[str]] = None) -> Dict[str, Dict[str, elif field_tuple[0].startswith('I'): current_pkg['install_date'] = field_tuple[1].strip() elif field_tuple[0].startswith('Va'): - if field_tuple[1].strip().lower() == 'none': + if not_signed and field_tuple[1].strip().lower() == 'none': pkgs['not_signed'][current_pkg['name']] = current_pkg del current_pkg['name'] - else: + elif signed: pkgs['signed'][current_pkg['name']] = current_pkg del current_pkg['name'] current_pkg = {} - if pkgs['signed'] or pkgs['not_signed']: + if thread_ignored and (pkgs['signed'] or pkgs['not_signed']): thread_ignored.join() - if ignored['pkgs']: + if ignored: to_del = set() for key in ('signed', 'not_signed'): if pkgs.get(key): for pkg in pkgs[key].keys(): - if pkg in ignored['pkgs']: + if pkg in ignored: to_del.add(pkg) for pkg in to_del: @@ -219,20 +228,23 @@ def sign_key(key: str, root_password: Optional[str]) -> SystemProcess: def list_ignored_packages(config_path: str = '/etc/pacman.conf') -> Set[str]: - pacman_conf = new_subprocess(['cat', config_path]) - ignored = set() - grep = new_subprocess(['grep', '-Eo', r'\s*#*\s*ignorepkg\s*=\s*.+'], stdin=pacman_conf.stdout) - for o in grep.stdout: - if o: - line = o.decode().strip() + try: + pacman_conf = new_subprocess(['cat', config_path]) + grep = new_subprocess(['grep', '-Eo', r'\s*#*\s*ignorepkg\s*=\s*.+'], stdin=pacman_conf.stdout) + for o in grep.stdout: + if o: + line = o.decode().strip() - if not line.startswith('#'): - ignored.add(line.split('=')[1].strip()) + if not line.startswith('#'): + ignored.add(line.split('=')[1].strip()) - pacman_conf.terminate() - grep.terminate() - return ignored + pacman_conf.terminate() + grep.terminate() + return ignored + except: + traceback.print_exc() + return ignored def check_missing(names: Set[str]) -> Set[str]: @@ -1037,3 +1049,24 @@ def find_one_match(name: str) -> Optional[str]: if matches and len(matches) == 1: return matches[0] + + +def map_available_packages() -> Optional[Dict[str, Any]]: + output = run_cmd('pacman -Sl') + + if output: + res = dict() + for line in output.split('\n'): + line_strip = line.strip() + + if line_strip: + package_data = line.split(' ') + + if len(package_data) >= 3: + pkgname = package_data[1].strip() + + if pkgname: + res[pkgname] = {'v': package_data[2].strip(), + 'r': package_data[0].strip(), + 'i': len(package_data) == 4 and 'installed' in package_data[3]} + return res diff --git a/bauh/gems/arch/resources/locale/ca b/bauh/gems/arch/resources/locale/ca index 2a5dda49..d1995e91 100644 --- a/bauh/gems/arch/resources/locale/ca +++ b/bauh/gems/arch/resources/locale/ca @@ -88,6 +88,8 @@ arch.config.suggest_optdep_uninstall=Uninstall optional dependencies arch.config.suggest_optdep_uninstall.tip=If the optional dependencies associated with uninstalled packages should be suggested for uninstallation. Only the optional dependencies that are not dependencies of other packages will be suggested. arch.config.suggest_unneeded_uninstall=Uninstall unneeded dependencies arch.config.suggest_unneeded_uninstall.tip=If the dependencies apparently no longer necessary associated with the uninstalled packages should be suggested for uninstallation. When this property is enabled it automatically disables the property {}. +arch.config.suggestions_exp=Suggestions expiration +arch.config.suggestions_exp.tip=It defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. arch.config.sync_dbs=Synchronize packages databases arch.config.sync_dbs.tip=Synchronizes the package databases once a day before the first package installation, upgrade or downgrade. This option helps to prevent errors during these operations. arch.config.sync_dbs_start.tip=Synchronizes the package databases during the initialization once a day diff --git a/bauh/gems/arch/resources/locale/de b/bauh/gems/arch/resources/locale/de index a286c61e..577f3181 100644 --- a/bauh/gems/arch/resources/locale/de +++ b/bauh/gems/arch/resources/locale/de @@ -88,6 +88,8 @@ arch.config.suggest_optdep_uninstall=Uninstall optional dependencies arch.config.suggest_optdep_uninstall.tip=If the optional dependencies associated with uninstalled packages should be suggested for uninstallation. Only the optional dependencies that are not dependencies of other packages will be suggested. arch.config.suggest_unneeded_uninstall=Uninstall unneeded dependencies arch.config.suggest_unneeded_uninstall.tip=If the dependencies apparently no longer necessary associated with the uninstalled packages should be suggested for uninstallation. When this property is enabled it automatically disables the property {}. +arch.config.suggestions_exp=Suggestions expiration +arch.config.suggestions_exp.tip=It defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. arch.config.sync_dbs=Synchronize packages databases arch.config.sync_dbs.tip=Synchronizes the package databases once a day before the first package installation, upgrade or downgrade. This option helps to prevent errors during these operations. arch.config.sync_dbs_start.tip=Synchronizes the package databases during the initialization once a day diff --git a/bauh/gems/arch/resources/locale/en b/bauh/gems/arch/resources/locale/en index a81625e8..d68de2e8 100644 --- a/bauh/gems/arch/resources/locale/en +++ b/bauh/gems/arch/resources/locale/en @@ -88,6 +88,8 @@ arch.config.suggest_optdep_uninstall=Uninstall optional dependencies arch.config.suggest_optdep_uninstall.tip=If the optional dependencies associated with uninstalled packages should be suggested for uninstallation. Only the optional dependencies that are not dependencies of other packages will be suggested. arch.config.suggest_unneeded_uninstall=Uninstall unneeded dependencies arch.config.suggest_unneeded_uninstall.tip=If the dependencies apparently no longer necessary associated with the uninstalled packages should be suggested for uninstallation. When this property is enabled it automatically disables the property {}. +arch.config.suggestions_exp=Suggestions expiration +arch.config.suggestions_exp.tip=It defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. arch.config.sync_dbs=Synchronize packages databases arch.config.sync_dbs.tip=Synchronizes the package databases once a day before the first package installation, upgrade or downgrade. This option helps to prevent errors during these operations. arch.config.sync_dbs_start.tip=Synchronizes the package databases during the initialization once a day diff --git a/bauh/gems/arch/resources/locale/es b/bauh/gems/arch/resources/locale/es index 751e2224..4a9faebd 100644 --- a/bauh/gems/arch/resources/locale/es +++ b/bauh/gems/arch/resources/locale/es @@ -88,6 +88,8 @@ arch.config.suggest_optdep_uninstall=Desinstalar dependencias opcionales arch.config.suggest_optdep_uninstall.tip=Si las dependencias opcionales asociadas con los paquetes desinstalados deben se sugeridas para desinstalación. Solo se sugerirán las dependencias opcionales que no sean dependencias de otros paquetes. arch.config.suggest_unneeded_uninstall=Desinstalar las dependencias innecesarias arch.config.suggest_unneeded_uninstall.tip=Si las dependencias aparentemente ya no necesarias asociadas con los paquetes desinstalados deben ser sugeridas para desinstalación. Cuando esta propiedad está habilitada, automáticamente deshabilita la propiedad {}. +arch.config.suggestions_exp=Expiración de sugerencias +arch.config.suggestions_exp.tip=Define el período (en horas) en el que la sugerencias almacenadas en disco seran consideradas actualizadas. Use 0 si desea siempre actualizarlas. arch.config.sync_dbs=Sincronizar las bases de paquetes arch.config.sync_dbs.tip=Sincroniza las bases de paquetes una vez al día antes de la primera instalación, actualización o reversión de paquete. Esta opción ayuda a prevenir errores durante estas operaciones. arch.config.sync_dbs_start.tip=Sincroniza las bases de paquetes durante la inicialización una vez al día diff --git a/bauh/gems/arch/resources/locale/fr b/bauh/gems/arch/resources/locale/fr index 6e22de0d..d9e51342 100644 --- a/bauh/gems/arch/resources/locale/fr +++ b/bauh/gems/arch/resources/locale/fr @@ -88,6 +88,8 @@ arch.config.suggest_optdep_uninstall=Désinstaller les dépendances optionnelles arch.config.suggest_optdep_uninstall.tip=Si les dépendances optionnelles liées au paquets désinstallés dévraient être suggérés pour désinstallation. Seules les dépendances n'étant pas liées à d'autres paquets seront suggérées. arch.config.suggest_unneeded_uninstall=Désinstaller les dépendances inutiles arch.config.suggest_unneeded_uninstall.tip=Si les dépendances apparemment inutiles liées au paquets désinstallés dévraient être suggérés pour désinstallation. Activer cette propriété désactive automatiquement {}. +arch.config.suggestions_exp=Suggestions expiration +arch.config.suggestions_exp.tip=It defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. arch.config.sync_dbs=Synchroniser les bases de données des paquets arch.config.sync_dbs.tip=Synchronise les bases de données du paquet une fois par jour avant sa première installation, mise à jour ou downgrade. Cette option aide à éviter les erreurs durant ces opérations. arch.config.sync_dbs_start.tip=Synchronise les bases de données du paquet durant l'initialisation journalière diff --git a/bauh/gems/arch/resources/locale/it b/bauh/gems/arch/resources/locale/it index 1700da7b..7dd9e9f7 100644 --- a/bauh/gems/arch/resources/locale/it +++ b/bauh/gems/arch/resources/locale/it @@ -88,6 +88,8 @@ arch.config.suggest_optdep_uninstall=Uninstall optional dependencies arch.config.suggest_optdep_uninstall.tip=If the optional dependencies associated with uninstalled packages should be suggested for uninstallation. Only the optional dependencies that are not dependencies of other packages will be suggested. arch.config.suggest_unneeded_uninstall=Uninstall unneeded dependencies arch.config.suggest_unneeded_uninstall.tip=If the dependencies apparently no longer necessary associated with the uninstalled packages should be suggested for uninstallation. When this property is enabled it automatically disables the property {}. +arch.config.suggestions_exp=Suggestions expiration +arch.config.suggestions_exp.tip=It defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. arch.config.sync_dbs=Synchronize packages databases arch.config.sync_dbs.tip=Synchronizes the package databases once a day before the first package installation, upgrade or downgrade. This option helps to prevent errors during these operations. arch.config.sync_dbs_start.tip=Synchronizes the package databases during the initialization once a day diff --git a/bauh/gems/arch/resources/locale/pt b/bauh/gems/arch/resources/locale/pt index f1c3d537..9823ae76 100644 --- a/bauh/gems/arch/resources/locale/pt +++ b/bauh/gems/arch/resources/locale/pt @@ -87,6 +87,8 @@ arch.config.suggest_optdep_uninstall=Desinstalar dependências opcionais arch.config.suggest_optdep_uninstall.tip=Se as dependências opcionais associadas a pacotes desinstalados devem ser sugeridas para desinstalação. Somente as dependências opcionais que não sejam dependências de outros pacotes serão sugeridas. arch.config.suggest_unneeded_uninstall=Desinstalar dependências desnecessárias arch.config.suggest_unneeded_uninstall.tip=Se as dependências aparentemente não mais necessárias associadas aos pacotes desinstalados devem ser sugeridas para desinstalação. Quando essa proprieda está habilitada automaticamente desabilita a propriedade {}. +arch.config.suggestions_exp=Validade de sugestões +arch.config.suggestions_exp.tip=Define o período (em horas) no qual as sugestões armazenadas em disco serão consideradas atualizadas. Use 0 se quiser que elas sempre sejam atualizadas. arch.config.sync_dbs=Sincronizar bases de pacotes arch.config.sync_dbs.tip=Sincroniza as bases de pacotes uma vez ao dia antes da primeira instalação, atualização ou reversão de pacote. Essa opção ajuda a evitar erros durante essa operações. arch.config.sync_dbs_start.tip=Sincroniza as bases de pacotes durante a inicialização uma vez ao dia diff --git a/bauh/gems/arch/resources/locale/ru b/bauh/gems/arch/resources/locale/ru index 6ae73ab6..5d32fd16 100644 --- a/bauh/gems/arch/resources/locale/ru +++ b/bauh/gems/arch/resources/locale/ru @@ -88,6 +88,8 @@ arch.config.suggest_optdep_uninstall=Uninstall optional dependencies arch.config.suggest_optdep_uninstall.tip=If the optional dependencies associated with uninstalled packages should be suggested for uninstallation. Only the optional dependencies that are not dependencies of other packages will be suggested. arch.config.suggest_unneeded_uninstall=Uninstall unneeded dependencies arch.config.suggest_unneeded_uninstall.tip=If the dependencies apparently no longer necessary associated with the uninstalled packages should be suggested for uninstallation. When this property is enabled it automatically disables the property {}. +arch.config.suggestions_exp=Suggestions expiration +arch.config.suggestions_exp.tip=It defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. arch.config.sync_dbs=Синхронизация баз данных пакетов arch.config.sync_dbs.tip=Синхронизируйте базы данных пакетов один раз в день перед первой установкой, обновлением или понижением версии пакета. Этот параметр помогает предотвратить ошибки во время этих операций. arch.config.sync_dbs_start.tip=Синхронизирует базы данных пакетов во время инициализации один раз в день diff --git a/bauh/gems/arch/resources/locale/tr b/bauh/gems/arch/resources/locale/tr index b193ff2e..9a943128 100644 --- a/bauh/gems/arch/resources/locale/tr +++ b/bauh/gems/arch/resources/locale/tr @@ -88,6 +88,8 @@ arch.config.suggest_optdep_uninstall=Uninstall optional dependencies arch.config.suggest_optdep_uninstall.tip=If the optional dependencies associated with uninstalled packages should be suggested for uninstallation. Only the optional dependencies that are not dependencies of other packages will be suggested. arch.config.suggest_unneeded_uninstall=Uninstall unneeded dependencies arch.config.suggest_unneeded_uninstall.tip=If the dependencies apparently no longer necessary associated with the uninstalled packages should be suggested for uninstallation. When this property is enabled it automatically disables the property {}. +arch.config.suggestions_exp=Suggestions expiration +arch.config.suggestions_exp.tip=It defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. arch.config.sync_dbs=Paket veritabanlarını senkronize et arch.config.sync_dbs.tip=İlk veritabanını kurmadan, yükseltmeden veya indirmeden önce paket veritabanlarını günde bir kez senkronize eder. Bu seçenek, bu işlemler sırasında hataların önlenmesine yardımcı olur. arch.config.sync_dbs_start.tip=Başlatma sırasında paket veritabanlarını günde bir kez senkronize eder diff --git a/bauh/gems/arch/suggestions.py b/bauh/gems/arch/suggestions.py new file mode 100644 index 00000000..731c92c9 --- /dev/null +++ b/bauh/gems/arch/suggestions.py @@ -0,0 +1,211 @@ +import os +import traceback +from datetime import datetime, timedelta +from logging import Logger +from pathlib import Path +from threading import Thread +from typing import Optional, Dict + +from bauh.api.abstract.handler import TaskManager +from bauh.api.abstract.model import SuggestionPriority +from bauh.api.http import HttpClient +from bauh.commons.boot import CreateConfigFile +from bauh.gems.arch import ARCH_CACHE_DIR, get_icon_path +from bauh.view.util.translation import I18n + + +class RepositorySuggestionsDownloader(Thread): + + _file_suggestions: Optional[str] = None + _file_suggestions_ts: Optional[str] = None + _url_suggestions: Optional[str] = None + + @classmethod + def file_suggestions(cls) -> str: + if cls._file_suggestions is None: + cls._file_suggestions = f'{ARCH_CACHE_DIR}/suggestions.txt' + + return cls._file_suggestions + + @classmethod + def file_suggestions_timestamp(cls) -> str: + if cls._file_suggestions_ts is None: + cls._file_suggestions_ts = f'{cls.file_suggestions()}.ts' + + return cls._file_suggestions_ts + + @classmethod + def url_suggestions(cls) -> str: + if cls._url_suggestions is None: + cls._url_suggestions = 'https://raw.githubusercontent.com/vinifmor/bauh-files' \ + '/master/arch/suggestions.txt' + + return cls._url_suggestions + + def __init__(self, logger: Logger, http_client: HttpClient, i18n: I18n, + create_config: Optional[CreateConfigFile] = None): + super(RepositorySuggestionsDownloader, self).__init__() + self._log = logger + self.i18n = i18n + self.http_client = http_client + self._taskman: Optional[TaskManager] = None + self.create_config = create_config + self.task_id = 'arch.suggs' + + def register_task(self, taskman: Optional[TaskManager]): + self._taskman = taskman + if taskman: + self._taskman.register_task(id_=self.task_id, label=self.i18n['task.download_suggestions'], + icon_path=get_icon_path()) + + @property + def taskman(self) -> TaskManager: + if self._taskman is None: + self._taskman = TaskManager() + + return self._taskman + + @classmethod + def should_download(cls, arch_config: dict, logger: Logger, only_positive_exp: bool = False) -> bool: + try: + exp_hours = int(arch_config['suggestions_exp']) + except ValueError: + logger.error(f"The Arch configuration property 'suggestions_exp' has a non int value set: " + f"{arch_config['suggestions']['expiration']}") + return not only_positive_exp + + if exp_hours <= 0: + logger.info("Suggestions cache is disabled") + return not only_positive_exp + + if not os.path.exists(cls.file_suggestions()): + logger.info(f"'{cls.file_suggestions()}' not found. It must be downloaded") + return True + + if not os.path.exists(cls.file_suggestions()): + logger.info(f"'{cls.file_suggestions()}' not found. The suggestions file must be downloaded.") + return True + + with open(cls.file_suggestions_timestamp()) as f: + timestamp_str = f.read() + + try: + suggestions_timestamp = datetime.fromtimestamp(float(timestamp_str)) + except: + logger.error(f'Could not parse the Arch cached suggestions timestamp: {timestamp_str}') + traceback.print_exc() + return True + + update = suggestions_timestamp + timedelta(hours=exp_hours) <= datetime.utcnow() + return update + + def _save(self, text: str, timestamp: float): + self._log.info(f"Caching suggestions to '{self.file_suggestions()}'") + + cache_dir = os.path.dirname(self.file_suggestions()) + + try: + Path(cache_dir).mkdir(parents=True, exist_ok=True) + cache_dir_ok = True + except OSError: + self._log.error(f"Could not create cache directory '{cache_dir}'") + traceback.print_exc() + cache_dir_ok = False + + if cache_dir_ok: + try: + with open(self.file_suggestions(), 'w+') as f: + f.write(text) + except: + self._log.error(f"An exception happened while writing the file '{self.file_suggestions()}'") + traceback.print_exc() + + try: + with open(self.file_suggestions_timestamp(), 'w+') as f: + f.write(str(timestamp)) + except: + self._log.error(f"An exception happened while writing the file '{self.file_suggestions_timestamp()}'") + traceback.print_exc() + + def parse_suggestions(self, suggestions_str: str) -> Dict[str, SuggestionPriority]: + output = dict() + for line in suggestions_str.split('\n'): + clean_line = line.strip() + + if clean_line: + line_split = clean_line.split(':', 1) + + if len(line_split) == 2: + try: + prio = int(line_split[0]) + except ValueError: + self._log.warning(f"Could not parse Arch package suggestion: {line}") + continue + + output[line_split[1]] = SuggestionPriority(prio) + + return output + + def read_cached(self) -> Optional[Dict[str, SuggestionPriority]]: + self._log.info(f"Reading cached suggestions file '{self.file_suggestions()}'") + + try: + with open(self.file_suggestions()) as f: + sugs_str = f.read() + except FileNotFoundError: + self._log.warning(f"Cached suggestions file does not exist ({self.file_suggestions()})") + return + + if not sugs_str: + self._log.warning(f"Cached suggestions file '{self.file_suggestions()}' is empty") + return + + return self.parse_suggestions(sugs_str) + + def download(self) -> Optional[Dict[str, SuggestionPriority]]: + self.taskman.update_progress(self.task_id, progress=1, substatus=None) + + self._log.info(f"Downloading suggestions from {self.url_suggestions()}") + res = self.http_client.get(self.url_suggestions()) + + suggestions = None + if res.status_code == 200 and res.text: + self.taskman.update_progress(self.task_id, progress=50, substatus=None) + suggestions = self.parse_suggestions(res.text) + + if suggestions: + self._save(text=res.text, timestamp=datetime.utcnow().timestamp()) + else: + self._log.warning("No Arch suggestions to cache") + else: + self._log.warning(f"Could not retrieve Arch suggestions. " + f"Response (status={res.status_code}, text={res.text})") + + self.taskman.update_progress(self.task_id, progress=100, substatus=None) + self.taskman.finish_task(self.task_id) + return suggestions + + def read(self, arch_config: dict) -> Optional[Dict[str, int]]: + if self.should_download(arch_config=arch_config, logger=self._log): + return self.download() + + return self.read_cached() + + def run(self): + if self.create_config: + if self.create_config.is_alive(): + self.taskman.update_progress(self.task_id, 0, + self.i18n['task.waiting_task'].format(self.create_config.task_name)) + self.create_config.join() + + if not self.should_download(arch_config=self.create_config.config, logger=self._log, + only_positive_exp=False): + self.taskman.update_progress(self.task_id, 100, self.i18n['task.canceled']) + self.taskman.finish_task(self.task_id) + return + + self.download() + else: + self._log.error(f"No {CreateConfigFile.__class__.__name__} instance set. Aborting..") + self.taskman.update_progress(self.task_id, 100, self.i18n['error']) + self.taskman.finish_task(self.task_id)