diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aaf7b59..00ba46fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

- - upgrade: only removing packages after downloading the required ones (when multi-threaded download is enabled) + - upgrade: + - upgrading firstly the keyring packages declared in **SyncFirst** (**/etc/pacman.conf**) to avoid pacman downloading issues + - only removing packages after downloading the required ones ### Fixes - Arch diff --git a/bauh/api/abstract/controller.py b/bauh/api/abstract/controller.py index 91fd2989..3b7d5774 100644 --- a/bauh/api/abstract/controller.py +++ b/bauh/api/abstract/controller.py @@ -58,6 +58,7 @@ class UpgradeRequirements: self.to_remove = to_remove # when an upgrading package conflicts with a not upgrading package ( check all the non-upgrading packages deps an add here [including those selected to upgrade as well] self.to_upgrade = to_upgrade self.cannot_upgrade = cannot_upgrade + self.context = {} # caches relevant data to actually perform the upgrade class TransactionResult: diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index c33bee57..93453c41 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -11,7 +11,7 @@ from datetime import datetime from math import floor from pathlib import Path from threading import Thread -from typing import List, Set, Type, Tuple, Dict, Iterable +from typing import List, Set, Type, Tuple, Dict, Iterable, Optional import requests @@ -768,25 +768,93 @@ class ArchManager(SoftwareManager): return files - def _upgrade_repo_pkgs(self, pkgs: List[str], handler: ProcessHandler, root_password: str, arch_config: dict, overwrite_files: bool = False, - status_handler: TransactionStatusHandler = None, skip_download: bool = False, sizes: Dict[str, int] = None, already_downloaded: int = 0) -> bool: + def list_related(self, pkgs: Iterable[str], all_pkgs: Iterable[str], data: Dict[str, dict], related: Set[str], provided_map: Dict[str, Set[str]]) -> Set[str]: + related.update(pkgs) - if not skip_download and self._should_download_packages(arch_config): + deps = set() + + for pkg in pkgs: + pkg_deps = data[pkg]['d'] + + if pkg_deps: + deps.update({d for d in pkg_deps if d not in related}) + + if deps: + if not provided_map: + for p in all_pkgs: + for provided in data[p].get('p', {p}): + sources = provided_map.get(provided, set()) + provided_map[provided] = sources + sources.add(p) + + added_sources = set() + for dep in deps: + sources = provided_map.get(dep) + + if sources: + for source in sources: + if source not in related: + related.add(source) + added_sources.add(source) + + if added_sources: + self.list_related(added_sources, all_pkgs, data, related, provided_map) + + return related + + def _upgrade_repo_pkgs(self, to_upgrade: List[str], to_remove: Optional[Set[str]], handler: ProcessHandler, root_password: str, + multithread_download: bool, pkgs_data: Dict[str, dict], overwrite_files: bool = False, + status_handler: TransactionStatusHandler = None, sizes: Dict[str, int] = None, download: bool = True, + check_syncfirst: bool = True) -> bool: + + to_sync_first = None + if check_syncfirst: + to_sync_first = [p for p in pacman.get_packages_to_sync_first() if p.endswith('-keyring') and p in to_upgrade] + + if to_sync_first: + self.logger.info("Upgrading keyrings marked as 'SyncFirst'") + if not self._upgrade_repo_pkgs(to_upgrade=to_sync_first, + to_remove=None, + handler=handler, + root_password=root_password, + sizes=sizes, + download=True, + multithread_download=multithread_download, + pkgs_data=pkgs_data, + check_syncfirst=False): + return False + + to_upgrade_remaining = [p for p in to_upgrade if p not in to_sync_first] if to_sync_first else to_upgrade + + # pre-downloading all packages before removing any + if download: try: - downloaded = self._download_packages(pkgs, handler, root_password, sizes) + downloaded = self._download_packages(pkgnames=to_upgrade_remaining, + handler=handler, + root_password=root_password, + sizes=sizes, + multithreaded=multithread_download) + + if downloaded < len(to_upgrade_remaining): + self._show_upgrade_download_failed(handler.watcher) + return False + except ArchDownloadException: + self._show_upgrade_download_failed(handler.watcher) return False - else: - downloaded = already_downloaded + + if to_remove and not self._remove_transaction_packages(to_remove, handler, root_password): + return False try: if status_handler: output_handler = status_handler else: - output_handler = TransactionStatusHandler(handler.watcher, self.i18n, len(pkgs), self.logger, downloading=downloaded) + output_handler = TransactionStatusHandler(handler.watcher, self.i18n, len(to_upgrade), self.logger, downloading=len(to_upgrade_remaining)) output_handler.start() - success, upgrade_output = handler.handle_simple(pacman.upgrade_several(pkgnames=pkgs, + self.logger.info("Upgrading {} repository packages: {}".format(len(to_upgrade), ', '.join(to_upgrade))) + success, upgrade_output = handler.handle_simple(pacman.upgrade_several(pkgnames=to_upgrade_remaining, root_password=root_password, overwrite_conflicting_files=overwrite_files), output_handler=output_handler.handle,) @@ -797,10 +865,10 @@ class ArchManager(SoftwareManager): output_handler.join() handler.watcher.print("Repository packages successfully upgraded") handler.watcher.change_substatus(self.i18n['arch.upgrade.caching_pkgs_data']) - repo_map = pacman.map_repositories(pkgs) + repo_map = pacman.map_repositories(to_upgrade_remaining) pkg_map = {} - for name in pkgs: + for name in to_upgrade_remaining: repo = repo_map.get(name) pkg_map[name] = ArchPackage(name=name, repository=repo, @@ -817,13 +885,16 @@ class ArchManager(SoftwareManager): confirmation_label=self.i18n['arch.upgrade.conflicting_files.stop'], components=files): - return self._upgrade_repo_pkgs(pkgs=pkgs, + return self._upgrade_repo_pkgs(to_upgrade=to_upgrade_remaining, handler=handler, root_password=root_password, overwrite_files=True, status_handler=output_handler, - skip_download=True, - arch_config=arch_config, + multithread_download=multithread_download, + download=False, + check_syncfirst=False, + pkgs_data=pkgs_data, + to_remove=None, sizes=sizes) else: output_handler.stop_working() @@ -842,27 +913,26 @@ class ArchManager(SoftwareManager): traceback.print_exc() return False - def _remove_transaction_packages(self, requirements: UpgradeRequirements, handler: ProcessHandler, root_password: str) -> bool: - to_remove_names = {r.pkg.name for r in requirements.to_remove} + def _remove_transaction_packages(self, to_remove: Set[str], handler: ProcessHandler, root_password: str) -> bool: output_handler = TransactionStatusHandler(watcher=handler.watcher, i18n=self.i18n, pkgs_to_sync=0, logger=self.logger, - pkgs_to_remove=len(to_remove_names)) + pkgs_to_remove=len(to_remove)) output_handler.start() try: - success = handler.handle(pacman.remove_several(pkgnames=to_remove_names, root_password=root_password, skip_checks=True), + success = handler.handle(pacman.remove_several(pkgnames=to_remove, root_password=root_password, skip_checks=True), output_handler=output_handler.handle) if not success: - self.logger.error("Could not remove packages: {}".format(', '.join(to_remove_names))) + self.logger.error("Could not remove packages: {}".format(', '.join(to_remove))) output_handler.stop_working() output_handler.join() return False return True except: - self.logger.error("An error occured while removing packages: {}".format(', '.join(to_remove_names))) + self.logger.error("An error occured while removing packages: {}".format(', '.join(to_remove))) traceback.print_exc() output_handler.stop_working() output_handler.join() @@ -899,45 +969,19 @@ class ArchManager(SoftwareManager): arch_config = read_config() self._sync_databases(arch_config=arch_config, root_password=root_password, handler=handler) - # for now if the multithreaded-download is disabled packages will be removed before the download: - multithreaded_download = self._should_download_packages(arch_config) - - if requirements.to_remove and (not repo_pkgs or not multithreaded_download): - if not self._remove_transaction_packages(requirements, handler, root_password): - return False - if repo_pkgs: - repo_pkgs_names = [p.name for p in repo_pkgs] - - downloaded = -1 - if requirements.to_remove and multithreaded_download: # pre-downloading all packages before removing any - try: - downloaded = self._download_packages(repo_pkgs_names, handler, root_password, pkg_sizes) - - if downloaded < len(repo_pkgs_names): - self._show_upgrade_download_failed(handler.watcher) - return False - - except ArchDownloadException: - self._show_upgrade_download_failed(handler.watcher) - return False - - if not self._remove_transaction_packages(requirements, handler, root_password): - return False - - watcher.change_status('{}...'.format(self.i18n['arch.upgrade.upgrade_repo_pkgs'])) - self.logger.info("Upgrading {} repository packages: {}".format(len(repo_pkgs_names), - ', '.join(repo_pkgs_names))) - - if not self._upgrade_repo_pkgs(pkgs=repo_pkgs_names, + if not self._upgrade_repo_pkgs(to_upgrade=[p.name for p in repo_pkgs], + to_remove={r.pkg.name for r in requirements.to_remove} if requirements.to_remove else None, handler=handler, root_password=root_password, - arch_config=arch_config, - skip_download=downloaded >= 0, - already_downloaded=0 if downloaded < 0 else downloaded, + multithread_download=self._multithreaded_download_enabled(arch_config), + pkgs_data=requirements.context['data'], sizes=pkg_sizes): return False + elif requirements.to_remove and not self._remove_transaction_packages({r.pkg.name for r in requirements.to_remove}, handler, root_password): + return False + if aur_pkgs: watcher.change_status('{}...'.format(self.i18n['arch.upgrade.upgrade_aur_pkgs'])) for pkg in aur_pkgs: @@ -1450,10 +1494,10 @@ class ArchManager(SoftwareManager): return {dep} downloaded = 0 - if self._should_download_packages(context.config): + if self._multithreaded_download_enabled(context.config): try: pkg_sizes = pacman.map_download_sizes(repo_dep_names) - downloaded = self._download_packages(repo_dep_names, context.handler, context.root_password, pkg_sizes) + downloaded = self._download_packages(repo_dep_names, context.handler, context.root_password, pkg_sizes, multithreaded=True) except ArchDownloadException: return False @@ -1756,21 +1800,36 @@ class ArchManager(SoftwareManager): return True - def _should_download_packages(self, arch_config: dict) -> bool: + def _multithreaded_download_enabled(self, arch_config: dict) -> bool: return bool(arch_config['repositories_mthread_download']) \ and self.context.file_downloader.is_multithreaded() \ and pacman.is_mirrors_available() - def _download_packages(self, pkgnames: List[str], handler: ProcessHandler, root_password: str, sizes: Dict[str, int] = None) -> int: - download_service = MultithreadedDownloadService(file_downloader=self.context.file_downloader, - http_client=self.http_client, - logger=self.context.logger, - i18n=self.i18n) - self.logger.info("Initializing multi-threaded download for {} package(s)".format(len(pkgnames))) - return download_service.download_packages(pkgs=pkgnames, - handler=handler, - sizes=sizes, - root_password=root_password) + def _download_packages(self, pkgnames: List[str], handler: ProcessHandler, root_password: str, sizes: Dict[str, int] = None, multithreaded: bool = True) -> int: + if multithreaded: + download_service = MultithreadedDownloadService(file_downloader=self.context.file_downloader, + http_client=self.http_client, + logger=self.context.logger, + i18n=self.i18n) + self.logger.info("Initializing multi-threaded download for {} repository package(s)".format(len(pkgnames))) + return download_service.download_packages(pkgs=pkgnames, + handler=handler, + sizes=sizes, + root_password=root_password) + else: + self.logger.info("Downloading {} repository package(s)".format(len(pkgnames))) + output_handler = TransactionStatusHandler(handler.watcher, self.i18n, len(pkgnames), self.logger) + output_handler.start() + try: + success, _ = handler.handle_simple(pacman.download(root_password, *pkgnames), output_handler=output_handler.handle) + + if success: + return len(pkgnames) + else: + raise ArchDownloadException() + except: + traceback.print_exc() + raise ArchDownloadException() def _install(self, context: TransactionContext) -> bool: check_install_output = [] @@ -1830,13 +1889,13 @@ class ArchManager(SoftwareManager): to_install.append(pkgpath) downloaded = 0 - if self._should_download_packages(context.config): + if self._multithreaded_download_enabled(context.config): to_download = [p for p in to_install if not p.startswith('/')] if to_download: try: pkg_sizes = pacman.map_download_sizes(to_download) - downloaded = self._download_packages(to_download, context.handler, context.root_password, pkg_sizes) + downloaded = self._download_packages(to_download, context.handler, context.root_password, pkg_sizes, multithreaded=True) except ArchDownloadException: return False diff --git a/bauh/gems/arch/pacman.py b/bauh/gems/arch/pacman.py index cc33fdd3..1c7b7aa0 100644 --- a/bauh/gems/arch/pacman.py +++ b/bauh/gems/arch/pacman.py @@ -18,6 +18,7 @@ RE_DOWNLOAD_SIZE = re.compile(r'Download Size\s*:\s*([0-9,\.]+)\s(\w+)\n?', re.I RE_UPDATE_REQUIRED_FIELDS = re.compile(r'(\bProvides\b|\bInstalled Size\b|\bConflicts With\b)\s*:\s(.+)\n') RE_REMOVE_TRANSITIVE_DEPS = re.compile(r'removing\s([\w\-_]+)\s.+required\sby\s([\w\-_]+)\n?') RE_AVAILABLE_MIRRORS = re.compile(r'.+\s+OK\s+.+\s+(\d+:\d+)\s+.+(http.+)') +RE_PACMAN_SYNC_FIRST = re.compile(r'SyncFirst\s*=\s*(.+)') def is_available() -> bool: @@ -733,6 +734,12 @@ def upgrade_several(pkgnames: Iterable[str], root_password: str, overwrite_confl error_phrases={'error: failed to prepare transaction', 'error: failed to commit transaction', 'error: target not found'}) +def download(root_password: str, *pkgnames: str) -> SimpleProcess: + return SimpleProcess(cmd=['pacman', '-Swdd', *pkgnames, '--noconfirm'], + root_password=root_password, + error_phrases={'error: failed to prepare transaction', 'error: failed to commit transaction', 'error: target not found'}) + + def remove_several(pkgnames: Iterable[str], root_password: str, skip_checks: bool = False) -> SystemProcess: cmd = ['pacman', '-R', *pkgnames, '--noconfirm'] @@ -1119,3 +1126,13 @@ def get_mirrors_branch() -> str: _, output = system.run(['pacman-mirrors', '-G']) return output.strip() + +def get_packages_to_sync_first() -> Set[str]: + if os.path.exists('/etc/pacman.conf'): + with open('/etc/pacman.conf') as f: + to_sync_first = RE_PACMAN_SYNC_FIRST.findall(f.read()) + + if to_sync_first: + return {s.strip() for s in to_sync_first[0].split(' ') if s and s.strip()} + + return set() diff --git a/bauh/gems/arch/updates.py b/bauh/gems/arch/updates.py index 11bacc94..66cebfce 100644 --- a/bauh/gems/arch/updates.py +++ b/bauh/gems/arch/updates.py @@ -424,6 +424,7 @@ class UpdatesSummarizer: to_sync.update(context.to_install.keys()) res.to_install = [self._map_requirement(p, context, to_install=True, to_sync=to_sync) for p in context.to_install.values()] + res.context['data'] = context.pkgs_data return res def __update_context_based_on_to_remove(self, context: UpdateRequirementsContext):