From 78912d5eee64bafc1a55b58cbc1f27c8cdd4eea8 Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Thu, 6 Aug 2020 11:53:52 -0300 Subject: [PATCH] [arch] fix -> some conflict resolution scenarios when upgrading several packages --- CHANGELOG.md | 4 + bauh/gems/arch/dependencies.py | 4 +- bauh/gems/arch/pacman.py | 68 ++++++++--- bauh/gems/arch/updates.py | 204 ++++++++++++++++++++++++++------- bauh/view/qt/thread.py | 6 +- 5 files changed, 228 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dc1effb..dd035d76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Arch - not able to upgrade a package that explicitly defines a conflict with itself (e.g: grub) - Downloading some AUR packages sources twice when multi-threaded download is enabled + - upgrade summary: + - not displaying all packages that must be uninstalled + - displaying "required size" for packages that must be uninstalled + - some conflict resolution scenarios when upgrading several packages - UI - crashing when nothing can be upgraded diff --git a/bauh/gems/arch/dependencies.py b/bauh/gems/arch/dependencies.py index 24404504..99f21c8a 100644 --- a/bauh/gems/arch/dependencies.py +++ b/bauh/gems/arch/dependencies.py @@ -456,7 +456,9 @@ class DependenciesAnalyser: return missing_deps def map_all_required_by(self, pkgnames: Iterable[str], to_ignore: Set[str]) -> Set[str]: - to_ignore.update(pkgnames) + if pkgnames: + to_ignore.update(pkgnames) + all_requirements = {req for reqs in pacman.map_required_by(pkgnames).values() for req in reqs if req not in to_ignore} if all_requirements: diff --git a/bauh/gems/arch/pacman.py b/bauh/gems/arch/pacman.py index 903d605e..cc33fdd3 100644 --- a/bauh/gems/arch/pacman.py +++ b/bauh/gems/arch/pacman.py @@ -4,8 +4,7 @@ from threading import Thread from typing import List, Set, Tuple, Dict, Iterable from bauh.commons import system -from bauh.commons.system import run_cmd, new_subprocess, new_root_subprocess, SystemProcess, SimpleProcess, \ - ProcessHandler +from bauh.commons.system import run_cmd, new_subprocess, new_root_subprocess, SystemProcess, SimpleProcess from bauh.commons.util import size_to_byte from bauh.gems.arch.exceptions import PackageNotFoundException @@ -539,16 +538,6 @@ def upgrade_system(root_password: str) -> SimpleProcess: return SimpleProcess(cmd=['pacman', '-Syyu', '--noconfirm'], root_password=root_password) -def get_dependencies_to_remove(pkgs: Iterable[str], root_password: str) -> Dict[str, str]: - proc = SimpleProcess(cmd=['pacman', '-R', *pkgs, '--confirm'], root_password=root_password) - success, output = ProcessHandler().handle_simple(proc) - - if not output: - return {} - - return {t[1]: t[0] for t in RE_REMOVE_TRANSITIVE_DEPS.findall(output)} - - def fill_provided_map(key: str, val: str, output: dict): current_val = output.get(key) @@ -744,8 +733,12 @@ def upgrade_several(pkgnames: Iterable[str], root_password: str, overwrite_confl error_phrases={'error: failed to prepare transaction', 'error: failed to commit transaction', 'error: target not found'}) -def remove_several(pkgnames: Iterable[str], root_password: str) -> SystemProcess: +def remove_several(pkgnames: Iterable[str], root_password: str, skip_checks: bool = False) -> SystemProcess: cmd = ['pacman', '-R', *pkgnames, '--noconfirm'] + + if skip_checks: + cmd.append('-dd') + if root_password: return SystemProcess(new_root_subprocess(cmd, root_password), wrong_error_phrase='warning:') else: @@ -865,6 +858,51 @@ def map_all_deps(names: Iterable[str], only_installed: bool = False) -> Dict[str return res +def map_required_dependencies(*names: str) -> Dict[str, Set[str]]: + output = run_cmd('pacman -Qi {}'.format(' '.join(names) if names else '')) + + if output: + res = {} + latest_name, deps, latest_field = None, None, None + + for l in output.split('\n'): + if l: + if l[0] != ' ': + line = l.strip() + field_sep_idx = line.index(':') + field = line[0:field_sep_idx].strip() + + if field == 'Name': + val = line[field_sep_idx + 1:].strip() + latest_name = val + deps = None + elif field == 'Depends On': + val = line[field_sep_idx + 1:].strip() + + if deps is None: + deps = set() + + if val != 'None': + if ':' in val: + dep_info = val.split(':') + deps.add(dep_info[0].strip()) + else: + deps.update({dep.strip() for dep in val.split(' ') if dep}) + + elif latest_name and deps is not None: + res[latest_name] = deps + latest_name, deps, latest_field = None, None, None + + elif latest_name and deps is not None: + if ':' in l: + dep_info = l.split(':') + deps.add(dep_info[0].strip()) + else: + deps.update({dep.strip() for dep in l.split(' ') if dep}) + + return res + + def get_cache_dir() -> str: dir_pattern = re.compile(r'.*CacheDir\s*=\s*.+') @@ -887,8 +925,8 @@ def get_cache_dir() -> str: return '/var/cache/pacman/pkg' -def map_required_by(names: Iterable[str]) -> Dict[str, Set[str]]: - output = run_cmd('pacman -Qi {}'.format(' '.join(names))) +def map_required_by(names: Iterable[str] = None) -> Dict[str, Set[str]]: + output = run_cmd('pacman -Qi {}'.format(' '.join(names) if names else '')) if output: res = {} diff --git a/bauh/gems/arch/updates.py b/bauh/gems/arch/updates.py index ee010f8d..11bacc94 100644 --- a/bauh/gems/arch/updates.py +++ b/bauh/gems/arch/updates.py @@ -10,6 +10,7 @@ from bauh.gems.arch.aur import AURClient from bauh.gems.arch.dependencies import DependenciesAnalyser from bauh.gems.arch.exceptions import PackageNotFoundException from bauh.gems.arch.model import ArchPackage +from bauh.gems.arch.pacman import RE_DEP_OPERATORS from bauh.view.util.translation import I18n @@ -181,45 +182,13 @@ class UpdatesSummarizer: root_conflict = self._filter_and_map_conflicts(context) - sub_conflict = pacman.get_dependencies_to_remove(root_conflict.keys(), context.root_password) if root_conflict else None - - to_remove_map = {} - if sub_conflict: - for dep, source in sub_conflict.items(): - if dep not in to_remove_map and (not blacklist or dep not in blacklist): - req = ArchPackage(name=dep, installed=True, i18n=self.i18n) - to_remove_map[dep] = req - reason = "{} '{}'".format(self.i18n['arch.info.depends on'].capitalize(), source) - context.to_remove[dep] = UpgradeRequirement(req, reason) - if root_conflict: for dep, source in root_conflict.items(): - if dep not in to_remove_map and (not blacklist or dep not in blacklist): + if dep not in context.to_remove and (not blacklist or dep not in blacklist): req = ArchPackage(name=dep, installed=True, i18n=self.i18n) - to_remove_map[dep] = req reason = "{} '{}'".format(self.i18n['arch.info.conflicts with'].capitalize(), source) context.to_remove[dep] = UpgradeRequirement(req, reason) - if to_remove_map: - for name in to_remove_map.keys(): # upgrading lists - if name in context.pkgs_data: - del context.pkgs_data[name] - - if name in context.aur_to_update: - del context.aur_to_update[name] - - if name in context.repo_to_update: - del context.repo_to_update[name] - - removed_size = pacman.get_installed_size([*to_remove_map.keys()]) - - if removed_size: - for name, size in removed_size.items(): - if size is not None: - req = context.to_remove.get(name) - if req: - req.extra_size = size - def _map_and_add_package(self, pkg_data: Tuple[str, str], idx: int, output: dict): version = None @@ -334,7 +303,7 @@ class UpdatesSummarizer: context.aur_index.update(names) self.logger.info("AUR index loaded on the context") - def _map_requirement(self, pkg: ArchPackage, context: UpdateRequirementsContext, installed_sizes: Dict[str, int] = None, to_install: bool = False, to_upgrade: Set[str] = None) -> UpgradeRequirement: + def _map_requirement(self, pkg: ArchPackage, context: UpdateRequirementsContext, installed_sizes: Dict[str, int] = None, to_install: bool = False, to_sync: Set[str] = None) -> UpgradeRequirement: requirement = UpgradeRequirement(pkg) if pkg.repository != 'aur': @@ -347,10 +316,32 @@ class UpdatesSummarizer: if current_size is not None and data['s']: requirement.extra_size = data['s'] - current_size - if to_install and context.pkgs_data and context.to_update: + required_by = set() + + if to_install and to_sync and context.pkgs_data: names = context.pkgs_data[pkg.name].get('p', {pkg.name}) - required_by = ','.join([p for p in to_upgrade if p != pkg.name and context.pkgs_data[p]['d'] and any([n for n in names if n in context.pkgs_data[p]['d']])]) - requirement.reason = '{}: {}'.format(self.i18n['arch.info.required by'].capitalize(), required_by if required_by else '?') + to_sync_deps_cache = {} + for p in to_sync: + if p != pkg.name and p in context.pkgs_data: + deps = to_sync_deps_cache.get(p) + + if deps is None: + deps = context.pkgs_data[p]['d'] + + if deps is None: + deps = set() + else: + deps = {RE_DEP_OPERATORS.split(d)[0] for d in deps} + + to_sync_deps_cache[p] = deps + + if deps: + for n in names: + if n in deps: + required_by.add(p) + break + + requirement.reason = '{}: {}'.format(self.i18n['arch.info.required by'].capitalize(), ','.join(required_by) if required_by else '?') return requirement @@ -403,6 +394,8 @@ class UpdatesSummarizer: self.logger.error("Package '{}' not found".format(e.name)) return + self.__update_context_based_on_to_remove(context) + if context.to_update: installed_sizes = pacman.get_installed_size(list(context.to_update.keys())) @@ -427,7 +420,140 @@ class UpdatesSummarizer: res.cannot_upgrade = [d for d in context.cannot_upgrade.values()] if context.to_install: - to_upgrade = {r.pkg.name for r in res.to_upgrade} if res.to_upgrade else None - res.to_install = [self._map_requirement(p, context, to_install=True, to_upgrade=to_upgrade) for p in context.to_install.values()] + to_sync = {r.pkg.name for r in res.to_upgrade} if res.to_upgrade else {} + to_sync.update(context.to_install.keys()) + res.to_install = [self._map_requirement(p, context, to_install=True, to_sync=to_sync) for p in context.to_install.values()] return res + + def __update_context_based_on_to_remove(self, context: UpdateRequirementsContext): + if context.to_remove: + to_remove_provided = {} + # filtering all package to synchronization from the transaction context + to_sync = {*(context.to_update.keys() if context.to_update else set()), *(context.to_install.keys() if context.to_install else set())} + if to_sync: # checking if any packages to sync on the context rely on the 'to remove' ones + to_remove_provided.update(pacman.map_provided(remote=False, pkgs=context.to_remove.keys())) + to_remove_from_sync = {} # will store all packages that should be removed + + for pname in to_sync: + if pname in context.pkgs_data: + deps = context.pkgs_data[pname].get('d') + + if deps: + required = set() + + for pkg in context.to_remove: + for provided in to_remove_provided[pkg]: + if provided in deps: + required.add(pkg) + break + + if required: + to_remove_from_sync[pname] = required + else: + self.logger.warning("Conflict resolution: package '{}' marked to synchronization has no data loaded") + + if to_remove_from_sync: # removing all these packages and their dependents from the context + self._add_to_remove(to_sync, to_remove_from_sync, context) + + # checking if the installed packages that are not in the transaction context rely on the current packages to be removed: + current_to_remove = {*context.to_remove.keys()} + required_by_installed = self.deps_analyser.map_all_required_by(current_to_remove, {*to_sync}) + + if required_by_installed: + # updating provided context: + provided_not_mapped = set() + for pkg in current_to_remove.difference({*to_remove_provided.keys()}): + if pkg not in context.pkgs_data: + provided_not_mapped.add(pkg) + else: + provided = context.pkgs_data[pkg].get('p') + if provided: + to_remove_provided[pkg] = provided + else: + provided_not_mapped.add(pkg) + + if provided_not_mapped: + to_remove_provided.update(pacman.map_provided(remote=False, pkgs=provided_not_mapped)) + + deps_no_data = {dep for dep in required_by_installed if dep in context.pkgs_data} + deps_nodata_deps = pacman.map_required_dependencies(*deps_no_data) if deps_no_data else {} + + reverse_to_remove_provided = {p: name for name, provided in to_remove_provided for p in provided} + + for pkg in required_by_installed: + if pkg not in context.to_remove: + if pkg in context.pkgs_data: + dep_deps = context.pkgs_data[pkg].get('d') + else: + dep_deps = deps_nodata_deps.get(pkg) + + if dep_deps: + source = ', '.join((reverse_to_remove_provided[d] for d in dep_deps if d in reverse_to_remove_provided)) + reason = "{} '{}'".format(self.i18n['arch.info.depends on'].capitalize(), source if source else '?') + context.to_remove[pkg] = UpgradeRequirement(pkg=ArchPackage(name=pkg, + installed=True, + i18n=self.i18n), + reason=reason) + + for name in context.to_remove: # upgrading lists + if name in context.pkgs_data: + del context.pkgs_data[name] + + if name in context.aur_to_update: + del context.aur_to_update[name] + + if name in context.repo_to_update: + del context.repo_to_update[name] + + removed_size = pacman.get_installed_size([*context.to_remove.keys()]) + + if removed_size: + for name, size in removed_size.items(): + if size is not None: + req = context.to_remove.get(name) + if req: + req.extra_size = size + + def _add_to_remove(self, pkgs_to_sync: Set[str], names: Dict[str, Set[str]], context: UpdateRequirementsContext, to_ignore: Set[str] = None): + blacklist = to_ignore if to_ignore else set() + blacklist.update(names) + + dependents = {} + for pname in pkgs_to_sync: + if pname not in blacklist: + data = context.pkgs_data.get(pname) + + if data: + deps = data.get('d') + + if deps: + for n in names: + if n in deps: + all_deps = dependents.get(n, set()) + all_deps.update(pname) + dependents[n] = all_deps + + else: + self.logger.warning("Package '{}' to sync could not be removed from the transaction context because its data was not loaded") + + for n in names: + if n in context.pkgs_data: + if n not in context.to_remove: + depends_on = names.get(n) + if depends_on: + reason = "{} '{}'".format(self.i18n['arch.info.depends on'].capitalize(), ', '.join(depends_on)) + else: + reason = '?' + + context.to_remove[n] = UpgradeRequirement(pkg=ArchPackage(name=n, + installed=True, + i18n=self.i18n), + reason=reason) + + all_deps = dependents.get(n) + + if all_deps: + self._add_to_remove(pkgs_to_sync, {dep: {n} for dep in all_deps}, context, blacklist) + else: + self.logger.warning("Package '{}' could not be removed from the transaction context because its data was not loaded") diff --git a/bauh/view/qt/thread.py b/bauh/view/qt/thread.py index 7a561002..10a66c95 100644 --- a/bauh/view/qt/thread.py +++ b/bauh/view/qt/thread.py @@ -192,7 +192,7 @@ class UpgradeSelected(AsyncAction): self.manager = manager self.i18n = i18n - def _req_as_option(self, req: UpgradeRequirement, tooltip: bool = True, custom_tooltip: str = None) -> InputOption: + def _req_as_option(self, req: UpgradeRequirement, tooltip: bool = True, custom_tooltip: str = None, required_size: bool = True) -> InputOption: if req.pkg.installed: icon_path = req.pkg.get_disk_icon_path() @@ -204,7 +204,7 @@ class UpgradeSelected(AsyncAction): size_str = '{}: {}'.format(self.i18n['size'].capitalize(), '?' if req.extra_size is None else get_human_size_str(req.extra_size)) - if req.extra_size != req.required_size: + if required_size and req.extra_size != req.required_size: size_str += ' ( {}: {} )'.format(self.i18n['action.update.pkg.required_size'].capitalize(), '?' if req.required_size is None else get_human_size_str(req.required_size)) @@ -250,7 +250,7 @@ class UpgradeSelected(AsyncAction): return FormComponent(label=lb, components=comps), (required_size, extra_size) def _gen_to_remove_form(self, reqs: List[UpgradeRequirement]) -> FormComponent: - opts = [self._req_as_option(r, False, r.reason) for r in reqs] + opts = [self._req_as_option(r, False, r.reason, required_size=False) for r in reqs] comps = [MultipleSelectComponent(label='', options=opts, default_options=set(opts))] required_size, extra_size = self._sum_pkgs_size(reqs)