[arch] fix -> some conflict resolution scenarios when upgrading several packages

CHANGELOG.md

@@ -16,6 +16,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Arch
     - not able to upgrade a package that explicitly defines a conflict with itself (e.g: grub)
     - Downloading some AUR packages sources twice when multi-threaded download is enabled
+    - upgrade summary:
+        - not displaying all packages that must be uninstalled
+        - displaying "required size" for packages that must be uninstalled
+    - some conflict resolution scenarios when upgrading several packages
 - UI
     - crashing when nothing can be upgraded
 

bauh/gems/arch/dependencies.py

@@ -456,7 +456,9 @@ class DependenciesAnalyser:
         return missing_deps
 
     def map_all_required_by(self, pkgnames: Iterable[str], to_ignore: Set[str]) -> Set[str]:
-        to_ignore.update(pkgnames)
+        if pkgnames:
+            to_ignore.update(pkgnames)
+
         all_requirements = {req for reqs in pacman.map_required_by(pkgnames).values() for req in reqs if req not in to_ignore}
 
         if all_requirements:

bauh/gems/arch/pacman.py

@@ -4,8 +4,7 @@ from threading import Thread
 from typing import List, Set, Tuple, Dict, Iterable
 
 from bauh.commons import system
-from bauh.commons.system import run_cmd, new_subprocess, new_root_subprocess, SystemProcess, SimpleProcess, \
+from bauh.commons.system import run_cmd, new_subprocess, new_root_subprocess, SystemProcess, SimpleProcess
-    ProcessHandler
 from bauh.commons.util import size_to_byte
 from bauh.gems.arch.exceptions import PackageNotFoundException
 
@@ -539,16 +538,6 @@ def upgrade_system(root_password: str) -> SimpleProcess:
     return SimpleProcess(cmd=['pacman', '-Syyu', '--noconfirm'], root_password=root_password)
 
 
-def get_dependencies_to_remove(pkgs: Iterable[str], root_password: str) -> Dict[str, str]:
-    proc = SimpleProcess(cmd=['pacman', '-R', *pkgs, '--confirm'], root_password=root_password)
-    success, output = ProcessHandler().handle_simple(proc)
-
-    if not output:
-        return {}
-
-    return {t[1]: t[0] for t in RE_REMOVE_TRANSITIVE_DEPS.findall(output)}
-
-
 def fill_provided_map(key: str, val: str, output: dict):
     current_val = output.get(key)
 
@@ -744,8 +733,12 @@ def upgrade_several(pkgnames: Iterable[str], root_password: str, overwrite_confl
                          error_phrases={'error: failed to prepare transaction', 'error: failed to commit transaction', 'error: target not found'})
 
 
-def remove_several(pkgnames: Iterable[str], root_password: str) -> SystemProcess:
+def remove_several(pkgnames: Iterable[str], root_password: str, skip_checks: bool = False) -> SystemProcess:
     cmd = ['pacman', '-R', *pkgnames, '--noconfirm']
+
+    if skip_checks:
+        cmd.append('-dd')
+
     if root_password:
         return SystemProcess(new_root_subprocess(cmd, root_password), wrong_error_phrase='warning:')
     else:
@@ -865,6 +858,51 @@ def map_all_deps(names: Iterable[str], only_installed: bool = False) -> Dict[str
         return res
 
 
+def map_required_dependencies(*names: str) -> Dict[str, Set[str]]:
+    output = run_cmd('pacman -Qi {}'.format(' '.join(names) if names else ''))
+
+    if output:
+        res = {}
+        latest_name, deps, latest_field = None, None, None
+
+        for l in output.split('\n'):
+            if l:
+                if l[0] != ' ':
+                    line = l.strip()
+                    field_sep_idx = line.index(':')
+                    field = line[0:field_sep_idx].strip()
+
+                    if field == 'Name':
+                        val = line[field_sep_idx + 1:].strip()
+                        latest_name = val
+                        deps = None
+                    elif field == 'Depends On':
+                        val = line[field_sep_idx + 1:].strip()
+
+                        if deps is None:
+                            deps = set()
+
+                        if val != 'None':
+                            if ':' in val:
+                                dep_info = val.split(':')
+                                deps.add(dep_info[0].strip())
+                            else:
+                                deps.update({dep.strip() for dep in val.split(' ') if dep})
+
+                    elif latest_name and deps is not None:
+                        res[latest_name] = deps
+                        latest_name, deps, latest_field = None, None, None
+
+                elif latest_name and deps is not None:
+                    if ':' in l:
+                        dep_info = l.split(':')
+                        deps.add(dep_info[0].strip())
+                    else:
+                        deps.update({dep.strip() for dep in l.split(' ') if dep})
+
+        return res
+
+
 def get_cache_dir() -> str:
     dir_pattern = re.compile(r'.*CacheDir\s*=\s*.+')
 
@@ -887,8 +925,8 @@ def get_cache_dir() -> str:
             return '/var/cache/pacman/pkg'
 
 
-def map_required_by(names: Iterable[str]) -> Dict[str, Set[str]]:
+def map_required_by(names: Iterable[str] = None) -> Dict[str, Set[str]]:
-    output = run_cmd('pacman -Qi {}'.format(' '.join(names)))
+    output = run_cmd('pacman -Qi {}'.format(' '.join(names) if names else ''))
 
     if output:
         res = {}

bauh/gems/arch/updates.py

@@ -10,6 +10,7 @@ from bauh.gems.arch.aur import AURClient
 from bauh.gems.arch.dependencies import DependenciesAnalyser
 from bauh.gems.arch.exceptions import PackageNotFoundException
 from bauh.gems.arch.model import ArchPackage
+from bauh.gems.arch.pacman import RE_DEP_OPERATORS
 from bauh.view.util.translation import I18n
 
 
@@ -181,45 +182,13 @@ class UpdatesSummarizer:
 
         root_conflict = self._filter_and_map_conflicts(context)
 
-        sub_conflict = pacman.get_dependencies_to_remove(root_conflict.keys(), context.root_password) if root_conflict else None
-
-        to_remove_map = {}
-        if sub_conflict:
-            for dep, source in sub_conflict.items():
-                if dep not in to_remove_map and (not blacklist or dep not in blacklist):
-                    req = ArchPackage(name=dep, installed=True, i18n=self.i18n)
-                    to_remove_map[dep] = req
-                    reason = "{} '{}'".format(self.i18n['arch.info.depends on'].capitalize(), source)
-                    context.to_remove[dep] = UpgradeRequirement(req, reason)
-
         if root_conflict:
             for dep, source in root_conflict.items():
-                if dep not in to_remove_map and (not blacklist or dep not in blacklist):
+                if dep not in context.to_remove and (not blacklist or dep not in blacklist):
                     req = ArchPackage(name=dep, installed=True, i18n=self.i18n)
-                    to_remove_map[dep] = req
                     reason = "{} '{}'".format(self.i18n['arch.info.conflicts with'].capitalize(), source)
                     context.to_remove[dep] = UpgradeRequirement(req, reason)
 
-        if to_remove_map:
-            for name in to_remove_map.keys():  # upgrading lists
-                if name in context.pkgs_data:
-                    del context.pkgs_data[name]
-
-                if name in context.aur_to_update:
-                    del context.aur_to_update[name]
-
-                if name in context.repo_to_update:
-                    del context.repo_to_update[name]
-
-            removed_size = pacman.get_installed_size([*to_remove_map.keys()])
-
-            if removed_size:
-                for name, size in removed_size.items():
-                    if size is not None:
-                        req = context.to_remove.get(name)
-                        if req:
-                            req.extra_size = size
-
     def _map_and_add_package(self, pkg_data: Tuple[str, str], idx: int, output: dict):
         version = None
 
@@ -334,7 +303,7 @@ class UpdatesSummarizer:
                 context.aur_index.update(names)
                 self.logger.info("AUR index loaded on the context")
 
-    def _map_requirement(self, pkg: ArchPackage, context: UpdateRequirementsContext, installed_sizes: Dict[str, int] = None, to_install: bool = False, to_upgrade: Set[str] = None) -> UpgradeRequirement:
+    def _map_requirement(self, pkg: ArchPackage, context: UpdateRequirementsContext, installed_sizes: Dict[str, int] = None, to_install: bool = False, to_sync: Set[str] = None) -> UpgradeRequirement:
         requirement = UpgradeRequirement(pkg)
 
         if pkg.repository != 'aur':
@@ -347,10 +316,32 @@ class UpdatesSummarizer:
             if current_size is not None and data['s']:
                 requirement.extra_size = data['s'] - current_size
 
-            if to_install and context.pkgs_data and context.to_update:
+            required_by = set()
+
+            if to_install and to_sync and context.pkgs_data:
                 names = context.pkgs_data[pkg.name].get('p', {pkg.name})
-                required_by = ','.join([p for p in to_upgrade if p != pkg.name and context.pkgs_data[p]['d'] and any([n for n in names if n in context.pkgs_data[p]['d']])])
+                to_sync_deps_cache = {}
-                requirement.reason = '{}: {}'.format(self.i18n['arch.info.required by'].capitalize(), required_by if required_by else '?')
+                for p in to_sync:
+                    if p != pkg.name and p in context.pkgs_data:
+                        deps = to_sync_deps_cache.get(p)
+
+                        if deps is None:
+                            deps = context.pkgs_data[p]['d']
+
+                            if deps is None:
+                                deps = set()
+                            else:
+                                deps = {RE_DEP_OPERATORS.split(d)[0] for d in deps}
+
+                            to_sync_deps_cache[p] = deps
+
+                        if deps:
+                            for n in names:
+                                if n in deps:
+                                    required_by.add(p)
+                                    break
+
+                requirement.reason = '{}: {}'.format(self.i18n['arch.info.required by'].capitalize(), ','.join(required_by) if required_by else '?')
 
         return requirement
 
@@ -403,6 +394,8 @@ class UpdatesSummarizer:
             self.logger.error("Package '{}' not found".format(e.name))
             return
 
+        self.__update_context_based_on_to_remove(context)
+
         if context.to_update:
             installed_sizes = pacman.get_installed_size(list(context.to_update.keys()))
 
@@ -427,7 +420,140 @@ class UpdatesSummarizer:
             res.cannot_upgrade = [d for d in context.cannot_upgrade.values()]
 
         if context.to_install:
-            to_upgrade = {r.pkg.name for r in res.to_upgrade} if res.to_upgrade else None
+            to_sync = {r.pkg.name for r in res.to_upgrade} if res.to_upgrade else {}
-            res.to_install = [self._map_requirement(p, context, to_install=True, to_upgrade=to_upgrade) for p in context.to_install.values()]
+            to_sync.update(context.to_install.keys())
+            res.to_install = [self._map_requirement(p, context, to_install=True, to_sync=to_sync) for p in context.to_install.values()]
 
         return res
+
+    def __update_context_based_on_to_remove(self, context: UpdateRequirementsContext):
+        if context.to_remove:
+            to_remove_provided = {}
+            # filtering all package to synchronization from the transaction context
+            to_sync = {*(context.to_update.keys() if context.to_update else set()), *(context.to_install.keys() if context.to_install else set())}
+            if to_sync:  # checking if any packages to sync on the context rely on the 'to remove' ones
+                to_remove_provided.update(pacman.map_provided(remote=False, pkgs=context.to_remove.keys()))
+                to_remove_from_sync = {}  # will store all packages that should be removed
+
+                for pname in to_sync:
+                    if pname in context.pkgs_data:
+                        deps = context.pkgs_data[pname].get('d')
+
+                        if deps:
+                            required = set()
+
+                            for pkg in context.to_remove:
+                                for provided in to_remove_provided[pkg]:
+                                    if provided in deps:
+                                        required.add(pkg)
+                                        break
+
+                            if required:
+                                to_remove_from_sync[pname] = required
+                    else:
+                        self.logger.warning("Conflict resolution: package '{}' marked to synchronization has no data loaded")
+
+                if to_remove_from_sync:  # removing all these packages and their dependents from the context
+                    self._add_to_remove(to_sync, to_remove_from_sync, context)
+
+            # checking if the installed packages that are not in the transaction context rely on the current packages to be removed:
+            current_to_remove = {*context.to_remove.keys()}
+            required_by_installed = self.deps_analyser.map_all_required_by(current_to_remove, {*to_sync})
+
+            if required_by_installed:
+                # updating provided context:
+                provided_not_mapped = set()
+                for pkg in current_to_remove.difference({*to_remove_provided.keys()}):
+                    if pkg not in context.pkgs_data:
+                        provided_not_mapped.add(pkg)
+                    else:
+                        provided = context.pkgs_data[pkg].get('p')
+                        if provided:
+                            to_remove_provided[pkg] = provided
+                        else:
+                            provided_not_mapped.add(pkg)
+
+                if provided_not_mapped:
+                    to_remove_provided.update(pacman.map_provided(remote=False, pkgs=provided_not_mapped))
+
+                deps_no_data = {dep for dep in required_by_installed if dep in context.pkgs_data}
+                deps_nodata_deps = pacman.map_required_dependencies(*deps_no_data) if deps_no_data else {}
+
+                reverse_to_remove_provided = {p: name for name, provided in to_remove_provided for p in provided}
+
+                for pkg in required_by_installed:
+                    if pkg not in context.to_remove:
+                        if pkg in context.pkgs_data:
+                            dep_deps = context.pkgs_data[pkg].get('d')
+                        else:
+                            dep_deps = deps_nodata_deps.get(pkg)
+
+                        if dep_deps:
+                            source = ', '.join((reverse_to_remove_provided[d] for d in dep_deps if d in reverse_to_remove_provided))
+                            reason = "{} '{}'".format(self.i18n['arch.info.depends on'].capitalize(), source if source else '?')
+                            context.to_remove[pkg] = UpgradeRequirement(pkg=ArchPackage(name=pkg,
+                                                                                        installed=True,
+                                                                                        i18n=self.i18n),
+                                                                        reason=reason)
+
+            for name in context.to_remove:  # upgrading lists
+                if name in context.pkgs_data:
+                    del context.pkgs_data[name]
+
+                if name in context.aur_to_update:
+                    del context.aur_to_update[name]
+
+                if name in context.repo_to_update:
+                    del context.repo_to_update[name]
+
+            removed_size = pacman.get_installed_size([*context.to_remove.keys()])
+
+            if removed_size:
+                for name, size in removed_size.items():
+                    if size is not None:
+                        req = context.to_remove.get(name)
+                        if req:
+                            req.extra_size = size
+
+    def _add_to_remove(self, pkgs_to_sync: Set[str], names: Dict[str, Set[str]], context: UpdateRequirementsContext, to_ignore: Set[str] = None):
+        blacklist = to_ignore if to_ignore else set()
+        blacklist.update(names)
+
+        dependents = {}
+        for pname in pkgs_to_sync:
+            if pname not in blacklist:
+                data = context.pkgs_data.get(pname)
+
+                if data:
+                    deps = data.get('d')
+
+                    if deps:
+                        for n in names:
+                            if n in deps:
+                                all_deps = dependents.get(n, set())
+                                all_deps.update(pname)
+                                dependents[n] = all_deps
+
+                else:
+                    self.logger.warning("Package '{}' to sync could not be removed from the transaction context because its data was not loaded")
+
+        for n in names:
+            if n in context.pkgs_data:
+                if n not in context.to_remove:
+                    depends_on = names.get(n)
+                    if depends_on:
+                        reason = "{} '{}'".format(self.i18n['arch.info.depends on'].capitalize(), ', '.join(depends_on))
+                    else:
+                        reason = '?'
+
+                    context.to_remove[n] = UpgradeRequirement(pkg=ArchPackage(name=n,
+                                                                              installed=True,
+                                                                              i18n=self.i18n),
+                                                              reason=reason)
+
+                all_deps = dependents.get(n)
+
+                if all_deps:
+                    self._add_to_remove(pkgs_to_sync, {dep: {n} for dep in all_deps}, context, blacklist)
+            else:
+                self.logger.warning("Package '{}' could not be removed from the transaction context because its data was not loaded")

bauh/view/qt/thread.py

@@ -192,7 +192,7 @@ class UpgradeSelected(AsyncAction):
         self.manager = manager
         self.i18n = i18n
 
-    def _req_as_option(self, req: UpgradeRequirement, tooltip: bool = True, custom_tooltip: str = None) -> InputOption:
+    def _req_as_option(self, req: UpgradeRequirement, tooltip: bool = True, custom_tooltip: str = None, required_size: bool = True) -> InputOption:
         if req.pkg.installed:
             icon_path = req.pkg.get_disk_icon_path()
 
@@ -204,7 +204,7 @@ class UpgradeSelected(AsyncAction):
 
         size_str = '{}: {}'.format(self.i18n['size'].capitalize(),
                                    '?' if req.extra_size is None else get_human_size_str(req.extra_size))
-        if req.extra_size != req.required_size:
+        if required_size and req.extra_size != req.required_size:
             size_str += ' ( {}: {} )'.format(self.i18n['action.update.pkg.required_size'].capitalize(),
                                              '?' if req.required_size is None else get_human_size_str(req.required_size))
 
@@ -250,7 +250,7 @@ class UpgradeSelected(AsyncAction):
         return FormComponent(label=lb, components=comps), (required_size, extra_size)
 
     def _gen_to_remove_form(self, reqs: List[UpgradeRequirement]) -> FormComponent:
-        opts = [self._req_as_option(r, False, r.reason) for r in reqs]
+        opts = [self._req_as_option(r, False, r.reason, required_size=False) for r in reqs]
         comps = [MultipleSelectComponent(label='', options=opts, default_options=set(opts))]
         required_size, extra_size = self._sum_pkgs_size(reqs)