[fix][arch] actually fixing some installed 'not-signed' repository packages

bauh/gems/arch/controller.py

@@ -23,7 +23,7 @@ from bauh.api.abstract.model import PackageUpdate, PackageHistory, SoftwarePacka
 from bauh.api.abstract.view import MessageType, FormComponent, InputOption, SingleSelectComponent, SelectViewType, \
     ViewComponent, PanelComponent, MultipleSelectComponent, TextInputComponent, TextComponent
 from bauh.api.constants import TEMP_DIR
-from bauh.commons import user
+from bauh.commons import user, internet
 from bauh.commons.category import CategoriesDownloader
 from bauh.commons.config import save_config
 from bauh.commons.html import bold
@@ -284,21 +284,21 @@ class ArchManager(SoftwareManager):
         database.register_sync(self.logger)
         return True
 
-    def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader):
-        app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories)
-        app.downgrade_enabled = downgrade_enabled
+    def _upgrade_search_result(self, apidata: dict, installed_pkgs: Dict[str, ArchPackage], downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader):
+        pkg = installed_pkgs.get(apidata['Name'])
 
-        if app.installed:
-            res.installed.append(app)
+        if not pkg:
+            pkg = self.mapper.map_api_data(apidata, None, self.categories)
+            pkg.downgrade_enabled = downgrade_enabled
 
-            if disk_loader:
-                disk_loader.fill(app)
+        if pkg.installed:
+            res.installed.append(pkg)
         else:
-            res.new.append(app)
+            res.new.append(pkg)
 
-        Thread(target=self.mapper.fill_package_build, args=(app,), daemon=True).start()
+        Thread(target=self.mapper.fill_package_build, args=(pkg,), daemon=True).start()
 
-    def _search_in_repos_and_fill(self, words: str, disk_loader: DiskCacheLoader, read_installed: Thread, installed: dict, res: SearchResult):
+    def _search_in_repos_and_fill(self, words: str, disk_loader: DiskCacheLoader, read_installed: Thread, installed: List[ArchPackage], res: SearchResult):
         repo_search = pacman.search(words)
 
         if not repo_search:  # the package may not be mapped on the databases anymore
@@ -320,27 +320,26 @@ class ArchManager(SoftwareManager):
             if repo_pkgs:
                 read_installed.join()
 
+                repo_installed = {p.name: p for p in installed if p.repository != 'aur'} if installed else {}
+
                 for pkg in repo_pkgs:
-                    if installed['signed'] and pkg.name in installed['signed']:
-                        pkg.installed = True
-
-                        if disk_loader:
-                            disk_loader.fill(pkg)
-
-                        res.installed.append(pkg)
+                    pkg_installed = repo_installed.get(pkg.name)
+                    if pkg_installed:
+                        res.installed.append(pkg_installed)
                     else:
                         pkg.installed = False
                         res.new.append(pkg)
 
-    def _search_in_aur_and_fill(self, words: str, disk_loader: DiskCacheLoader, read_installed: Thread, installed: dict, res: SearchResult):
+    def _search_in_aur_and_fill(self, words: str, disk_loader: DiskCacheLoader, read_installed: Thread, installed: List[ArchPackage], res: SearchResult):
         api_res = self.aur_client.search(words)
 
         if api_res and api_res.get('results'):
             read_installed.join()
+            aur_installed = {p.name: p for p in installed if p.repository == 'aur'}
 
             downgrade_enabled = git.is_enabled()
             for pkgdata in api_res['results']:
-                self._upgrade_search_result(pkgdata, installed, downgrade_enabled, res, disk_loader)
+                self._upgrade_search_result(pkgdata, aur_installed, downgrade_enabled, res, disk_loader)
 
         else:  # if there are no results from the API (it could be because there were too many), tries the names index:
             if self.index_aur:
@@ -361,10 +360,11 @@ class ArchManager(SoftwareManager):
 
                 if pkgsinfo:
                     read_installed.join()
+                    aur_installed = {p.name: p for p in installed if p.repository == 'aur'}
                     downgrade_enabled = git.is_enabled()
 
                     for pkgdata in pkgsinfo:
-                        self._upgrade_search_result(pkgdata, installed, downgrade_enabled, res, disk_loader)
+                        self._upgrade_search_result(pkgdata, aur_installed, downgrade_enabled, res, disk_loader)
 
     def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1, is_url: bool = False) -> SearchResult:
         if is_url:
@@ -375,11 +375,11 @@ class ArchManager(SoftwareManager):
         if not any([arch_config['repositories'], arch_config['aur']]):
             return SearchResult([], [], 0)
 
-        installed = {}
-        repo_map = pacman.map_repositories()
-        read_installed = Thread(target=lambda: installed.update(pacman.map_installed(repositories=arch_config['repositories'],
-                                                                                     aur=arch_config['aur'],
-                                                                                     repo_map=repo_map)), daemon=True)
+        installed = []
+        read_installed = Thread(target=lambda: installed.extend(self.read_installed(disk_loader=disk_loader,
+                                                                                    only_apps=False,
+                                                                                    limit=-1,
+                                                                                    internet_available=True).installed), daemon=True)
         read_installed.start()
 
         res = SearchResult([], [], 0)
@@ -404,30 +404,30 @@ class ArchManager(SoftwareManager):
         res.total = len(res.installed) + len(res.new)
         return res
 
-    def _fill_aur_pkgs(self, not_signed: dict, pkgs: list, disk_loader: DiskCacheLoader, internet_available: bool):
+    def _fill_aur_pkgs(self, aur_pkgs: dict, output: list, disk_loader: DiskCacheLoader, internet_available: bool):
         downgrade_enabled = git.is_enabled()
 
         if internet_available:
             try:
-                pkgsinfo = self.aur_client.get_info(not_signed.keys())
+                pkgsinfo = self.aur_client.get_info(aur_pkgs.keys())
 
                 if pkgsinfo:
                     for pkgdata in pkgsinfo:
-                        pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories)
+                        pkg = self.mapper.map_api_data(pkgdata, aur_pkgs, self.categories)
                         pkg.downgrade_enabled = downgrade_enabled
 
                         if disk_loader:
                             disk_loader.fill(pkg)
                             pkg.status = PackageStatus.READY
 
-                        pkgs.append(pkg)
+                        output.append(pkg)
 
                 return
             except requests.exceptions.ConnectionError:
                 self.logger.warning('Could not retrieve installed AUR packages API data. It seems the internet connection is off.')
                 self.logger.info("Reading only local AUR packages data")
 
-        for name, data in not_signed.items():
+        for name, data in aur_pkgs.items():
             pkg = ArchPackage(name=name, version=data.get('version'),
                               latest_version=data.get('version'), description=data.get('description'),
                               installed=True, repository='aur', i18n=self.i18n)
@@ -439,34 +439,38 @@ class ArchManager(SoftwareManager):
                 disk_loader.fill(pkg)
                 pkg.status = PackageStatus.READY
 
-            pkgs.append(pkg)
+            output.append(pkg)
 
     def _fill_repo_updates(self, updates: dict):
         updates.update(pacman.list_repository_updates())
 
-    def _fill_repo_pkgs(self, signed: dict, pkgs: list, repo_map: Dict[str, str], disk_loader: DiskCacheLoader):
+    def _fill_repo_pkgs(self, repo_pkgs: dict, pkgs: list, disk_loader: DiskCacheLoader):
         updates = {}
 
         thread_updates = Thread(target=self._fill_repo_updates, args=(updates,), daemon=True)
         thread_updates.start()
 
-        if len(repo_map) != len(signed):
-            self.logger.warning("Not mapped all signed packages repositories. Mapped: {}. Total: {}".format(len(repo_map), len(signed)))
+        repo_map = pacman.map_repositories(repo_pkgs)
+        if len(repo_map) != len(repo_pkgs):
+            self.logger.warning("Not mapped all signed packages repositories. Mapped: {}. Total: {}".format(len(repo_map), len(repo_pkgs)))
 
         thread_updates.join()
 
         self.logger.info("Repository updates found" if updates else "No repository updates found")
 
-        for name, data in signed.items():
+        for name, data in repo_pkgs.items():
+            pkgversion = data.get('version')
+            pkgrepo = repo_map.get(name)
             pkg = ArchPackage(name=name,
-                              version=data.get('version'),
-                              latest_version=data.get('version'),
+                              version=pkgversion,
+                              latest_version=pkgversion,
                               description=data.get('description'),
+                              maintainer=pkgrepo,
                               i18n=self.i18n,
                               installed=True,
-                              repository=repo_map.get(name))
-            pkg.categories = self.categories.get(pkg.name)
-            pkg.downgrade_enabled = True
+                              repository=pkgrepo,
+                              categories=self.categories.get(name))
+            pkg.downgrade_enabled = False
 
             if updates:
                 update_version = updates.get(pkg.name)
@@ -483,20 +487,40 @@ class ArchManager(SoftwareManager):
     def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult:
         self.aur_client.clean_caches()
         arch_config = read_config()
-        repo_map = pacman.map_repositories()
-        installed = pacman.map_installed(repo_map=repo_map, repositories=arch_config['repositories'], aur=arch_config['aur'])
+
+        installed = pacman.map_installed()
+
+        aur_pkgs, repo_pkgs = None, None
+
+        if arch_config['repositories'] and installed['signed']:
+            repo_pkgs = installed['signed']
+
+        if installed['not_signed']:
+            if self.index_aur:
+                self.index_aur.join()
+
+            aur_index = self.aur_client.read_index()
+
+            for pkg in {*installed['not_signed']}:
+                if pkg not in aur_index:
+                    if repo_pkgs is not None:
+                        repo_pkgs[pkg] = installed['not_signed'][pkg]
+
+                    del installed['not_signed'][pkg]
+
+            aur_pkgs = installed['not_signed']
 
         pkgs = []
-        if installed and (installed['not_signed'] or installed['signed']):
+        if repo_pkgs or aur_pkgs:
             map_threads = []
 
-            if installed['not_signed']:
-                t = Thread(target=self._fill_aur_pkgs, args=(installed['not_signed'], pkgs, disk_loader, internet_available), daemon=True)
+            if aur_pkgs:
+                t = Thread(target=self._fill_aur_pkgs, args=(aur_pkgs, pkgs, disk_loader, internet_available), daemon=True)
                 t.start()
                 map_threads.append(t)
 
-            if installed['signed']:
-                t = Thread(target=self._fill_repo_pkgs, args=(installed['signed'], pkgs, repo_map, disk_loader), daemon=True)
+            if repo_pkgs:
+                t = Thread(target=self._fill_repo_pkgs, args=(repo_pkgs, pkgs, disk_loader), daemon=True)
                 t.start()
                 map_threads.append(t)
 
@@ -733,7 +757,16 @@ class ArchManager(SoftwareManager):
                 handler.watcher.print("Repository packages successfully upgraded")
                 handler.watcher.change_substatus(self.i18n['arch.upgrade.caching_pkgs_data'])
                 repo_map = pacman.map_repositories(pkgs)
-                disk.save_several(pkgs, repo_map=repo_map, overwrite=True, maintainer=None)
+
+                pkg_map = {}
+                for name in pkgs:
+                    repo = repo_map.get(name)
+                    pkg_map[name] = ArchPackage(name=name,
+                                                repository=repo,
+                                                maintainer=repo,
+                                                categories=self.categories.get(name))
+
+                disk.save_several(pkg_map, overwrite=True, maintainer=None)
                 return True
             elif 'conflicting files' in upgrade_output:
                 files = self._map_conflicting_file(upgrade_output)
@@ -1373,18 +1406,6 @@ class ArchManager(SoftwareManager):
 
         return False
 
-    def _map_unknown_missing_deps(self, deps: List[str], watcher: ProcessWatcher, check_subdeps: bool = True) -> List[Tuple[str, str]]:
-        depnames = {RE_SPLIT_VERSION.split(dep)[0] for dep in deps}
-        dep_repos = self._map_repos(depnames)
-
-        if len(depnames) != len(dep_repos):  # checking if a dependency could not be found in any repository
-            for dep in depnames:
-                if dep not in dep_repos:
-                    message.show_dep_not_found(dep, self.i18n, watcher)
-                    return
-
-        return self.deps_analyser.map_known_missing_deps(dep_repos, watcher, check_subdeps)
-
     def _ask_and_install_missing_deps(self, context: TransactionContext,  missing_deps: List[Tuple[str, str]]) -> bool:
         context.watcher.change_substatus(self.i18n['arch.missing_deps_found'].format(bold(context.name)))
 
@@ -1626,19 +1647,31 @@ class ArchManager(SoftwareManager):
 
         if installed:
             context.watcher.change_substatus(self.i18n['status.caching_data'].format(bold(context.name)))
-            pkgnames = {context.name}
-            repo_map = {context.name: context.repository}
 
+            cache_map = {context.name: ArchPackage(name=context.name,
+                                                   repository=context.repository,
+                                                   maintainer=context.maintainer,
+                                                   categories=self.categories.get(context.name))}
             if context.missing_deps:
-                for dep in context.missing_deps:
-                    pkgnames.add(dep[0])
-                    repo_map[dep[0]] = dep[1]
+                aur_deps = {dep[0] for dep in context.missing_deps if dep[1] == 'aur'}
 
-            disk.save_several(pkgnames=pkgnames,
-                              repo_map=repo_map,
-                              maintainer=context.maintainer,
-                              overwrite=True,
-                              categories=self.categories)
+                if aur_deps:
+                    aur_data = self.aur_client.get_info(aur_deps)
+
+                    if aur_data:
+                        aur_data = {info['Name']: info for info in aur_data}
+                    else:
+                        aur_data = {n: {} for n in aur_deps}
+                else:
+                    aur_data = None
+
+                for dep in context.missing_deps:
+                    cache_map[dep[0]] = ArchPackage(name=dep[0],
+                                                    repository=dep[1],
+                                                    maintainer=dep[1] if dep[1] != 'aur' else (aur_data[dep[0]].get('Maintainer') if aur_data else None),
+                                                    categories=self.categories.get(context.name))
+
+            disk.save_several(pkgs=cache_map, maintainer=None, overwrite=True)
 
             self._update_progress(context, 100)
 
@@ -1848,7 +1881,8 @@ class ArchManager(SoftwareManager):
         arch_config = read_config(update_file=True)
 
         if arch_config['aur'] or arch_config['repositories']:
-            ArchDiskCacheUpdater(task_manager, arch_config, self.i18n, self.context.logger).start()
+            ArchDiskCacheUpdater(task_man=task_manager, arch_config=arch_config, i18n=self.i18n, logger=self.context.logger,
+                                 controller=self, internet_available=internet_available).start()
 
         if arch_config['aur']:
             ArchCompilationOptimizer(arch_config, self.i18n, self.context.logger, task_manager).start()
@@ -2085,18 +2119,15 @@ class ArchManager(SoftwareManager):
 
     def upgrade_system(self, root_password: str, watcher: ProcessWatcher) -> bool:
         repo_map = pacman.map_repositories()
-        installed = pacman.map_installed(repo_map=repo_map, repositories=True, aur=False)
+        installed = self.read_installed(limit=-1, only_apps=False, pkg_types=None, internet_available=internet.is_available(), disk_loader=None).installed
 
-        if not installed or not installed['signed']:
+        if not installed:
             watcher.show_message(title=self.i18n['arch.custom_action.upgrade_system'],
                                  body=self.i18n['arch.custom_action.upgrade_system.no_updates'],
                                  type_=MessageType.INFO)
             return False
 
-        pkgs = []
-        self._fill_repo_pkgs(installed['signed'], pkgs, None)
-
-        to_update = [p for p in pkgs if p.update]
+        to_update = [p for p in installed if p.repository != 'aur' and p.update]
 
         if not to_update:
             watcher.show_message(title=self.i18n['arch.custom_action.upgrade_system'],