[arch] improvement: preventing AUR's installed packages from not being mapped in cases the communication with the API fails

CHANGELOG.md

@@ -7,7 +7,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [0.9.27]
 
 ### Improvements
-- code refactoring (String formatting method)
+- Arch:
+  - preventing AUR's installed packages from not being mapped in cases the communication with the API fails
+  - code refactoring (String formatting method)
 
 ### Fixes
 - Arch

bauh/gems/arch/aur.py

@@ -112,12 +112,29 @@ class AURClient:
     def search(self, words: str) -> dict:
         return self.http_client.get_json(URL_SEARCH + words)
 
-    def get_info(self, names: Iterable[str]) -> List[dict]:
+    def get_info(self, names: Iterable[str]) -> Optional[List[dict]]:
         try:
             res = self.http_client.get_json(URL_INFO + self._map_names_as_queries(names))
-            return res['results'] if res and res.get('results') else []
+        except requests.exceptions.ConnectionError:
-        except:
+            self.logger.warning('Could not retrieve installed AUR packages API data. It seems the internet connection is off.')
-            return []
+            return
+
+        if res is None:
+            self.logger.warning("Call to AUR API's info endpoint has failed")
+            return
+
+        error = res.get('error')
+
+        if error:
+            self.logger.warning(f"AUR API's info endpoint returned an unexpected error: {error}")
+            return
+
+        results = res.get('results')
+
+        if results is not None:
+            return results
+
+        self.logger.warning(f"AUR API's info endpoint returned an unexpected response: {res}")
 
     def map_provided(self, pkgname: str, pkgver: str, provided: Optional[Iterable[str]] = None, strip_epoch: bool = True) -> Set[str]:
         all_provided = {pkgname, f"{pkgname}={pkgver.split('-')[0] if strip_epoch else pkgver}"}
@@ -130,31 +147,34 @@ class AURClient:
         return all_provided
 
     def gen_updates_data(self, names: Iterable[str]) -> Generator[Tuple[str, dict], None, None]:
-        for package in self.get_info(names):
+        pkgs_info = self.get_info(names)
-            pkgname, pkgver = package['Name'], package['Version'].split('-')[0]
 
-            deps = set()
+        if pkgs_info:
+            for package in pkgs_info:
+                pkgname, pkgver = package['Name'], package['Version'].split('-')[0]
 
-            for dtype in ('Depends', 'MakeDepends', 'CheckDepends'):
+                deps = set()
-                dep_set = package.get(dtype)
-                if dep_set:
-                    deps.update(dep_set)
 
-            conflicts = set()
+                for dtype in ('Depends', 'MakeDepends', 'CheckDepends'):
-            pkg_conflicts = package.get('Conflicts')
+                    dep_set = package.get(dtype)
+                    if dep_set:
+                        deps.update(dep_set)
 
-            if pkg_conflicts:
+                conflicts = set()
-                conflicts.update(pkg_conflicts)
+                pkg_conflicts = package.get('Conflicts')
 
-            yield pkgname, {
+                if pkg_conflicts:
-                'v': pkgver,
+                    conflicts.update(pkg_conflicts)
-                'b': package.get('PackageBase', pkgname),
+
-                'r': 'aur',
+                yield pkgname, {
-                'p': self.map_provided(pkgname=pkgname, pkgver=pkgver, provided=package.get('Provides'), strip_epoch=False),
+                    'v': pkgver,
-                'd': deps,
+                    'b': package.get('PackageBase', pkgname),
-                'c': conflicts,
+                    'r': 'aur',
-                'ds': None,
+                    'p': self.map_provided(pkgname=pkgname, pkgver=pkgver, provided=package.get('Provides'), strip_epoch=False),
-                's': None}
+                    'd': deps,
+                    'c': conflicts,
+                    'ds': None,
+                    's': None}
 
     def get_src_info(self, name: str, real_name: Optional[str] = None) -> dict:
         srcinfo = self.srcinfo_cache.get(name)
@@ -215,7 +235,7 @@ class AURClient:
         return self.extract_required_dependencies(info)
 
     def _map_names_as_queries(self, names: Iterable[str]) -> str:
-        return '&'.join(['arg[]={}'.format(urllib.parse.quote(n)) for n in names])
+        return '&'.join((f'arg[]={urllib.parse.quote(n)}' for n in names))
 
     def read_local_index(self) -> dict:
         self.logger.info('Checking if the cached AUR index file exists')

bauh/gems/arch/controller.py

@@ -14,7 +14,6 @@ from pwd import getpwnam
 from threading import Thread
 from typing import List, Set, Type, Tuple, Dict, Iterable, Optional, Collection, Generator
 
-import requests
 from dateutil.parser import parse as parse_date
 
 from bauh import __app_name__
@@ -412,67 +411,72 @@ class ArchManager(SoftwareManager):
         res.update_total()
         return res
 
-    def _fill_aur_pkgs(self, aur_pkgs: dict, output: List[ArchPackage], disk_loader: DiskCacheLoader, internet_available: bool,
+    def _fill_aur_pkgs_offline(self, aur_pkgs: dict,  arch_config: dict, output: List[ArchPackage], disk_loader: Optional[DiskCacheLoader]):
+        self.logger.info("Reading cached data from installed AUR packages")
+
+        editable_pkgbuilds = self._read_editable_pkgbuilds() if arch_config['edit_aur_pkgbuild'] is not False else None
+        for name, data in aur_pkgs.items():
+            pkg = ArchPackage(name=name, version=data.get('version'),
+                              latest_version=data.get('version'), description=data.get('description'),
+                              installed=True, repository='aur', i18n=self.i18n)
+
+            pkg.categories = self.categories.get(pkg.name)
+            pkg.pkgbuild_editable = pkg.name in editable_pkgbuilds if editable_pkgbuilds is not None else None
+
+            if disk_loader:
+                disk_loader.fill(pkg)
+
+            pkg.status = PackageStatus.READY
+            output.append(pkg)
+
+    def _fill_aur_pkgs(self, aur_pkgs: dict, output: List[ArchPackage], disk_loader: Optional[DiskCacheLoader], internet_available: bool,
                        arch_config: dict, rebuild_check: Optional[Thread], rebuild_ignored: Optional[Thread], rebuild_output: Optional[Dict[str, Set[str]]]):
 
-        if internet_available:
+        if not internet_available:
-            try:
+            self._fill_aur_pkgs_offline(aur_pkgs=aur_pkgs, arch_config=arch_config,
-                pkgsinfo = self.aur_client.get_info(aur_pkgs.keys())
+                                        output=output, disk_loader=disk_loader)
-            except requests.exceptions.ConnectionError:
+            return
-                self.logger.warning('Could not retrieve installed AUR packages API data. It seems the internet connection is off.')
-                self.logger.info("Reading only local AUR packages data")
-                return
 
-            if pkgsinfo:
+        pkgsinfo = self.aur_client.get_info(aur_pkgs.keys())
-                editable_pkgbuilds = self._read_editable_pkgbuilds() if arch_config['edit_aur_pkgbuild'] is not False else None
-
-                ignore_rebuild_check = None
-                if rebuild_ignored and rebuild_output is not None:
-                    rebuild_ignored.join()
-                    ignore_rebuild_check = rebuild_output['ignored']
-
-                to_rebuild = None
-                if rebuild_check and rebuild_output is not None:
-                    self.logger.info("Waiting for rebuild-detector")
-                    rebuild_check.join()
-                    to_rebuild = rebuild_output['to_rebuild']
-
-                for pkgdata in pkgsinfo:
-                    pkg = self.aur_mapper.map_api_data(pkgdata, aur_pkgs, self.categories)
-                    pkg.pkgbuild_editable = pkg.name in editable_pkgbuilds if editable_pkgbuilds is not None else None
-
-                    if pkg.installed:
-                        if disk_loader:
-                            disk_loader.fill(pkg, sync=True)
-
-                        pkg.update = self._check_aur_package_update(pkg=pkg,
-                                                                    installed_data=aur_pkgs.get(pkg.name, {}),
-                                                                    api_data=pkgdata)
-                        pkg.aur_update = pkg.update  # used in 'set_rebuild_check'
-
-                        if ignore_rebuild_check is not None:
-                            pkg.allow_rebuild = pkg.name not in ignore_rebuild_check
-
-                        if to_rebuild and not pkg.update and pkg.name in to_rebuild:
-                            pkg.require_rebuild = True
-
-                        pkg.update_state()
-
-                    pkg.status = PackageStatus.READY
-                    output.append(pkg)
 
+        if pkgsinfo is None:
+            self._fill_aur_pkgs_offline(aur_pkgs=aur_pkgs, arch_config=arch_config, output=output, disk_loader=disk_loader)
+        elif not pkgsinfo:
+            self.logger.warning("No data found for the supposed installed AUR packages returned from AUR API's info endpoint")
         else:
             editable_pkgbuilds = self._read_editable_pkgbuilds() if arch_config['edit_aur_pkgbuild'] is not False else None
-            for name, data in aur_pkgs.items():
-                pkg = ArchPackage(name=name, version=data.get('version'),
-                                  latest_version=data.get('version'), description=data.get('description'),
-                                  installed=True, repository='aur', i18n=self.i18n)
 
-                pkg.categories = self.categories.get(pkg.name)
+            ignore_rebuild_check = None
+            if rebuild_ignored and rebuild_output is not None:
+                rebuild_ignored.join()
+                ignore_rebuild_check = rebuild_output['ignored']
+
+            to_rebuild = None
+            if rebuild_check and rebuild_output is not None:
+                self.logger.info("Waiting for rebuild-detector")
+                rebuild_check.join()
+                to_rebuild = rebuild_output['to_rebuild']
+
+            for pkgdata in pkgsinfo:
+                pkg = self.aur_mapper.map_api_data(pkgdata, aur_pkgs, self.categories)
                 pkg.pkgbuild_editable = pkg.name in editable_pkgbuilds if editable_pkgbuilds is not None else None
 
-                if disk_loader:
+                if pkg.installed:
-                    disk_loader.fill(pkg)
+                    if disk_loader:
+                        disk_loader.fill(pkg, sync=True)
+
+                    pkg.update = self._check_aur_package_update(pkg=pkg,
+                                                                installed_data=aur_pkgs.get(pkg.name, {}),
+                                                                api_data=pkgdata)
+                    pkg.aur_update = pkg.update  # used in 'set_rebuild_check'
+
+                    if ignore_rebuild_check is not None:
+                        pkg.allow_rebuild = pkg.name not in ignore_rebuild_check
+
+                    if to_rebuild and not pkg.update and pkg.name in to_rebuild:
+                        pkg.require_rebuild = True
+
+                    pkg.update_state()
 
                 pkg.status = PackageStatus.READY
                 output.append(pkg)
@@ -1811,9 +1815,13 @@ class ArchManager(SoftwareManager):
 
         if len(pkgnames) != len(pkg_repos):  # checking if any dep not found in the distro repos are from AUR
             norepos = {p for p in pkgnames if p not in pkg_repos}
-            for pkginfo in self.aur_client.get_info(norepos):
+
-                if pkginfo.get('Name') in norepos:
+            aur_info = self.aur_client.get_info(norepos)
-                    pkg_repos[pkginfo['Name']] = 'aur'
+
+            if aur_info:
+                for pkginfo in aur_info:
+                    if pkginfo.get('Name') in norepos:
+                        pkg_repos[pkginfo['Name']] = 'aur'
 
         return pkg_repos