diff --git a/CHANGELOG.md b/CHANGELOG.md index 03c1b544..a43f0337 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [0.9.27] ### Improvements -- code refactoring (String formatting method) +- Arch: + - preventing AUR's installed packages from not being mapped in cases the communication with the API fails + - code refactoring (String formatting method) ### Fixes - Arch diff --git a/bauh/gems/arch/aur.py b/bauh/gems/arch/aur.py index 09129fe8..7a3b62e2 100644 --- a/bauh/gems/arch/aur.py +++ b/bauh/gems/arch/aur.py @@ -112,12 +112,29 @@ class AURClient: def search(self, words: str) -> dict: return self.http_client.get_json(URL_SEARCH + words) - def get_info(self, names: Iterable[str]) -> List[dict]: + def get_info(self, names: Iterable[str]) -> Optional[List[dict]]: try: res = self.http_client.get_json(URL_INFO + self._map_names_as_queries(names)) - return res['results'] if res and res.get('results') else [] - except: - return [] + except requests.exceptions.ConnectionError: + self.logger.warning('Could not retrieve installed AUR packages API data. It seems the internet connection is off.') + return + + if res is None: + self.logger.warning("Call to AUR API's info endpoint has failed") + return + + error = res.get('error') + + if error: + self.logger.warning(f"AUR API's info endpoint returned an unexpected error: {error}") + return + + results = res.get('results') + + if results is not None: + return results + + self.logger.warning(f"AUR API's info endpoint returned an unexpected response: {res}") def map_provided(self, pkgname: str, pkgver: str, provided: Optional[Iterable[str]] = None, strip_epoch: bool = True) -> Set[str]: all_provided = {pkgname, f"{pkgname}={pkgver.split('-')[0] if strip_epoch else pkgver}"} @@ -130,31 +147,34 @@ class AURClient: return all_provided def gen_updates_data(self, names: Iterable[str]) -> Generator[Tuple[str, dict], None, None]: - for package in self.get_info(names): - pkgname, pkgver = package['Name'], package['Version'].split('-')[0] + pkgs_info = self.get_info(names) - deps = set() + if pkgs_info: + for package in pkgs_info: + pkgname, pkgver = package['Name'], package['Version'].split('-')[0] - for dtype in ('Depends', 'MakeDepends', 'CheckDepends'): - dep_set = package.get(dtype) - if dep_set: - deps.update(dep_set) + deps = set() - conflicts = set() - pkg_conflicts = package.get('Conflicts') + for dtype in ('Depends', 'MakeDepends', 'CheckDepends'): + dep_set = package.get(dtype) + if dep_set: + deps.update(dep_set) - if pkg_conflicts: - conflicts.update(pkg_conflicts) + conflicts = set() + pkg_conflicts = package.get('Conflicts') - yield pkgname, { - 'v': pkgver, - 'b': package.get('PackageBase', pkgname), - 'r': 'aur', - 'p': self.map_provided(pkgname=pkgname, pkgver=pkgver, provided=package.get('Provides'), strip_epoch=False), - 'd': deps, - 'c': conflicts, - 'ds': None, - 's': None} + if pkg_conflicts: + conflicts.update(pkg_conflicts) + + yield pkgname, { + 'v': pkgver, + 'b': package.get('PackageBase', pkgname), + 'r': 'aur', + 'p': self.map_provided(pkgname=pkgname, pkgver=pkgver, provided=package.get('Provides'), strip_epoch=False), + 'd': deps, + 'c': conflicts, + 'ds': None, + 's': None} def get_src_info(self, name: str, real_name: Optional[str] = None) -> dict: srcinfo = self.srcinfo_cache.get(name) @@ -215,7 +235,7 @@ class AURClient: return self.extract_required_dependencies(info) def _map_names_as_queries(self, names: Iterable[str]) -> str: - return '&'.join(['arg[]={}'.format(urllib.parse.quote(n)) for n in names]) + return '&'.join((f'arg[]={urllib.parse.quote(n)}' for n in names)) def read_local_index(self) -> dict: self.logger.info('Checking if the cached AUR index file exists') diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 942d631d..8af0c9c8 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -14,7 +14,6 @@ from pwd import getpwnam from threading import Thread from typing import List, Set, Type, Tuple, Dict, Iterable, Optional, Collection, Generator -import requests from dateutil.parser import parse as parse_date from bauh import __app_name__ @@ -412,67 +411,72 @@ class ArchManager(SoftwareManager): res.update_total() return res - def _fill_aur_pkgs(self, aur_pkgs: dict, output: List[ArchPackage], disk_loader: DiskCacheLoader, internet_available: bool, + def _fill_aur_pkgs_offline(self, aur_pkgs: dict, arch_config: dict, output: List[ArchPackage], disk_loader: Optional[DiskCacheLoader]): + self.logger.info("Reading cached data from installed AUR packages") + + editable_pkgbuilds = self._read_editable_pkgbuilds() if arch_config['edit_aur_pkgbuild'] is not False else None + for name, data in aur_pkgs.items(): + pkg = ArchPackage(name=name, version=data.get('version'), + latest_version=data.get('version'), description=data.get('description'), + installed=True, repository='aur', i18n=self.i18n) + + pkg.categories = self.categories.get(pkg.name) + pkg.pkgbuild_editable = pkg.name in editable_pkgbuilds if editable_pkgbuilds is not None else None + + if disk_loader: + disk_loader.fill(pkg) + + pkg.status = PackageStatus.READY + output.append(pkg) + + def _fill_aur_pkgs(self, aur_pkgs: dict, output: List[ArchPackage], disk_loader: Optional[DiskCacheLoader], internet_available: bool, arch_config: dict, rebuild_check: Optional[Thread], rebuild_ignored: Optional[Thread], rebuild_output: Optional[Dict[str, Set[str]]]): - if internet_available: - try: - pkgsinfo = self.aur_client.get_info(aur_pkgs.keys()) - except requests.exceptions.ConnectionError: - self.logger.warning('Could not retrieve installed AUR packages API data. It seems the internet connection is off.') - self.logger.info("Reading only local AUR packages data") - return + if not internet_available: + self._fill_aur_pkgs_offline(aur_pkgs=aur_pkgs, arch_config=arch_config, + output=output, disk_loader=disk_loader) + return - if pkgsinfo: - editable_pkgbuilds = self._read_editable_pkgbuilds() if arch_config['edit_aur_pkgbuild'] is not False else None - - ignore_rebuild_check = None - if rebuild_ignored and rebuild_output is not None: - rebuild_ignored.join() - ignore_rebuild_check = rebuild_output['ignored'] - - to_rebuild = None - if rebuild_check and rebuild_output is not None: - self.logger.info("Waiting for rebuild-detector") - rebuild_check.join() - to_rebuild = rebuild_output['to_rebuild'] - - for pkgdata in pkgsinfo: - pkg = self.aur_mapper.map_api_data(pkgdata, aur_pkgs, self.categories) - pkg.pkgbuild_editable = pkg.name in editable_pkgbuilds if editable_pkgbuilds is not None else None - - if pkg.installed: - if disk_loader: - disk_loader.fill(pkg, sync=True) - - pkg.update = self._check_aur_package_update(pkg=pkg, - installed_data=aur_pkgs.get(pkg.name, {}), - api_data=pkgdata) - pkg.aur_update = pkg.update # used in 'set_rebuild_check' - - if ignore_rebuild_check is not None: - pkg.allow_rebuild = pkg.name not in ignore_rebuild_check - - if to_rebuild and not pkg.update and pkg.name in to_rebuild: - pkg.require_rebuild = True - - pkg.update_state() - - pkg.status = PackageStatus.READY - output.append(pkg) + pkgsinfo = self.aur_client.get_info(aur_pkgs.keys()) + if pkgsinfo is None: + self._fill_aur_pkgs_offline(aur_pkgs=aur_pkgs, arch_config=arch_config, output=output, disk_loader=disk_loader) + elif not pkgsinfo: + self.logger.warning("No data found for the supposed installed AUR packages returned from AUR API's info endpoint") else: editable_pkgbuilds = self._read_editable_pkgbuilds() if arch_config['edit_aur_pkgbuild'] is not False else None - for name, data in aur_pkgs.items(): - pkg = ArchPackage(name=name, version=data.get('version'), - latest_version=data.get('version'), description=data.get('description'), - installed=True, repository='aur', i18n=self.i18n) - pkg.categories = self.categories.get(pkg.name) + ignore_rebuild_check = None + if rebuild_ignored and rebuild_output is not None: + rebuild_ignored.join() + ignore_rebuild_check = rebuild_output['ignored'] + + to_rebuild = None + if rebuild_check and rebuild_output is not None: + self.logger.info("Waiting for rebuild-detector") + rebuild_check.join() + to_rebuild = rebuild_output['to_rebuild'] + + for pkgdata in pkgsinfo: + pkg = self.aur_mapper.map_api_data(pkgdata, aur_pkgs, self.categories) pkg.pkgbuild_editable = pkg.name in editable_pkgbuilds if editable_pkgbuilds is not None else None - if disk_loader: - disk_loader.fill(pkg) + if pkg.installed: + if disk_loader: + disk_loader.fill(pkg, sync=True) + + pkg.update = self._check_aur_package_update(pkg=pkg, + installed_data=aur_pkgs.get(pkg.name, {}), + api_data=pkgdata) + pkg.aur_update = pkg.update # used in 'set_rebuild_check' + + if ignore_rebuild_check is not None: + pkg.allow_rebuild = pkg.name not in ignore_rebuild_check + + if to_rebuild and not pkg.update and pkg.name in to_rebuild: + pkg.require_rebuild = True + + pkg.update_state() pkg.status = PackageStatus.READY output.append(pkg) @@ -1811,9 +1815,13 @@ class ArchManager(SoftwareManager): if len(pkgnames) != len(pkg_repos): # checking if any dep not found in the distro repos are from AUR norepos = {p for p in pkgnames if p not in pkg_repos} - for pkginfo in self.aur_client.get_info(norepos): - if pkginfo.get('Name') in norepos: - pkg_repos[pkginfo['Name']] = 'aur' + + aur_info = self.aur_client.get_info(norepos) + + if aur_info: + for pkginfo in aur_info: + if pkginfo.get('Name') in norepos: + pkg_repos[pkginfo['Name']] = 'aur' return pkg_repos