From e56ecc8f49b06674c9ddbeb67777c8665e2f7155 Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Tue, 24 Sep 2019 12:26:33 -0300 Subject: [PATCH] fix: not internet connection scenario when reading installed and searching --- bauh/api/abstract/controller.py | 3 +- bauh/api/http.py | 6 +++- bauh/commons/internet.py | 16 +++++++++ bauh/gems/arch/controller.py | 47 +++++++++++++++++++------- bauh/gems/arch/model.py | 11 +++---- bauh/gems/arch/worker.py | 15 +++++---- bauh/gems/flatpak/controller.py | 25 +++++++++----- bauh/gems/snap/controller.py | 11 ++++--- bauh/view/core/controller.py | 58 +++++++++++++++++++++------------ bauh/view/qt/thread.py | 24 ++++++++++---- bauh/view/qt/window.py | 10 ++++-- bauh/view/resources/locale/en | 2 +- bauh/view/resources/locale/es | 2 +- bauh/view/resources/locale/pt | 2 +- 14 files changed, 160 insertions(+), 72 deletions(-) create mode 100644 bauh/commons/internet.py diff --git a/bauh/api/abstract/controller.py b/bauh/api/abstract/controller.py index cdaee2bb..abb3d4b6 100644 --- a/bauh/api/abstract/controller.py +++ b/bauh/api/abstract/controller.py @@ -47,12 +47,13 @@ class SoftwareManager(ABC): pass @abstractmethod - def read_installed(self, disk_loader: DiskCacheLoader, limit: int, only_apps: bool, pkg_types: Set[Type[SoftwarePackage]]) -> SearchResult: + def read_installed(self, disk_loader: DiskCacheLoader, limit: int, only_apps: bool, pkg_types: Set[Type[SoftwarePackage]], internet_available: bool) -> SearchResult: """ :param disk_loader: a running disk loader thread that loads application data from the disk asynchronously :param limit: the max number of packages to be retrieved. <= 1 should retrieve everything :param only_apps: if only application packages should be retrieved :param pkg_types: use 'None' to bring any or specify some + :param internet_available: if there is internet connection :return: """ pass diff --git a/bauh/api/http.py b/bauh/api/http.py index 97bbdc1d..22b15e99 100644 --- a/bauh/api/http.py +++ b/bauh/api/http.py @@ -28,7 +28,11 @@ class HttpClient: if self.sleep > 0: time.sleep(self.sleep) - except: + except Exception as e: + if isinstance(e, requests.exceptions.ConnectionError): + self.logger.error('Internet seems to be off') + raise + self.logger.error("Could not retrieve data from '{}'".format(url)) traceback.print_exc() continue diff --git a/bauh/commons/internet.py b/bauh/commons/internet.py new file mode 100644 index 00000000..28239edb --- /dev/null +++ b/bauh/commons/internet.py @@ -0,0 +1,16 @@ +import logging + +import requests + +from bauh.api.http import HttpClient + + +def is_available(client: HttpClient, logger: logging.Logger) -> bool: + try: + client.get('https://google.com') + return True + except requests.exceptions.ConnectionError: + if logger: + logger.warning('Internet connection seems to be off') + return False + diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 9941473b..4009b401 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -7,6 +7,8 @@ from pathlib import Path from threading import Thread from typing import List, Set, Type +import requests + from bauh.api.abstract.controller import SearchResult, SoftwareManager, ApplicationContext from bauh.api.abstract.disk import DiskCacheLoader from bauh.api.abstract.handler import ProcessWatcher @@ -103,19 +105,40 @@ class ArchManager(SoftwareManager): res.total = len(res.installed) + len(res.new) return res - def _fill_aur_pkgs(self, not_signed: dict, apps: list, disk_loader: DiskCacheLoader): + def _fill_aur_pkgs(self, not_signed: dict, pkgs: list, disk_loader: DiskCacheLoader, internet_available: bool): downgrade_enabled = git.is_enabled() - pkgsinfo = self.aur_client.get_info(not_signed.keys()) - if pkgsinfo: - for pkgdata in pkgsinfo: - pkg = self.mapper.map_api_data(pkgdata, not_signed) - pkg.downgrade_enabled = downgrade_enabled - if disk_loader: - disk_loader.fill(pkg) - pkg.status = PackageStatus.READY + if internet_available: + try: + pkgsinfo = self.aur_client.get_info(not_signed.keys()) - apps.append(pkg) + if pkgsinfo: + for pkgdata in pkgsinfo: + pkg = self.mapper.map_api_data(pkgdata, not_signed) + pkg.downgrade_enabled = downgrade_enabled + + if disk_loader: + disk_loader.fill(pkg) + pkg.status = PackageStatus.READY + + pkgs.append(pkg) + + return + except requests.exceptions.ConnectionError: + self.logger.warning('Could not retrieve installed AUR packages API data. It seems the internet connection is off.') + self.logger.info("Reading only local AUR packages data") + + for name, data in not_signed.items(): + pkg = ArchPackage(name=name, version=data.get('version'), + latest_version=data.get('version'), description=data.get('description'), + installed=True, mirror='aur') + pkg.downgrade_enabled = downgrade_enabled + + if disk_loader: + disk_loader.fill(pkg) + pkg.status = PackageStatus.READY + + pkgs.append(pkg) def _fill_mirror_pkgs(self, mirrors: dict, apps: list): # TODO @@ -126,13 +149,13 @@ class ArchManager(SoftwareManager): app.update = False # TODO apps.append(app) - def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None) -> SearchResult: + def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult: installed = pacman.list_and_map_installed() apps = [] if installed and installed['not_signed']: self.dcache_updater.join() - self._fill_aur_pkgs(installed['not_signed'], apps, disk_loader) + self._fill_aur_pkgs(installed['not_signed'], apps, disk_loader, internet_available) return SearchResult(apps, None, len(apps)) diff --git a/bauh/gems/arch/model.py b/bauh/gems/arch/model.py index 49458771..d8a0730c 100644 --- a/bauh/gems/arch/model.py +++ b/bauh/gems/arch/model.py @@ -7,6 +7,8 @@ from bauh.commons import resource from bauh.gems.arch import ROOT_DIR +CACHED_ATTRS = {'command', 'icon_path', 'mirror', 'maintainer', 'desktop_entry'} + class ArchPackage(SoftwarePackage): @@ -77,20 +79,17 @@ class ArchPackage(SoftwarePackage): cache = {} # required attrs to cache - for a in {'command', 'icon_path', 'mirror'}: + for a in CACHED_ATTRS: val = getattr(self, a) if val: cache[a] = val - if self.desktop_entry: - cache['desktop_entry'] = self.desktop_entry - return cache def fill_cached_data(self, data: dict): if data: - for a in {'command', 'icon_path', 'mirror', 'desktop_entry'}: + for a in CACHED_ATTRS: val = data.get(a) if val: setattr(self, a, val) @@ -100,7 +99,7 @@ class ArchPackage(SoftwarePackage): def can_be_run(self) -> bool: # only returns if there is a desktop entry set for the application to avoid running command-line applications - return bool(self.command) and bool(self.desktop_entry) + return bool(self.desktop_entry) and bool(self.command) def get_publisher(self): return self.maintainer diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index cd2a412b..8d091f91 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -31,13 +31,16 @@ class AURIndexUpdater(Thread): def run(self): while True: self.logger.info('Pre-indexing AUR packages in memory') - res = self.http_client.get(URL_INDEX) + try: + res = self.http_client.get(URL_INDEX) - if res and res.text: - self.man.names_index = {n.replace('-', '').replace('_', '').replace('.', ''): n for n in res.text.split('\n') if n and not n.startswith('#')} - self.logger.info('Pre-indexed {} AUR package names in memory'.format(len(self.man.names_index))) - else: - self.logger.warning('No data returned from: {}'.format(URL_INDEX)) + if res and res.text: + self.man.names_index = {n.replace('-', '').replace('_', '').replace('.', ''): n for n in res.text.split('\n') if n and not n.startswith('#')} + self.logger.info('Pre-indexed {} AUR package names in memory'.format(len(self.man.names_index))) + else: + self.logger.warning('No data returned from: {}'.format(URL_INDEX)) + except ConnectionError: + self.logger.warning('No internet connection: could not pre-index packages') time.sleep(5 * 60) # updates every 5 minutes diff --git a/bauh/gems/flatpak/controller.py b/bauh/gems/flatpak/controller.py index 37974116..34e37b81 100644 --- a/bauh/gems/flatpak/controller.py +++ b/bauh/gems/flatpak/controller.py @@ -1,4 +1,3 @@ -import subprocess from datetime import datetime from threading import Thread from typing import List, Set, Type @@ -27,7 +26,7 @@ class FlatpakManager(SoftwareManager): def get_managed_types(self) -> Set["type"]: return {FlatpakApplication} - def _map_to_model(self, app_json: dict, installed: bool, disk_loader: DiskCacheLoader) -> FlatpakApplication: + def _map_to_model(self, app_json: dict, installed: bool, disk_loader: DiskCacheLoader, internet: bool = True) -> FlatpakApplication: app = FlatpakApplication(**app_json) app.installed = installed @@ -39,7 +38,9 @@ class FlatpakManager(SoftwareManager): if not app.runtime: if disk_loader: disk_loader.fill(app) # preloading cached disk data - FlatpakAsyncDataLoader(app=app, api_cache=self.api_cache, manager=self, context=self.context).start() + + if internet: + FlatpakAsyncDataLoader(app=app, api_cache=self.api_cache, manager=self, context=self.context).start() else: app.fill_cached_data(api_data) @@ -73,22 +74,28 @@ class FlatpakManager(SoftwareManager): def _add_updates(self, version: str, output: list): output.append(flatpak.list_updates_as_str(version)) - def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None) -> SearchResult: + def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult: version = flatpak.get_version() updates = [] - thread_updates = Thread(target=self._add_updates, args=(version, updates)) - thread_updates.start() + + if internet_available: + thread_updates = Thread(target=self._add_updates, args=(version, updates)) + thread_updates.start() + else: + thread_updates = None installed = flatpak.list_installed(version) models = [] if installed: - thread_updates.join() + if thread_updates: + thread_updates.join() for app_json in installed: - model = self._map_to_model(app_json=app_json, installed=True, disk_loader=disk_loader) - model.update = app_json['ref'] in updates[0] + model = self._map_to_model(app_json=app_json, installed=True, + disk_loader=disk_loader, internet=internet_available) + model.update = app_json['ref'] in updates[0] if updates else None models.append(model) return SearchResult(models, None, len(models)) diff --git a/bauh/gems/snap/controller.py b/bauh/gems/snap/controller.py index c3603118..70a11c8a 100644 --- a/bauh/gems/snap/controller.py +++ b/bauh/gems/snap/controller.py @@ -8,6 +8,7 @@ from bauh.api.abstract.disk import DiskCacheLoader from bauh.api.abstract.handler import ProcessWatcher from bauh.api.abstract.model import SoftwarePackage, PackageHistory, PackageUpdate, PackageSuggestion, \ SuggestionPriority +from bauh.commons import internet from bauh.commons.system import SystemProcess, ProcessHandler from bauh.gems.snap import snap, suggestions from bauh.gems.snap.constants import SNAP_API_URL @@ -26,7 +27,7 @@ class SnapManager(SoftwareManager): self.http_client = context.http_client self.logger = context.logger - def map_json(self, app_json: dict, installed: bool, disk_loader: DiskCacheLoader) -> SnapApplication: + def map_json(self, app_json: dict, installed: bool, disk_loader: DiskCacheLoader, internet: bool = True) -> SnapApplication: app = SnapApplication(publisher=app_json.get('publisher'), rev=app_json.get('rev'), notes=app_json.get('notes'), @@ -49,7 +50,8 @@ class SnapManager(SoftwareManager): if disk_loader and app.installed: disk_loader.fill(app) - SnapAsyncDataLoader(app=app, api_cache=self.api_cache, manager=self, context=self.context).start() + if internet: + SnapAsyncDataLoader(app=app, api_cache=self.api_cache, manager=self, context=self.context).start() else: app.fill_cached_data(api_data) @@ -79,9 +81,10 @@ class SnapManager(SoftwareManager): else: return SearchResult([], [], 0) - def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None) -> SearchResult: + def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult: if snap.is_snapd_running(): - installed = [self.map_json(app_json, installed=True, disk_loader=disk_loader) for app_json in snap.read_installed()] + internet_available = internet.is_available(self.http_client, self.logger) + installed = [self.map_json(app_json, installed=True, disk_loader=disk_loader, internet=internet_available) for app_json in snap.read_installed()] return SearchResult(installed, None, len(installed)) else: return SearchResult([], None, 0) diff --git a/bauh/view/core/controller.py b/bauh/view/core/controller.py index 80226e70..d867c64c 100755 --- a/bauh/view/core/controller.py +++ b/bauh/view/core/controller.py @@ -8,6 +8,8 @@ from bauh.api.abstract.controller import SoftwareManager, SearchResult, Applicat from bauh.api.abstract.disk import DiskCacheLoader from bauh.api.abstract.handler import ProcessWatcher from bauh.api.abstract.model import SoftwarePackage, PackageUpdate, PackageHistory, PackageSuggestion, PackageAction +from bauh.api.exception import NoInternetException +from bauh.commons import internet SUGGESTIONS_LIMIT = 5 @@ -93,27 +95,30 @@ class GenericSoftwareManager(SoftwareManager): res = SearchResult([], [], 0) - norm_word = word.strip().lower() - disk_loader = self.disk_loader_factory.new() - disk_loader.start() + if internet.is_available(self.context.http_client, self.context.logger): + norm_word = word.strip().lower() + disk_loader = self.disk_loader_factory.new() + disk_loader.start() - threads = [] + threads = [] - for man in self.managers: - t = Thread(target=self._search, args=(norm_word, man, disk_loader, res)) - t.start() - threads.append(t) + for man in self.managers: + t = Thread(target=self._search, args=(norm_word, man, disk_loader, res)) + t.start() + threads.append(t) - for t in threads: - t.join() + for t in threads: + t.join() - if disk_loader: - disk_loader.stop_working() - disk_loader.join() + if disk_loader: + disk_loader.stop_working() + disk_loader.join() - res.installed = self._sort(res.installed, norm_word) - res.new = self._sort(res.new, norm_word) - res.total = len(res.installed) + len(res.new) + res.installed = self._sort(res.installed, norm_word) + res.new = self._sort(res.new, norm_word) + res.total = len(res.installed) + len(res.new) + else: + raise NoInternetException() tf = time.time() self.logger.info('Took {0:.2f} seconds'.format(tf - ti)) @@ -130,10 +135,21 @@ class GenericSoftwareManager(SoftwareManager): def can_work(self) -> bool: return True - def read_installed(self, disk_loader: DiskCacheLoader = None, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None) -> SearchResult: + def _is_internet_available(self, res: dict): + res['available'] = internet.is_available(self.context.http_client, self.context.logger) + + def _get_internet_check(self, res: dict) -> Thread: + t = Thread(target=self._is_internet_available, args=(res,)) + t.start() + return t + + def read_installed(self, disk_loader: DiskCacheLoader = None, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult: ti = time.time() self._wait_to_be_ready() + internet_available = {} + thread_internet_check = self._get_internet_check(internet_available) + res = SearchResult([], None, 0) disk_loader = None @@ -145,8 +161,9 @@ class GenericSoftwareManager(SoftwareManager): disk_loader = self.disk_loader_factory.new() disk_loader.start() + thread_internet_check.join() mti = time.time() - man_res = man.read_installed(disk_loader=disk_loader, pkg_types=None) + man_res = man.read_installed(disk_loader=disk_loader, pkg_types=None, internet_available=internet_available['available']) mtf = time.time() self.logger.info(man.__class__.__name__ + " took {0:.2f} seconds".format(mtf - mti)) @@ -163,8 +180,9 @@ class GenericSoftwareManager(SoftwareManager): disk_loader = self.disk_loader_factory.new() disk_loader.start() + thread_internet_check.join() mti = time.time() - man_res = man.read_installed(disk_loader=disk_loader, pkg_types=None) + man_res = man.read_installed(disk_loader=disk_loader, pkg_types=None, internet_available=internet_available['available']) mtf = time.time() self.logger.info(man.__class__.__name__ + " took {0:.2f} seconds".format(mtf - mti)) @@ -310,7 +328,7 @@ class GenericSoftwareManager(SoftwareManager): suggestions.extend(man_sugs) def list_suggestions(self, limit: int) -> List[PackageSuggestion]: - if self.managers: + if self.managers and internet.is_available(self.context.http_client, self.context.logger): suggestions, threads = [], [] for man in self.managers: t = Thread(target=self._fill_suggestions, args=(suggestions, man, SUGGESTIONS_LIMIT)) diff --git a/bauh/view/qt/thread.py b/bauh/view/qt/thread.py index 677d5746..f7995e5b 100644 --- a/bauh/view/qt/thread.py +++ b/bauh/view/qt/thread.py @@ -1,3 +1,4 @@ +import re import time from datetime import datetime, timedelta from typing import List, Type, Set @@ -14,6 +15,8 @@ from bauh.api.exception import NoInternetException from bauh.view.qt import commons from bauh.view.qt.view_model import PackageView +RE_VERSION_IN_NAME = re.compile(r'\s+version\s+[\w\.]+\s*$') + class AsyncAction(QThread, ProcessWatcher): @@ -87,7 +90,10 @@ class UpdateSelectedApps(AsyncAction): if self.apps_to_update: updated, updated_types = 0, set() for app in self.apps_to_update: - self.change_status('{} {} {}...'.format(self.locale_keys['manage_window.status.upgrading'], app.model.name, app.model.version)) + + name = app.model.name if not RE_VERSION_IN_NAME.findall(app.model.name) else app.model.name.split('version')[0].strip() + + self.change_status('{} {} {}...'.format(self.locale_keys['manage_window.status.upgrading'], name, app.model.version)) success = bool(self.manager.update(app.model, self.root_password, self)) if not success: @@ -223,14 +229,18 @@ class SearchPackages(AsyncAction): self.manager = manager def run(self): - apps_found = [] + search_res = {'pkgs_found': [], 'error': None} if self.word: - res = self.manager.search(self.word) - apps_found.extend(res.installed) - apps_found.extend(res.new) - self.notify_finished(apps_found) - self.word = None + try: + res = self.manager.search(self.word) + search_res['pkgs_found'].extend(res.installed) + search_res['pkgs_found'].extend(res.new) + except NoInternetException: + search_res['error'] = 'internet.required' + finally: + self.notify_finished(search_res) + self.word = None class InstallPackage(AsyncAction): diff --git a/bauh/view/qt/window.py b/bauh/view/qt/window.py index 62165c8e..a377b22f 100755 --- a/bauh/view/qt/window.py +++ b/bauh/view/qt/window.py @@ -846,10 +846,14 @@ class ManageWindow(QWidget): self.thread_search.word = word self.thread_search.start() - def _finish_search(self, pkgs_found: List[SoftwarePackage]): + def _finish_search(self, res: dict): self.finish_action() - self.ref_bt_upgrade.setVisible(False) - self.update_pkgs(pkgs_found, as_installed=False, ignore_updates=True) + + if not res['error']: + self.ref_bt_upgrade.setVisible(False) + self.update_pkgs(res['pkgs_found'], as_installed=False, ignore_updates=True) + else: + dialog.show_message(title=self.i18n['warning'].capitalize(), body=self.i18n[res['error']], type_=MessageType.WARNING) def install(self, pkg: PackageView): pwd = None diff --git a/bauh/view/resources/locale/en b/bauh/view/resources/locale/en index 83244c5f..1c8be86c 100644 --- a/bauh/view/resources/locale/en +++ b/bauh/view/resources/locale/en @@ -1,4 +1,4 @@ -manage_window.title=Applications Manager +manage_window.title=my applications manage_window.columns.latest_version=Latest Version manage_window.columns.update=Upgrade ? manage_window.apps_table.row.actions.info=Information diff --git a/bauh/view/resources/locale/es b/bauh/view/resources/locale/es index 89a0da1e..996be9b5 100644 --- a/bauh/view/resources/locale/es +++ b/bauh/view/resources/locale/es @@ -1,4 +1,4 @@ -manage_window.title=Administrador de Aplicativos +manage_window.title=mis aplicativos manage_window.columns.latest_version=Ultima Versión manage_window.columns.update=Actualizar ? manage_window.columns.installed=Instalado diff --git a/bauh/view/resources/locale/pt b/bauh/view/resources/locale/pt index ddd035ed..3918145d 100644 --- a/bauh/view/resources/locale/pt +++ b/bauh/view/resources/locale/pt @@ -1,4 +1,4 @@ -manage_window.title=Gerenciador de Aplicativos +manage_window.title=meus aplicativos manage_window.columns.latest_version=Última Versão manage_window.columns.update=Atualizar ? manage_window.columns.installed=Instalado