diff --git a/CHANGELOG.md b/CHANGELOG.md index e1db3adb..d4a38c0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Features - General - new parameter `--suggestions`: forces loading software suggestions after the initialization process [#260](https://github.com/vinifmor/bauh/issues/260) + - allowing custom suggestions / curated software to be mapped by Linux distributions (more on [README.md](https://github.com/vinifmor/bauh#suggestions)) [#260](https://github.com/vinifmor/bauh/issues/260) ### Improvements - Arch @@ -46,6 +47,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Flatpak - not all selected runtime partials to upgrade are actually requested to be upgraded +- Web + - not reading from the cached suggestions file after the first request + - not detecting some generated apps as installed + - UI - double suggestions loading call when no app is returned diff --git a/README.md b/README.md index 5da3ba51..c5d3f9d9 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Key features - [Native Web applications](#type_web) 7. [General settings](#settings) - [Forbidden packaging formats](#forbidden_gems) + - [Custom suggestions / curated software](#suggestions) 8. [Directory structure, caching and logs](#dirs) 9. [Custom themes](#custom_themes) 10. [Tray icons](#tray_icons) @@ -213,7 +214,6 @@ bauh is officially distributed through [PyPi](https://pypi.org/project/bauh) and - Downloaded database files are stored at `~/.cache/bauh/appimage` (or `/var/cache/bauh/appimage` for **root**) as **apps.db** and **releases.db** - Databases are updated during the initialization process if they are considered outdated - The configuration file is located at `~/.config/bauh/appimage.yml` (or `/etc/bauh/appimage.yml` for **root**) and it allows the following customizations: -- Applications with ignored updates are defined at `~/.config/bauh/appimage/updates_ignored.txt` (or `/etc/bauh/appimage/updates_ignored.txt` for **root**) ``` database: @@ -222,6 +222,9 @@ suggestions: expiration: 24 # defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. Default: 24. ``` +- Applications with ignored updates are defined at `~/.config/bauh/appimage/updates_ignored.txt` (or `/etc/bauh/appimage/updates_ignored.txt` for **root**) +- Cached package suggestions: `~/.cache/bauh/web/suggestions.txt` (or `/var/cache/bauh/web/suggestions.yml` for **root**) + ##### Arch packages/AUR @@ -289,6 +292,7 @@ aur_rebuild_detector: true # it checks if packages built with old library versio prefer_repository_provider: true # when there is just one repository provider for a given a dependency and several from AUR, it will be automatically picked. suggestions_exp: 24 # it defines the period (in hours) in which the suggestions stored in disc will be considered up to date. Use 0 if you always want to update them. ``` +- Cached package suggestions: `~/.cache/bauh/arch/suggestions.txt` (or `/var/cache/bauh/arch/suggestions.yml` for **root**) ##### Debian packages - Basic actions supported: **search**, **install**, **uninstall**, **upgrade** @@ -300,7 +304,7 @@ suggestions_exp: 24 # it defines the period (in hours) in which the suggestions - **purge**: removes the packages and all related configuration files - Files: - runnable applications index: `~/.cache/bauh/debian/apps_idx.json` (or `/var/cache/bauh/debian/apps_idx.json` for **root**) - - package suggestions: `~/.cache/bauh/debian/suggestions.txt` (or `/var/cache/bauh/debian/suggestions.txt` for **root**) + - cached package suggestions: `~/.cache/bauh/debian/suggestions.txt` (or `/var/cache/bauh/debian/suggestions.txt` for **root**) - configuration: `~/.config/bauh/debian.yml` or `/etc/bauh/debian.yml` - `index_apps.exp`: time period (**in minutes**) in which the installed applications cache is considered up-to-date during startup (default: `1440` -> 24 hours) - `sync_pkgs.time`: time period (**in minutes**) in which the packages synchronization must be done on startup (default: `1440` -> 24 hours) @@ -317,7 +321,7 @@ installation_level: null # defines a default installation level: "user" or "syst ``` - Custom actions supported: - **Full update**: it completely updates the Flatpak apps and components. Useful if you are having issues with runtime updates. - +- Cached package suggestions: `~/.cache/bauh/flatpak/suggestions.txt` (or `/var/cache/bauh/flatpak/suggestions.txt` for **root**) #### Snap @@ -330,6 +334,7 @@ installation_level: null # defines a default installation level: "user" or "syst install_channel: false # it allows to select an available channel during the application installation. Default: false categories_exp: 24 # It defines the expiration time (in HOURS) of the Snaps categories mapping file stored in disc. Use 0 so that it is always updated during initialization. ``` +- Cached package suggestions: `~/.cache/bauh/snap/suggestions.txt` (or `/var/cache/bauh/snap/suggestions.txt` for **root**) #### Native Web applications @@ -387,7 +392,7 @@ environment: suggestions: cache_exp: 24 # defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated. Default: 24. ``` - +- Cached package suggestions: `~/.cache/bauh/web/suggestions.txt` (or `/var/cache/bauh/web/suggestions.yml` for **root**) #### General settings @@ -463,6 +468,30 @@ appimage # flatpak # 'sharps' can be used to ignore a given line (comment) ``` +##### Custom suggestions / curated software +- The software suggestions are download from [bauh-files](https://github.com/vinifmor/bauh-files) by default + - [appimage](https://github.com/vinifmor/bauh-files/blob/master/appimage/suggestions.txt) + - [arch](https://github.com/vinifmor/bauh-files/blob/master/appimage/suggestions.txt) + - [debian](https://github.com/vinifmor/bauh-files/blob/master/debian/suggestions_v1.txt) + - [flatpak](https://github.com/vinifmor/bauh-files/blob/master/flatpak/suggestions.txt) + - [snap](https://github.com/vinifmor/bauh-files/blob/master/snap/suggestions.txt) + - [web](https://github.com/vinifmor/bauh-files/blob/master/web/env/v2/suggestions.yml) + +- Most of the files follow the pattern: `{priority_number}:${id or name}` + - Priority numbers: 0 (LOW), 1 (MEDIUM), 2 (HIGH), 3 (TOP) + - The priority number is used to sort the retrieved suggestions + +- If Linux distributions want to provide their custom suggestions files: + - Create the file `/etc/bauh/suggestions.conf` + - The content is basically a mapping for each gem to a url or local file (absolute path). + - Example: + ``` + arch=https://mydomain.com/arch/suggestions.txt # remote file + appimage=/etc/bauh/appimage/suggestions.txt # local file (absolute path) + # snap = my mapping # comments with a '#' are allowed + ``` + - If a given gem name is omitted, its suggestions will be downloaded from the default location. + #### Directory structure, caching and logs - `~/.config/bauh` (or `/etc/bauh` for **root**): stores configuration files - `~/.cache/bauh` (or `/var/cache/bauh` for **root**): stores data about your installed applications, databases, indexes, etc. Files are stored here to provide a faster initialization and data recovery. diff --git a/bauh/api/abstract/context.py b/bauh/api/abstract/context.py index 2b41a891..3fe3f901 100644 --- a/bauh/api/abstract/context.py +++ b/bauh/api/abstract/context.py @@ -1,5 +1,6 @@ import logging import sys +from typing import Optional, Dict from bauh.api.abstract.cache import MemoryCacheFactory from bauh.api.abstract.disk import DiskCacheLoaderFactory @@ -15,7 +16,7 @@ class ApplicationContext: cache_factory: MemoryCacheFactory, disk_loader_factory: DiskCacheLoaderFactory, logger: logging.Logger, file_downloader: FileDownloader, distro: str, app_name: str, app_version: str, internet_checker: InternetChecker, root_user: bool, screen_width: int = -1, - screen_height: int = -1): + screen_height: int = -1, suggestions_mapping: Optional[Dict[str, str]] = None): """ :param download_icons: if packages icons should be downloaded :param http_client: a shared instance of http client @@ -31,6 +32,7 @@ class ApplicationContext: :param internet_checker :param screen_width :param screen_height + :param suggestions_mapping """ self.download_icons = download_icons self.http_client = http_client @@ -51,6 +53,7 @@ class ApplicationContext: self.internet_checker = internet_checker self.screen_width = screen_width self.screen_height = screen_height + self._suggestions_mapping = suggestions_mapping def is_system_x86_64(self): return self.arch_x86_64 @@ -60,3 +63,13 @@ class ApplicationContext: def is_internet_available(self) -> bool: return self.internet_checker.is_available() + + def get_suggestion_url(self, module: str, default: Optional[str] = None) -> Optional[str]: + if self._suggestions_mapping: + module_split = module.split(f'{self.app_name}.gems.') + + if len(module_split) > 1: + gem_name = module_split[1].split('.')[0] + return self._suggestions_mapping.get(gem_name, default) + + return default diff --git a/bauh/cli/app.py b/bauh/cli/app.py index 2400d5e6..ad88fb53 100644 --- a/bauh/cli/app.py +++ b/bauh/cli/app.py @@ -52,6 +52,7 @@ def main(): app_name=__app_name__, app_version=__version__, internet_checker=InternetChecker(offline=False), + suggestions_mapping=None, # TODO not needed at the moment root_user=user.is_root()) managers = gems.load_managers(context=context, locale=i18n.current_key, config=app_config, diff --git a/bauh/commons/custom_types.py b/bauh/commons/custom_types.py new file mode 100644 index 00000000..2ee4ef3c --- /dev/null +++ b/bauh/commons/custom_types.py @@ -0,0 +1,20 @@ +from typing import Optional, Any + + +class Value: + + def __init__(self, value: Optional[Any] = None): + self.value = value + + def __repr__(self): + return str(self.value) + + def __str__(self): + return self.__repr__() + + def __eq__(self, other): + if isinstance(other, Value): + return self.value == other.value + + def __hash__(self): + return hash(self.value) diff --git a/bauh/commons/suggestions.py b/bauh/commons/suggestions.py new file mode 100644 index 00000000..c7565e2c --- /dev/null +++ b/bauh/commons/suggestions.py @@ -0,0 +1,35 @@ +from logging import Logger +from typing import Dict, Optional, Tuple + +from bauh.api.abstract.model import SuggestionPriority + + +def parse(suggestions_str: str, logger: Optional[Logger] = None, type_: Optional[str] = None, + splitter: str = '=') \ + -> Dict[str, SuggestionPriority]: + output = dict() + + for line in suggestions_str.split('\n'): + clean_line = line.strip() + + if clean_line: + line_split = clean_line.split(splitter, 1) + + if len(line_split) == 2: + prio_str, name = line_split[0].strip(), line_split[1].strip() + + if prio_str and name: + try: + prio = int(line_split[0]) + except ValueError: + if logger: + logger.warning(f"Could not parse {type_ + ' ' if type_ else ''}suggestion: {line}") + continue + + output[line_split[1]] = SuggestionPriority(prio) + + return output + + +def sort_by_priority(names_prios: Dict[str, SuggestionPriority]) -> Tuple[str, ...]: + return tuple(pair[1] for pair in sorted(((names_prios[n], n) for n in names_prios), reverse=True)) diff --git a/bauh/gems/appimage/__init__.py b/bauh/gems/appimage/__init__.py index 24133d96..4812d5cc 100644 --- a/bauh/gems/appimage/__init__.py +++ b/bauh/gems/appimage/__init__.py @@ -9,7 +9,6 @@ from bauh.commons import resource ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) APPIMAGE_SHARED_DIR = f'{SHARED_FILES_DIR}/appimage' INSTALLATION_DIR = f'{APPIMAGE_SHARED_DIR}/installed' -SUGGESTIONS_FILE = f'https://raw.githubusercontent.com/vinifmor/{__app_name__}-files/master/appimage/suggestions.txt' CONFIG_FILE = f'{CONFIG_DIR}/appimage.yml' APPIMAGE_CONFIG_DIR = f'{CONFIG_DIR}/appimage' UPDATES_IGNORED_FILE = f'{APPIMAGE_CONFIG_DIR}/updates_ignored.txt' @@ -19,8 +18,6 @@ APPIMAGE_CACHE_DIR = f'{CACHE_DIR}/appimage' DATABASE_APPS_FILE = f'{APPIMAGE_CACHE_DIR}/apps.db' DATABASE_RELEASES_FILE = f'{APPIMAGE_CACHE_DIR}/releases.db' DATABASES_TS_FILE = f'{APPIMAGE_CACHE_DIR}/dbs.ts' -SUGGESTIONS_CACHED_FILE = f'{APPIMAGE_CACHE_DIR}/suggestions.txt' -SUGGESTIONS_CACHED_TS_FILE = f'{APPIMAGE_CACHE_DIR}/suggestions.ts' DOWNLOAD_DIR = f'{TEMP_DIR}/appimage/download' diff --git a/bauh/gems/appimage/controller.py b/bauh/gems/appimage/controller.py index 1abc57ae..88ba7f98 100644 --- a/bauh/gems/appimage/controller.py +++ b/bauh/gems/appimage/controller.py @@ -7,7 +7,6 @@ import sqlite3 import subprocess import traceback from datetime import datetime -from math import floor from pathlib import Path from typing import Set, Type, List, Tuple, Optional, Iterable, Generator @@ -83,6 +82,7 @@ class AppImageManager(SoftwareManager, SettingsController): self._action_self_install: Optional[CustomSoftwareAction] = None self._app_github: Optional[str] = None self._search_unfilled_attrs: Optional[Tuple[str, ...]] = None + self._suggestions_downloader: Optional[AppImageSuggestionsDownloader] = None def install_file(self, root_password: Optional[str], watcher: ProcessWatcher) -> bool: max_width = 350 @@ -746,9 +746,11 @@ class AppImageManager(SoftwareManager, SettingsController): create_config=create_config, http_client=self.context.http_client, logger=self.context.logger).start() - AppImageSuggestionsDownloader(taskman=task_manager, i18n=self.context.i18n, - http_client=self.context.http_client, logger=self.context.logger, - create_config=create_config).start() + if not self.suggestions_downloader.is_custom_local_file_mapped(): + self.suggestions_downloader.taskman = task_manager + self.suggestions_downloader.create_config = create_config + self.suggestions_downloader.register_task() + self.suggestions_downloader.start() def list_updates(self, internet_available: bool) -> List[PackageUpdate]: res = self.read_installed(disk_loader=None, internet_available=internet_available) @@ -768,23 +770,24 @@ class AppImageManager(SoftwareManager, SettingsController): return [self.i18n['appimage.warning.missing_db_files'].format(appimage=bold('AppImage'))] def list_suggestions(self, limit: int, filter_installed: bool) -> Optional[List[PackageSuggestion]]: - res = [] + if limit == 0: + return connection = self._get_db_connection(DATABASE_APPS_FILE) if connection: - suggestions = AppImageSuggestionsDownloader(appimage_config=self.configman.get_config(), logger=self.logger, - i18n=self.i18n, http_client=self.http_client, - taskman=TaskManager()).read() + self.suggestions_downloader.taskman = TaskManager() + suggestions = tuple(self.suggestions_downloader.read()) if not suggestions: - self.logger.warning("Could not read suggestions") - return res + self.logger.warning("Could not read AppImage suggestions") + return else: - self.logger.info("Mapping suggestions") + self.logger.info("Mapping AppImage suggestions") try: if filter_installed: - installed = {i.name.lower() for i in self.read_installed(disk_loader=None, connection=connection).installed} + installed = {i.name.lower() for i in self.read_installed(disk_loader=None, + connection=connection).installed} else: installed = None @@ -795,7 +798,7 @@ class AppImageManager(SoftwareManager, SettingsController): name = lsplit[1].strip() - if limit <= 0 or len(sugs_map) < limit: + if limit < 0 or len(sugs_map) < limit: if not installed or not name.lower() in installed: sugs_map[name] = SuggestionPriority(int(lsplit[0])) else: @@ -804,17 +807,18 @@ class AppImageManager(SoftwareManager, SettingsController): cursor = connection.cursor() cursor.execute(query.FIND_APPS_BY_NAME_FULL.format(','.join([f"'{s}'" for s in sugs_map.keys()]))) + res = [] for t in cursor.fetchall(): app = AppImage(*t, i18n=self.i18n) res.append(PackageSuggestion(app, sugs_map[app.name.lower()])) - self.logger.info(f"Mapped {len(res)} suggestions") + + self.logger.info(f"Mapped {len(res)} AppImage suggestions") + return res except: traceback.print_exc() finally: connection.close() - return res - def is_default_enabled(self) -> bool: return True @@ -1080,3 +1084,18 @@ class AppImageManager(SoftwareManager, SettingsController): self._app_github = f'vinifmor/{self.context.app_name}' return self._app_github + + @property + def suggestions_downloader(self) -> AppImageSuggestionsDownloader: + if not self._suggestions_downloader: + file_url = self.context.get_suggestion_url(self.__module__) + self._suggestions_downloader = AppImageSuggestionsDownloader(taskman=TaskManager(), + i18n=self.context.i18n, + http_client=self.context.http_client, + logger=self.context.logger, + file_url=file_url) + + if self._suggestions_downloader.is_custom_local_file_mapped(): + self.logger.info(f"Local AppImage suggestions file mapped: {file_url}") + + return self._suggestions_downloader diff --git a/bauh/gems/appimage/worker.py b/bauh/gems/appimage/worker.py index ea6ede95..95d8a3ae 100644 --- a/bauh/gems/appimage/worker.py +++ b/bauh/gems/appimage/worker.py @@ -8,7 +8,7 @@ import traceback from datetime import datetime, timedelta from pathlib import Path from threading import Thread -from typing import Optional, List +from typing import Optional, Generator import requests @@ -17,8 +17,7 @@ from bauh.api.http import HttpClient from bauh.commons.boot import CreateConfigFile from bauh.commons.html import bold from bauh.gems.appimage import get_icon_path, INSTALLATION_DIR, SYMLINKS_DIR, util, DATABASES_TS_FILE, \ - APPIMAGE_CACHE_DIR, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES, SUGGESTIONS_FILE, \ - SUGGESTIONS_CACHED_TS_FILE, SUGGESTIONS_CACHED_FILE + APPIMAGE_CACHE_DIR, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES from bauh.gems.appimage.model import AppImage from bauh.view.util.translation import I18n @@ -286,7 +285,9 @@ class SymlinksVerifier(Thread): class AppImageSuggestionsDownloader(Thread): - def __init__(self, logger: logging.Logger, http_client: HttpClient, i18n: I18n, taskman: TaskManager, create_config: Optional[CreateConfigFile] = None, appimage_config: Optional[dict] = None): + def __init__(self, logger: logging.Logger, http_client: HttpClient, i18n: I18n, file_url: Optional[str], + create_config: Optional[CreateConfigFile] = None, appimage_config: Optional[dict] = None, + taskman: Optional[TaskManager] = None): super(AppImageSuggestionsDownloader, self).__init__(daemon=True) self.create_config = create_config self.logger = logger @@ -295,133 +296,189 @@ class AppImageSuggestionsDownloader(Thread): self.taskman = taskman self.config = appimage_config self.task_id = 'appim.suggestions' - self.taskman.register_task(id_=self.task_id, label=i18n['task.download_suggestions'], icon_path=get_icon_path()) + self._cached_file_path: Optional[str] = None + self._cached_ts_file_path: Optional[str] = None + + if file_url: + self._file_url = file_url + else: + self._file_url = f'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/suggestions.txt' + + @property + def cached_file_path(self) -> str: + if not self._cached_file_path: + self._cached_file_path = f'{APPIMAGE_CACHE_DIR}/suggestions.txt' + + return self._cached_file_path + + @property + def cached_ts_file_path(self) -> str: + if not self._cached_ts_file_path: + self._cached_ts_file_path = f'{APPIMAGE_CACHE_DIR}/suggestions.ts' + + return self._cached_ts_file_path + + def register_task(self): + self.taskman.register_task(id_=self.task_id, + label=self.i18n['task.download_suggestions'], icon_path=get_icon_path()) + + def is_custom_local_file_mapped(self) -> bool: + return self._file_url and self._file_url.startswith('/') def should_download(self, appimage_config: dict) -> bool: + if not self._file_url: + self.logger.error("No AppImage suggestions file URL defined") + return False + + if self.is_custom_local_file_mapped(): + return False + try: exp_hours = int(appimage_config['suggestions']['expiration']) except: - self.logger.error("An exception happened while trying to parse 'suggestions.expiration'") + self.logger.error("An exception happened while trying to parse the AppImage 'suggestions.expiration'") traceback.print_exc() return True if exp_hours <= 0: - self.logger.info("Suggestions cache is disabled") + self.logger.info("The AppImage suggestions cache is disabled") return True - if not os.path.exists(SUGGESTIONS_CACHED_FILE): - self.logger.info("'{}' not found. It must be downloaded".format(SUGGESTIONS_CACHED_FILE)) + if not os.path.exists(self.cached_file_path): + self.logger.info(f"File {self.cached_file_path} not found. It must be downloaded") return True - if not os.path.exists(SUGGESTIONS_CACHED_TS_FILE): - self.logger.info("'{}' not found. The suggestions file must be downloaded.") + if not os.path.exists(self.cached_ts_file_path): + self.logger.info(f"File {self.cached_ts_file_path} not found. The suggestions file must be downloaded.") return True - with open(SUGGESTIONS_CACHED_TS_FILE) as f: + with open(self.cached_ts_file_path) as f: timestamp_str = f.read() try: suggestions_timestamp = datetime.fromtimestamp(float(timestamp_str)) except: - self.logger.error('Could not parse the cached suggestions timestamp: {}'.format(timestamp_str)) + self.logger.error(f'Could not parse the cached AppImage suggestions timestamp: {timestamp_str}') traceback.print_exc() return True update = suggestions_timestamp + timedelta(hours=exp_hours) <= datetime.utcnow() return update - def read(self) -> Optional[List[str]]: - self.logger.info("Checking if suggestions should be downloaded") + def read(self) -> Generator[str, None, None]: + if not self._file_url: + self.logger.error("No AppImage suggestions file URL defined") + yield from () + + self.logger.info("Checking if AppImage suggestions should be downloaded") if self.should_download(self.config): suggestions_timestamp = datetime.utcnow().timestamp() suggestions_str = self.download() Thread(target=self.cache_suggestions, args=(suggestions_str, suggestions_timestamp), daemon=True).start() else: - self.logger.info("Reading cached suggestions from '{}'".format(SUGGESTIONS_CACHED_FILE)) - with open(SUGGESTIONS_CACHED_FILE) as f: + if self.is_custom_local_file_mapped(): + file_path, log_ref = self._file_url, 'local' + else: + file_path, log_ref = self.cached_file_path, 'cached' + + self.logger.info(f"Reading {log_ref} AppImage suggestions from {file_path}") + with open(file_path) as f: suggestions_str = f.read() - return self.map_suggestions(suggestions_str) if suggestions_str else None + yield from self.map_suggestions(suggestions_str) if suggestions_str else () def cache_suggestions(self, text: str, timestamp: float): - self.logger.info("Caching suggestions to '{}'".format(SUGGESTIONS_FILE)) + self.logger.info(f"Caching AppImage suggestions to {self.cached_file_path}") - cache_dir = os.path.dirname(SUGGESTIONS_CACHED_FILE) + cache_dir = os.path.dirname(self.cached_file_path) try: Path(cache_dir).mkdir(parents=True, exist_ok=True) cache_dir_ok = True except OSError: - self.logger.error("Could not create cache directory '{}'".format(cache_dir)) + self.logger.error(f"Could not create the caching directory {cache_dir}") traceback.print_exc() cache_dir_ok = False if cache_dir_ok: try: - with open(SUGGESTIONS_CACHED_FILE, 'w+') as f: + with open(self.cached_file_path, 'w+') as f: f.write(text) except: - self.logger.error("An exception happened while writing the file '{}'".format(SUGGESTIONS_FILE)) + self.logger.error(f"An exception happened while writing AppImage suggestions to {self.cached_file_path}") traceback.print_exc() try: - with open(SUGGESTIONS_CACHED_TS_FILE, 'w+') as f: + with open(self.cached_ts_file_path, 'w+') as f: f.write(str(timestamp)) except: - self.logger.error("An exception happened while writing the file '{}'".format(SUGGESTIONS_CACHED_TS_FILE)) + self.logger.error(f"An exception happened while writing the cached AppImage suggestions timestamp " + f"to {self.cached_ts_file_path}") traceback.print_exc() def download(self) -> Optional[str]: - self.logger.info("Downloading suggestions from {}".format(SUGGESTIONS_FILE)) + if not self._file_url: + self.logger.error("No AppImage suggestions file URL defined") + return + + if self.is_custom_local_file_mapped(): + self.logger.warning("Local AppImage suggestions file mapped. Nothing will be downloaded") + return + + self.logger.info(f"Downloading AppImage suggestions from {self._file_url}") try: - res = self.http_client.get(SUGGESTIONS_FILE) + res = self.http_client.get(self._file_url) except requests.exceptions.ConnectionError: - self.logger.warning("Could not download suggestion from '{}'".format(SUGGESTIONS_FILE)) + self.logger.warning(f"Could not download suggestion from {self._file_url}") return if not res: - self.logger.warning("Could not download suggestion from '{}'".format(SUGGESTIONS_FILE)) + self.logger.warning(f"Could not download suggestion from {self._file_url}") return if not res.text: - self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_FILE)) + self.logger.warning(f"No AppImage suggestion found in {self._file_url}") return return res.text - def map_suggestions(self, text: str) -> List[str]: - return [line for line in text.split('\n') if line] + def map_suggestions(self, text: str) -> Generator[str, None, None]: + return (line for line in text.split('\n') if line) def run(self): - if self.create_config: - self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name))) - self.create_config.join() - self.config = self.create_config.config - ti = time.time() - self.taskman.update_progress(self.task_id, 1, None) - self.logger.info("Checking if suggestions should be downloaded") - should_download = self.should_download(self.config) - self.taskman.update_progress(self.task_id, 30, None) + if not self.is_custom_local_file_mapped(): + if self.create_config: + wait_msg = self.i18n['task.waiting_task'].format(bold(self.create_config.task_name)) + self.taskman.update_progress(self.task_id, 0, wait_msg) + self.create_config.join() + self.config = self.create_config.config - try: - if should_download: - suggestions_timestamp = datetime.utcnow().timestamp() - suggestions_str = self.download() - self.taskman.update_progress(self.task_id, 70, None) + ti = time.time() + self.taskman.update_progress(self.task_id, 1, None) - if suggestions_str: - self.cache_suggestions(suggestions_str, suggestions_timestamp) - else: - self.logger.info("Cached suggestions are up-to-date") - except: - self.logger.error("An unexpected exception happened") - traceback.print_exc() + self.logger.info("Checking if AppImage suggestions should be downloaded") + should_download = self.should_download(self.config) + self.taskman.update_progress(self.task_id, 30, None) + + try: + if should_download: + suggestions_timestamp = datetime.utcnow().timestamp() + suggestions_str = self.download() + self.taskman.update_progress(self.task_id, 70, None) + + if suggestions_str: + self.cache_suggestions(suggestions_str, suggestions_timestamp) + else: + self.logger.info("Cached AppImage suggestions are up-to-date") + except: + self.logger.error("An unexpected exception happened while downloading AppImage suggestions") + traceback.print_exc() self.taskman.update_progress(self.task_id, 100, None) self.taskman.finish_task(self.task_id) tf = time.time() - self.logger.info("Took {0:.9f} seconds to download suggestions".format(tf - ti)) + self.logger.info(f"Took {tf - ti:.9f} seconds to download suggestions") diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 22a66838..1506e792 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -31,6 +31,7 @@ from bauh.commons import system from bauh.commons.boot import CreateConfigFile from bauh.commons.category import CategoriesDownloader from bauh.commons.html import bold +from bauh.commons.suggestions import sort_by_priority from bauh.commons.system import SystemProcess, ProcessHandler, new_subprocess, run_cmd, SimpleProcess from bauh.commons.util import datetime_as_milis from bauh.commons.view_utils import new_select @@ -2798,9 +2799,10 @@ class ArchManager(SoftwareManager, SettingsController): self.index_aur = AURIndexUpdater(context=self.context, taskman=task_manager, create_config=create_config) # must always execute to properly determine the installed packages (even that AUR is disabled) self.index_aur.start() - self.suggestions_downloader.create_config = create_config - self.suggestions_downloader.register_task(task_manager) - self.suggestions_downloader.start() + if not self.suggestions_downloader.is_custom_local_file_mapped(): + self.suggestions_downloader.create_config = create_config + self.suggestions_downloader.register_task(task_manager) + self.suggestions_downloader.start() refresh_mirrors = RefreshMirrors(taskman=task_manager, i18n=self.i18n, root_password=root_password, logger=self.logger, create_config=create_config) @@ -3727,7 +3729,7 @@ class ArchManager(SoftwareManager, SettingsController): def list_suggestions(self, limit: int, filter_installed: bool) -> Optional[List[PackageSuggestion]]: if limit == 0: - return [] + return arch_config = self.configman.get_config() @@ -3751,7 +3753,7 @@ class ArchManager(SoftwareManager, SettingsController): if not name_priority: self.logger.info("No Arch package suggestions found") - return [] + return self.logger.info(f"Found {len(name_priority)} named Arch package suggestions") @@ -3760,7 +3762,7 @@ class ArchManager(SoftwareManager, SettingsController): if not available_packages: self.logger.error("No available Arch package found. It will not be possible to return suggestions") - return [] + return fill_ignored.join() @@ -3775,7 +3777,7 @@ class ArchManager(SoftwareManager, SettingsController): if not available_suggestions: self.logger.info("No Arch package suggestion to return") - return [] + return if filter_installed: ignored_updates = set() @@ -3784,11 +3786,9 @@ class ArchManager(SoftwareManager, SettingsController): else: ignored_updates, thread_fill_ignored_updates = None, None - # sorting by priority - suggestion_by_priority = tuple(pair[1] for pair in sorted(((name_priority[n], n) for n in available_suggestions), - reverse=True)) + suggestion_by_priority = sort_by_priority({n: name_priority[n] for n in available_suggestions}) - if 0 < limit < len(available_suggestions): + if available_suggestions and 0 < limit < len(available_suggestions): suggestion_by_priority = suggestion_by_priority[0:limit] self.logger.info(f'Available Arch package suggestions: {len(suggestion_by_priority)}') @@ -3844,8 +3844,14 @@ class ArchManager(SoftwareManager, SettingsController): @property def suggestions_downloader(self) -> RepositorySuggestionsDownloader: if not self._suggestions_downloader: + file_url = self.context.get_suggestion_url(self.__module__) + self._suggestions_downloader = RepositorySuggestionsDownloader(logger=self.logger, http_client=self.http_client, - i18n=self.i18n) + i18n=self.i18n, + file_url=file_url) + + if self._suggestions_downloader.is_custom_local_file_mapped(): + self.logger.info(f"Local Arch suggestions file mapped: {file_url}") return self._suggestions_downloader diff --git a/bauh/gems/arch/suggestions.py b/bauh/gems/arch/suggestions.py index 731c92c9..b87f9acf 100644 --- a/bauh/gems/arch/suggestions.py +++ b/bauh/gems/arch/suggestions.py @@ -12,13 +12,13 @@ from bauh.api.http import HttpClient from bauh.commons.boot import CreateConfigFile from bauh.gems.arch import ARCH_CACHE_DIR, get_icon_path from bauh.view.util.translation import I18n +from bauh.commons.suggestions import parse class RepositorySuggestionsDownloader(Thread): _file_suggestions: Optional[str] = None _file_suggestions_ts: Optional[str] = None - _url_suggestions: Optional[str] = None @classmethod def file_suggestions(cls) -> str: @@ -34,22 +34,16 @@ class RepositorySuggestionsDownloader(Thread): return cls._file_suggestions_ts - @classmethod - def url_suggestions(cls) -> str: - if cls._url_suggestions is None: - cls._url_suggestions = 'https://raw.githubusercontent.com/vinifmor/bauh-files' \ - '/master/arch/suggestions.txt' - - return cls._url_suggestions - def __init__(self, logger: Logger, http_client: HttpClient, i18n: I18n, - create_config: Optional[CreateConfigFile] = None): + create_config: Optional[CreateConfigFile] = None, file_url: Optional[str] = None): super(RepositorySuggestionsDownloader, self).__init__() self._log = logger self.i18n = i18n self.http_client = http_client self._taskman: Optional[TaskManager] = None self.create_config = create_config + self._file_url = file_url if file_url else 'https://raw.githubusercontent.com/vinifmor/bauh-files' \ + '/master/arch/suggestions.txt' self.task_id = 'arch.suggs' def register_task(self, taskman: Optional[TaskManager]): @@ -65,38 +59,50 @@ class RepositorySuggestionsDownloader(Thread): return self._taskman - @classmethod - def should_download(cls, arch_config: dict, logger: Logger, only_positive_exp: bool = False) -> bool: + def should_download(self, arch_config: dict, only_positive_exp: bool = False) -> bool: + if not self._file_url: + self._log.error("No Arch suggestions file URL defined") + return False + + if self._file_url.startswith('/'): + return False + try: exp_hours = int(arch_config['suggestions_exp']) except ValueError: - logger.error(f"The Arch configuration property 'suggestions_exp' has a non int value set: " - f"{arch_config['suggestions']['expiration']}") + self._log.error(f"The Arch configuration property 'suggestions_exp' has a non int value set: " + f"{arch_config['suggestions']['expiration']}") return not only_positive_exp if exp_hours <= 0: - logger.info("Suggestions cache is disabled") + self._log.info("Suggestions cache is disabled") return not only_positive_exp - if not os.path.exists(cls.file_suggestions()): - logger.info(f"'{cls.file_suggestions()}' not found. It must be downloaded") + if not os.path.exists(self.file_suggestions()): + self._log.info(f"'{self.file_suggestions()}' not found. It must be downloaded") return True - if not os.path.exists(cls.file_suggestions()): - logger.info(f"'{cls.file_suggestions()}' not found. The suggestions file must be downloaded.") + if not os.path.exists(self.file_suggestions_timestamp()): + self._log.info(f"'{self.file_suggestions()}' not found. The suggestions file must be downloaded.") return True - with open(cls.file_suggestions_timestamp()) as f: + with open(self.file_suggestions_timestamp()) as f: timestamp_str = f.read() try: suggestions_timestamp = datetime.fromtimestamp(float(timestamp_str)) except: - logger.error(f'Could not parse the Arch cached suggestions timestamp: {timestamp_str}') + self._log.error(f'Could not parse the Arch cached suggestions timestamp: {timestamp_str}') traceback.print_exc() return True update = suggestions_timestamp + timedelta(hours=exp_hours) <= datetime.utcnow() + + if update: + self._log.info("The cached suggestions file is no longer valid") + else: + self._log.info("The cached suggestions file is up-to-date") + return update def _save(self, text: str, timestamp: float): @@ -127,56 +133,42 @@ class RepositorySuggestionsDownloader(Thread): self._log.error(f"An exception happened while writing the file '{self.file_suggestions_timestamp()}'") traceback.print_exc() - def parse_suggestions(self, suggestions_str: str) -> Dict[str, SuggestionPriority]: - output = dict() - for line in suggestions_str.split('\n'): - clean_line = line.strip() + def read_cached(self, custom_file: Optional[str] = None) -> Optional[Dict[str, SuggestionPriority]]: + if custom_file: + file_path, log_ref = custom_file, 'local' + else: + file_path, log_ref = self.file_suggestions(), 'cached' - if clean_line: - line_split = clean_line.split(':', 1) - - if len(line_split) == 2: - try: - prio = int(line_split[0]) - except ValueError: - self._log.warning(f"Could not parse Arch package suggestion: {line}") - continue - - output[line_split[1]] = SuggestionPriority(prio) - - return output - - def read_cached(self) -> Optional[Dict[str, SuggestionPriority]]: - self._log.info(f"Reading cached suggestions file '{self.file_suggestions()}'") + self._log.info(f"Reading {log_ref} Arch suggestions file '{file_path}'") try: - with open(self.file_suggestions()) as f: + with open(file_path) as f: sugs_str = f.read() except FileNotFoundError: - self._log.warning(f"Cached suggestions file does not exist ({self.file_suggestions()})") + self._log.warning(f"{log_ref.capitalize()} suggestions file does not exist ({file_path})") return if not sugs_str: - self._log.warning(f"Cached suggestions file '{self.file_suggestions()}' is empty") + self._log.warning(f"{log_ref.capitalize()} suggestions file '{file_path}' is empty") return - return self.parse_suggestions(sugs_str) + return parse(sugs_str, self._log, 'Arch') def download(self) -> Optional[Dict[str, SuggestionPriority]]: self.taskman.update_progress(self.task_id, progress=1, substatus=None) - self._log.info(f"Downloading suggestions from {self.url_suggestions()}") - res = self.http_client.get(self.url_suggestions()) + self._log.info(f"Downloading suggestions from {self._file_url}") + res = self.http_client.get(self._file_url) suggestions = None if res.status_code == 200 and res.text: self.taskman.update_progress(self.task_id, progress=50, substatus=None) - suggestions = self.parse_suggestions(res.text) + suggestions = parse(res.text, self._log, 'Arch') if suggestions: self._save(text=res.text, timestamp=datetime.utcnow().timestamp()) else: - self._log.warning("No Arch suggestions to cache") + self._log.warning(f"Could not parse any Arch suggestion from {self._file_suggestions_ts}") else: self._log.warning(f"Could not retrieve Arch suggestions. " f"Response (status={res.status_code}, text={res.text})") @@ -186,10 +178,17 @@ class RepositorySuggestionsDownloader(Thread): return suggestions def read(self, arch_config: dict) -> Optional[Dict[str, int]]: - if self.should_download(arch_config=arch_config, logger=self._log): - return self.download() + if self._file_url: + if self.is_custom_local_file_mapped(): + return self.read_cached(custom_file=self._file_url) - return self.read_cached() + if self.should_download(arch_config=arch_config): + return self.download() + + return self.read_cached() + + def is_custom_local_file_mapped(self) -> bool: + return self._file_url and self._file_url.startswith('/') def run(self): if self.create_config: @@ -198,8 +197,7 @@ class RepositorySuggestionsDownloader(Thread): self.i18n['task.waiting_task'].format(self.create_config.task_name)) self.create_config.join() - if not self.should_download(arch_config=self.create_config.config, logger=self._log, - only_positive_exp=False): + if not self.should_download(arch_config=self.create_config.config, only_positive_exp=False): self.taskman.update_progress(self.task_id, 100, self.i18n['task.canceled']) self.taskman.finish_task(self.task_id) return diff --git a/bauh/gems/debian/controller.py b/bauh/gems/debian/controller.py index 8bc8fc64..3e8dbc49 100644 --- a/bauh/gems/debian/controller.py +++ b/bauh/gems/debian/controller.py @@ -508,7 +508,7 @@ class DebianPackageManager(SoftwareManager, SettingsController): root_password=root_password, aptitude=self.aptitude) sync_pkgs.start() - if DebianSuggestionsDownloader.should_download(deb_config, self._log, only_positive_exp=True): + if self.suggestions_downloader.should_download(deb_config, only_positive_exp=True): self.suggestions_downloader.register_task(task_manager) self.suggestions_downloader.start() @@ -887,8 +887,13 @@ class DebianPackageManager(SoftwareManager, SettingsController): @property def suggestions_downloader(self) -> DebianSuggestionsDownloader: if not self._suggestions_downloader: + file_url = self.context.get_suggestion_url(self.__module__) self._suggestions_downloader = DebianSuggestionsDownloader(i18n=self._i18n, logger=self._log, - http_client=self.context.http_client) + http_client=self.context.http_client, + file_url=file_url) + + if self._suggestions_downloader.is_local_suggestions_file(): + self._log.info(f"Local Debian suggestions file mapped: {file_url}") return self._suggestions_downloader diff --git a/bauh/gems/debian/suggestions.py b/bauh/gems/debian/suggestions.py index da361ebf..878a6a54 100644 --- a/bauh/gems/debian/suggestions.py +++ b/bauh/gems/debian/suggestions.py @@ -9,6 +9,7 @@ from typing import Optional, Dict from bauh.api.abstract.handler import TaskManager from bauh.api.abstract.model import SuggestionPriority from bauh.api.http import HttpClient +from bauh.commons.suggestions import parse from bauh.gems.debian import DEBIAN_ICON_PATH, DEBIAN_CACHE_DIR from bauh.view.util.translation import I18n @@ -33,20 +34,18 @@ class DebianSuggestionsDownloader(Thread): return cls._file_suggestions_ts - @classmethod - def url_suggestions(cls) -> str: - if cls._url_suggestions is None: - cls._url_suggestions = 'https://raw.githubusercontent.com/vinifmor/bauh-files' \ - '/master/debian/suggestions.txt' - - return cls._url_suggestions - - def __init__(self, logger: Logger, http_client: HttpClient, i18n: I18n): + def __init__(self, logger: Logger, http_client: HttpClient, i18n: I18n, file_url: Optional[str]): super(DebianSuggestionsDownloader, self).__init__() self._log = logger self.i18n = i18n self.http_client = http_client self._taskman: Optional[TaskManager] = None + + if file_url: + self._file_url = file_url + else: + self._file_url = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/debian/suggestions_v1.txt' + self.task_id = 'debian.suggs' def register_task(self, taskman: Optional[TaskManager]): @@ -55,6 +54,9 @@ class DebianSuggestionsDownloader(Thread): self._taskman.register_task(id_=self.task_id, label=self.i18n['task.download_suggestions'], icon_path=DEBIAN_ICON_PATH) + def is_local_suggestions_file(self) -> bool: + return self._file_url and self._file_url.startswith('/') + @property def taskman(self) -> TaskManager: if self._taskman is None: @@ -62,34 +64,40 @@ class DebianSuggestionsDownloader(Thread): return self._taskman - @classmethod - def should_download(cls, debian_config: dict, logger: Logger, only_positive_exp: bool = False) -> bool: + def should_download(self, debian_config: dict, only_positive_exp: bool = False) -> bool: + if not self._file_url: + self._log.error("No Debian suggestions file URL defined") + return False + + if self.is_local_suggestions_file(): + return False + try: exp_hours = int(debian_config['suggestions.exp']) except ValueError: - logger.error(f"The Debian configuration property 'suggestions.expiration' has a non int value set: " - f"{debian_config['suggestions']['expiration']}") + self._log.error(f"The Debian configuration property 'suggestions.expiration' has a non int value set: " + f"{debian_config['suggestions']['expiration']}") return not only_positive_exp if exp_hours <= 0: - logger.info("Suggestions cache is disabled") + self._log.info("Suggestions cache is disabled") return not only_positive_exp - if not os.path.exists(cls.file_suggestions()): - logger.info(f"'{cls.file_suggestions()}' not found. It must be downloaded") + if not os.path.exists(self.file_suggestions()): + self._log.info(f"'{self.file_suggestions()}' not found. It must be downloaded") return True - if not os.path.exists(cls.file_suggestions()): - logger.info(f"'{cls.file_suggestions()}' not found. The suggestions file must be downloaded.") + if not os.path.exists(self.file_suggestions()): + self._log.info(f"'{self.file_suggestions()}' not found. The suggestions file must be downloaded.") return True - with open(cls.file_suggestions_timestamp()) as f: + with open(self.file_suggestions_timestamp()) as f: timestamp_str = f.read() try: suggestions_timestamp = datetime.fromtimestamp(float(timestamp_str)) except: - logger.error(f'Could not parse the Debian cached suggestions timestamp: {timestamp_str}') + self._log.error(f'Could not parse the Debian cached suggestions timestamp: {timestamp_str}') traceback.print_exc() return True @@ -97,6 +105,13 @@ class DebianSuggestionsDownloader(Thread): return update def _save(self, text: str, timestamp: float): + if not self._file_url: + self._log.error("No Debian suggestions file URL defined") + return + + if self.is_local_suggestions_file(): + return False + self._log.info(f"Caching suggestions to '{self.file_suggestions()}'") cache_dir = os.path.dirname(self.file_suggestions()) @@ -124,51 +139,49 @@ class DebianSuggestionsDownloader(Thread): self._log.error(f"An exception happened while writing the file '{self.file_suggestions_timestamp()}'") traceback.print_exc() - def parse_suggestions(self, suggestions_str: str) -> Dict[str, SuggestionPriority]: - output = dict() - for line in suggestions_str.split('\n'): - clean_line = line.strip() - - if clean_line: - line_split = clean_line.split(':', 1) - - if len(line_split) == 2: - try: - prio = int(line_split[0]) - except ValueError: - self._log.warning(f"Could not parse Debian package suggestion: {line}") - continue - - output[line_split[1]] = SuggestionPriority(prio) - - return output - def read_cached(self) -> Optional[Dict[str, SuggestionPriority]]: - self._log.info(f"Reading cached suggestions file '{self.file_suggestions()}'") + if not self._file_url: + self._log.error("No Debian suggestions file URL defined") + return + + if self.is_local_suggestions_file(): + file_path, log_ref = self._file_url, 'local' + else: + file_path, log_ref = self.file_suggestions(), 'cached' + + self._log.info(f"Reading {log_ref} suggestions file {file_path}") try: - with open(self.file_suggestions()) as f: + with open(file_path) as f: sugs_str = f.read() except FileNotFoundError: - self._log.warning(f"Cached suggestions file does not exist ({self.file_suggestions()})") + self._log.warning(f"The {log_ref} suggestions file does not exist ({file_path})") + return + except OSError: + self._log.warning(f"Could not read from the {log_ref} suggestions file ({file_path})") + traceback.print_exc() return if not sugs_str: - self._log.warning(f"Cached suggestions file '{self.file_suggestions()}' is empty") + self._log.warning(f"The {log_ref} suggestions file '{file_path}' is empty") return - return self.parse_suggestions(sugs_str) + return parse(sugs_str, self._log, 'Debian') def download(self) -> Optional[Dict[str, SuggestionPriority]]: + if not self._file_url: + self._log.error("No Debian suggestions file URL defined") + return + self.taskman.update_progress(self.task_id, progress=1, substatus=None) - self._log.info(f"Downloading suggestions from {self.url_suggestions()}") - res = self.http_client.get(self.url_suggestions()) + self._log.info(f"Downloading Debian suggestions from {self._file_url}") + res = self.http_client.get(self._file_url) suggestions = None if res.status_code == 200 and res.text: self.taskman.update_progress(self.task_id, progress=50, substatus=None) - suggestions = self.parse_suggestions(res.text) + suggestions = parse(res.text, self._log, 'Debian') if suggestions: self._save(text=res.text, timestamp=datetime.utcnow().timestamp()) @@ -183,10 +196,14 @@ class DebianSuggestionsDownloader(Thread): return suggestions def read(self, debian_config: dict) -> Optional[Dict[str, int]]: - if self.should_download(debian_config=debian_config, logger=self._log): - return self.download() + if not self._file_url: + self._log.error("No Debian suggestions file URL defined") + return - return self.read_cached() + if self.is_local_suggestions_file() or not self.should_download(debian_config=debian_config): + return self.read_cached() + + return self.download() def run(self): self.download() diff --git a/bauh/gems/flatpak/__init__.py b/bauh/gems/flatpak/__init__.py index a775678b..a577ba88 100644 --- a/bauh/gems/flatpak/__init__.py +++ b/bauh/gems/flatpak/__init__.py @@ -9,7 +9,6 @@ from bauh.api.paths import CONFIG_DIR from bauh.commons import resource ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) -SUGGESTIONS_FILE = f'https://raw.githubusercontent.com/vinifmor/{__app_name__}-files/master/flatpak/suggestions.txt' CONFIG_FILE = f'{CONFIG_DIR}/flatpak.yml' FLATPAK_CONFIG_DIR = f'{CONFIG_DIR}/flatpak' UPDATES_IGNORED_FILE = f'{FLATPAK_CONFIG_DIR}/updates_ignored.txt' diff --git a/bauh/gems/flatpak/controller.py b/bauh/gems/flatpak/controller.py index 674aebf0..a2cc9112 100644 --- a/bauh/gems/flatpak/controller.py +++ b/bauh/gems/flatpak/controller.py @@ -2,7 +2,6 @@ import os import re import traceback from datetime import datetime -from math import floor from operator import attrgetter from pathlib import Path from threading import Thread @@ -16,13 +15,14 @@ from bauh.api.abstract.controller import SearchResult, SoftwareManager, Applicat from bauh.api.abstract.disk import DiskCacheLoader from bauh.api.abstract.handler import ProcessWatcher, TaskManager from bauh.api.abstract.model import PackageHistory, PackageUpdate, SoftwarePackage, PackageSuggestion, \ - SuggestionPriority, PackageStatus, CustomSoftwareAction + PackageStatus, CustomSoftwareAction, SuggestionPriority from bauh.api.abstract.view import MessageType, FormComponent, SingleSelectComponent, InputOption, SelectViewType, \ PanelComponent, ViewComponentAlignment +from bauh.commons import suggestions from bauh.commons.boot import CreateConfigFile from bauh.commons.html import strip_html, bold from bauh.commons.system import ProcessHandler -from bauh.gems.flatpak import flatpak, SUGGESTIONS_FILE, CONFIG_FILE, UPDATES_IGNORED_FILE, FLATPAK_CONFIG_DIR, \ +from bauh.gems.flatpak import flatpak, CONFIG_FILE, UPDATES_IGNORED_FILE, FLATPAK_CONFIG_DIR, \ EXPORTS_PATH, \ get_icon_path, VERSION_1_5, VERSION_1_2, VERSION_1_12 from bauh.gems.flatpak.config import FlatpakConfigManager @@ -48,6 +48,7 @@ class FlatpakManager(SoftwareManager, SettingsController): self.logger = context.logger self.configman = FlatpakConfigManager() self._action_full_update: Optional[CustomSoftwareAction] = None + self._suggestions_file_url: Optional[str] = None def get_managed_types(self) -> Set["type"]: return {FlatpakApplication} @@ -583,47 +584,99 @@ class FlatpakManager(SoftwareManager, SettingsController): def list_warnings(self, internet_available: bool) -> Optional[List[str]]: pass - def list_suggestions(self, limit: int, filter_installed: bool) -> Optional[List[PackageSuggestion]]: - cli_version = flatpak.get_version() - res = [] + def _read_local_suggestions_file(self) -> Optional[str]: + try: + with open(self.suggestions_file_url) as f: + suggestions_str = f.read() - self.logger.info("Downloading the suggestions file {}".format(SUGGESTIONS_FILE)) - file = self.http_client.get(SUGGESTIONS_FILE) + return suggestions_str + except FileNotFoundError: + self.logger.error(f"Local Flatpak suggestions file not found: {self.suggestions_file_url}") + except OSError: + self.logger.error(f"Could not read local Flatpak suggestions file: {self.suggestions_file_url}") + traceback.print_exc() - if not file or not file.text: - self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_FILE)) - return res + def _download_remote_suggestions_file(self) -> Optional[str]: + self.logger.info(f"Downloading the Flatpak suggestions from {self.suggestions_file_url}") + file = self.http_client.get(self.suggestions_file_url) + + if file: + return file.text + + def _fill_suggestion(self, appid: str, priority: SuggestionPriority, flatpak_version: Version, remote: str, + output: List[PackageSuggestion]): + app_json = flatpak.search(flatpak_version, appid, remote, app_id=True) + + if app_json: + model = PackageSuggestion(self._map_to_model(app_json[0], False, None)[0], priority) + self.suggestions_cache.add(appid, model) + output.append(model) else: - self.logger.info("Mapping suggestions") - remote_level = self._get_search_remote() - installed = {i.id for i in self.read_installed(disk_loader=None).installed} if filter_installed else None + self.logger.warning(f"Could not find Flatpak suggestions '{appid}'") - for line in file.text.split('\n'): - if line: - if limit <= 0 or len(res) < limit: - sug = line.split('=') - appid = sug[1].strip() + def list_suggestions(self, limit: int, filter_installed: bool) -> Optional[List[PackageSuggestion]]: + if limit == 0: + return - if installed and appid in installed: - continue + if self.is_local_suggestions_file_mapped(): + suggestions_str = self._read_local_suggestions_file() + else: + suggestions_str = self._download_remote_suggestions_file() - priority = SuggestionPriority(int(sug[0])) + if suggestions_str is None: + return - cached_sug = self.suggestions_cache.get(appid) + if not suggestions_str: + self.logger.warning(f"No Flatpak suggestion found in {self.suggestions_file_url}") + return - if cached_sug: - res.append(cached_sug) - else: - app_json = flatpak.search(cli_version, appid, remote_level, app_id=True) + ids_prios = suggestions.parse(suggestions_str, self.logger, 'Flatpak') - if app_json: - model = PackageSuggestion(self._map_to_model(app_json[0], False, None)[0], priority) - self.suggestions_cache.add(appid, model) - res.append(model) - else: - break + if not ids_prios: + self.logger.warning(f"No Flatpak suggestion could be parsed from {self.suggestions_file_url}") + return + + suggestion_by_priority = suggestions.sort_by_priority(ids_prios) + + if filter_installed: + installed = {i.id for i in self.read_installed(disk_loader=None).installed} + + if installed: + suggestion_by_priority = tuple(id_ for id_ in suggestion_by_priority if id_ not in installed) + + if suggestion_by_priority and 0 < limit < len(suggestion_by_priority): + suggestion_by_priority = suggestion_by_priority[0:limit] + + self.logger.info(f'Available Flatpak suggestions: {len(suggestion_by_priority)}') + + if not suggestion_by_priority: + return + + flatpak_version = flatpak.get_version() + remote = self._get_search_remote() + + self.logger.info("Mapping Flatpak suggestions") + res, fill_suggestions = [], [] + cached_count = 0 + + for appid in suggestion_by_priority: + cached_instance = self.suggestions_cache.get(appid) + + if cached_instance: + res.append(cached_instance) + cached_count += 1 + else: + fill = Thread(target=self._fill_suggestion, args=(appid, ids_prios[appid], flatpak_version, + remote, res)) + fill.start() + fill_suggestions.append(fill) + + for fill in fill_suggestions: + fill.join() + + if cached_count > 0: + self.logger.info(f"Returning {cached_count} cached Flatpak suggestions") - res.sort(key=lambda s: s.priority.value, reverse=True) return res def is_default_enabled(self) -> bool: @@ -796,3 +849,21 @@ class FlatpakManager(SoftwareManager, SettingsController): requires_root=False) return self._action_full_update + + @property + def suggestions_file_url(self) -> str: + if self._suggestions_file_url is None: + file_url = self.context.get_suggestion_url(self.__module__) + + if not file_url: + file_url = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/flatpak/suggestions.txt' + + self._suggestions_file_url = file_url + + if file_url.startswith('/'): + self.logger.info(f"Local Flatpak suggestions file mapped: {file_url}") + + return self._suggestions_file_url + + def is_local_suggestions_file_mapped(self) -> bool: + return self.suggestions_file_url.startswith('/') diff --git a/bauh/gems/snap/__init__.py b/bauh/gems/snap/__init__.py index 582ce071..978c1d33 100644 --- a/bauh/gems/snap/__init__.py +++ b/bauh/gems/snap/__init__.py @@ -8,7 +8,6 @@ SNAP_CACHE_DIR = f'{CACHE_DIR}/snap' CONFIG_FILE = f'{CONFIG_DIR}/snap.yml' CATEGORIES_FILE_PATH = f'{SNAP_CACHE_DIR}/categories.txt' URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/categories.txt' -SUGGESTIONS_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/suggestions.txt' def get_icon_path() -> str: diff --git a/bauh/gems/snap/controller.py b/bauh/gems/snap/controller.py index 02d47f83..7cabfd17 100644 --- a/bauh/gems/snap/controller.py +++ b/bauh/gems/snap/controller.py @@ -13,12 +13,13 @@ from bauh.api.abstract.model import SoftwarePackage, PackageHistory, PackageUpda from bauh.api.abstract.view import SingleSelectComponent, SelectViewType, InputOption, PanelComponent, \ FormComponent, TextInputComponent from bauh.api.exception import NoInternetException +from bauh.commons import suggestions from bauh.commons.boot import CreateConfigFile from bauh.commons.category import CategoriesDownloader from bauh.commons.html import bold from bauh.commons.system import SystemProcess, ProcessHandler, new_root_subprocess from bauh.commons.view_utils import new_select, get_human_size_str -from bauh.gems.snap import snap, URL_CATEGORIES_FILE, CATEGORIES_FILE_PATH, SUGGESTIONS_FILE, \ +from bauh.gems.snap import snap, URL_CATEGORIES_FILE, CATEGORIES_FILE_PATH, \ get_icon_path, snapd from bauh.gems.snap.config import SnapConfigManager from bauh.gems.snap.model import SnapApplication @@ -42,6 +43,7 @@ class SnapManager(SoftwareManager, SettingsController): self.suggestions_cache = context.cache_factory.new() self.info_path = None self.configman = SnapConfigManager() + self._suggestions_url: Optional[str] = None def _fill_categories(self, app: SnapApplication): categories = self.categories.get(app.name.lower()) @@ -315,7 +317,8 @@ class SnapManager(SoftwareManager, SettingsController): self.logger.warning(f'It seems Snap API is not available. Search output: {output}') return [self.i18n['snap.notifications.api.unavailable'].format(bold('Snaps'), bold('Snap'))] - def _fill_suggestion(self, name: str, priority: SuggestionPriority, snapd_client: SnapdClient, out: List[PackageSuggestion]): + def _fill_suggestion(self, name: str, priority: SuggestionPriority, snapd_client: SnapdClient, + out: List[PackageSuggestion]): res = snapd_client.find_by_name(name) if res: @@ -361,46 +364,87 @@ class SnapManager(SoftwareManager, SettingsController): app.status = PackageStatus.READY return app + def _read_local_suggestions_file(self) -> Optional[str]: + try: + with open(self.suggestions_url) as f: + suggestions_str = f.read() + + return suggestions_str + except FileNotFoundError: + self.logger.error(f"Local Snap suggestions file not found: {self.suggestions_url}") + except OSError: + self.logger.error(f"Could not read local Snap suggestions file: {self.suggestions_url}") + traceback.print_exc() + + def _download_remote_suggestions_file(self) -> Optional[str]: + self.logger.info(f"Downloading the Snap suggestions from {self.suggestions_url}") + file = self.http_client.get(self.suggestions_url) + + if file: + return file.text + def list_suggestions(self, limit: int, filter_installed: bool) -> Optional[List[PackageSuggestion]]: - res = [] + if limit == 0 or not snapd.is_running(): + return - if snapd.is_running(): - self.logger.info(f'Downloading suggestions file {SUGGESTIONS_FILE}') - file = self.http_client.get(SUGGESTIONS_FILE) + if self.is_local_suggestions_file_mapped(): + suggestions_str = self._read_local_suggestions_file() + else: + suggestions_str = self._download_remote_suggestions_file() - if not file or not file.text: - self.logger.warning(f"No suggestion found in {SUGGESTIONS_FILE}") - return res + if suggestions_str is None: + return + + if not suggestions_str: + self.logger.warning(f"No Snap suggestion found in {self.suggestions_url}") + return + + ids_prios = suggestions.parse(suggestions_str, self.logger, 'Snap') + + if not ids_prios: + self.logger.warning(f"No Snap suggestion could be parsed from {self.suggestions_url}") + return + + suggestion_by_priority = suggestions.sort_by_priority(ids_prios) + snapd_client = SnapdClient(self.logger) + + if filter_installed: + installed = {s['name'].lower() for s in snapd_client.list_all_snaps()} + + if installed: + suggestion_by_priority = tuple(n for n in suggestion_by_priority if n not in installed) + + if suggestion_by_priority and 0 < limit < len(suggestion_by_priority): + suggestion_by_priority = suggestion_by_priority[0:limit] + + self.logger.info(f'Available Snap suggestions: {len(suggestion_by_priority)}') + + if not suggestion_by_priority: + return + + self.logger.info("Mapping Snap suggestions") + + instances, threads = [], [] + + res, cached_count = [], 0 + for name in suggestion_by_priority: + cached_sug = self.suggestions_cache.get(name) + + if cached_sug: + res.append(cached_sug) + cached_count += 1 else: - self.logger.info('Mapping suggestions') + t = Thread(target=self._fill_suggestion, args=(name, ids_prios[name], snapd_client, res)) + t.start() + threads.append(t) + time.sleep(0.001) # to avoid being blocked - suggestions, threads = [], [] - snapd_client = SnapdClient(self.logger) - installed = {s['name'].lower() for s in snapd_client.list_all_snaps()} + for t in threads: + t.join() - for l in file.text.split('\n'): - if l: - if limit <= 0 or len(suggestions) < limit: - sug = l.strip().split('=') - name = sug[1] + if cached_count > 0: + self.logger.info(f"Returning {cached_count} cached Snap suggestions") - if not installed or name not in installed: - cached_sug = self.suggestions_cache.get(name) - - if cached_sug: - res.append(cached_sug) - else: - t = Thread(target=self._fill_suggestion, args=(name, SuggestionPriority(int(sug[0])), snapd_client, res)) - t.start() - threads.append(t) - time.sleep(0.001) # to avoid being blocked - else: - break - - for t in threads: - t.join() - - res.sort(key=lambda s: s.priority.value, reverse=True) return res def is_default_enabled(self) -> bool: @@ -509,3 +553,21 @@ class SnapManager(SoftwareManager, SettingsController): raise Exception('aborted') else: return select.get_selected() + + @property + def suggestions_url(self) -> str: + if not self._suggestions_url: + file_url = self.context.get_suggestion_url(self.__module__) + + if not file_url: + file_url = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/suggestions.txt' + + self._suggestions_url = file_url + + if file_url.startswith('/'): + self.logger.info(f"Local Snap suggestions file mapped: {file_url}") + + return self._suggestions_url + + def is_local_suggestions_file_mapped(self) -> bool: + return self.suggestions_url.startswith('/') diff --git a/bauh/gems/web/__init__.py b/bauh/gems/web/__init__.py index dbb27a29..28802793 100644 --- a/bauh/gems/web/__init__.py +++ b/bauh/gems/web/__init__.py @@ -22,12 +22,9 @@ URL_ENVIRONMENT_SETTINGS = f'https://raw.githubusercontent.com/vinifmor/bauh-fil DESKTOP_ENTRY_PATH_PATTERN = f'{DESKTOP_ENTRIES_DIR}/{__app_name__}.web.' + '{name}.desktop' URL_FIX_PATTERN = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/env/v2/fix/{domain}/{electron_branch}/fix.js" URL_PROPS_PATTERN = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/env/v2/fix/{domain}/{electron_branch}/properties" -URL_SUGGESTIONS = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/env/v2/suggestions.yml" UA_CHROME = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36' TEMP_PATH = f'{TEMP_DIR}/web' SEARCH_INDEX_FILE = f'{WEB_CACHE_DIR}/index.yml' -SUGGESTIONS_CACHE_FILE = f'{WEB_CACHE_DIR}/suggestions.yml' -SUGGESTIONS_CACHE_TS_FILE = map_timestamp_file(SUGGESTIONS_CACHE_FILE) CONFIG_FILE = f'{CONFIG_DIR}/web.yml' ENVIRONMENT_SETTINGS_CACHED_FILE = f'{WEB_CACHE_DIR}/environment.yml' ENVIRONMENT_SETTINGS_TS_FILE = f'{WEB_CACHE_DIR}/environment.ts' diff --git a/bauh/gems/web/controller.py b/bauh/gems/web/controller.py index 7844df6a..7de6698e 100644 --- a/bauh/gems/web/controller.py +++ b/bauh/gems/web/controller.py @@ -6,10 +6,9 @@ import re import shutil import subprocess import traceback -from math import floor from pathlib import Path from threading import Thread -from typing import List, Type, Set, Tuple, Optional, Dict, Generator, Iterable +from typing import List, Type, Set, Tuple, Optional, Dict, Generator, Iterable, Pattern import requests import yaml @@ -32,9 +31,8 @@ from bauh.commons.boot import CreateConfigFile from bauh.commons.html import bold from bauh.commons.system import ProcessHandler, get_dir_size, SimpleProcess from bauh.commons.view_utils import get_human_size_str -from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \ - SUGGESTIONS_CACHE_FILE, ROOT_DIR, TEMP_PATH, FIX_FILE_PATH, ELECTRON_CACHE_DIR, \ - get_icon_path, URL_PROPS_PATTERN +from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, \ + ROOT_DIR, TEMP_PATH, FIX_FILE_PATH, ELECTRON_CACHE_DIR, UA_CHROME, get_icon_path, URL_PROPS_PATTERN from bauh.gems.web.config import WebConfigManager from bauh.gems.web.environment import EnvironmentUpdater, EnvironmentComponent from bauh.gems.web.model import WebApplication @@ -44,18 +42,18 @@ from bauh.gems.web.worker import SuggestionsManager, UpdateEnvironmentSettings, try: from bs4 import BeautifulSoup, SoupStrainer + BS4_AVAILABLE = True except: BS4_AVAILABLE = False - try: import lxml + LXML_AVAILABLE = True except: LXML_AVAILABLE = False -RE_PROTOCOL_STRIP = re.compile(r'[a-zA-Z]+://') RE_SEVERAL_SPACES = re.compile(r'\s+') RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:.]') @@ -64,7 +62,7 @@ DEFAULT_LANGUAGE_HEADER = 'en-US, en' class WebApplicationManager(SoftwareManager, SettingsController): - def __init__(self, context: ApplicationContext, suggestions_loader: Optional[SuggestionsLoader] = None): + def __init__(self, context: ApplicationContext): super(WebApplicationManager, self).__init__(context=context) self.http_client = context.http_client self.env_updater = EnvironmentUpdater(logger=context.logger, http_client=context.http_client, @@ -73,11 +71,13 @@ class WebApplicationManager(SoftwareManager, SettingsController): self.i18n = context.i18n self.logger = context.logger self.env_thread = None - self.suggestions_loader = suggestions_loader + self.suggestions_loader: Optional[SuggestionsLoader] = None + self._suggestions_manager: Optional[SuggestionsManager] = None self.suggestions = {} self.configman = WebConfigManager() self.idxman = SearchIndexManager(logger=context.logger) self._custom_actions: Optional[Iterable[CustomSoftwareAction]] = None + self._re_protocol_strip: Optional[Pattern] = None def get_accept_language_header(self) -> str: try: @@ -174,7 +174,7 @@ class WebApplicationManager(SoftwareManager, SettingsController): if icon_url: return icon_url - def _get_app_description(self, url: str, soup: "BeautifulSoup") -> str: + def _get_app_description(self, url: str, soup: "BeautifulSoup") -> str: description = None desc_tag = soup.head.find('meta', attrs={'name': 'description'}) @@ -227,13 +227,17 @@ class WebApplicationManager(SoftwareManager, SettingsController): return props except Exception as e: - self.logger.warning(f"Error when trying to retrieve custom installation properties for {props_url}: {e.__class__.__name__}") + self.logger.warning( + f"Error when trying to retrieve custom installation properties for {props_url}: {e.__class__.__name__}") def _map_electron_branch(self, version: str) -> str: return f"electron_{'_'.join(version.split('.')[0:-1])}_X" - def _strip_url_protocol(self, url: str) -> str: - return RE_PROTOCOL_STRIP.split(url)[1].strip().lower() + def strip_url_protocol(self, url: str) -> str: + if not self._re_protocol_strip: + self._re_protocol_strip = re.compile(r'^[a-zA-Z]+://(www\.)?') + + return self._re_protocol_strip.split(url)[-1].strip().lower() def serialize_to_disk(self, pkg: SoftwarePackage, icon_bytes: Optional[bytes], only_icon: bool): super(WebApplicationManager, self).serialize_to_disk(pkg=pkg, icon_bytes=None, only_icon=False) @@ -242,7 +246,8 @@ class WebApplicationManager(SoftwareManager, SettingsController): headers = {'Accept-language': self.get_accept_language_header(), 'User-Agent': UA_CHROME} try: - return self.http_client.get(url, headers=headers, ignore_ssl=True, single_call=True, session=False, allow_redirects=True) + return self.http_client.get(url, headers=headers, ignore_ssl=True, single_call=True, session=False, + allow_redirects=True) except Exception as e: self.logger.warning(f"Could not GET '{url}'. Exception: {e.__class__.__name__}") @@ -263,9 +268,9 @@ class WebApplicationManager(SoftwareManager, SettingsController): if is_url: url = words[0:-1] if words.endswith('/') else words - url_no_protocol = self._strip_url_protocol(url) + url_no_protocol = self.strip_url_protocol(url) - installed_matches = [app for app in installed if self._strip_url_protocol(app.url) == url_no_protocol] + installed_matches = [app for app in installed if self.strip_url_protocol(app.url) == url_no_protocol] if installed_matches: res.installed.extend(installed_matches) @@ -320,25 +325,28 @@ class WebApplicationManager(SoftwareManager, SettingsController): self.logger.info("Query '{}' was not found in the suggestion's index".format(words)) res.installed.extend(installed_matches) else: - if not os.path.exists(SUGGESTIONS_CACHE_FILE): + cached_file_path = self.suggestions_manager.get_cached_file_path() + + if not os.path.exists(cached_file_path): # if the suggestions cache was not found, it will not be possible to retrieve the matched apps # so only the installed matches will be returned - self.logger.warning("Suggestion cached file {} was not found".format(SUGGESTIONS_CACHE_FILE)) + self.logger.warning(f"Suggestion file {cached_file_path} was not found") res.installed.extend(installed_matches) else: - with open(SUGGESTIONS_CACHE_FILE) as f: + with open(cached_file_path) as f: cached_suggestions = yaml.safe_load(f.read()) if not cached_suggestions: # if no suggestion is found, it will not be possible to retrieve the matched apps # so only the installed matches will be returned - self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_CACHE_FILE)) + self.logger.warning(f"No suggestion found in {cached_file_path}") res.installed.extend(installed_matches) else: - matched_suggestions = [cached_suggestions[key] for key in index_match_keys if cached_suggestions.get(key)] + matched_suggestions = [cached_suggestions[key] for key in index_match_keys if + cached_suggestions.get(key)] if not matched_suggestions: - self.logger.warning("No suggestion found for the search index keys: {}".format(index_match_keys)) + self.logger.warning(f"No suggestion found for the query index keys: {index_match_keys}") res.installed.extend(installed_matches) else: matched_suggestions.sort(key=lambda s: s.get('priority', 0), reverse=True) @@ -376,7 +384,8 @@ class WebApplicationManager(SoftwareManager, SettingsController): return res - def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = True) -> SearchResult: + def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int = -1, only_apps: bool = False, + pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = True) -> SearchResult: res = SearchResult([], [], 0) if os.path.exists(INSTALLED_PATH): @@ -393,12 +402,14 @@ class WebApplicationManager(SoftwareManager, SettingsController): def upgrade(self, requirements: UpgradeRequirements, root_password: Optional[str], watcher: ProcessWatcher) -> bool: pass - def uninstall(self, pkg: WebApplication, root_password: Optional[str], watcher: ProcessWatcher, disk_loader: DiskCacheLoader) -> TransactionResult: + def uninstall(self, pkg: WebApplication, root_password: Optional[str], watcher: ProcessWatcher, + disk_loader: DiskCacheLoader) -> TransactionResult: self.logger.info("Checking if {} installation directory {} exists".format(pkg.name, pkg.installation_dir)) if not os.path.exists(pkg.installation_dir): watcher.show_message(title=self.i18n['error'], - body=self.i18n['web.uninstall.error.install_dir.not_found'].format(bold(pkg.installation_dir)), + body=self.i18n['web.uninstall.error.install_dir.not_found'].format( + bold(pkg.installation_dir)), type_=MessageType.ERROR) return TransactionResult.fail() @@ -464,7 +475,8 @@ class WebApplicationManager(SoftwareManager, SettingsController): def get_info(self, pkg: WebApplication) -> dict: if pkg.installed: - info = {'0{}_{}'.format(idx + 1, att): getattr(pkg, att) for idx, att in enumerate(('url', 'description', 'version', 'categories', 'installation_dir', 'desktop_entry'))} + info = {'0{}_{}'.format(idx + 1, att): getattr(pkg, att) for idx, att in + enumerate(('url', 'description', 'version', 'categories', 'installation_dir', 'desktop_entry'))} info['07_exec_file'] = pkg.get_exec_path() info['08_icon_path'] = pkg.get_disk_icon_path() @@ -481,12 +493,14 @@ class WebApplicationManager(SoftwareManager, SettingsController): return info else: - return {'0{}_{}'.format(idx + 1, att): getattr(pkg, att) for idx, att in enumerate(('url', 'description', 'version', 'categories'))} + return {'0{}_{}'.format(idx + 1, att): getattr(pkg, att) for idx, att in + enumerate(('url', 'description', 'version', 'categories'))} def get_history(self, pkg: SoftwarePackage) -> PackageHistory: pass - def _ask_install_options(self, app: WebApplication, watcher: ProcessWatcher, pre_validated: bool) -> Tuple[bool, List[str]]: + def _ask_install_options(self, app: WebApplication, watcher: ProcessWatcher, pre_validated: bool) -> Tuple[ + bool, List[str]]: watcher.change_substatus(self.i18n['web.install.substatus.options']) max_width = 350 @@ -703,9 +717,11 @@ class WebApplicationManager(SoftwareManager, SettingsController): self.logger.error("An exception has happened when downloading {}".format(pkg.icon_url)) traceback.print_exc() else: - self.logger.warning('Could no retrieve the icon {} defined for the suggestion {}'.format(pkg.icon_url, pkg.name)) + self.logger.warning( + 'Could no retrieve the icon {} defined for the suggestion {}'.format(pkg.icon_url, pkg.name)) except: - self.logger.warning('An exception happened when trying to retrieve the icon {} for the suggestion {}'.format(pkg.icon_url, + self.logger.warning( + 'An exception happened when trying to retrieve the icon {} for the suggestion {}'.format(pkg.icon_url, pkg.name)) traceback.print_exc() @@ -749,7 +765,7 @@ class WebApplicationManager(SoftwareManager, SettingsController): electron_version = str(next((c for c in env_components if c.id == 'electron')).version) - url_domain, electron_branch = self._strip_url_protocol(pkg.url), self._map_electron_branch(electron_version) + url_domain, electron_branch = self.strip_url_protocol(pkg.url), self._map_electron_branch(electron_version) fix = self._get_fix_for(url_domain=url_domain, electron_branch=electron_branch) if fix: @@ -886,7 +902,8 @@ class WebApplicationManager(SoftwareManager, SettingsController): return TransactionResult(success=True, installed=[pkg], removed=[]) - def install(self, pkg: WebApplication, root_password: Optional[str], disk_loader: DiskCacheLoader, watcher: ProcessWatcher) -> TransactionResult: + def install(self, pkg: WebApplication, root_password: Optional[str], disk_loader: DiskCacheLoader, + watcher: ProcessWatcher) -> TransactionResult: continue_install, install_options = self._ask_install_options(pkg, watcher, pre_validated=True) if not continue_install: @@ -967,9 +984,7 @@ class WebApplicationManager(SoftwareManager, SettingsController): create_config=create_config, i18n=self.i18n).start() - self.suggestions_loader = SuggestionsLoader(manager=SuggestionsManager(logger=self.logger, - http_client=self.http_client, - i18n=self.i18n), + self.suggestions_loader = SuggestionsLoader(manager=self.suggestions_manager, logger=self.logger, i18n=self.i18n, taskman=task_manager, @@ -1008,7 +1023,8 @@ class WebApplicationManager(SoftwareManager, SettingsController): app.description = self._get_app_description(app.url, soup) try: - find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url, session=False)) + find_url = not app.icon_url or ( + app.icon_url and not self.http_client.exists(app.icon_url, session=False)) except (requests.exceptions.ConnectionError, requests.exceptions.ConnectTimeout): find_url = None @@ -1049,6 +1065,9 @@ class WebApplicationManager(SoftwareManager, SettingsController): output.update(self.configman.get_config()) def list_suggestions(self, limit: int, filter_installed: bool) -> Optional[List[PackageSuggestion]]: + if limit == 0: + return + web_config = {} thread_config = Thread(target=self._fill_config_async, args=(web_config,)) @@ -1057,10 +1076,19 @@ class WebApplicationManager(SoftwareManager, SettingsController): if self.suggestions: suggestions = self.suggestions elif self.suggestions_loader: - self.suggestions_loader.join(5) + if self.suggestions_loader.is_alive(): + self.suggestions_loader.join(5) + suggestions = self.suggestions + elif self.suggestions_manager.is_custom_local_file_mapped(): + suggestions = self.suggestions_manager.read_cached() else: - suggestions = SuggestionsManager(logger=self.logger, http_client=self.http_client, i18n=self.i18n).download() + thread_config.join() + + if self.suggestions_manager.should_download(web_config): + suggestions = self.suggestions_manager.download() + else: + suggestions = self.suggestions_manager.read_cached() # cleaning memory self.suggestions_loader = None @@ -1071,22 +1099,24 @@ class WebApplicationManager(SoftwareManager, SettingsController): suggestion_list.sort(key=lambda s: s.get('priority', 0), reverse=True) if filter_installed: - installed = {self._strip_url_protocol(i.url) for i in self.read_installed(disk_loader=None).installed} + installed = {self.strip_url_protocol(i.url) for i in self.read_installed(disk_loader=None).installed} else: installed = None env_settings, res = None, [] for s in suggestion_list: - if limit <= 0 or len(res) < limit: + if limit < 0 or len(res) < limit: if installed: - surl = self._strip_url_protocol(s['url']) + surl = self.strip_url_protocol(s['url']) if surl in installed: continue if env_settings is None: # reading settings if not already loaded - thread_config.join() + if thread_config.is_alive(): + thread_config.join() + env_settings = self.env_updater.read_settings(web_config=web_config) res.append(self._map_suggestion(s, env_settings)) @@ -1107,7 +1137,8 @@ class WebApplicationManager(SoftwareManager, SettingsController): return res - def execute_custom_action(self, action: CustomSoftwareAction, pkg: SoftwarePackage, root_password: Optional[str], watcher: ProcessWatcher) -> bool: + def execute_custom_action(self, action: CustomSoftwareAction, pkg: SoftwarePackage, root_password: Optional[str], + watcher: ProcessWatcher) -> bool: pass def is_default_enabled(self) -> bool: @@ -1127,7 +1158,8 @@ class WebApplicationManager(SoftwareManager, SettingsController): print('{}[bauh][web] Directory {} deleted{}'.format(Fore.YELLOW, ENV_PATH, Fore.RESET)) except: if logs: - print('{}[bauh][web] An exception has happened when deleting {}{}'.format(Fore.RED, ENV_PATH, Fore.RESET)) + print('{}[bauh][web] An exception has happened when deleting {}{}'.format(Fore.RED, ENV_PATH, + Fore.RESET)) traceback.print_exc() def get_settings(self) -> Optional[Generator[SettingsView, None, None]]: @@ -1140,13 +1172,16 @@ class WebApplicationManager(SoftwareManager, SettingsController): id_='electron_branch') native_opts = [ - InputOption(label=self.i18n['web.settings.nativefier.env'].capitalize(), value=False, tooltip=self.i18n['web.settings.nativefier.env.tooltip'].format(app=self.context.app_name)), - InputOption(label=self.i18n['web.settings.nativefier.system'].capitalize(), value=True, tooltip=self.i18n['web.settings.nativefier.system.tooltip']) + InputOption(label=self.i18n['web.settings.nativefier.env'].capitalize(), value=False, + tooltip=self.i18n['web.settings.nativefier.env.tooltip'].format(app=self.context.app_name)), + InputOption(label=self.i18n['web.settings.nativefier.system'].capitalize(), value=True, + tooltip=self.i18n['web.settings.nativefier.system.tooltip']) ] select_nativefier = SingleSelectComponent(label="Nativefier", options=native_opts, - default_option=[o for o in native_opts if o.value == web_config['environment']['system']][0], + default_option=[o for o in native_opts if + o.value == web_config['environment']['system']][0], type_=SelectViewType.COMBO, tooltip=self.i18n['web.settings.nativefier.tip'], alignment=ViewComponentAlignment.CENTER, @@ -1155,7 +1190,8 @@ class WebApplicationManager(SoftwareManager, SettingsController): env_settings_exp = TextInputComponent(label=self.i18n['web.settings.cache_exp'], tooltip=self.i18n['web.settings.cache_exp.tip'], capitalize_label=False, - value=int(web_config['environment']['cache_exp']) if isinstance(web_config['environment']['cache_exp'], int) else '', + value=int(web_config['environment']['cache_exp']) if isinstance( + web_config['environment']['cache_exp'], int) else '', only_int=True, id_='web_cache_exp') @@ -1221,3 +1257,13 @@ class WebApplicationManager(SoftwareManager, SettingsController): ) yield from self._custom_actions + + @property + def suggestions_manager(self): + if self._suggestions_manager is None: + self._suggestions_manager = SuggestionsManager(logger=self.logger, + http_client=self.http_client, + i18n=self.i18n, + file_url=self.context.get_suggestion_url(self.__module__)) + + return self._suggestions_manager diff --git a/bauh/gems/web/suggestions.py b/bauh/gems/web/suggestions.py index 5797958c..90256ee1 100644 --- a/bauh/gems/web/suggestions.py +++ b/bauh/gems/web/suggestions.py @@ -3,23 +3,44 @@ import traceback from datetime import datetime, timedelta from logging import Logger from pathlib import Path +from typing import Optional import requests import yaml from bauh.api.http import HttpClient -from bauh.gems.web import URL_SUGGESTIONS, SUGGESTIONS_CACHE_FILE, SUGGESTIONS_CACHE_TS_FILE +from bauh.commons.util import map_timestamp_file +from bauh.gems.web import WEB_CACHE_DIR from bauh.view.util.translation import I18n class SuggestionsManager: - def __init__(self, http_client: HttpClient, logger: Logger, i18n: I18n): + def __init__(self, http_client: HttpClient, logger: Logger, i18n: I18n, file_url: Optional[str]): self.http_client = http_client self.logger = logger self.i18n = i18n + if file_url: + self._file_url = file_url + else: + self._file_url = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/env/v2/suggestions.yml" + self._cached_file_path = f'{WEB_CACHE_DIR}/suggestions.yml' + self._cached_file_ts_path = map_timestamp_file(self._cached_file_path) + + @property + def file_url(self) -> Optional[str]: + return self._file_url + + def is_custom_local_file_mapped(self) -> bool: + return self._file_url and self._file_url.startswith('/') def should_download(self, web_config: dict) -> bool: + if not self._file_url: + return False + + if self.is_custom_local_file_mapped(): + return False + exp = web_config['suggestions']['cache_exp'] if web_config['suggestions']['cache_exp'] is None: self.logger.info("No cache expiration defined for suggestions") @@ -28,28 +49,28 @@ class SuggestionsManager: try: exp = int(exp) except ValueError: - self.logger.error("Error while parsing the 'suggestions.cache_exp' ({}) settings property".format(exp)) + self.logger.error(f"Error while parsing the 'suggestions.cache_exp' ({exp}) settings property") return True if exp <= 0: - self.logger.info("No cache expiration defined for suggestions ({})".format(exp)) + self.logger.info(f"No cache expiration defined for suggestions ({exp})") return True - if not os.path.exists(SUGGESTIONS_CACHE_FILE): - self.logger.info("No suggestions cached file found '{}'".format(SUGGESTIONS_CACHE_FILE)) + if not os.path.exists(self._cached_file_path): + self.logger.info(f"No suggestions cached file found '{self._cached_file_path}'") return True - if not os.path.exists(SUGGESTIONS_CACHE_TS_FILE): - self.logger.info("No suggestions cache file timestamp found '{}'".format(SUGGESTIONS_CACHE_TS_FILE)) + if not os.path.exists(self._cached_file_ts_path): + self.logger.info(f"No suggestions cache file timestamp found '{self._cached_file_ts_path}'") return True - with open(SUGGESTIONS_CACHE_TS_FILE) as f: + with open(self._cached_file_ts_path) as f: timestamp_str = f.read() try: sugs_timestamp = datetime.fromtimestamp(float(timestamp_str)) except: - self.logger.error("Could not parse the cached suggestions file timestamp: {}".format(timestamp_str)) + self.logger.error(f"Could not parse the cached suggestions file timestamp: {timestamp_str}") return True expired = sugs_timestamp + timedelta(days=exp) <= datetime.utcnow() @@ -61,17 +82,25 @@ class SuggestionsManager: self.logger.info("Cached suggestions file is up to date") return False + def get_cached_file_path(self) -> str: + return self._file_url if self.is_custom_local_file_mapped() else self._cached_file_path + def read_cached(self, check_file: bool = True) -> dict: - if check_file and not os.path.exists(SUGGESTIONS_CACHE_FILE): - self.logger.warning("Cached suggestions file does not exist ({})".format(SUGGESTIONS_CACHE_FILE)) + if self.is_custom_local_file_mapped(): + file_path, log_ref = self._file_url, 'local' + else: + file_path, log_ref = self._cached_file_path, 'cached' + + if check_file and not os.path.exists(file_path): + self.logger.warning(f"{log_ref.capitalize()} suggestions file does not exist ({file_path})") return {} - self.logger.info("Reading cached suggestions file '{}'".format(SUGGESTIONS_CACHE_FILE)) - with open(SUGGESTIONS_CACHE_FILE) as f: + self.logger.info(f"Reading {log_ref} suggestions file '{file_path}'") + with open(file_path) as f: sugs_str = f.read() if not sugs_str: - self.logger.warning("Cached suggestions file '{}' is empty".format(SUGGESTIONS_CACHE_FILE)) + self.logger.warning(f"{log_ref.capitalize()} suggestions file '{file_path}' is empty") return {} try: @@ -82,14 +111,14 @@ class SuggestionsManager: return {} def download(self) -> dict: - self.logger.info("Reading suggestions from {}".format(URL_SUGGESTIONS)) + self.logger.info(f"Reading suggestions from {self._file_url}") try: - suggestions = self.http_client.get_yaml(URL_SUGGESTIONS, session=False) + suggestions = self.http_client.get_yaml(self._file_url, session=False) if suggestions: - self.logger.info("{} suggestions successfully read".format(len(suggestions))) + self.logger.info(f"{len(suggestions)} suggestions successfully read") else: - self.logger.warning("Could not read suggestions from {}".format(URL_SUGGESTIONS)) + self.logger.warning(f"Could not read suggestions from {self._file_url}") except (requests.exceptions.ConnectionError, requests.exceptions.ConnectTimeout): self.logger.warning("Internet seems to be off: it was not possible to retrieve the suggestions") @@ -102,31 +131,36 @@ class SuggestionsManager: if not suggestions: return - self.logger.info('Caching {} suggestions to the disk'.format(len(suggestions))) - suggestions_file_dir = os.path.dirname(SUGGESTIONS_CACHE_FILE) + if self.is_custom_local_file_mapped(): + return + + self.logger.info(f'Caching {len(suggestions)} suggestions to the disk') + suggestions_file_dir = os.path.dirname(self._cached_file_path) + try: Path(suggestions_file_dir).mkdir(parents=True, exist_ok=True) except OSError: - self.logger.error("Could not generate the directory {}".format(suggestions_file_dir)) + self.logger.error(f"Could not generate the directory {suggestions_file_dir}") traceback.print_exc() return try: - with open(SUGGESTIONS_CACHE_FILE, 'w+') as f: + with open(self._cached_file_path, 'w+') as f: f.write(yaml.safe_dump(suggestions)) except: - self.logger.error("Could write to {}".format(SUGGESTIONS_CACHE_FILE)) + self.logger.error(f"Could write to {self._cached_file_path}") traceback.print_exc() return - self.logger.info("{} suggestions successfully cached to file '{}'".format(len(suggestions), SUGGESTIONS_CACHE_FILE)) + self.logger.info(f"{len(suggestions)} suggestions successfully cached to file '{self._cached_file_path}'") try: - with open(SUGGESTIONS_CACHE_TS_FILE, 'w+') as f: + with open(self._cached_file_ts_path, 'w+') as f: f.write(str(timestamp)) except: - self.logger.error("Could not write to {}".format(SUGGESTIONS_CACHE_TS_FILE)) + self.logger.error(f"Could not write to {self._cached_file_ts_path}") traceback.print_exc() return - self.logger.info("Suggestions cached file timestamp ({}) successfully saved at '{}'".format(timestamp, SUGGESTIONS_CACHE_TS_FILE)) + self.logger.info(f"Suggestions cached file timestamp ({timestamp}) " + f"successfully saved at '{self._cached_file_ts_path}'") diff --git a/bauh/gems/web/worker.py b/bauh/gems/web/worker.py index bcff6488..df0f3916 100644 --- a/bauh/gems/web/worker.py +++ b/bauh/gems/web/worker.py @@ -35,27 +35,33 @@ class SuggestionsLoader(Thread): def run(self): ti = time.time() - self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name))) - self.create_config.join() - - self.taskman.update_progress(self.task_id, 10, None) - - if not self.internet_connection: - self.logger.warning("No internet connection. Only cached suggestions can be loaded") - self.suggestions = self.manager.read_cached(check_file=True) - elif not self.manager.should_download(self.create_config.config): - self.suggestions = self.manager.read_cached(check_file=False) + if self.manager.is_custom_local_file_mapped(): + self.taskman.update_progress(self.task_id, 50, None) + self.logger.info(f"Local Web suggestions file mapped: {self.manager.file_url}") + self.suggestions = self.manager.read_cached() else: - try: - timestamp = datetime.utcnow().timestamp() - self.suggestions = self.manager.download() + wait_msg = self.i18n['task.waiting_task'].format(bold(self.create_config.task_name)) + self.taskman.update_progress(self.task_id, 0, wait_msg) + self.create_config.join() - if self.suggestions: - self.taskman.update_progress(self.task_id, 50, self.i18n['web.task.suggestions.saving']) - self.manager.save_to_disk(self.suggestions, timestamp) - except: - self.logger.error("Unexpected exception") - traceback.print_exc() + self.taskman.update_progress(self.task_id, 10, None) + + if not self.internet_connection: + self.logger.warning("No internet connection. Only cached suggestions can be loaded") + self.suggestions = self.manager.read_cached() + elif not self.manager.should_download(self.create_config.config): + self.suggestions = self.manager.read_cached(check_file=False) + else: + try: + timestamp = datetime.utcnow().timestamp() + self.suggestions = self.manager.download() + + if self.suggestions: + self.taskman.update_progress(self.task_id, 50, self.i18n['web.task.suggestions.saving']) + self.manager.save_to_disk(self.suggestions, timestamp) + except: + self.logger.error("Unexpected exception") + traceback.print_exc() if self.suggestions_callback: self.taskman.update_progress(self.task_id, 75, None) @@ -73,7 +79,8 @@ class SuggestionsLoader(Thread): class SearchIndexGenerator(Thread): - def __init__(self, taskman: TaskManager, idxman: SearchIndexManager, suggestions_loader: SuggestionsLoader, i18n: I18n, logger: logging.Logger): + def __init__(self, taskman: TaskManager, idxman: SearchIndexManager, suggestions_loader: SuggestionsLoader, + i18n: I18n, logger: logging.Logger): super(SearchIndexGenerator, self).__init__(daemon=True) self.taskman = taskman self.idxman = idxman @@ -85,8 +92,11 @@ class SearchIndexGenerator(Thread): def run(self): ti = time.time() - self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.suggestions_loader.task_name))) - self.suggestions_loader.join() + + if self.suggestions_loader.is_alive(): + wait_msg = self.i18n['task.waiting_task'].format(bold(self.suggestions_loader.task_name)) + self.taskman.update_progress(self.task_id, 0, wait_msg) + self.suggestions_loader.join() if self.suggestions_loader.suggestions: self.taskman.update_progress(self.task_id, 1, None) diff --git a/bauh/manage.py b/bauh/manage.py index 9e9bddf9..789ea3d3 100644 --- a/bauh/manage.py +++ b/bauh/manage.py @@ -13,6 +13,7 @@ from bauh.context import generate_i18n, DEFAULT_I18N_KEY, new_qt_application from bauh.view.core import gems from bauh.view.core.controller import GenericSoftwareManager from bauh.view.core.downloader import AdaptableFileDownloader +from bauh.view.core.suggestions import read_suggestions_mapping from bauh.view.qt.prepare import PreparePanel from bauh.view.qt.settings import SettingsWindow from bauh.view.qt.window import ManageWindow @@ -48,6 +49,7 @@ def new_manage_panel(app_args: Namespace, app_config: dict, logger: logging.Logg app_name=__app_name__, app_version=__version__, internet_checker=InternetChecker(offline=app_args.offline), + suggestions_mapping=read_suggestions_mapping(), root_user=user.is_root()) managers = gems.load_managers(context=context, locale=i18n.current_key, config=app_config, diff --git a/bauh/view/core/controller.py b/bauh/view/core/controller.py index 6a910506..66f9acaa 100755 --- a/bauh/view/core/controller.py +++ b/bauh/view/core/controller.py @@ -1,3 +1,4 @@ +import os import re import shutil import time diff --git a/bauh/view/core/suggestions.py b/bauh/view/core/suggestions.py new file mode 100644 index 00000000..280f3a88 --- /dev/null +++ b/bauh/view/core/suggestions.py @@ -0,0 +1,32 @@ +import os +from typing import Optional, Dict +from bauh import __app_name__ + + +def read_suggestions_mapping() -> Optional[Dict[str, str]]: + file_path = f'/etc/{__app_name__}/suggestions.conf' + + if os.path.isfile(file_path): + try: + with open(file_path) as f: + file_content = f.read() + except FileNotFoundError: + return + + if not file_content: + return + + mapping = {} + for line in file_content.split('\n'): + line_strip = line.strip() + + if not line_strip.startswith('#'): + gem_file = line_strip.split('=') + + if len(gem_file) == 2: + gem_name, file_url = gem_file[0].strip(), gem_file[1].strip() + + if gem_name and file_url: + mapping[gem_name] = file_url + + return mapping if mapping else None diff --git a/tests/gems/web/test_controller.py b/tests/gems/web/test_controller.py index 442ffe24..12dc76eb 100644 --- a/tests/gems/web/test_controller.py +++ b/tests/gems/web/test_controller.py @@ -45,3 +45,20 @@ class WebApplicationManagerTest(TestCase): returned = self.manager.get_accept_language_header() self.assertEqual(f'en-IN, en', returned) getdefaultlocale.assert_called_once() + + def test_strip_url_protocol__http_no_www(self): + res = self.manager.strip_url_protocol('http://test.com') + self.assertEqual('test.com', res) + + def test_strip_url_protocol__http_with_www(self): + res = self.manager.strip_url_protocol('http://www.test.com') + self.assertEqual('test.com', res) + + def test_strip_url_protocol__https_no_www(self): + res = self.manager.strip_url_protocol('https://test.com') + self.assertEqual('test.com', res) + + def test_strip_url_protocol__https_with_www(self): + res = self.manager.strip_url_protocol('https://www.test.com') + self.assertEqual('test.com', res) +