From 7a3245ae5e6abb53668f3978377f471e4eb6eb9c Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Mon, 1 Feb 2021 17:09:30 -0300 Subject: [PATCH] [appimage] improvement -> caching suggestions to disk --- CHANGELOG.md | 2 + README.md | 2 + bauh/gems/appimage/__init__.py | 10 +- bauh/gems/appimage/config.py | 3 + bauh/gems/appimage/controller.py | 24 ++-- bauh/gems/appimage/resources/locale/ca | 1 + bauh/gems/appimage/resources/locale/de | 1 + bauh/gems/appimage/resources/locale/en | 1 + bauh/gems/appimage/resources/locale/es | 1 + bauh/gems/appimage/resources/locale/fr | 1 + bauh/gems/appimage/resources/locale/it | 1 + bauh/gems/appimage/resources/locale/pt | 1 + bauh/gems/appimage/resources/locale/ru | 1 + bauh/gems/appimage/resources/locale/tr | 1 + bauh/gems/appimage/worker.py | 162 +++++++++++++++++++++++-- bauh/view/core/controller.py | 2 +- 16 files changed, 191 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e519e135..f1aa588d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [0.9.14] 2021 ### Improvements +- AppImage + - caching suggestions to disk. The cache expiration can be controlled through the new property settings `suggestions.expiration` (in hours. Default: 24). - Web - nativefier URL moved to [bauh-files](https://github.com/vinifmor/bauh-files/blob/master/web/env/v1/environment.yml) diff --git a/README.md b/README.md index b570f895..4d923458 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,8 @@ bauh is officially distributed through [PyPi](https://pypi.org/project/bauh) and ``` database: expiration: 60 # defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it. +suggestions: + expiration: 24 # defines the period (in hours) in which the cached suggestions will be considered up to date. Use 0 if you always want to update them. ``` diff --git a/bauh/gems/appimage/__init__.py b/bauh/gems/appimage/__init__.py index d64ddf62..aef041c2 100644 --- a/bauh/gems/appimage/__init__.py +++ b/bauh/gems/appimage/__init__.py @@ -14,11 +14,13 @@ CONFIG_DIR = '{}/appimage'.format(CONFIG_PATH) UPDATES_IGNORED_FILE = '{}/updates_ignored.txt'.format(CONFIG_DIR) SYMLINKS_DIR = '{}/.local/bin'.format(str(Path.home())) URL_COMPRESSED_DATABASES = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/dbs.tar.gz' -DATABASES_DIR = '{}/appimage'.format(CACHE_PATH) -DATABASE_APPS_FILE = '{}/apps.db'.format(DATABASES_DIR) -DATABASE_RELEASES_FILE = '{}/releases.db'.format(DATABASES_DIR) -DATABASES_TS_FILE = '{}/dbs.ts'.format(DATABASES_DIR) +APPIMAGE_CACHE_PATH = '{}/appimage'.format(CACHE_PATH) +DATABASE_APPS_FILE = '{}/apps.db'.format(APPIMAGE_CACHE_PATH) +DATABASE_RELEASES_FILE = '{}/releases.db'.format(APPIMAGE_CACHE_PATH) +DATABASES_TS_FILE = '{}/dbs.ts'.format(APPIMAGE_CACHE_PATH) DESKTOP_ENTRIES_PATH = '{}/.local/share/applications'.format(str(Path.home())) +SUGGESTIONS_CACHED_FILE = '{}/suggestions.txt'.format(APPIMAGE_CACHE_PATH) +SUGGESTIONS_CACHED_TS_FILE = '{}/suggestions.ts'.format(APPIMAGE_CACHE_PATH) def get_icon_path() -> str: diff --git a/bauh/gems/appimage/config.py b/bauh/gems/appimage/config.py index 2ced2afb..9f508d46 100644 --- a/bauh/gems/appimage/config.py +++ b/bauh/gems/appimage/config.py @@ -11,5 +11,8 @@ class AppImageConfigManager(YAMLConfigManager): return { 'database': { 'expiration': 60 + }, + 'suggestions': { + 'expiration': 24 } } diff --git a/bauh/gems/appimage/controller.py b/bauh/gems/appimage/controller.py index 1a8cae55..f61a29bf 100644 --- a/bauh/gems/appimage/controller.py +++ b/bauh/gems/appimage/controller.py @@ -27,13 +27,13 @@ from bauh.commons import resource from bauh.commons.boot import CreateConfigFile from bauh.commons.html import bold from bauh.commons.system import SystemProcess, new_subprocess, ProcessHandler, run_cmd, SimpleProcess -from bauh.gems.appimage import query, INSTALLATION_PATH, LOCAL_PATH, SUGGESTIONS_FILE, ROOT_DIR, \ +from bauh.gems.appimage import query, INSTALLATION_PATH, LOCAL_PATH, ROOT_DIR, \ CONFIG_DIR, UPDATES_IGNORED_FILE, util, get_default_manual_installation_file_dir, DATABASE_APPS_FILE, \ - DATABASE_RELEASES_FILE, DESKTOP_ENTRIES_PATH, DATABASES_DIR, get_icon_path + DATABASE_RELEASES_FILE, DESKTOP_ENTRIES_PATH, APPIMAGE_CACHE_PATH, get_icon_path from bauh.gems.appimage.config import AppImageConfigManager from bauh.gems.appimage.model import AppImage from bauh.gems.appimage.util import replace_desktop_entry_exec_command -from bauh.gems.appimage.worker import DatabaseUpdater, SymlinksVerifier +from bauh.gems.appimage.worker import DatabaseUpdater, SymlinksVerifier, AppImageSuggestionsDownloader RE_DESKTOP_ICON = re.compile(r'Icon\s*=\s*.+\n') RE_ICON_ENDS_WITH = re.compile(r'.+\.(png|svg)$') @@ -629,6 +629,10 @@ class AppImageManager(SoftwareManager): create_config=create_config, http_client=self.context.http_client, logger=self.context.logger).start() + AppImageSuggestionsDownloader(taskman=task_manager, i18n=self.context.i18n, + http_client=self.context.http_client, logger=self.context.logger, + create_config=create_config).start() + def list_updates(self, internet_available: bool) -> List[PackageUpdate]: res = self.read_installed(disk_loader=None, internet_available=internet_available) @@ -641,7 +645,7 @@ class AppImageManager(SoftwareManager): return updates def list_warnings(self, internet_available: bool) -> List[str]: - dbfiles = glob.glob('{}/*.db'.format(DATABASES_DIR)) + dbfiles = glob.glob('{}/*.db'.format(APPIMAGE_CACHE_PATH)) if not dbfiles or len({f for f in (DATABASE_APPS_FILE, DATABASE_RELEASES_FILE) if f in dbfiles}) != 2: return [self.i18n['appimage.warning.missing_db_files'].format(appimage=bold('AppImage'))] @@ -652,16 +656,16 @@ class AppImageManager(SoftwareManager): connection = self._get_db_connection(DATABASE_APPS_FILE) if connection: - file = self.http_client.get(SUGGESTIONS_FILE) + suggestions = AppImageSuggestionsDownloader(appimage_config=self.configman.get_config(), logger=self.logger, + i18n=self.i18n, http_client=self.http_client, + taskman=TaskManager()).read() - if not file or not file.text: - self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_FILE)) + if not suggestions: + self.logger.warning("Could not read suggestions") return res else: self.logger.info("Mapping suggestions") try: - sugs = [l for l in file.text.split('\n') if l] - if filter_installed: installed = {i.name.lower() for i in self.read_installed(disk_loader=None, connection=connection).installed} else: @@ -669,7 +673,7 @@ class AppImageManager(SoftwareManager): sugs_map = {} - for s in sugs: + for s in suggestions: lsplit = s.split('=') name = lsplit[1].strip() diff --git a/bauh/gems/appimage/resources/locale/ca b/bauh/gems/appimage/resources/locale/ca index 34b228f7..39080488 100644 --- a/bauh/gems/appimage/resources/locale/ca +++ b/bauh/gems/appimage/resources/locale/ca @@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files appimage.update_database.uncompressing=Uncompressing files appimage.task.db_update=Actualització de bases de dades appimage.task.db_update.checking=Checking for updates +appimage.task.suggestions=Downloading suggestions appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=No s’ha pogut suprimir el directori d’instal·lació de l’aplicació {} appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. diff --git a/bauh/gems/appimage/resources/locale/de b/bauh/gems/appimage/resources/locale/de index 2cccb9bf..a6fd80ef 100644 --- a/bauh/gems/appimage/resources/locale/de +++ b/bauh/gems/appimage/resources/locale/de @@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files appimage.update_database.uncompressing=Uncompressing files appimage.task.db_update=Updating databases appimage.task.db_update.checking=Checking for updates +appimage.task.suggestions=Downloading suggestions appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=Anwendungordner {} konnte nicht entfernt werden appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. diff --git a/bauh/gems/appimage/resources/locale/en b/bauh/gems/appimage/resources/locale/en index 04217331..cc7168d8 100644 --- a/bauh/gems/appimage/resources/locale/en +++ b/bauh/gems/appimage/resources/locale/en @@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files appimage.update_database.uncompressing=Uncompressing files appimage.task.db_update=Updating databases appimage.task.db_update.checking=Checking for updates +appimage.task.suggestions=Downloading suggestions appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=Could not remove the application installation directory {} appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. diff --git a/bauh/gems/appimage/resources/locale/es b/bauh/gems/appimage/resources/locale/es index 047ca4fd..816f7a35 100644 --- a/bauh/gems/appimage/resources/locale/es +++ b/bauh/gems/appimage/resources/locale/es @@ -37,6 +37,7 @@ appimage.update_database.downloading=Descargando archivos de la base de datos appimage.update_database.uncompressing=Descomprindo archivos appimage.task.db_update=Actualizando base de datos appimage.task.db_update.checking=Buscando actualizaciones +appimage.task.suggestions=Descargando sugerencias appimage.task.symlink_check=Verificando links simbólicos appimage.uninstall.error.remove_folder=No se pudo eliminar el directorio de instalación de la aplicación {} appimage.warning.missing_db_files=No se encontraron archivos de la base de datos {appimage}. Algunas acciones no funcionarán correctamente. diff --git a/bauh/gems/appimage/resources/locale/fr b/bauh/gems/appimage/resources/locale/fr index 0730d376..f050641e 100644 --- a/bauh/gems/appimage/resources/locale/fr +++ b/bauh/gems/appimage/resources/locale/fr @@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files appimage.update_database.uncompressing=Uncompressing files appimage.task.db_update=Mise à jour des bases de données appimage.task.db_update.checking=Checking for updates +appimage.task.suggestions=Downloading suggestions appimage.task.symlink_check=Verification des symlinks appimage.uninstall.error.remove_folder=Impossible de supprimer le répertoire d'installation {} appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. diff --git a/bauh/gems/appimage/resources/locale/it b/bauh/gems/appimage/resources/locale/it index 72b78692..351654fe 100644 --- a/bauh/gems/appimage/resources/locale/it +++ b/bauh/gems/appimage/resources/locale/it @@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files appimage.update_database.uncompressing=Uncompressing files appimage.task.db_update=Aggiornamento dei database appimage.task.db_update.checking=Checking for updates +appimage.task.suggestions=Downloading suggestions appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=Impossibile rimuovere la directory di installazione dell'applicazione {} appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. diff --git a/bauh/gems/appimage/resources/locale/pt b/bauh/gems/appimage/resources/locale/pt index f4357e77..f12eb9c4 100644 --- a/bauh/gems/appimage/resources/locale/pt +++ b/bauh/gems/appimage/resources/locale/pt @@ -37,6 +37,7 @@ appimage.update_database.downloading=Baixando arquivos do banco de dados appimage.update_database.uncompressing=Descomprimindo arquivos appimage.task.db_update=Atualizando base de dados appimage.task.db_update.checking=Checando atualizações +appimage.task.suggestions=Baixando sugestões appimage.task.symlink_check=Verificando links simbólicos appimage.uninstall.error.remove_folder=Não foi possível remover o diretóri ode instalação do aplicativo {} appimage.warning.missing_db_files=Arquivos do banco de dados {appimage} não foram encontrados. Algumas ações não funcionarão corretamente. diff --git a/bauh/gems/appimage/resources/locale/ru b/bauh/gems/appimage/resources/locale/ru index adeb3fd1..c586acb0 100644 --- a/bauh/gems/appimage/resources/locale/ru +++ b/bauh/gems/appimage/resources/locale/ru @@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files appimage.update_database.uncompressing=Uncompressing files appimage.task.db_update=Обновление базы данных appimage.task.db_update.checking=Checking for updates +appimage.task.suggestions=Downloading suggestions appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=Не удалось удалить каталог приложения {} appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. diff --git a/bauh/gems/appimage/resources/locale/tr b/bauh/gems/appimage/resources/locale/tr index 3173b2f4..98f66087 100644 --- a/bauh/gems/appimage/resources/locale/tr +++ b/bauh/gems/appimage/resources/locale/tr @@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files appimage.update_database.uncompressing=Uncompressing files appimage.task.db_update=Veritabanları güncelleniyor appimage.task.db_update.checking=Checking for updates +appimage.task.suggestions=Downloading suggestions appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder={} uygulama kurulum dizini kaldırılamadı appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. diff --git a/bauh/gems/appimage/worker.py b/bauh/gems/appimage/worker.py index 4ebb3920..caef42bd 100644 --- a/bauh/gems/appimage/worker.py +++ b/bauh/gems/appimage/worker.py @@ -8,20 +8,23 @@ import traceback from datetime import datetime, timedelta from pathlib import Path from threading import Thread -from typing import Optional +from typing import Optional, List + +import requests from bauh.api.abstract.handler import TaskManager, ProcessWatcher from bauh.api.http import HttpClient from bauh.commons.boot import CreateConfigFile from bauh.commons.html import bold from bauh.gems.appimage import get_icon_path, INSTALLATION_PATH, SYMLINKS_DIR, util, DATABASES_TS_FILE, \ - DATABASES_DIR, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES + APPIMAGE_CACHE_PATH, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES, SUGGESTIONS_FILE, \ + SUGGESTIONS_CACHED_TS_FILE, SUGGESTIONS_CACHED_FILE from bauh.gems.appimage.model import AppImage from bauh.view.util.translation import I18n class DatabaseUpdater(Thread): - COMPRESS_FILE_PATH = '{}/db.tar.gz'.format(DATABASES_DIR) + COMPRESS_FILE_PATH = '{}/db.tar.gz'.format(APPIMAGE_CACHE_PATH) def __init__(self, i18n: I18n, http_client: HttpClient, logger: logging.Logger, taskman: TaskManager, watcher: Optional[ProcessWatcher] = None, appimage_config: Optional[dict] = None, create_config: Optional[CreateConfigFile] = None): @@ -49,10 +52,10 @@ class DatabaseUpdater(Thread): self.logger.info("No expiration time configured for the AppImage database") return True - files = {*glob.glob('{}/*'.format(DATABASES_DIR))} + files = {*glob.glob('{}/*'.format(APPIMAGE_CACHE_PATH))} if not files: - self.logger.warning('No database files on {}'.format(DATABASES_DIR)) + self.logger.warning('No database files on {}'.format(APPIMAGE_CACHE_PATH)) return True if DATABASES_TS_FILE not in files: @@ -102,7 +105,7 @@ class DatabaseUpdater(Thread): self.logger.warning('Could not download the database file {}'.format(URL_COMPRESSED_DATABASES)) return False - Path(DATABASES_DIR).mkdir(parents=True, exist_ok=True) + Path(APPIMAGE_CACHE_PATH).mkdir(parents=True, exist_ok=True) with open(self.COMPRESS_FILE_PATH, 'wb+') as f: f.write(res.content) @@ -110,7 +113,7 @@ class DatabaseUpdater(Thread): self.logger.info("Database file saved at {}".format(self.COMPRESS_FILE_PATH)) self._update_task_progress(50, self.i18n['appimage.update_database.deleting_old']) - old_db_files = glob.glob(DATABASES_DIR + '/*.db') + old_db_files = glob.glob(APPIMAGE_CACHE_PATH + '/*.db') if old_db_files: self.logger.info('Deleting old database files') @@ -124,7 +127,7 @@ class DatabaseUpdater(Thread): try: tf = tarfile.open(self.COMPRESS_FILE_PATH) - tf.extractall(DATABASES_DIR) + tf.extractall(APPIMAGE_CACHE_PATH) self.logger.info('Successfully uncompressed file {}'.format(self.COMPRESS_FILE_PATH)) except: self.logger.error('Could not extract file {}'.format(self.COMPRESS_FILE_PATH)) @@ -279,3 +282,146 @@ class SymlinksVerifier(Thread): self.logger.info("No AppImage applications found. Aborting") self.taskman.update_progress(self.task_id, 100, '') self.taskman.finish_task(self.task_id) + + +class AppImageSuggestionsDownloader(Thread): + + def __init__(self, logger: logging.Logger, http_client: HttpClient, i18n: I18n, taskman: TaskManager, create_config: Optional[CreateConfigFile] = None, appimage_config: Optional[dict] = None): + super(AppImageSuggestionsDownloader, self).__init__(daemon=True) + self.create_config = create_config + self.logger = logger + self.i18n = i18n + self.http_client = http_client + self.taskman = taskman + self.config = appimage_config + self.task_id = 'appim.suggestions' + self.taskman.register_task(id_=self.task_id, label=i18n['appimage.task.suggestions'], icon_path=get_icon_path()) + + def should_download(self, appimage_config: dict) -> bool: + try: + exp_hours = int(appimage_config['suggestions']['expiration']) + except: + self.logger.error("An exception happened while trying to parse 'suggestions.expiration'") + traceback.print_exc() + return True + + if exp_hours <= 0: + self.logger.info("Suggestions cache is disabled") + return True + + if not os.path.exists(SUGGESTIONS_CACHED_FILE): + self.logger.info("'{}' not found. It must be downloaded".format(SUGGESTIONS_CACHED_FILE)) + return True + + if not os.path.exists(SUGGESTIONS_CACHED_TS_FILE): + self.logger.info("'{}' not found. The suggestions file must be downloaded.") + return True + + with open(SUGGESTIONS_CACHED_TS_FILE) as f: + timestamp_str = f.read() + + try: + suggestions_timestamp = datetime.fromtimestamp(float(timestamp_str)) + except: + self.logger.error('Could not parse the cached suggestions timestamp: {}'.format(timestamp_str)) + traceback.print_exc() + return True + + update = suggestions_timestamp + timedelta(hours=exp_hours) <= datetime.utcnow() + return update + + def read(self) -> Optional[List[str]]: + self.logger.info("Checking if suggestions should be downloaded") + if self.should_download(self.config): + suggestions_timestamp = datetime.utcnow().timestamp() + suggestions_str = self.download() + + Thread(target=self.cache_suggestions, args=(suggestions_str, suggestions_timestamp), daemon=True).start() + else: + self.logger.info("Reading cached suggestions from '{}'".format(SUGGESTIONS_CACHED_FILE)) + with open(SUGGESTIONS_CACHED_FILE) as f: + suggestions_str = f.read() + + return self.map_suggestions(suggestions_str) if suggestions_str else None + + def cache_suggestions(self, text: str, timestamp: float): + self.logger.info("Caching suggestions to '{}'".format(SUGGESTIONS_FILE)) + + cache_dir = os.path.dirname(SUGGESTIONS_CACHED_FILE) + + try: + Path(cache_dir).mkdir(parents=True, exist_ok=True) + cache_dir_ok = True + except OSError: + self.logger.error("Could not create cache directory '{}'".format(cache_dir)) + traceback.print_exc() + cache_dir_ok = False + + if cache_dir_ok: + try: + with open(SUGGESTIONS_CACHED_FILE, 'w+') as f: + f.write(text) + except: + self.logger.error("An exception happened while writing the file '{}'".format(SUGGESTIONS_FILE)) + traceback.print_exc() + + try: + with open(SUGGESTIONS_CACHED_TS_FILE, 'w+') as f: + f.write(str(timestamp)) + except: + self.logger.error("An exception happened while writing the file '{}'".format(SUGGESTIONS_CACHED_TS_FILE)) + traceback.print_exc() + + def download(self) -> Optional[str]: + self.logger.info("Downloading suggestions from {}".format(SUGGESTIONS_FILE)) + + try: + res = self.http_client.get(SUGGESTIONS_FILE) + except requests.exceptions.ConnectionError: + self.logger.warning("Could not download suggestion from '{}'".format(SUGGESTIONS_FILE)) + return + + if not res: + self.logger.warning("Could not download suggestion from '{}'".format(SUGGESTIONS_FILE)) + return + + if not res.text: + self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_FILE)) + return + + return res.text + + def map_suggestions(self, text: str) -> List[str]: + return [line for line in text.split('\n') if line] + + def run(self): + if self.create_config: + self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name))) + self.create_config.join() + self.config = self.create_config.config + + ti = time.time() + self.taskman.update_progress(self.task_id, 1, None) + + self.logger.info("Checking if suggestions should be downloaded") + should_download = self.should_download(self.config) + self.taskman.update_progress(self.task_id, 30, None) + + try: + if should_download: + suggestions_timestamp = datetime.utcnow().timestamp() + suggestions_str = self.download() + self.taskman.update_progress(self.task_id, 70, None) + + if suggestions_str: + self.cache_suggestions(suggestions_str, suggestions_timestamp) + else: + self.logger.info("Cached suggestions are up-to-date") + except: + self.logger.error("An unexpected exception happened") + traceback.print_exc() + + self.taskman.update_progress(self.task_id, 100, None) + self.taskman.finish_task(self.task_id) + tf = time.time() + self.logger.info("Took {0:.9f} seconds to download suggestions".format(tf - ti)) diff --git a/bauh/view/core/controller.py b/bauh/view/core/controller.py index 01b3c169..bc4b4753 100755 --- a/bauh/view/core/controller.py +++ b/bauh/view/core/controller.py @@ -477,7 +477,7 @@ class GenericSoftwareManager(SoftwareManager): mti = time.time() man_sugs = man.list_suggestions(limit=limit, filter_installed=filter_installed) mtf = time.time() - self.logger.info(man.__class__.__name__ + ' took {0:.2f} seconds'.format(mtf - mti)) + self.logger.info(man.__class__.__name__ + ' took {0:.5f} seconds'.format(mtf - mti)) if man_sugs: if 0 < limit < len(man_sugs):