diff --git a/CHANGELOG.md b/CHANGELOG.md index 1caf713a..af1ebab5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,10 +55,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

- Web - - now the environment settings are cached for 24 hours. This period can be controlled through the new settings property **environment.update_interval** (in minutes -> default: 1440 = 24 hours. Use **0** so that they are always updated). + - now the environment settings are cached in disk for 24 hours. This period can be controlled through the new settings property **environment.cache_exp** (in minutes -> default: 1440 = 24 hours. Use **0** so that they are always updated).

+ + - now the suggestions are cached in disk for 24 hours. This period can be controlled through the new settings property **suggestions.cache_exp** (in hours -> default: 24 hours. Use **0** so that they are always updated). - displaying the "Indexing suggestions" task during the initialization process - all types now display an initialization task "Checking configuration file" responsible to create/update the respective configuration file diff --git a/README.md b/README.md index 70a944bb..4e8a789d 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,9 @@ environment: version: null # set a custom Electron version here (e.g: '6.1.4') system: false # set it to 'true' if you want to use the nativefier version globally installed on your system cache_exp: 1440 # defines the period (in minutes) in which the stored environment settings are considered valid. Use 0 so that they are always updated. Default: 1440 (24 hours) + +suggestions: + cache_exp: 24 # defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated. Default: 24. ``` - Required dependencies: - Arch-based systems: **python-lxml**, **python-beautifulsoup4** @@ -363,6 +366,7 @@ Priority: - Disable the application types you do not want to deal with - If you don't care about restarting the app every time a new supported package technology is installed, enable `single_dependency_checking`. This can reduce the application response time, since it won't need to recheck if the required technologies are available on your system every time a given action is executed. - If you don't mind to see the applications icons, you can disable them via `download: icons: false`. The application may have a slight response improvement, since it will reduce the IO and parallelism within it. +- For a faster initialization process, consider raising the values of settings properties associated with disk caching. ### Files and Logs - Installation logs and temporary files are saved at **/tmp/bauh** (or **/tmp/bauh_root** if you launch it as root) diff --git a/bauh/gems/web/__init__.py b/bauh/gems/web/__init__.py index e530425e..727c3eb5 100644 --- a/bauh/gems/web/__init__.py +++ b/bauh/gems/web/__init__.py @@ -3,9 +3,11 @@ from pathlib import Path from bauh.api.constants import DESKTOP_ENTRIES_DIR, CONFIG_PATH, TEMP_DIR, CACHE_PATH from bauh.commons import resource +from bauh.commons.util import map_timestamp_file ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) WEB_PATH = '{}/.local/share/bauh/web'.format(Path.home()) +WEB_CACHE_PATH = '{}/web'.format(CACHE_PATH) INSTALLED_PATH = '{}/installed'.format(WEB_PATH) ENV_PATH = '{}/env'.format(WEB_PATH) FIXES_PATH = '{}/fixes'.format(WEB_PATH) @@ -26,12 +28,13 @@ URL_FIX_PATTERN = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/ URL_SUGGESTIONS = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/env/v1/suggestions.yml" UA_CHROME = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36' TEMP_PATH = '{}/web'.format(TEMP_DIR) -SEARCH_INDEX_FILE = '{}/index.yml'.format(TEMP_PATH) -SUGGESTIONS_CACHE_FILE = '{}/suggestions.txt'.format(TEMP_PATH) +SEARCH_INDEX_FILE = '{}/index.yml'.format(WEB_CACHE_PATH) +SUGGESTIONS_CACHE_FILE = '{}/suggestions.yml'.format(WEB_CACHE_PATH) +SUGGESTIONS_CACHE_TS_FILE = map_timestamp_file(SUGGESTIONS_CACHE_FILE) CONFIG_FILE = '{}/web.yml'.format(CONFIG_PATH) URL_NATIVEFIER = 'https://github.com/jiahaog/nativefier/archive/v{version}.tar.gz' -ENVIRONMENT_SETTINGS_CACHED_FILE = '{}/web/environment.yml'.format(CACHE_PATH) -ENVIRONMENT_SETTINGS_TS_FILE = '{}/web/environment.ts'.format(CACHE_PATH) +ENVIRONMENT_SETTINGS_CACHED_FILE = '{}/environment.yml'.format(WEB_CACHE_PATH) +ENVIRONMENT_SETTINGS_TS_FILE = '{}/environment.ts'.format(WEB_CACHE_PATH) def get_icon_path() -> str: diff --git a/bauh/gems/web/config.py b/bauh/gems/web/config.py index 82ace5d1..19846c1f 100644 --- a/bauh/gems/web/config.py +++ b/bauh/gems/web/config.py @@ -13,5 +13,8 @@ class WebConfigManager(YAMLConfigManager): 'system': False, 'electron': {'version': None}, 'cache_exp': 1440 + }, + 'suggestions': { + 'cache_exp': 24 } } diff --git a/bauh/gems/web/controller.py b/bauh/gems/web/controller.py index db2ff42a..d77a00a6 100644 --- a/bauh/gems/web/controller.py +++ b/bauh/gems/web/controller.py @@ -830,19 +830,21 @@ class WebApplicationManager(SoftwareManager): create_config=create_config, i18n=self.i18n).start() - self.suggestions_loader = SuggestionsLoader(manager=SuggestionsManager(logger=self.logger, - http_client=self.http_client, - i18n=self.i18n), - logger=self.logger, - i18n=self.i18n, - taskman=task_manager, - suggestions_callback=self._assign_suggestions) - self.suggestions_loader.start() + self.suggestions_loader = SuggestionsLoader(manager=SuggestionsManager(logger=self.logger, + http_client=self.http_client, + i18n=self.i18n), + logger=self.logger, + i18n=self.i18n, + taskman=task_manager, + suggestions_callback=self._assign_suggestions, + create_config=create_config, + internet_connection=internet_available) + self.suggestions_loader.start() - SearchIndexGenerator(taskman=task_manager, - suggestions_loader=self.suggestions_loader, - i18n=self.i18n, - logger=self.logger).start() + SearchIndexGenerator(taskman=task_manager, + suggestions_loader=self.suggestions_loader, + i18n=self.i18n, + logger=self.logger).start() def list_updates(self, internet_available: bool) -> List[PackageUpdate]: pass @@ -1024,8 +1026,17 @@ class WebApplicationManager(SoftwareManager): max_width=max_width, id_='web_cache_exp') + sugs_exp = TextInputComponent(label=self.i18n['web.settings.suggestions.cache_exp'], + tooltip=self.i18n['web.settings.suggestions.cache_exp.tip'], + capitalize_label=False, + value=int(web_config['suggestions']['cache_exp']) if isinstance( + web_config['suggestions']['cache_exp'], int) else '', + only_int=True, + max_width=max_width, + id_='web_sugs_exp') + form_env = FormComponent(label=self.i18n['web.settings.nativefier.env'].capitalize(), - components=[input_electron, select_nativefier, env_settings_exp]) + components=[input_electron, select_nativefier, env_settings_exp, sugs_exp]) return PanelComponent([form_env]) @@ -1046,6 +1057,7 @@ class WebApplicationManager(SoftwareManager): web_config['environment']['system'] = system_nativefier web_config['environment']['cache_exp'] = form_env.get_component('web_cache_exp').get_int_value() + web_config['suggestions']['cache_exp'] = form_env.get_component('web_sugs_exp').get_int_value() try: self.configman.save_config(web_config) diff --git a/bauh/gems/web/resources/locale/ca b/bauh/gems/web/resources/locale/ca index 0365d8be..afd1879f 100644 --- a/bauh/gems/web/resources/locale/ca +++ b/bauh/gems/web/resources/locale/ca @@ -72,6 +72,8 @@ web.settings.nativefier.env=environment web.settings.nativefier.system.tooltip=The Nativefier version installed on your system will be used to install applications web.settings.nativefier.system=system web.settings.nativefier.tip=Defines which Nativefier version should be used to generate the Web applications +web.settings.suggestions.cache_exp=Suggestions expiration +web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated web.task.download_settings=Updating environment settings web.task.search_index=Indexing suggestions web.task.suggestions=Downloading suggestions diff --git a/bauh/gems/web/resources/locale/en b/bauh/gems/web/resources/locale/en index 0365d8be..18273835 100644 --- a/bauh/gems/web/resources/locale/en +++ b/bauh/gems/web/resources/locale/en @@ -72,6 +72,8 @@ web.settings.nativefier.env=environment web.settings.nativefier.system.tooltip=The Nativefier version installed on your system will be used to install applications web.settings.nativefier.system=system web.settings.nativefier.tip=Defines which Nativefier version should be used to generate the Web applications +web.settings.suggestions.cache_exp=Suggestions expiration +web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated. web.task.download_settings=Updating environment settings web.task.search_index=Indexing suggestions web.task.suggestions=Downloading suggestions diff --git a/bauh/gems/web/resources/locale/es b/bauh/gems/web/resources/locale/es index 03e99768..954867e2 100644 --- a/bauh/gems/web/resources/locale/es +++ b/bauh/gems/web/resources/locale/es @@ -63,7 +63,7 @@ web.install.substatus.checking_fixes=Verificando si hay correcciones publicadas web.install.substatus.options=Esperando las opciones de instalación web.install.substatus.shortcut=Generando un atajo de menú web.settings.cache_exp=Expiración de configuraciones de ambiente -web.settings.cache_exp.tip=Define el período (en minutos) en el que las configuraciones del ambiente almacenadas se consideran válidas. Utilice 0 para que estén siempre actualizadas. +web.settings.cache_exp.tip=Define el período (en minutos) en el que las configuraciones del ambiente almacenadas son consideradas válidas. Utilice 0 para que siempre sean actualizadas. web.settings.electron.version.label=Versión del Electron web.settings.electron.version.tooltip=Define una versión alternativa del Electron para renderizar las nuevas aplicaciones instaladas web.settings.env.nativefier.system.not_installed={} parece no estar instalado en su sistema @@ -72,6 +72,8 @@ web.settings.nativefier.env=ambiente web.settings.nativefier.system.tooltip=Se utilizará la versión de Nativefier instalada en su sistema para instalar aplicaciones web.settings.nativefier.system=sistema web.settings.nativefier.tip=Define qué versión de Nativefier debe usarse para generar las aplicaciones Web +web.settings.suggestions.cache_exp=Expiración de sugerencias +web.settings.suggestions.cache_exp.tip=Define el período (en HORAS) en el que las sugerencias almacenadas en el disco son consideradas actualizadas durante el proceso de inicialización. Utilice 0 para que siempre sean actualizadas. web.task.download_settings=Actualizando configuraciones de ambiente web.task.search_index=Indexando sugerencias web.task.suggestions=Descargando sugerencias diff --git a/bauh/gems/web/resources/locale/fr b/bauh/gems/web/resources/locale/fr index 5e0ea4d3..d65cef97 100644 --- a/bauh/gems/web/resources/locale/fr +++ b/bauh/gems/web/resources/locale/fr @@ -72,6 +72,8 @@ web.settings.nativefier.env=environnement web.settings.nativefier.system.tooltip=La version de Nativefier installée sur votre système va être utilisée pour installer des applications web.settings.nativefier.system=système web.settings.nativefier.tip=Definit quelle version de Nativefier doit être utilisée pour géner les applications Web +web.settings.suggestions.cache_exp=Suggestions expiration +web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated web.task.download_settings=Mise à jour des paramètres d'environnement web.task.search_index=Indexing suggestions web.task.suggestions=Téléchargement de suggestions diff --git a/bauh/gems/web/resources/locale/it b/bauh/gems/web/resources/locale/it index 0365d8be..afd1879f 100644 --- a/bauh/gems/web/resources/locale/it +++ b/bauh/gems/web/resources/locale/it @@ -72,6 +72,8 @@ web.settings.nativefier.env=environment web.settings.nativefier.system.tooltip=The Nativefier version installed on your system will be used to install applications web.settings.nativefier.system=system web.settings.nativefier.tip=Defines which Nativefier version should be used to generate the Web applications +web.settings.suggestions.cache_exp=Suggestions expiration +web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated web.task.download_settings=Updating environment settings web.task.search_index=Indexing suggestions web.task.suggestions=Downloading suggestions diff --git a/bauh/gems/web/resources/locale/pt b/bauh/gems/web/resources/locale/pt index 3310e350..dd66cb73 100644 --- a/bauh/gems/web/resources/locale/pt +++ b/bauh/gems/web/resources/locale/pt @@ -71,6 +71,8 @@ web.settings.nativefier.env=ambiente web.settings.nativefier.system.tooltip=A versão do Nativefier instalada no seu sistema será utilizada para instalar aplicativos web.settings.nativefier.system=sistema web.settings.nativefier.tip=Define qual versão do Nativefier será utilizada para gerar os aplicativos Web +web.settings.suggestions.cache_exp=Expiração de sugestões +web.settings.suggestions.cache_exp.tip=Define o período (em HORAS) em que as sugestões armazenadas em disco são consideradas atualizadas durante a inicialização. Use 0 para que elas sejam sempre atualizadas. web.task.download_settings=Atualizando configurações de ambiente web.task.search_index=Indexando sugestões web.task.suggestions=Baixando sugestões diff --git a/bauh/gems/web/resources/locale/ru b/bauh/gems/web/resources/locale/ru index d14ec477..e97c601f 100644 --- a/bauh/gems/web/resources/locale/ru +++ b/bauh/gems/web/resources/locale/ru @@ -72,6 +72,8 @@ web.settings.nativefier.env.tooltip=Будет использоваться ве web.settings.nativefier.system=Из системы web.settings.nativefier.system.tooltip=Будет использоваться версия Nativefier установленная в Вашей системе web.settings.nativefier.tip=Определяет, какие Nativefier версии должны быдут использованы для установки веб-приложений +web.settings.suggestions.cache_exp=Suggestions expiration +web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated web.task.download_settings=Updating environment settings web.task.search_index=Indexing suggestions web.task.suggestions=Downloading suggestions diff --git a/bauh/gems/web/resources/locale/tr b/bauh/gems/web/resources/locale/tr index 62a4d4ad..ac8dd89f 100644 --- a/bauh/gems/web/resources/locale/tr +++ b/bauh/gems/web/resources/locale/tr @@ -72,6 +72,8 @@ web.settings.nativefier.env=çevre web.settings.nativefier.system.tooltip=Sisteminizde yüklü olan Nativefier sürümü uygulamaları yüklemek için kullanılacaktır web.settings.nativefier.system=sistem web.settings.nativefier.tip=Web uygulamalarını oluşturmak için hangi Nativefier sürümünün kullanılması gerektiğini tanımlar +web.settings.suggestions.cache_exp=Suggestions expiration +web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated web.task.download_settings=Ortam ayarları güncelleniyor web.task.search_index=Indexing suggestions web.task.suggestions=Downloading suggestions diff --git a/bauh/gems/web/suggestions.py b/bauh/gems/web/suggestions.py index ecd0bf7f..5797958c 100644 --- a/bauh/gems/web/suggestions.py +++ b/bauh/gems/web/suggestions.py @@ -1,5 +1,6 @@ import os import traceback +from datetime import datetime, timedelta from logging import Logger from pathlib import Path @@ -7,7 +8,7 @@ import requests import yaml from bauh.api.http import HttpClient -from bauh.gems.web import URL_SUGGESTIONS, SUGGESTIONS_CACHE_FILE +from bauh.gems.web import URL_SUGGESTIONS, SUGGESTIONS_CACHE_FILE, SUGGESTIONS_CACHE_TS_FILE from bauh.view.util.translation import I18n @@ -18,6 +19,68 @@ class SuggestionsManager: self.logger = logger self.i18n = i18n + def should_download(self, web_config: dict) -> bool: + exp = web_config['suggestions']['cache_exp'] + if web_config['suggestions']['cache_exp'] is None: + self.logger.info("No cache expiration defined for suggestions") + return True + + try: + exp = int(exp) + except ValueError: + self.logger.error("Error while parsing the 'suggestions.cache_exp' ({}) settings property".format(exp)) + return True + + if exp <= 0: + self.logger.info("No cache expiration defined for suggestions ({})".format(exp)) + return True + + if not os.path.exists(SUGGESTIONS_CACHE_FILE): + self.logger.info("No suggestions cached file found '{}'".format(SUGGESTIONS_CACHE_FILE)) + return True + + if not os.path.exists(SUGGESTIONS_CACHE_TS_FILE): + self.logger.info("No suggestions cache file timestamp found '{}'".format(SUGGESTIONS_CACHE_TS_FILE)) + return True + + with open(SUGGESTIONS_CACHE_TS_FILE) as f: + timestamp_str = f.read() + + try: + sugs_timestamp = datetime.fromtimestamp(float(timestamp_str)) + except: + self.logger.error("Could not parse the cached suggestions file timestamp: {}".format(timestamp_str)) + return True + + expired = sugs_timestamp + timedelta(days=exp) <= datetime.utcnow() + + if expired: + self.logger.info("Cached suggestions file has expired.") + return True + else: + self.logger.info("Cached suggestions file is up to date") + return False + + def read_cached(self, check_file: bool = True) -> dict: + if check_file and not os.path.exists(SUGGESTIONS_CACHE_FILE): + self.logger.warning("Cached suggestions file does not exist ({})".format(SUGGESTIONS_CACHE_FILE)) + return {} + + self.logger.info("Reading cached suggestions file '{}'".format(SUGGESTIONS_CACHE_FILE)) + with open(SUGGESTIONS_CACHE_FILE) as f: + sugs_str = f.read() + + if not sugs_str: + self.logger.warning("Cached suggestions file '{}' is empty".format(SUGGESTIONS_CACHE_FILE)) + return {} + + try: + return yaml.safe_load(sugs_str) + except: + self.logger.error("An unexpected exception happened") + traceback.print_exc() + return {} + def download(self) -> dict: self.logger.info("Reading suggestions from {}".format(URL_SUGGESTIONS)) try: @@ -35,7 +98,7 @@ class SuggestionsManager: self.logger.info("Finished") return suggestions - def save_to_disk(self, suggestions: dict): + def save_to_disk(self, suggestions: dict, timestamp: float): if not suggestions: return @@ -51,9 +114,19 @@ class SuggestionsManager: try: with open(SUGGESTIONS_CACHE_FILE, 'w+') as f: f.write(yaml.safe_dump(suggestions)) - - self.logger.info("{} suggestions successfully cached to file '{}'".format(len(suggestions), SUGGESTIONS_CACHE_FILE)) except: self.logger.error("Could write to {}".format(SUGGESTIONS_CACHE_FILE)) traceback.print_exc() return + + self.logger.info("{} suggestions successfully cached to file '{}'".format(len(suggestions), SUGGESTIONS_CACHE_FILE)) + + try: + with open(SUGGESTIONS_CACHE_TS_FILE, 'w+') as f: + f.write(str(timestamp)) + except: + self.logger.error("Could not write to {}".format(SUGGESTIONS_CACHE_TS_FILE)) + traceback.print_exc() + return + + self.logger.info("Suggestions cached file timestamp ({}) successfully saved at '{}'".format(timestamp, SUGGESTIONS_CACHE_TS_FILE)) diff --git a/bauh/gems/web/worker.py b/bauh/gems/web/worker.py index a96c9cea..91b86f94 100644 --- a/bauh/gems/web/worker.py +++ b/bauh/gems/web/worker.py @@ -1,6 +1,7 @@ import logging import time import traceback +from datetime import datetime from threading import Thread from typing import Optional @@ -18,7 +19,8 @@ from bauh.view.util.translation import I18n class SuggestionsLoader(Thread): def __init__(self, taskman: TaskManager, manager: SuggestionsManager, - i18n: I18n, logger: logging.Logger, suggestions_callback, suggestions: Optional[dict] = None): + i18n: I18n, logger: logging.Logger, suggestions_callback, create_config: CreateConfigFile, + internet_connection: bool, suggestions: Optional[dict] = None): super(SuggestionsLoader, self).__init__(daemon=True) self.taskman = taskman self.task_id = 'web_sugs' @@ -27,30 +29,47 @@ class SuggestionsLoader(Thread): self.i18n = i18n self.logger = logger self.suggestions = suggestions + self.create_config = create_config + self.internet_connection = internet_connection self.task_name = self.i18n['web.task.suggestions'] self.taskman.register_task(self.task_id, self.task_name, get_icon_path()) def run(self): ti = time.time() + self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name))) + self.create_config.join() + self.taskman.update_progress(self.task_id, 10, None) - try: - self.suggestions = self.manager.download() - if self.suggestions_callback: + if not self.internet_connection: + self.logger.warning("No internet connection. Only cached suggestions can be loaded") + self.suggestions = self.manager.read_cached(check_file=True) + elif not self.manager.should_download(self.create_config.config): + self.suggestions = self.manager.read_cached(check_file=False) + else: + try: + timestamp = datetime.utcnow().timestamp() + self.suggestions = self.manager.download() + + if self.suggestions: + self.taskman.update_progress(self.task_id, 50, self.i18n['web.task.suggestions.saving']) + self.manager.save_to_disk(self.suggestions, timestamp) + except: + self.logger.error("Unexpected exception") + traceback.print_exc() + + if self.suggestions_callback: + self.taskman.update_progress(self.task_id, 75, None) + try: self.suggestions_callback(self.suggestions) + except: + self.logger.error("Unexpected exception") + traceback.print_exc() - if self.suggestions: - self.taskman.update_progress(self.task_id, 50, self.i18n['web.task.suggestions.saving']) - self.manager.save_to_disk(self.suggestions) - - except: - self.logger.error("Unexpected exception") - traceback.print_exc() - finally: - self.taskman.update_progress(self.task_id, 100, None) - self.taskman.finish_task(self.task_id) - tf = time.time() - self.logger.info("Finished. Took {0:.2f} seconds".format(tf - ti)) + self.taskman.update_progress(self.task_id, 100, None) + self.taskman.finish_task(self.task_id) + tf = time.time() + self.logger.info("Finished. Took {0:.2f} seconds".format(tf - ti)) class SearchIndexGenerator(Thread):