[web] improvement -> suggestions are cached in disk for 24 hours

CHANGELOG.md

@@ -55,10 +55,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     </p>
 
 - Web
-    - now the environment settings are cached for 24 hours. This period can be controlled through the new settings property **environment.update_interval** (in minutes -> default: 1440 = 24 hours. Use **0** so that they are always updated).
+    - now the environment settings are cached in disk for 24 hours. This period can be controlled through the new settings property **environment.cache_exp** (in minutes -> default: 1440 = 24 hours. Use **0** so that they are always updated).
     <p align="center">
             <img src="https://raw.githubusercontent.com/vinifmor/bauh-files/master/pictures/releases/0.9.11/web_env_exp.png">
     </p>
+    
+    - now the suggestions are cached in disk for 24 hours. This period can be controlled through the new settings property **suggestions.cache_exp** (in hours -> default: 24 hours. Use **0** so that they are always updated).
     - displaying the "Indexing suggestions" task during the initialization process
 
 - all types now display an initialization task "Checking configuration file" responsible to create/update the respective configuration file

README.md

@@ -269,6 +269,9 @@ environment:
     version: null  # set a custom Electron version here (e.g: '6.1.4')
   system: false  # set it to 'true' if you want to use the nativefier version globally installed on your system 
   cache_exp: 1440 # defines the period (in minutes) in which the stored environment settings are considered valid. Use 0 so that they are always updated. Default: 1440 (24 hours)
+
+suggestions:
+    cache_exp: 24  # defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated. Default: 24.
 ```
 - Required dependencies: 
     - Arch-based systems: **python-lxml**, **python-beautifulsoup4**
@@ -363,6 +366,7 @@ Priority:
 - Disable the application types you do not want to deal with
 - If you don't care about restarting the app every time a new supported package technology is installed, enable `single_dependency_checking`. This can reduce the application response time, since it won't need to recheck if the required technologies are available on your system every time a given action is executed.
 - If you don't mind to see the applications icons, you can disable them via `download: icons: false`. The application may have a slight response improvement, since it will reduce the IO and parallelism within it.
+- For a faster initialization process, consider raising the values of settings properties associated with disk caching.
 
 ### Files and Logs
 - Installation logs and temporary files are saved at **/tmp/bauh** (or **/tmp/bauh_root** if you launch it as root)

bauh/gems/web/__init__.py

@@ -3,9 +3,11 @@ from pathlib import Path
 
 from bauh.api.constants import DESKTOP_ENTRIES_DIR, CONFIG_PATH, TEMP_DIR, CACHE_PATH
 from bauh.commons import resource
+from bauh.commons.util import map_timestamp_file
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 WEB_PATH = '{}/.local/share/bauh/web'.format(Path.home())
+WEB_CACHE_PATH = '{}/web'.format(CACHE_PATH)
 INSTALLED_PATH = '{}/installed'.format(WEB_PATH)
 ENV_PATH = '{}/env'.format(WEB_PATH)
 FIXES_PATH = '{}/fixes'.format(WEB_PATH)
@@ -26,12 +28,13 @@ URL_FIX_PATTERN = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/
 URL_SUGGESTIONS = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/env/v1/suggestions.yml"
 UA_CHROME = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
 TEMP_PATH = '{}/web'.format(TEMP_DIR)
-SEARCH_INDEX_FILE = '{}/index.yml'.format(TEMP_PATH)
+SEARCH_INDEX_FILE = '{}/index.yml'.format(WEB_CACHE_PATH)
-SUGGESTIONS_CACHE_FILE = '{}/suggestions.txt'.format(TEMP_PATH)
+SUGGESTIONS_CACHE_FILE = '{}/suggestions.yml'.format(WEB_CACHE_PATH)
+SUGGESTIONS_CACHE_TS_FILE = map_timestamp_file(SUGGESTIONS_CACHE_FILE)
 CONFIG_FILE = '{}/web.yml'.format(CONFIG_PATH)
 URL_NATIVEFIER = 'https://github.com/jiahaog/nativefier/archive/v{version}.tar.gz'
-ENVIRONMENT_SETTINGS_CACHED_FILE = '{}/web/environment.yml'.format(CACHE_PATH)
+ENVIRONMENT_SETTINGS_CACHED_FILE = '{}/environment.yml'.format(WEB_CACHE_PATH)
-ENVIRONMENT_SETTINGS_TS_FILE = '{}/web/environment.ts'.format(CACHE_PATH)
+ENVIRONMENT_SETTINGS_TS_FILE = '{}/environment.ts'.format(WEB_CACHE_PATH)
 
 
 def get_icon_path() -> str:

bauh/gems/web/config.py

@@ -13,5 +13,8 @@ class WebConfigManager(YAMLConfigManager):
                 'system': False,
                 'electron': {'version': None},
                 'cache_exp': 1440
+            },
+            'suggestions': {
+                'cache_exp': 24
             }
         }

bauh/gems/web/controller.py

@@ -830,19 +830,21 @@ class WebApplicationManager(SoftwareManager):
                                       create_config=create_config,
                                       i18n=self.i18n).start()
 
-            self.suggestions_loader = SuggestionsLoader(manager=SuggestionsManager(logger=self.logger,
+        self.suggestions_loader = SuggestionsLoader(manager=SuggestionsManager(logger=self.logger,
-                                                                                   http_client=self.http_client,
+                                                                               http_client=self.http_client,
-                                                                                   i18n=self.i18n),
+                                                                               i18n=self.i18n),
-                                                        logger=self.logger,
+                                                    logger=self.logger,
-                                                        i18n=self.i18n,
+                                                    i18n=self.i18n,
-                                                        taskman=task_manager,
+                                                    taskman=task_manager,
-                                                        suggestions_callback=self._assign_suggestions)
+                                                    suggestions_callback=self._assign_suggestions,
-            self.suggestions_loader.start()
+                                                    create_config=create_config,
+                                                    internet_connection=internet_available)
+        self.suggestions_loader.start()
 
-            SearchIndexGenerator(taskman=task_manager,
+        SearchIndexGenerator(taskman=task_manager,
-                                 suggestions_loader=self.suggestions_loader,
+                             suggestions_loader=self.suggestions_loader,
-                                 i18n=self.i18n,
+                             i18n=self.i18n,
-                                 logger=self.logger).start()
+                             logger=self.logger).start()
 
     def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
         pass
@@ -1024,8 +1026,17 @@ class WebApplicationManager(SoftwareManager):
                                               max_width=max_width,
                                               id_='web_cache_exp')
 
+        sugs_exp = TextInputComponent(label=self.i18n['web.settings.suggestions.cache_exp'],
+                                      tooltip=self.i18n['web.settings.suggestions.cache_exp.tip'],
+                                      capitalize_label=False,
+                                      value=int(web_config['suggestions']['cache_exp']) if isinstance(
+                                          web_config['suggestions']['cache_exp'], int) else '',
+                                      only_int=True,
+                                      max_width=max_width,
+                                      id_='web_sugs_exp')
+
         form_env = FormComponent(label=self.i18n['web.settings.nativefier.env'].capitalize(),
-                                 components=[input_electron, select_nativefier, env_settings_exp])
+                                 components=[input_electron, select_nativefier, env_settings_exp, sugs_exp])
 
         return PanelComponent([form_env])
 
@@ -1046,6 +1057,7 @@ class WebApplicationManager(SoftwareManager):
 
         web_config['environment']['system'] = system_nativefier
         web_config['environment']['cache_exp'] = form_env.get_component('web_cache_exp').get_int_value()
+        web_config['suggestions']['cache_exp'] = form_env.get_component('web_sugs_exp').get_int_value()
 
         try:
             self.configman.save_config(web_config)

bauh/gems/web/resources/locale/ca

@@ -72,6 +72,8 @@ web.settings.nativefier.env=environment
 web.settings.nativefier.system.tooltip=The Nativefier version installed on your system will be used to install applications
 web.settings.nativefier.system=system
 web.settings.nativefier.tip=Defines which Nativefier version should be used to generate the Web applications
+web.settings.suggestions.cache_exp=Suggestions expiration
+web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated
 web.task.download_settings=Updating environment settings
 web.task.search_index=Indexing suggestions
 web.task.suggestions=Downloading suggestions

bauh/gems/web/resources/locale/en

@@ -72,6 +72,8 @@ web.settings.nativefier.env=environment
 web.settings.nativefier.system.tooltip=The Nativefier version installed on your system will be used to install applications
 web.settings.nativefier.system=system
 web.settings.nativefier.tip=Defines which Nativefier version should be used to generate the Web applications
+web.settings.suggestions.cache_exp=Suggestions expiration
+web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated.
 web.task.download_settings=Updating environment settings
 web.task.search_index=Indexing suggestions
 web.task.suggestions=Downloading suggestions

bauh/gems/web/resources/locale/es

@@ -63,7 +63,7 @@ web.install.substatus.checking_fixes=Verificando si hay correcciones publicadas
 web.install.substatus.options=Esperando las opciones de instalación
 web.install.substatus.shortcut=Generando un atajo de menú
 web.settings.cache_exp=Expiración de configuraciones de ambiente
-web.settings.cache_exp.tip=Define el período (en minutos) en el que las configuraciones del ambiente almacenadas se consideran válidas. Utilice 0 para que estén siempre actualizadas.
+web.settings.cache_exp.tip=Define el período (en minutos) en el que las configuraciones del ambiente almacenadas son consideradas válidas. Utilice 0 para que siempre sean actualizadas.
 web.settings.electron.version.label=Versión del Electron
 web.settings.electron.version.tooltip=Define una versión alternativa del Electron para renderizar las nuevas aplicaciones instaladas
 web.settings.env.nativefier.system.not_installed={} parece no estar instalado en su sistema
@@ -72,6 +72,8 @@ web.settings.nativefier.env=ambiente
 web.settings.nativefier.system.tooltip=Se utilizará la versión de Nativefier instalada en su sistema para instalar aplicaciones
 web.settings.nativefier.system=sistema
 web.settings.nativefier.tip=Define qué versión de Nativefier debe usarse para generar las aplicaciones Web
+web.settings.suggestions.cache_exp=Expiración de sugerencias
+web.settings.suggestions.cache_exp.tip=Define el período (en HORAS) en el que las sugerencias almacenadas en el disco son consideradas actualizadas durante el proceso de inicialización. Utilice 0 para que siempre sean actualizadas.
 web.task.download_settings=Actualizando configuraciones de ambiente
 web.task.search_index=Indexando sugerencias
 web.task.suggestions=Descargando sugerencias

bauh/gems/web/resources/locale/fr

@@ -72,6 +72,8 @@ web.settings.nativefier.env=environnement
 web.settings.nativefier.system.tooltip=La version de Nativefier installée sur votre système va être utilisée pour installer des applications
 web.settings.nativefier.system=système
 web.settings.nativefier.tip=Definit quelle version de Nativefier doit être utilisée pour géner les applications Web
+web.settings.suggestions.cache_exp=Suggestions expiration
+web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated
 web.task.download_settings=Mise à jour des paramètres d'environnement
 web.task.search_index=Indexing suggestions
 web.task.suggestions=Téléchargement de suggestions

bauh/gems/web/resources/locale/it

@@ -72,6 +72,8 @@ web.settings.nativefier.env=environment
 web.settings.nativefier.system.tooltip=The Nativefier version installed on your system will be used to install applications
 web.settings.nativefier.system=system
 web.settings.nativefier.tip=Defines which Nativefier version should be used to generate the Web applications
+web.settings.suggestions.cache_exp=Suggestions expiration
+web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated
 web.task.download_settings=Updating environment settings
 web.task.search_index=Indexing suggestions
 web.task.suggestions=Downloading suggestions

bauh/gems/web/resources/locale/pt

@@ -71,6 +71,8 @@ web.settings.nativefier.env=ambiente
 web.settings.nativefier.system.tooltip=A versão do Nativefier instalada no seu sistema será utilizada para instalar aplicativos
 web.settings.nativefier.system=sistema
 web.settings.nativefier.tip=Define qual versão do Nativefier será utilizada para gerar os aplicativos Web
+web.settings.suggestions.cache_exp=Expiração de sugestões
+web.settings.suggestions.cache_exp.tip=Define o período (em HORAS) em que as sugestões armazenadas em disco são consideradas atualizadas durante a inicialização. Use 0 para que elas sejam sempre atualizadas.
 web.task.download_settings=Atualizando configurações de ambiente
 web.task.search_index=Indexando sugestões
 web.task.suggestions=Baixando sugestões

bauh/gems/web/resources/locale/ru

@@ -72,6 +72,8 @@ web.settings.nativefier.env.tooltip=Будет использоваться ве
 web.settings.nativefier.system=Из системы
 web.settings.nativefier.system.tooltip=Будет использоваться версия Nativefier установленная в Вашей системе
 web.settings.nativefier.tip=Определяет, какие Nativefier версии должны быдут использованы для установки веб-приложений
+web.settings.suggestions.cache_exp=Suggestions expiration
+web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated
 web.task.download_settings=Updating environment settings
 web.task.search_index=Indexing suggestions
 web.task.suggestions=Downloading suggestions

bauh/gems/web/resources/locale/tr

@@ -72,6 +72,8 @@ web.settings.nativefier.env=çevre
 web.settings.nativefier.system.tooltip=Sisteminizde yüklü olan Nativefier sürümü uygulamaları yüklemek için kullanılacaktır
 web.settings.nativefier.system=sistem
 web.settings.nativefier.tip=Web uygulamalarını oluşturmak için hangi Nativefier sürümünün kullanılması gerektiğini tanımlar
+web.settings.suggestions.cache_exp=Suggestions expiration
+web.settings.suggestions.cache_exp.tip=It defines the period (in HOURS) in which suggestions stored on the disk are considered up to date during the initialization process. Use 0 so that they are always updated
 web.task.download_settings=Ortam ayarları güncelleniyor
 web.task.search_index=Indexing suggestions
 web.task.suggestions=Downloading suggestions

bauh/gems/web/suggestions.py

@@ -1,5 +1,6 @@
 import os
 import traceback
+from datetime import datetime, timedelta
 from logging import Logger
 from pathlib import Path
 
@@ -7,7 +8,7 @@ import requests
 import yaml
 
 from bauh.api.http import HttpClient
-from bauh.gems.web import URL_SUGGESTIONS, SUGGESTIONS_CACHE_FILE
+from bauh.gems.web import URL_SUGGESTIONS, SUGGESTIONS_CACHE_FILE, SUGGESTIONS_CACHE_TS_FILE
 from bauh.view.util.translation import I18n
 
 
@@ -18,6 +19,68 @@ class SuggestionsManager:
         self.logger = logger
         self.i18n = i18n
 
+    def should_download(self, web_config: dict) -> bool:
+        exp = web_config['suggestions']['cache_exp']
+        if web_config['suggestions']['cache_exp'] is None:
+            self.logger.info("No cache expiration defined for suggestions")
+            return True
+
+        try:
+            exp = int(exp)
+        except ValueError:
+            self.logger.error("Error while parsing the 'suggestions.cache_exp' ({}) settings property".format(exp))
+            return True
+
+        if exp <= 0:
+            self.logger.info("No cache expiration defined for suggestions ({})".format(exp))
+            return True
+
+        if not os.path.exists(SUGGESTIONS_CACHE_FILE):
+            self.logger.info("No suggestions cached file found '{}'".format(SUGGESTIONS_CACHE_FILE))
+            return True
+
+        if not os.path.exists(SUGGESTIONS_CACHE_TS_FILE):
+            self.logger.info("No suggestions cache file timestamp found '{}'".format(SUGGESTIONS_CACHE_TS_FILE))
+            return True
+
+        with open(SUGGESTIONS_CACHE_TS_FILE) as f:
+            timestamp_str = f.read()
+
+        try:
+            sugs_timestamp = datetime.fromtimestamp(float(timestamp_str))
+        except:
+            self.logger.error("Could not parse the cached suggestions file timestamp: {}".format(timestamp_str))
+            return True
+
+        expired = sugs_timestamp + timedelta(days=exp) <= datetime.utcnow()
+
+        if expired:
+            self.logger.info("Cached suggestions file has expired.")
+            return True
+        else:
+            self.logger.info("Cached suggestions file is up to date")
+            return False
+
+    def read_cached(self, check_file: bool = True) -> dict:
+        if check_file and not os.path.exists(SUGGESTIONS_CACHE_FILE):
+            self.logger.warning("Cached suggestions file does not exist ({})".format(SUGGESTIONS_CACHE_FILE))
+            return {}
+
+        self.logger.info("Reading cached suggestions file '{}'".format(SUGGESTIONS_CACHE_FILE))
+        with open(SUGGESTIONS_CACHE_FILE) as f:
+            sugs_str = f.read()
+
+        if not sugs_str:
+            self.logger.warning("Cached suggestions file '{}' is empty".format(SUGGESTIONS_CACHE_FILE))
+            return {}
+
+        try:
+            return yaml.safe_load(sugs_str)
+        except:
+            self.logger.error("An unexpected exception happened")
+            traceback.print_exc()
+            return {}
+
     def download(self) -> dict:
         self.logger.info("Reading suggestions from {}".format(URL_SUGGESTIONS))
         try:
@@ -35,7 +98,7 @@ class SuggestionsManager:
         self.logger.info("Finished")
         return suggestions
 
-    def save_to_disk(self, suggestions: dict):
+    def save_to_disk(self, suggestions: dict, timestamp: float):
         if not suggestions:
             return
 
@@ -51,9 +114,19 @@ class SuggestionsManager:
         try:
             with open(SUGGESTIONS_CACHE_FILE, 'w+') as f:
                 f.write(yaml.safe_dump(suggestions))
-
-            self.logger.info("{} suggestions successfully cached to file '{}'".format(len(suggestions), SUGGESTIONS_CACHE_FILE))
         except:
             self.logger.error("Could write to {}".format(SUGGESTIONS_CACHE_FILE))
             traceback.print_exc()
             return
+
+        self.logger.info("{} suggestions successfully cached to file '{}'".format(len(suggestions), SUGGESTIONS_CACHE_FILE))
+
+        try:
+            with open(SUGGESTIONS_CACHE_TS_FILE, 'w+') as f:
+                f.write(str(timestamp))
+        except:
+            self.logger.error("Could not write to {}".format(SUGGESTIONS_CACHE_TS_FILE))
+            traceback.print_exc()
+            return
+
+        self.logger.info("Suggestions cached file timestamp ({}) successfully saved at '{}'".format(timestamp, SUGGESTIONS_CACHE_TS_FILE))

bauh/gems/web/worker.py

@@ -1,6 +1,7 @@
 import logging
 import time
 import traceback
+from datetime import datetime
 from threading import Thread
 from typing import Optional
 
@@ -18,7 +19,8 @@ from bauh.view.util.translation import I18n
 class SuggestionsLoader(Thread):
 
     def __init__(self, taskman: TaskManager, manager: SuggestionsManager,
-                 i18n: I18n, logger: logging.Logger, suggestions_callback, suggestions: Optional[dict] = None):
+                 i18n: I18n, logger: logging.Logger, suggestions_callback, create_config: CreateConfigFile,
+                 internet_connection: bool, suggestions: Optional[dict] = None):
         super(SuggestionsLoader, self).__init__(daemon=True)
         self.taskman = taskman
         self.task_id = 'web_sugs'
@@ -27,30 +29,47 @@ class SuggestionsLoader(Thread):
         self.i18n = i18n
         self.logger = logger
         self.suggestions = suggestions
+        self.create_config = create_config
+        self.internet_connection = internet_connection
         self.task_name = self.i18n['web.task.suggestions']
         self.taskman.register_task(self.task_id, self.task_name, get_icon_path())
 
     def run(self):
         ti = time.time()
+        self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name)))
+        self.create_config.join()
+
         self.taskman.update_progress(self.task_id, 10, None)
-        try:
-            self.suggestions = self.manager.download()
 
-            if self.suggestions_callback:
+        if not self.internet_connection:
+            self.logger.warning("No internet connection. Only cached suggestions can be loaded")
+            self.suggestions = self.manager.read_cached(check_file=True)
+        elif not self.manager.should_download(self.create_config.config):
+            self.suggestions = self.manager.read_cached(check_file=False)
+        else:
+            try:
+                timestamp = datetime.utcnow().timestamp()
+                self.suggestions = self.manager.download()
+
+                if self.suggestions:
+                    self.taskman.update_progress(self.task_id, 50, self.i18n['web.task.suggestions.saving'])
+                    self.manager.save_to_disk(self.suggestions, timestamp)
+            except:
+                self.logger.error("Unexpected exception")
+                traceback.print_exc()
+
+        if self.suggestions_callback:
+            self.taskman.update_progress(self.task_id, 75, None)
+            try:
                 self.suggestions_callback(self.suggestions)
+            except:
+                self.logger.error("Unexpected exception")
+                traceback.print_exc()
 
-            if self.suggestions:
+        self.taskman.update_progress(self.task_id, 100, None)
-                self.taskman.update_progress(self.task_id, 50, self.i18n['web.task.suggestions.saving'])
+        self.taskman.finish_task(self.task_id)
-                self.manager.save_to_disk(self.suggestions)
+        tf = time.time()
-
+        self.logger.info("Finished. Took {0:.2f} seconds".format(tf - ti))
-        except:
-            self.logger.error("Unexpected exception")
-            traceback.print_exc()
-        finally:
-            self.taskman.update_progress(self.task_id, 100, None)
-            self.taskman.finish_task(self.task_id)
-            tf = time.time()
-            self.logger.info("Finished. Took {0:.2f} seconds".format(tf - ti))
 
 
 class SearchIndexGenerator(Thread):