[appimage] improvement -> caching suggestions to disk

CHANGELOG.md

@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [0.9.14] 2021
 ### Improvements
+- AppImage
+    - caching suggestions to disk. The cache expiration can be controlled through the new property settings `suggestions.expiration` (in hours. Default: 24).
 - Web
     - nativefier URL moved to [bauh-files](https://github.com/vinifmor/bauh-files/blob/master/web/env/v1/environment.yml)
     

README.md

@@ -198,6 +198,8 @@ bauh is officially distributed through [PyPi](https://pypi.org/project/bauh) and
 ```
 database:
   expiration: 60  # defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it.
+suggestions:
+    expiration: 24  # defines the period (in hours) in which the cached suggestions will be considered up to date. Use 0 if you always want to update them.
 ```
 
 

bauh/gems/appimage/__init__.py

@@ -14,11 +14,13 @@ CONFIG_DIR = '{}/appimage'.format(CONFIG_PATH)
 UPDATES_IGNORED_FILE = '{}/updates_ignored.txt'.format(CONFIG_DIR)
 SYMLINKS_DIR = '{}/.local/bin'.format(str(Path.home()))
 URL_COMPRESSED_DATABASES = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/dbs.tar.gz'
-DATABASES_DIR = '{}/appimage'.format(CACHE_PATH)
+APPIMAGE_CACHE_PATH = '{}/appimage'.format(CACHE_PATH)
-DATABASE_APPS_FILE = '{}/apps.db'.format(DATABASES_DIR)
+DATABASE_APPS_FILE = '{}/apps.db'.format(APPIMAGE_CACHE_PATH)
-DATABASE_RELEASES_FILE = '{}/releases.db'.format(DATABASES_DIR)
+DATABASE_RELEASES_FILE = '{}/releases.db'.format(APPIMAGE_CACHE_PATH)
-DATABASES_TS_FILE = '{}/dbs.ts'.format(DATABASES_DIR)
+DATABASES_TS_FILE = '{}/dbs.ts'.format(APPIMAGE_CACHE_PATH)
 DESKTOP_ENTRIES_PATH = '{}/.local/share/applications'.format(str(Path.home()))
+SUGGESTIONS_CACHED_FILE = '{}/suggestions.txt'.format(APPIMAGE_CACHE_PATH)
+SUGGESTIONS_CACHED_TS_FILE = '{}/suggestions.ts'.format(APPIMAGE_CACHE_PATH)
 
 
 def get_icon_path() -> str:

bauh/gems/appimage/config.py

@@ -11,5 +11,8 @@ class AppImageConfigManager(YAMLConfigManager):
         return {
             'database': {
                 'expiration': 60
+            },
+            'suggestions': {
+                'expiration': 24
             }
         }

bauh/gems/appimage/controller.py

@@ -27,13 +27,13 @@ from bauh.commons import resource
 from bauh.commons.boot import CreateConfigFile
 from bauh.commons.html import bold
 from bauh.commons.system import SystemProcess, new_subprocess, ProcessHandler, run_cmd, SimpleProcess
-from bauh.gems.appimage import query, INSTALLATION_PATH, LOCAL_PATH, SUGGESTIONS_FILE, ROOT_DIR, \
+from bauh.gems.appimage import query, INSTALLATION_PATH, LOCAL_PATH, ROOT_DIR, \
     CONFIG_DIR, UPDATES_IGNORED_FILE, util, get_default_manual_installation_file_dir, DATABASE_APPS_FILE, \
-    DATABASE_RELEASES_FILE, DESKTOP_ENTRIES_PATH, DATABASES_DIR, get_icon_path
+    DATABASE_RELEASES_FILE, DESKTOP_ENTRIES_PATH, APPIMAGE_CACHE_PATH, get_icon_path
 from bauh.gems.appimage.config import AppImageConfigManager
 from bauh.gems.appimage.model import AppImage
 from bauh.gems.appimage.util import replace_desktop_entry_exec_command
-from bauh.gems.appimage.worker import DatabaseUpdater, SymlinksVerifier
+from bauh.gems.appimage.worker import DatabaseUpdater, SymlinksVerifier, AppImageSuggestionsDownloader
 
 RE_DESKTOP_ICON = re.compile(r'Icon\s*=\s*.+\n')
 RE_ICON_ENDS_WITH = re.compile(r'.+\.(png|svg)$')
@@ -629,6 +629,10 @@ class AppImageManager(SoftwareManager):
                             create_config=create_config, http_client=self.context.http_client,
                             logger=self.context.logger).start()
 
+            AppImageSuggestionsDownloader(taskman=task_manager, i18n=self.context.i18n,
+                                          http_client=self.context.http_client, logger=self.context.logger,
+                                          create_config=create_config).start()
+
     def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
         res = self.read_installed(disk_loader=None, internet_available=internet_available)
 
@@ -641,7 +645,7 @@ class AppImageManager(SoftwareManager):
         return updates
 
     def list_warnings(self, internet_available: bool) -> List[str]:
-        dbfiles = glob.glob('{}/*.db'.format(DATABASES_DIR))
+        dbfiles = glob.glob('{}/*.db'.format(APPIMAGE_CACHE_PATH))
 
         if not dbfiles or len({f for f in (DATABASE_APPS_FILE, DATABASE_RELEASES_FILE) if f in dbfiles}) != 2:
             return [self.i18n['appimage.warning.missing_db_files'].format(appimage=bold('AppImage'))]
@@ -652,16 +656,16 @@ class AppImageManager(SoftwareManager):
         connection = self._get_db_connection(DATABASE_APPS_FILE)
 
         if connection:
-            file = self.http_client.get(SUGGESTIONS_FILE)
+            suggestions = AppImageSuggestionsDownloader(appimage_config=self.configman.get_config(), logger=self.logger,
+                                                        i18n=self.i18n, http_client=self.http_client,
+                                                        taskman=TaskManager()).read()
 
-            if not file or not file.text:
+            if not suggestions:
-                self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_FILE))
+                self.logger.warning("Could not read suggestions")
                 return res
             else:
                 self.logger.info("Mapping suggestions")
                 try:
-                    sugs = [l for l in file.text.split('\n') if l]
-
                     if filter_installed:
                         installed = {i.name.lower() for i in self.read_installed(disk_loader=None, connection=connection).installed}
                     else:
@@ -669,7 +673,7 @@ class AppImageManager(SoftwareManager):
 
                     sugs_map = {}
 
-                    for s in sugs:
+                    for s in suggestions:
                         lsplit = s.split('=')
 
                         name = lsplit[1].strip()

bauh/gems/appimage/resources/locale/ca

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files
 appimage.task.db_update=Actualització de bases de dades
 appimage.task.db_update.checking=Checking for updates
+appimage.task.suggestions=Downloading suggestions
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=No s’ha pogut suprimir el directori d’instal·lació de l’aplicació {}
 appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

bauh/gems/appimage/resources/locale/de

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files
 appimage.task.db_update=Updating databases
 appimage.task.db_update.checking=Checking for updates
+appimage.task.suggestions=Downloading suggestions
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=Anwendungordner {} konnte nicht entfernt werden
 appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

bauh/gems/appimage/resources/locale/en

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files
 appimage.task.db_update=Updating databases
 appimage.task.db_update.checking=Checking for updates
+appimage.task.suggestions=Downloading suggestions
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=Could not remove the application installation directory {}
 appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

bauh/gems/appimage/resources/locale/es

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Descargando archivos de la base de datos
 appimage.update_database.uncompressing=Descomprindo archivos
 appimage.task.db_update=Actualizando base de datos
 appimage.task.db_update.checking=Buscando actualizaciones
+appimage.task.suggestions=Descargando sugerencias
 appimage.task.symlink_check=Verificando links simbólicos
 appimage.uninstall.error.remove_folder=No se pudo eliminar el directorio de instalación de la aplicación {}
 appimage.warning.missing_db_files=No se encontraron archivos de la base de datos {appimage}. Algunas acciones no funcionarán correctamente.

bauh/gems/appimage/resources/locale/fr

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files
 appimage.task.db_update=Mise à jour des bases de données
 appimage.task.db_update.checking=Checking for updates
+appimage.task.suggestions=Downloading suggestions
 appimage.task.symlink_check=Verification des symlinks
 appimage.uninstall.error.remove_folder=Impossible de supprimer le répertoire d'installation {}
 appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

bauh/gems/appimage/resources/locale/it

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files
 appimage.task.db_update=Aggiornamento dei database
 appimage.task.db_update.checking=Checking for updates
+appimage.task.suggestions=Downloading suggestions
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=Impossibile rimuovere la directory di installazione dell'applicazione {}
 appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

bauh/gems/appimage/resources/locale/pt

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Baixando arquivos do banco de dados
 appimage.update_database.uncompressing=Descomprimindo arquivos
 appimage.task.db_update=Atualizando base de dados
 appimage.task.db_update.checking=Checando atualizações
+appimage.task.suggestions=Baixando sugestões
 appimage.task.symlink_check=Verificando links simbólicos
 appimage.uninstall.error.remove_folder=Não foi possível remover o diretóri ode instalação do aplicativo {}
 appimage.warning.missing_db_files=Arquivos do banco de dados {appimage} não foram encontrados. Algumas ações não funcionarão corretamente.

bauh/gems/appimage/resources/locale/ru

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files
 appimage.task.db_update=Обновление базы данных
 appimage.task.db_update.checking=Checking for updates
+appimage.task.suggestions=Downloading suggestions
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=Не удалось удалить каталог приложения {}
 appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

bauh/gems/appimage/resources/locale/tr

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files
 appimage.task.db_update=Veritabanları güncelleniyor
 appimage.task.db_update.checking=Checking for updates
+appimage.task.suggestions=Downloading suggestions
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder={} uygulama kurulum dizini kaldırılamadı
 appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

bauh/gems/appimage/worker.py

@@ -8,20 +8,23 @@ import traceback
 from datetime import datetime, timedelta
 from pathlib import Path
 from threading import Thread
-from typing import Optional
+from typing import Optional, List
+
+import requests
 
 from bauh.api.abstract.handler import TaskManager, ProcessWatcher
 from bauh.api.http import HttpClient
 from bauh.commons.boot import CreateConfigFile
 from bauh.commons.html import bold
 from bauh.gems.appimage import get_icon_path, INSTALLATION_PATH, SYMLINKS_DIR, util, DATABASES_TS_FILE, \
-    DATABASES_DIR, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES
+    APPIMAGE_CACHE_PATH, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES, SUGGESTIONS_FILE, \
+    SUGGESTIONS_CACHED_TS_FILE, SUGGESTIONS_CACHED_FILE
 from bauh.gems.appimage.model import AppImage
 from bauh.view.util.translation import I18n
 
 
 class DatabaseUpdater(Thread):
-    COMPRESS_FILE_PATH = '{}/db.tar.gz'.format(DATABASES_DIR)
+    COMPRESS_FILE_PATH = '{}/db.tar.gz'.format(APPIMAGE_CACHE_PATH)
 
     def __init__(self, i18n: I18n, http_client: HttpClient, logger: logging.Logger, taskman: TaskManager,
                  watcher: Optional[ProcessWatcher] = None, appimage_config: Optional[dict] = None, create_config: Optional[CreateConfigFile] = None):
@@ -49,10 +52,10 @@ class DatabaseUpdater(Thread):
             self.logger.info("No expiration time configured for the AppImage database")
             return True
 
-        files = {*glob.glob('{}/*'.format(DATABASES_DIR))}
+        files = {*glob.glob('{}/*'.format(APPIMAGE_CACHE_PATH))}
 
         if not files:
-            self.logger.warning('No database files on {}'.format(DATABASES_DIR))
+            self.logger.warning('No database files on {}'.format(APPIMAGE_CACHE_PATH))
             return True
 
         if DATABASES_TS_FILE not in files:
@@ -102,7 +105,7 @@ class DatabaseUpdater(Thread):
             self.logger.warning('Could not download the database file {}'.format(URL_COMPRESSED_DATABASES))
             return False
 
-        Path(DATABASES_DIR).mkdir(parents=True, exist_ok=True)
+        Path(APPIMAGE_CACHE_PATH).mkdir(parents=True, exist_ok=True)
 
         with open(self.COMPRESS_FILE_PATH, 'wb+') as f:
             f.write(res.content)
@@ -110,7 +113,7 @@ class DatabaseUpdater(Thread):
         self.logger.info("Database file saved at {}".format(self.COMPRESS_FILE_PATH))
 
         self._update_task_progress(50, self.i18n['appimage.update_database.deleting_old'])
-        old_db_files = glob.glob(DATABASES_DIR + '/*.db')
+        old_db_files = glob.glob(APPIMAGE_CACHE_PATH + '/*.db')
 
         if old_db_files:
             self.logger.info('Deleting old database files')
@@ -124,7 +127,7 @@ class DatabaseUpdater(Thread):
 
         try:
             tf = tarfile.open(self.COMPRESS_FILE_PATH)
-            tf.extractall(DATABASES_DIR)
+            tf.extractall(APPIMAGE_CACHE_PATH)
             self.logger.info('Successfully uncompressed file {}'.format(self.COMPRESS_FILE_PATH))
         except:
             self.logger.error('Could not extract file {}'.format(self.COMPRESS_FILE_PATH))
@@ -279,3 +282,146 @@ class SymlinksVerifier(Thread):
         self.logger.info("No AppImage applications found. Aborting")
         self.taskman.update_progress(self.task_id, 100, '')
         self.taskman.finish_task(self.task_id)
+
+
+class AppImageSuggestionsDownloader(Thread):
+
+    def __init__(self, logger: logging.Logger, http_client: HttpClient, i18n: I18n, taskman: TaskManager, create_config: Optional[CreateConfigFile] = None,  appimage_config: Optional[dict] = None):
+        super(AppImageSuggestionsDownloader, self).__init__(daemon=True)
+        self.create_config = create_config
+        self.logger = logger
+        self.i18n = i18n
+        self.http_client = http_client
+        self.taskman = taskman
+        self.config = appimage_config
+        self.task_id = 'appim.suggestions'
+        self.taskman.register_task(id_=self.task_id, label=i18n['appimage.task.suggestions'], icon_path=get_icon_path())
+
+    def should_download(self, appimage_config: dict) -> bool:
+        try:
+            exp_hours = int(appimage_config['suggestions']['expiration'])
+        except:
+            self.logger.error("An exception happened while trying to parse 'suggestions.expiration'")
+            traceback.print_exc()
+            return True
+
+        if exp_hours <= 0:
+            self.logger.info("Suggestions cache is disabled")
+            return True
+
+        if not os.path.exists(SUGGESTIONS_CACHED_FILE):
+            self.logger.info("'{}' not found. It must be downloaded".format(SUGGESTIONS_CACHED_FILE))
+            return True
+
+        if not os.path.exists(SUGGESTIONS_CACHED_TS_FILE):
+            self.logger.info("'{}' not found. The suggestions file must be downloaded.")
+            return True
+
+        with open(SUGGESTIONS_CACHED_TS_FILE) as f:
+            timestamp_str = f.read()
+
+        try:
+            suggestions_timestamp = datetime.fromtimestamp(float(timestamp_str))
+        except:
+            self.logger.error('Could not parse the cached suggestions timestamp: {}'.format(timestamp_str))
+            traceback.print_exc()
+            return True
+
+        update = suggestions_timestamp + timedelta(hours=exp_hours) <= datetime.utcnow()
+        return update
+
+    def read(self) -> Optional[List[str]]:
+        self.logger.info("Checking if suggestions should be downloaded")
+        if self.should_download(self.config):
+            suggestions_timestamp = datetime.utcnow().timestamp()
+            suggestions_str = self.download()
+
+            Thread(target=self.cache_suggestions, args=(suggestions_str, suggestions_timestamp), daemon=True).start()
+        else:
+            self.logger.info("Reading cached suggestions from '{}'".format(SUGGESTIONS_CACHED_FILE))
+            with open(SUGGESTIONS_CACHED_FILE) as f:
+                suggestions_str = f.read()
+
+        return self.map_suggestions(suggestions_str) if suggestions_str else None
+
+    def cache_suggestions(self, text: str, timestamp: float):
+        self.logger.info("Caching suggestions to '{}'".format(SUGGESTIONS_FILE))
+
+        cache_dir = os.path.dirname(SUGGESTIONS_CACHED_FILE)
+
+        try:
+            Path(cache_dir).mkdir(parents=True, exist_ok=True)
+            cache_dir_ok = True
+        except OSError:
+            self.logger.error("Could not create cache directory '{}'".format(cache_dir))
+            traceback.print_exc()
+            cache_dir_ok = False
+
+        if cache_dir_ok:
+            try:
+                with open(SUGGESTIONS_CACHED_FILE, 'w+') as f:
+                    f.write(text)
+            except:
+                self.logger.error("An exception happened while writing the file '{}'".format(SUGGESTIONS_FILE))
+                traceback.print_exc()
+
+            try:
+                with open(SUGGESTIONS_CACHED_TS_FILE, 'w+') as f:
+                    f.write(str(timestamp))
+            except:
+                self.logger.error("An exception happened while writing the file '{}'".format(SUGGESTIONS_CACHED_TS_FILE))
+                traceback.print_exc()
+
+    def download(self) -> Optional[str]:
+        self.logger.info("Downloading suggestions from {}".format(SUGGESTIONS_FILE))
+
+        try:
+            res = self.http_client.get(SUGGESTIONS_FILE)
+        except requests.exceptions.ConnectionError:
+            self.logger.warning("Could not download suggestion from '{}'".format(SUGGESTIONS_FILE))
+            return
+
+        if not res:
+            self.logger.warning("Could not download suggestion from '{}'".format(SUGGESTIONS_FILE))
+            return
+
+        if not res.text:
+            self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_FILE))
+            return
+
+        return res.text
+
+    def map_suggestions(self, text: str) -> List[str]:
+        return [line for line in text.split('\n') if line]
+
+    def run(self):
+        if self.create_config:
+            self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name)))
+            self.create_config.join()
+            self.config = self.create_config.config
+
+        ti = time.time()
+        self.taskman.update_progress(self.task_id, 1, None)
+
+        self.logger.info("Checking if suggestions should be downloaded")
+        should_download = self.should_download(self.config)
+        self.taskman.update_progress(self.task_id, 30, None)
+
+        try:
+            if should_download:
+                suggestions_timestamp = datetime.utcnow().timestamp()
+                suggestions_str = self.download()
+                self.taskman.update_progress(self.task_id, 70, None)
+
+                if suggestions_str:
+                    self.cache_suggestions(suggestions_str, suggestions_timestamp)
+            else:
+                self.logger.info("Cached suggestions are up-to-date")
+        except:
+            self.logger.error("An unexpected exception happened")
+            traceback.print_exc()
+
+        self.taskman.update_progress(self.task_id, 100, None)
+        self.taskman.finish_task(self.task_id)
+        tf = time.time()
+        self.logger.info("Took {0:.9f} seconds to download suggestions".format(tf - ti))

bauh/view/core/controller.py

@@ -477,7 +477,7 @@ class GenericSoftwareManager(SoftwareManager):
             mti = time.time()
             man_sugs = man.list_suggestions(limit=limit, filter_installed=filter_installed)
             mtf = time.time()
-            self.logger.info(man.__class__.__name__ + ' took {0:.2f} seconds'.format(mtf - mti))
+            self.logger.info(man.__class__.__name__ + ' took {0:.5f} seconds'.format(mtf - mti))
 
             if man_sugs:
                 if 0 < limit < len(man_sugs):