[appimage] improvements -> database files only updated when expired

CHANGELOG.md

@@ -10,6 +10,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **git**: for AUR support.
 
 ### Improvements
+- AppImage
+    - database updater daemon dropped. Now the database is only downloaded/updated during the initialization process (if it is considered expired). Its expiration are controlled through the new settings property **database.expiration** (in minutes. default: 60 minutes. Use 0 so it is always updated).
+    <p align="center">
+            <img src="https://raw.githubusercontent.com/vinifmor/bauh-files/master/pictures/releases/0.9.11/appim_db_exp.png">
+    </p>
+    
+    - old settings properties were dropped (**db_updater.interval**, **db_updated.enabled**)
+    - database files (**apps.db** and **releases.db**) are now stored at **~/.cache/bauh/appimage**
+    - displaying a warning when the cached database files could not be found
+
 - Arch
     - AUR
         - upgrade checking now considers modification dates as well (needed because not all AUR packages follow versioning standards)

README.md

@@ -142,18 +142,16 @@ install_channel: false  # it allows to select an available channel during the ap
     - **Upgrade file**: allows to upgrade a manually installed AppImage file
 - Supported sources: [AppImageHub](https://appimage.github.io) (**applications with no releases published to GitHub are currently not available**)
 - Installed applications are store at **~/.local/share/bauh/appimage/installed**
-- Desktop entries ( menu shortcuts ) of the installed applications are stored at **~/.local/share/applications**
+- Desktop entries (menu shortcuts) of the installed applications are stored at **~/.local/share/applications**
 - Symlinks are created at **~/.local/bin**. They have the same name of the application (if the name already exists, it will be created as 'app_name-appimage'. e.g: 'rpcs3-appimage')
-- Downloaded database files are stored at **~/.local/share/bauh/appimage** as **apps.db** and **releases.db**
+- Downloaded database files are stored at **~/.cache/bauh/appimage** as **apps.db** and **releases.db**
-- Databases are always updated when bauh starts
+- Databases are updated during the initialization process if they are considered outdated (after 60 minutes by default, but this behavior can be changed via the AppImage settings)
-- Databases updater daemon running every 20 minutes (it can be customized via the configuration file described below)
 - All supported application names can be found at [apps.txt](https://github.com/vinifmor/bauh-files/blob/master/appimage/apps.txt)
 - Applications with ignored updates are defined at **~/.config/bauh/appimage/updates_ignored.txt**
 - The configuration file is located at **~/.config/bauh/appimage.yml** and it allows the following customizations:
 ```
-db_updater:
+database:
-  enabled: true  # if 'false': disables the daemon database updater (bauh will not be able to see if there are updates for your already installed AppImages)
+  expiration: 60  # defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it.
-  interval: 1200  # the databases update interval in SECONDS (1200 == 20 minutes)
 ```
 - Required dependencies
     - Arch-based systems: **sqlite**, **wget** (or **aria2**/**axel** for faster multi-threaded downloads)

bauh/api/abstract/cache.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Set
+from typing import Set, Optional
 
 
 class MemoryCache(ABC):
@@ -42,7 +42,7 @@ class MemoryCacheFactory(ABC):
     """
 
     @abstractmethod
-    def new(self, expiration: int) -> MemoryCache:
+    def new(self, expiration: Optional[int]) -> MemoryCache:
         """
         :param expiration: expiration time for the cache keys in seconds. Use -1 to disable this feature.
         :return:

bauh/api/abstract/controller.py

@@ -28,6 +28,10 @@ class SearchResult:
         self.new = new
         self.total = total
 
+    @classmethod
+    def empty(cls):
+        return cls(installed=[], new=[], total=0)
+
 
 class UpgradeRequirement:
 
@@ -116,7 +120,7 @@ class SoftwareManager(ABC):
         pass
 
     @abstractmethod
-    def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int, only_apps: bool, pkg_types: Set[Type[SoftwarePackage]], internet_available: bool) -> SearchResult:
+    def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int, only_apps: bool, pkg_types: Optional[Set[Type[SoftwarePackage]]], internet_available: bool) -> SearchResult:
         """
         :param disk_loader:  a running disk loader thread that loads application data from the disk asynchronously
         :param limit: the max number of packages to be retrieved. <= 1 should retrieve everything

bauh/gems/appimage/__init__.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 from typing import Optional
 
-from bauh.api.constants import CONFIG_PATH
+from bauh.api.constants import CONFIG_PATH, CACHE_PATH
 from bauh.commons import resource
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -13,6 +13,12 @@ CONFIG_FILE = '{}/appimage.yml'.format(CONFIG_PATH)
 CONFIG_DIR = '{}/appimage'.format(CONFIG_PATH)
 UPDATES_IGNORED_FILE = '{}/updates_ignored.txt'.format(CONFIG_DIR)
 SYMLINKS_DIR = '{}/.local/bin'.format(str(Path.home()))
+URL_COMPRESSED_DATABASES = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/dbs.tar.gz'
+DATABASES_DIR = '{}/appimage'.format(CACHE_PATH)
+DATABASE_APPS_FILE = '{}/apps.db'.format(DATABASES_DIR)
+DATABASE_RELEASES_FILE = '{}/releases.db'.format(DATABASES_DIR)
+DATABASES_TS_FILE = '{}/dbs.ts'.format(DATABASES_DIR)
+DESKTOP_ENTRIES_PATH = '{}/.local/share/applications'.format(str(Path.home()))
 
 
 def get_icon_path() -> str:

bauh/gems/appimage/config.py

@@ -4,9 +4,8 @@ from bauh.gems.appimage import CONFIG_FILE
 
 def read_config(update_file: bool = False) -> dict:
     default = {
-        'db_updater': {
+        'database': {
-            'interval': 60 * 20,
+            'expiration': 60
-            'enabled': True
         }
     }
     return read(CONFIG_FILE, default, update_file=update_file)

bauh/gems/appimage/controller.py

@@ -9,7 +9,6 @@ import traceback
 from datetime import datetime
 from math import floor
 from pathlib import Path
-from threading import Lock
 from typing import Set, Type, List, Tuple, Optional
 
 from colorama import Fore
@@ -29,17 +28,13 @@ from bauh.commons.config import save_config
 from bauh.commons.html import bold
 from bauh.commons.system import SystemProcess, new_subprocess, ProcessHandler, run_cmd, SimpleProcess
 from bauh.gems.appimage import query, INSTALLATION_PATH, LOCAL_PATH, SUGGESTIONS_FILE, CONFIG_FILE, ROOT_DIR, \
-    CONFIG_DIR, UPDATES_IGNORED_FILE, util, get_default_manual_installation_file_dir
+    CONFIG_DIR, UPDATES_IGNORED_FILE, util, get_default_manual_installation_file_dir, DATABASE_APPS_FILE, \
+    DATABASE_RELEASES_FILE, DESKTOP_ENTRIES_PATH, DATABASES_DIR
 from bauh.gems.appimage.config import read_config
 from bauh.gems.appimage.model import AppImage
 from bauh.gems.appimage.util import replace_desktop_entry_exec_command
 from bauh.gems.appimage.worker import DatabaseUpdater, SymlinksVerifier
 
-DB_APPS_PATH = '{}/{}'.format(str(Path.home()), '.local/share/bauh/appimage/apps.db')
-DB_RELEASES_PATH = '{}/{}'.format(str(Path.home()), '.local/share/bauh/appimage/releases.db')
-
-DESKTOP_ENTRIES_PATH = '{}/.local/share/applications'.format(str(Path.home()))
-
 RE_DESKTOP_ICON = re.compile(r'Icon\s*=\s*.+\n')
 RE_ICON_ENDS_WITH = re.compile(r'.+\.(png|svg)$')
 RE_APPIMAGE_NAME = re.compile(r'(.+)\.appimage', flags=re.IGNORECASE)
@@ -81,7 +76,6 @@ class AppImageManager(SoftwareManager):
         self.http_client = context.http_client
         self.logger = context.logger
         self.file_downloader = context.file_downloader
-        self.db_locks = {DB_APPS_PATH: Lock(), DB_RELEASES_PATH: Lock()}
         self.custom_actions = [CustomSoftwareAction(i18n_label_key='appimage.custom_action.install_file',
                                                     i18n_status_key='appimage.custom_action.install_file.status',
                                                     manager=self,
@@ -177,60 +171,74 @@ class AppImageManager(SoftwareManager):
 
     def _get_db_connection(self, db_path: str) -> sqlite3.Connection:
         if os.path.exists(db_path):
-            self.db_locks[db_path].acquire()
+            try:
-            return sqlite3.connect(db_path)
+                return sqlite3.connect(db_path)
+            except:
+                self.logger.error("Could not connect to database file '{}'".format(db_path))
+                traceback.print_exc()
         else:
-            self.logger.warning("Could not get a database connection. File '{}' not found".format(db_path))
+            self.logger.warning("Could not get a connection for database '{}'".format(db_path))
-
-    def _close_connection(self, db_path: str, con: sqlite3.Connection):
-        con.close()
-        self.db_locks[db_path].release()
 
     def _gen_app_key(self, app: AppImage):
         return '{}{}'.format(app.name.lower(), app.github.lower() if app.github else '')
 
     def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1, is_url: bool = False) -> SearchResult:
         if is_url:
-            return SearchResult([], [], 0)
+            return SearchResult.empty()
 
-        res = SearchResult([], [], 0)
+        apps_conn = self._get_db_connection(DATABASE_APPS_FILE)
-        connection = self._get_db_connection(DB_APPS_PATH)
 
-        if connection:
+        if not apps_conn:
-            try:
+            return SearchResult.empty()
-                cursor = connection.cursor()
-                cursor.execute(query.SEARCH_APPS_BY_NAME_OR_DESCRIPTION.format(words, words))
 
-                found_map = {}
+        not_installed, found_map = [], {}
-                idx = 0
-                for r in cursor.fetchall():
-                    app = AppImage(*r, i18n=self.i18n, custom_actions=self.custom_app_actions)
-                    res.new.append(app)
-                    found_map[self._gen_app_key(app)] = {'app': app, 'idx': idx}
-                    idx += 1
 
-            finally:
+        try:
-                self._close_connection(DB_APPS_PATH, connection)
+            cursor = apps_conn.cursor()
+            cursor.execute(query.SEARCH_APPS_BY_NAME_OR_DESCRIPTION.format(words, words))
 
-            if res.new:
+            idx = 0
-                installed = self.read_installed(disk_loader, limit, only_apps=False, pkg_types=None, internet_available=True).installed
+            for r in cursor.fetchall():
+                app = AppImage(*r, i18n=self.i18n, custom_actions=self.custom_app_actions)
+                not_installed.append(app)
+                found_map[self._gen_app_key(app)] = {'app': app, 'idx': idx}
+                idx += 1
+        except:
+            self.logger.error("An exception happened while querying the 'apps' database")
+            traceback.print_exc()
+            apps_conn.close()
+            return SearchResult.empty()
 
-                if installed:
+        installed_found = []
-                    for iapp in installed:
-                        key = self._gen_app_key(iapp)
 
-                        new_found = found_map.get(key)
+        if not_installed:
+            installed = self.read_installed(disk_loader=disk_loader, limit=limit,
+                                            only_apps=False,
+                                            pkg_types=None,
+                                            connection=apps_conn,
+                                            internet_available=True).installed
+            if installed:
+                for appim in installed:
+                    key = self._gen_app_key(appim)
 
-                        if new_found:
+                    new_found = found_map.get(key)
-                            del res.new[new_found['idx']]
-                            res.installed.append(iapp)
 
-        res.total = len(res.installed) + len(res.new)
+                    if new_found:
-        return res
+                        del not_installed[new_found['idx']]
+                        installed_found.append(appim)
 
-    def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False,
+        try:
-                       pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None, connection: sqlite3.Connection = None) -> SearchResult:
+            apps_conn.close()
-        res = SearchResult([], [], 0)
+        except:
+            self.logger.error("An exception happened when trying to close the connection to database file '{}'".format(DATABASE_APPS_FILE))
+            traceback.print_exc()
+
+        return SearchResult(new=not_installed, installed=installed_found, total=len(not_installed) + len(installed_found))
+
+    def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int = -1, only_apps: bool = False,
+                       pkg_types: Optional[Set[Type[SoftwarePackage]]] = None, internet_available: bool = None, connection: sqlite3.Connection = None) -> SearchResult:
+        installed_apps = []
+        res = SearchResult(installed_apps, [], 0)
 
         if os.path.exists(INSTALLATION_PATH):
             installed = run_cmd('ls {}*/data.json'.format(INSTALLATION_PATH), print_error=False)
@@ -243,19 +251,19 @@ class AppImageManager(SoftwareManager):
                             app = AppImage(installed=True, i18n=self.i18n, custom_actions=self.custom_app_actions, **json.loads(f.read()))
                             app.icon_url = app.icon_path
 
-                        res.installed.append(app)
+                        installed_apps.append(app)
                         names.add("'{}'".format(app.name.lower()))
 
-                if res.installed:
+                if installed_apps:
-                    con = self._get_db_connection(DB_APPS_PATH) if not connection else connection
+                    apps_con = self._get_db_connection(DATABASE_APPS_FILE) if not connection else connection
 
-                    if con:
+                    if apps_con:
                         try:
-                            cursor = con.cursor()
+                            cursor = apps_con.cursor()
                             cursor.execute(query.FIND_APPS_BY_NAME.format(','.join(names)))
 
                             for tup in cursor.fetchall():
-                                for app in res.installed:
+                                for app in installed_apps:
                                     if app.name.lower() == tup[0].lower() and (not app.github or app.github.lower() == tup[1].lower()):
                                         continuous_version = app.version == 'continuous'
                                         continuous_update = tup[2] == 'continuous'
@@ -276,15 +284,16 @@ class AppImageManager(SoftwareManager):
 
                                         break
                         except:
+                            self.logger.error("An exception happened while querying the database file {}".format(apps_con))
                             traceback.print_exc()
                         finally:
-                            if not connection:
+                            if not connection:  # the connection can only be closed if it was opened within this method
-                                self._close_connection(DB_APPS_PATH, con)
+                                apps_con.close()
 
                     ignored_updates = self._read_ignored_updates()
 
                     if ignored_updates:
-                        for app in res.installed:
+                        for app in installed_apps:
                             if app.supports_ignored_updates() and app.name in ignored_updates:
                                 app.updates_ignored = True
 
@@ -406,40 +415,53 @@ class AppImageManager(SoftwareManager):
         history = []
         res = PackageHistory(pkg, history, -1)
 
-        connection = self._get_db_connection(DB_APPS_PATH)
+        app_con = self._get_db_connection(DATABASE_APPS_FILE)
 
-        if connection:
+        if not app_con:
-            try:
+            return res
-                cursor = connection.cursor()
 
-                cursor.execute(query.FIND_APP_ID_BY_NAME_AND_GITHUB.format(pkg.name.lower(), pkg.github.lower() if pkg.github else ''))
+        try:
-                app_tuple = cursor.fetchone()
+            cursor = app_con.cursor()
 
-                if not app_tuple:
+            cursor.execute(query.FIND_APP_ID_BY_NAME_AND_GITHUB.format(pkg.name.lower(), pkg.github.lower() if pkg.github else ''))
-                    self.logger.warning("Could not retrieve {} from the database {}".format(pkg, DB_APPS_PATH))
+            app_tuple = cursor.fetchone()
-                    return res
-            finally:
-                self._close_connection(DB_APPS_PATH, connection)
 
-            connection = self._get_db_connection(DB_RELEASES_PATH)
+            if not app_tuple:
+                self.logger.warning("Could not retrieve {} from the database {}".format(pkg, DATABASE_APPS_FILE))
+                return res
+        except:
+            self.logger.error("An exception happened while querying the database file '{}'".format(DATABASE_APPS_FILE))
+            traceback.print_exc()
+            app_con.close()
+            return res
 
-            if connection:
+        app_con.close()
-                try:
-                    cursor = connection.cursor()
 
-                    releases = cursor.execute(query.FIND_RELEASES_BY_APP_ID.format(app_tuple[0]))
+        releases_con = self._get_db_connection(DATABASE_RELEASES_FILE)
 
-                    if releases:
+        if not releases_con:
-                        for idx, tup in enumerate(releases):
+            return res
-                            history.append({'0_version': tup[0], '1_published_at': datetime.strptime(tup[2], '%Y-%m-%dT%H:%M:%SZ') if tup[2] else '', '2_url_download': tup[1]})
 
-                            if res.pkg_status_idx == -1 and pkg.version == tup[0]:
+        try:
-                                res.pkg_status_idx = idx
+            cursor = releases_con.cursor()
 
-                finally:
+            releases = cursor.execute(query.FIND_RELEASES_BY_APP_ID.format(app_tuple[0]))
-                    self._close_connection(DB_RELEASES_PATH, connection)
 
-        return res
+            if releases:
+                for idx, tup in enumerate(releases):
+                    history.append({'0_version': tup[0],
+                                    '1_published_at': datetime.strptime(tup[2], '%Y-%m-%dT%H:%M:%SZ') if tup[
+                                        2] else '', '2_url_download': tup[1]})
+
+                    if res.pkg_status_idx == -1 and pkg.version == tup[0]:
+                        res.pkg_status_idx = idx
+
+                return res
+        except:
+            self.logger.error("An exception happened while querying the database file '{}'".format(DATABASE_RELEASES_FILE))
+            traceback.print_exc()
+        finally:
+            releases_con.close()
 
     def _find_desktop_file(self, folder: str) -> str:
         for r, d, files in os.walk(folder):
@@ -452,7 +474,7 @@ class AppImageManager(SoftwareManager):
             if RE_ICON_ENDS_WITH.match(f):
                 return f
 
-    def install(self, pkg: AppImage, root_password: str, disk_loader: DiskCacheLoader, watcher: ProcessWatcher) -> TransactionResult:
+    def install(self, pkg: AppImage, root_password: str, disk_loader: Optional[DiskCacheLoader], watcher: ProcessWatcher) -> TransactionResult:
         handler = ProcessHandler(watcher)
 
         out_dir = INSTALLATION_PATH + pkg.get_clean_name()
@@ -587,22 +609,17 @@ class AppImageManager(SoftwareManager):
         return False
 
     def prepare(self, task_manager: TaskManager, root_password: str, internet_available: bool):
-        local_config = read_config(update_file=True)
-        interval = local_config['db_updater']['interval'] or 20 * 60
-
-        updater = DatabaseUpdater(task_man=task_manager,
-                                  i18n=self.context.i18n,
-                                  http_client=self.context.http_client, logger=self.context.logger,
-                                  db_locks=self.db_locks, interval=interval,
-                                  internet_checker=self.context.internet_checker)
-        if local_config['db_updater']['enabled']:
-            updater.start()
-        elif internet_available:
-            updater.download_databases()  # only once
-
         symlink_check = SymlinksVerifier(taskman=task_manager, i18n=self.i18n, logger=self.logger)
         symlink_check.start()
 
+        if internet_available:
+            updater = DatabaseUpdater(task_man=task_manager, i18n=self.context.i18n,
+                                      http_client=self.context.http_client, logger=self.context.logger)
+
+            if updater.should_update(read_config()):
+                updater.register_task()
+                updater.start()
+
     def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
         res = self.read_installed(disk_loader=None, internet_available=internet_available)
 
@@ -615,12 +632,15 @@ class AppImageManager(SoftwareManager):
         return updates
 
     def list_warnings(self, internet_available: bool) -> List[str]:
-        pass
+        dbfiles = glob.glob('{}/*.db'.format(DATABASES_DIR))
+
+        if not dbfiles or len({f for f in (DATABASE_APPS_FILE, DATABASE_RELEASES_FILE) if f in dbfiles}) != 2:
+            return [self.i18n['appimage.warning.missing_db_files'].format(appimage=bold('AppImage'))]
 
     def list_suggestions(self, limit: int, filter_installed: bool) -> List[PackageSuggestion]:
         res = []
 
-        connection = self._get_db_connection(DB_APPS_PATH)
+        connection = self._get_db_connection(DATABASE_APPS_FILE)
 
         if connection:
             file = self.http_client.get(SUGGESTIONS_FILE)
@@ -661,7 +681,7 @@ class AppImageManager(SoftwareManager):
                 except:
                     traceback.print_exc()
                 finally:
-                    self._close_connection(DB_APPS_PATH, connection)
+                    connection.close()
 
         return res
 
@@ -705,34 +725,22 @@ class AppImageManager(SoftwareManager):
         config = read_config()
         max_width = floor(screen_width * 0.15)
 
-        enabled_opts = [InputOption(label=self.i18n['yes'].capitalize(), value=True),
+        opts = [
-                        InputOption(label=self.i18n['no'].capitalize(), value=False)]
+            TextInputComponent(label=self.i18n['appimage.config.database.expiration'],
-
+                               value=int(config['database']['expiration']) if isinstance(config['database']['expiration'], int) else '',
-        updater_opts = [
+                               tooltip=self.i18n['appimage.config.database.expiration.tip'],
-            SingleSelectComponent(label=self.i18n['appimage.config.db_updates.activated'],
-                                  options=enabled_opts,
-                                  default_option=[o for o in enabled_opts if o.value == config['db_updater']['enabled']][0],
-                                  max_per_line=len(enabled_opts),
-                                  type_=SelectViewType.RADIO,
-                                  tooltip=self.i18n['appimage.config.db_updates.activated.tip'],
-                                  max_width=max_width,
-                                  id_='up_enabled'),
-            TextInputComponent(label=self.i18n['interval'],
-                               value=str(config['db_updater']['interval']),
-                               tooltip=self.i18n['appimage.config.db_updates.interval.tip'],
                                only_int=True,
                                max_width=max_width,
-                               id_='up_int')
+                               id_='appim_db_exp')
         ]
 
-        return PanelComponent([FormComponent(updater_opts, self.i18n['appimage.config.db_updates'])])
+        return PanelComponent([FormComponent(opts, self.i18n['appimage.config.database'])])
 
     def save_settings(self, component: PanelComponent) -> Tuple[bool, Optional[List[str]]]:
         config = read_config()
 
         panel = component.components[0]
-        config['db_updater']['enabled'] = panel.get_component('up_enabled').get_selected()
+        config['database']['expiration'] = panel.get_component('appim_db_exp').get_int_value()
-        config['db_updater']['interval'] = panel.get_component('up_int').get_int_value()
 
         try:
             save_config(config, CONFIG_FILE)

bauh/gems/appimage/resources/locale/ca

@@ -1,7 +1,6 @@
-appimage.config.db_updates=Database update
+appimage.config.database=Database
-appimage.config.db_updates.activated=activated
+appimage.config.database.expiration=Expiration
-appimage.config.db_updates.activated.tip=It will be possible to check for updates related to the installed applications
+appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it.
-appimage.config.db_updates.interval.tip=Update interval in SECONDS
 appimage.custom_action.install_file=Install AppImage file
 appimage.custom_action.install_file.details=Installation details
 appimage.custom_action.install_file.invalid_file=You must define a valid AppImage file
@@ -34,5 +33,6 @@ appimage.install.permission=S’està concedint el permís d’execució a {}
 appimage.task.db_update=Actualització de bases de dades
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=No s’ha pogut suprimir el directori d’instal·lació de l’aplicació {}
+appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.
 gem.appimage.info=Aplicacions publicades a https://appimage.github.io/
 gem.appimage.label=AppImage

bauh/gems/appimage/resources/locale/de

@@ -1,7 +1,6 @@
-appimage.config.db_updates=Database update
+appimage.config.database=Database
-appimage.config.db_updates.activated=activated
+appimage.config.database.expiration=Expiration
-appimage.config.db_updates.activated.tip=It will be possible to check for updates related to the installed applications
+appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it.
-appimage.config.db_updates.interval.tip=Update interval in SECONDS
 appimage.custom_action.install_file=Install AppImage file
 appimage.custom_action.install_file.details=Installation details
 appimage.custom_action.install_file.invalid_file=You must define a valid AppImage file
@@ -34,5 +33,6 @@ appimage.install.permission=Ausführberechtigungen für {}
 appimage.task.db_update=Updating databases
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=Anwendungordner {} konnte nicht entfernt werden
+appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.
 gem.appimage.info=Anwendungen veröffentlicht auf https://appimage.github.io/
 gem.appimage.label=AppImage

bauh/gems/appimage/resources/locale/en

@@ -1,7 +1,6 @@
-appimage.config.db_updates=Database update
+appimage.config.database=Database
-appimage.config.db_updates.activated=activated
+appimage.config.database.expiration=Expiration
-appimage.config.db_updates.activated.tip=It will be possible to check for updates related to the installed applications
+appimage.config.database.expiration.tip=appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date. Use 0 if you always want to update it.
-appimage.config.db_updates.interval.tip=Update interval in SECONDS
 appimage.custom_action.install_file=Install AppImage file
 appimage.custom_action.install_file.details=Installation details
 appimage.custom_action.install_file.invalid_file=You must define a valid AppImage file
@@ -34,5 +33,6 @@ appimage.install.permission=Giving execution permission to {}
 appimage.task.db_update=Updating databases
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=Could not remove the application installation directory {}
+appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.
 gem.appimage.info=Applications published at https://appimage.github.io/
 gem.appimage.label=AppImage

bauh/gems/appimage/resources/locale/es

@@ -1,7 +1,6 @@
-appimage.config.db_updates=Actualización de la base de datos
+appimage.config.database=Base de datos
-appimage.config.db_updates.activated=activada
+appimage.config.database.expiration=Expiración
-appimage.config.db_updates.activated.tip=Será posible buscar actualizaciones relacionadas con las aplicaciones instaladas
+appimage.config.database.expiration.tip=Define el período (en minutos) en el que la base de datos será considerada actualizada durante el proceso de inicialización. Use 0 si siempre desea actualizarla.
-appimage.config.db_updates.interval.tip=Intervalo de actualización en SEGUNDOS
 appimage.custom_action.install_file=Instalar archivo AppImage
 appimage.custom_action.install_file.details=Detalles de instalación
 appimage.custom_action.install_file.invalid_file=Debe definir un archivo AppImage válido
@@ -34,5 +33,6 @@ appimage.install.permission=Concediendo permiso de ejecución a {}
 appimage.task.db_update=Actualizando base de datos
 appimage.task.symlink_check=Verificando links simbólicos
 appimage.uninstall.error.remove_folder=No se pudo eliminar el directorio de instalación de la aplicación {}
+appimage.warning.missing_db_files=No se encontraron archivos de la base de datos {appimage}. Algunas acciones no funcionarán correctamente.
 gem.appimage.info=Aplicativos publicados en https://appimage.github.io/
 gem.appimage.label=AppImage

bauh/gems/appimage/resources/locale/fr

@@ -1,7 +1,6 @@
-appimage.config.db_updates=Mise à jour de la base de données
+appimage.config.database=Database
-appimage.config.db_updates.activated=activé
+appimage.config.database.expiration=Expiration
-appimage.config.db_updates.activated.tip=La recherche de mises à jour liées aux applications installées sera possible
+appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date. Use 0 if you always want to update it.
-appimage.config.db_updates.interval.tip=Intervalle de mises à jour en SECONDES
 appimage.custom_action.install_file=Installer fichier AppImage
 appimage.custom_action.install_file.details=Détails d'installation
 appimage.custom_action.install_file.invalid_file=Vous devez définir un fichier AppImage valide
@@ -34,5 +33,6 @@ appimage.install.permission={} est maintenant exécutable
 appimage.task.db_update=Mise à jour des bases de données
 appimage.task.symlink_check=Verification des symlinks
 appimage.uninstall.error.remove_folder=Impossible de supprimer le répertoire d'installation {}
+appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.
 gem.appimage.info=Applications publiées à https://appimage.github.io/
 gem.appimage.label=AppImage

bauh/gems/appimage/resources/locale/it

@@ -1,7 +1,6 @@
-appimage.config.db_updates=Database update
+appimage.config.database=Database
-appimage.config.db_updates.activated=activated
+appimage.config.database.expiration=Expiration
-appimage.config.db_updates.activated.tip=It will be possible to check for updates related to the installed applications
+appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it.
-appimage.config.db_updates.interval.tip=Update interval in SECONDS
 appimage.custom_action.install_file=Install AppImage file
 appimage.custom_action.install_file.details=Installation details
 appimage.custom_action.install_file.invalid_file=You must define a valid AppImage file
@@ -34,5 +33,6 @@ appimage.install.permission=Gdare il permesso di esecuzione a {}
 appimage.task.db_update=Aggiornamento dei database
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=Impossibile rimuovere la directory di installazione dell'applicazione {}
+appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.
 gem.appimage.info=Applicazioni pubblicate su https://appimage.github.io/
 gem.appimage.label=AppImage

bauh/gems/appimage/resources/locale/pt

@@ -1,7 +1,6 @@
-appimage.config.db_updates=Atualização da base de dados
+appimage.config.database=Banco de dados
-appimage.config.db_updates.activated=ativada
+appimage.config.database.expiration=Expiração
-appimage.config.db_updates.activated.tip=Será possível verificar se há atualizações para os aplicativos instalados
+appimage.config.database.expiration.tip=Define o período (em minutos) no qual o banco de dados será considerado atualizado durante o processo de inicialização. Use 0 se você quiser que ele sempre seja atualizado.
-appimage.config.db_updates.interval.tip=Intervalo de atualização em SEGUNDOS
 appimage.custom_action.install_file=Instalar arquivo AppImage
 appimage.custom_action.install_file.details=Detalhes de instalação
 appimage.custom_action.install_file.invalid_file=Você precisa definir um arquivo AppImage válido
@@ -34,5 +33,6 @@ appimage.install.permission=Concedendo permissão de execução para {}
 appimage.task.db_update=Atualizando base de dados
 appimage.task.symlink_check=Verificando links simbólicos
 appimage.uninstall.error.remove_folder=Não foi possível remover o diretóri ode instalação do aplicativo {}
+appimage.warning.missing_db_files=Arquivos do banco de dados {appimage} não foram encontrados. Algumas ações não funcionarão corretamente.
 gem.appimage.info=Aplicativos publicados em https://appimage.github.io/
 gem.appimage.label=AppImage

bauh/gems/appimage/resources/locale/ru

@@ -1,7 +1,6 @@
-appimage.config.db_updates=Обновление базы данных
+appimage.config.database=Database
-appimage.config.db_updates.activated=Активировать
+appimage.config.database.expiration=Expiration
-appimage.config.db_updates.activated.tip=Позволит проверять обновления установленных приложений
+appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it.
-appimage.config.db_updates.interval.tip=Интервал обновления в секундах
 appimage.custom_action.install_file=Установить файл AppImage
 appimage.custom_action.install_file.details=Детали установки
 appimage.custom_action.install_file.invalid_file=Вы должны определить допустимый файл AppImage
@@ -34,5 +33,6 @@ appimage.install.permission=Установить права на выполне
 appimage.task.db_update=Обновление базы данных
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder=Не удалось удалить каталог приложения {}
+appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.
 gem.appimage.info=Приложения, опубликованные на https://appimage.github.io/
 gem.appimage.label=AppImage

bauh/gems/appimage/resources/locale/tr

@@ -1,7 +1,6 @@
-appimage.config.db_updates=Veritabanı güncelle
+appimage.config.database=Database
-appimage.config.db_updates.activated=Aktifleştirildi
+appimage.config.database.expiration=Expiration
-appimage.config.db_updates.activated.tip=Yüklü uygulamalarla ilgili güncellemeleri kontrol etmek mümkün olacak
+appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it.
-appimage.config.db_updates.interval.tip=SANİYE cinsinden güncelleme aralığı
 appimage.custom_action.install_file=AppImage dosyasını yükle
 appimage.custom_action.install_file.details=Kurulum detayları
 appimage.custom_action.install_file.invalid_file=Geçerli bir AppImage dosyası tanımlamanız gerekir
@@ -34,5 +33,6 @@ appimage.install.permission={} için yürütme izni veriliyor
 appimage.task.db_update=Veritabanları güncelleniyor
 appimage.task.symlink_check=Checking symlinks
 appimage.uninstall.error.remove_folder={} uygulama kurulum dizini kaldırılamadı
+appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.
 gem.appimage.info=https://appimage.github.io/ adresinde yayınlanan uygulamalar
 gem.appimage.label=AppImage

bauh/gems/appimage/worker.py

@@ -5,108 +5,151 @@ import os
 import tarfile
 import time
 import traceback
+from datetime import datetime, timedelta
 from pathlib import Path
 from threading import Thread
-
+from typing import Optional
-import requests
 
 from bauh.api.abstract.handler import TaskManager, ProcessWatcher
 from bauh.api.http import HttpClient
-from bauh.commons.internet import InternetChecker
+from bauh.gems.appimage import get_icon_path, INSTALLATION_PATH, SYMLINKS_DIR, util, DATABASES_TS_FILE, \
-from bauh.gems.appimage import LOCAL_PATH, get_icon_path, INSTALLATION_PATH, SYMLINKS_DIR, util
+    DATABASES_DIR, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES
 from bauh.gems.appimage.model import AppImage
 from bauh.view.util.translation import I18n
 
 
 class DatabaseUpdater(Thread):
-    URL_DB = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/dbs.tar.gz'
+    COMPRESS_FILE_PATH = '{}/db.tar.gz'.format(DATABASES_DIR)
-    COMPRESS_FILE_PATH = LOCAL_PATH + '/db.tar.gz'
 
-    def __init__(self, task_man: TaskManager, i18n: I18n, http_client: HttpClient,
+    def __init__(self, task_man: TaskManager, i18n: I18n, http_client: HttpClient, logger: logging.Logger):
-                 logger: logging.Logger, db_locks: dict, interval: int,
-                 internet_checker: InternetChecker):
         super(DatabaseUpdater, self).__init__(daemon=True)
         self.http_client = http_client
         self.logger = logger
-        self.db_locks = db_locks
-        self.sleep_time = interval
         self.i18n = i18n
         self.task_man = task_man
         self.task_id = 'appim_db'
-        self.internet_checker = internet_checker
+
+    def should_update(self, appimage_config: dict) -> bool:
+        ti = time.time()
+
+        try:
+            db_exp = int(appimage_config['database']['expiration'])
+        except ValueError:
+            self.logger.error("Could not parse settings property 'database.expiration': {}".format(appimage_config['database']['expiration']))
+            return True
+
+        if db_exp <= 0:
+            self.logger.info("No expiration time configured for the AppImage database")
+            return True
+
+        files = {*glob.glob('{}/*'.format(DATABASES_DIR))}
+
+        if not files:
+            self.logger.warning('No database files on {}'.format(DATABASES_DIR))
+            return True
+
+        if DATABASES_TS_FILE not in files:
+            self.logger.warning("No database timestamp file found ({})".format(DATABASES_TS_FILE))
+            return True
+
+        if DATABASE_APPS_FILE not in files:
+            self.logger.warning("Database file '{}' not found".format(DATABASE_APPS_FILE))
+            return True
+
+        if DATABASE_RELEASES_FILE not in files:
+            self.logger.warning("Database file '{}' not found".format(DATABASE_RELEASES_FILE))
+            return True
+
+        with open(DATABASES_TS_FILE) as f:
+            dbs_ts_str = f.read()
+
+        try:
+            dbs_timestamp = datetime.fromtimestamp(float(dbs_ts_str))
+        except:
+            self.logger.error('Could not parse the databases timestamp: {}'.format(dbs_ts_str))
+            traceback.print_exc()
+            return True
+
+        update = dbs_timestamp + timedelta(minutes=db_exp) <= datetime.utcnow()
+        self.logger.info('Finished. Took {0:.2f} seconds'.format(time.time() - ti))
+        return update
+
+    def register_task(self):
+        if self.task_man:
+            self.task_man.register_task(self.task_id, self.i18n['appimage.task.db_update'], get_icon_path())
 
     def _finish_task(self):
         if self.task_man:
             self.task_man.update_progress(self.task_id, 100, None)
             self.task_man.finish_task(self.task_id)
             self.task_man = None
+            self.logger.info("Finished")
+
+    def _update_task_progress(self, progress: float, substatus: Optional[str] = None):
+        if self.task_man:
+            self.task_man.update_progress(self.task_id, progress, substatus)
 
     def download_databases(self):
-        if self.task_man:
+        self._update_task_progress(1)  # TODO add substatus
-            self.task_man.register_task(self.task_id, self.i18n['appimage.task.db_update'], get_icon_path())
-            self.task_man.update_progress(self.task_id, 10, None)
-
-        try:
-            if not self.internet_checker.is_available():
-                self._finish_task()
-                return
-        except requests.exceptions.ConnectionError:
-            self.logger.warning('The internet connection seems to be off.')
-            self._finish_task()
-            return
-
         self.logger.info('Retrieving AppImage databases')
 
+        database_timestamp = datetime.utcnow().timestamp()
         try:
-            res = self.http_client.get(self.URL_DB, session=False)
+            res = self.http_client.get(URL_COMPRESSED_DATABASES, session=False)
         except Exception as e:
             self.logger.error("An error ocurred while downloading the AppImage database: {}".format(e.__class__.__name__))
             res = None
 
-        if res:
+        if not res:
-            Path(LOCAL_PATH).mkdir(parents=True, exist_ok=True)
+            self.logger.warning('Could not download the database file {}'.format(URL_COMPRESSED_DATABASES))
-
-            with open(self.COMPRESS_FILE_PATH, 'wb+') as f:
-                f.write(res.content)
-
-            self.logger.info("Database file saved at {}".format(self.COMPRESS_FILE_PATH))
-
-            old_db_files = glob.glob(LOCAL_PATH + '/*.db')
-
-            if old_db_files:
-                self.logger.info('Deleting old database files')
-                for f in old_db_files:
-                    self.db_locks[f].acquire()
-                    try:
-                        os.remove(f)
-                    finally:
-                        self.db_locks[f].release()
-
-                self.logger.info('Old database files deleted')
-
-            self.logger.info('Uncompressing {}'.format(self.COMPRESS_FILE_PATH))
-
-            try:
-                tf = tarfile.open(self.COMPRESS_FILE_PATH)
-                tf.extractall(LOCAL_PATH)
-                self.logger.info('Successfully uncompressed file {}'.format(self.COMPRESS_FILE_PATH))
-            except:
-                self.logger.error('Could not extract file {}'.format(self.COMPRESS_FILE_PATH))
-                traceback.print_exc()
-            finally:
-                self.logger.info('Deleting {}'.format(self.COMPRESS_FILE_PATH))
-                os.remove(self.COMPRESS_FILE_PATH)
-                self.logger.info('Successfully removed {}'.format(self.COMPRESS_FILE_PATH))
-            self._finish_task()
-        else:
-            self.logger.warning('Could not download the database file {}'.format(self.URL_DB))
             self._finish_task()
+            return
+
+        self._update_task_progress(25)  # TODO add substatus
+        Path(DATABASES_DIR).mkdir(parents=True, exist_ok=True)
+
+        with open(self.COMPRESS_FILE_PATH, 'wb+') as f:
+            f.write(res.content)
+
+        self.logger.info("Database file saved at {}".format(self.COMPRESS_FILE_PATH))
+
+        self._update_task_progress(50)  # TODO add substatus
+        old_db_files = glob.glob(DATABASES_DIR + '/*.db')
+
+        if old_db_files:
+            self.logger.info('Deleting old database files')
+            for f in old_db_files:
+                os.remove(f)
+
+            self.logger.info('Old database files deleted')
+
+        self._update_task_progress(75)  # TODO add substatus
+        self.logger.info('Uncompressing {}'.format(self.COMPRESS_FILE_PATH))
+
+        try:
+            tf = tarfile.open(self.COMPRESS_FILE_PATH)
+            tf.extractall(DATABASES_DIR)
+            self.logger.info('Successfully uncompressed file {}'.format(self.COMPRESS_FILE_PATH))
+        except:
+            self.logger.error('Could not extract file {}'.format(self.COMPRESS_FILE_PATH))
+            traceback.print_exc()
+        finally:
+            self.logger.info('Deleting {}'.format(self.COMPRESS_FILE_PATH))
+            os.remove(self.COMPRESS_FILE_PATH)
+            self.logger.info('File {} deleted'.format(self.COMPRESS_FILE_PATH))
+
+        self._update_task_progress(95)
+        self.logger.info("Saving database timestamp {}".format(database_timestamp))
+
+        with open(DATABASES_TS_FILE, 'w+') as f:
+            f.write(str(database_timestamp))
+
+        self.logger.info("Database timestamp saved")
+
+        self._finish_task()
 
     def run(self):
-        while True:
+        self.download_databases()
-            self.download_databases()
-            self.logger.info('Sleeping')
-            time.sleep(self.sleep_time)
 
 
 class SymlinksVerifier(Thread):

bauh/view/util/cache.py

@@ -1,6 +1,7 @@
 import datetime
 import time
 from threading import Lock, Thread
+from typing import Optional
 
 from bauh.api.abstract.cache import MemoryCache, MemoryCacheFactory
 
@@ -106,7 +107,7 @@ class DefaultMemoryCacheFactory(MemoryCacheFactory):
         self.expiration_time = expiration_time
         self.cleaner = cleaner
 
-    def new(self, expiration: int = None) -> MemoryCache:
+    def new(self, expiration: Optional[int] = None) -> MemoryCache:
         instance = DefaultMemoryCache(expiration if expiration is not None else self.expiration_time)
 
         if self.cleaner: