diff --git a/CHANGELOG.md b/CHANGELOG.md index dee08ea9..4595c307 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **git**: for AUR support. ### Improvements +- AppImage + - database updater daemon dropped. Now the database is only downloaded/updated during the initialization process (if it is considered expired). Its expiration are controlled through the new settings property **database.expiration** (in minutes. default: 60 minutes. Use 0 so it is always updated). +

+ +

+ + - old settings properties were dropped (**db_updater.interval**, **db_updated.enabled**) + - database files (**apps.db** and **releases.db**) are now stored at **~/.cache/bauh/appimage** + - displaying a warning when the cached database files could not be found + - Arch - AUR - upgrade checking now considers modification dates as well (needed because not all AUR packages follow versioning standards) diff --git a/README.md b/README.md index e907d30d..fe2da841 100644 --- a/README.md +++ b/README.md @@ -142,18 +142,16 @@ install_channel: false # it allows to select an available channel during the ap - **Upgrade file**: allows to upgrade a manually installed AppImage file - Supported sources: [AppImageHub](https://appimage.github.io) (**applications with no releases published to GitHub are currently not available**) - Installed applications are store at **~/.local/share/bauh/appimage/installed** -- Desktop entries ( menu shortcuts ) of the installed applications are stored at **~/.local/share/applications** +- Desktop entries (menu shortcuts) of the installed applications are stored at **~/.local/share/applications** - Symlinks are created at **~/.local/bin**. They have the same name of the application (if the name already exists, it will be created as 'app_name-appimage'. e.g: 'rpcs3-appimage') -- Downloaded database files are stored at **~/.local/share/bauh/appimage** as **apps.db** and **releases.db** -- Databases are always updated when bauh starts -- Databases updater daemon running every 20 minutes (it can be customized via the configuration file described below) +- Downloaded database files are stored at **~/.cache/bauh/appimage** as **apps.db** and **releases.db** +- Databases are updated during the initialization process if they are considered outdated (after 60 minutes by default, but this behavior can be changed via the AppImage settings) - All supported application names can be found at [apps.txt](https://github.com/vinifmor/bauh-files/blob/master/appimage/apps.txt) - Applications with ignored updates are defined at **~/.config/bauh/appimage/updates_ignored.txt** - The configuration file is located at **~/.config/bauh/appimage.yml** and it allows the following customizations: ``` -db_updater: - enabled: true # if 'false': disables the daemon database updater (bauh will not be able to see if there are updates for your already installed AppImages) - interval: 1200 # the databases update interval in SECONDS (1200 == 20 minutes) +database: + expiration: 60 # defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it. ``` - Required dependencies - Arch-based systems: **sqlite**, **wget** (or **aria2**/**axel** for faster multi-threaded downloads) diff --git a/bauh/api/abstract/cache.py b/bauh/api/abstract/cache.py index ac3c2534..c0bcabcf 100644 --- a/bauh/api/abstract/cache.py +++ b/bauh/api/abstract/cache.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Set +from typing import Set, Optional class MemoryCache(ABC): @@ -42,7 +42,7 @@ class MemoryCacheFactory(ABC): """ @abstractmethod - def new(self, expiration: int) -> MemoryCache: + def new(self, expiration: Optional[int]) -> MemoryCache: """ :param expiration: expiration time for the cache keys in seconds. Use -1 to disable this feature. :return: diff --git a/bauh/api/abstract/controller.py b/bauh/api/abstract/controller.py index 335814b9..aaa18d53 100644 --- a/bauh/api/abstract/controller.py +++ b/bauh/api/abstract/controller.py @@ -28,6 +28,10 @@ class SearchResult: self.new = new self.total = total + @classmethod + def empty(cls): + return cls(installed=[], new=[], total=0) + class UpgradeRequirement: @@ -116,7 +120,7 @@ class SoftwareManager(ABC): pass @abstractmethod - def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int, only_apps: bool, pkg_types: Set[Type[SoftwarePackage]], internet_available: bool) -> SearchResult: + def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int, only_apps: bool, pkg_types: Optional[Set[Type[SoftwarePackage]]], internet_available: bool) -> SearchResult: """ :param disk_loader: a running disk loader thread that loads application data from the disk asynchronously :param limit: the max number of packages to be retrieved. <= 1 should retrieve everything diff --git a/bauh/gems/appimage/__init__.py b/bauh/gems/appimage/__init__.py index b2242bf3..d64ddf62 100644 --- a/bauh/gems/appimage/__init__.py +++ b/bauh/gems/appimage/__init__.py @@ -2,7 +2,7 @@ import os from pathlib import Path from typing import Optional -from bauh.api.constants import CONFIG_PATH +from bauh.api.constants import CONFIG_PATH, CACHE_PATH from bauh.commons import resource ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -13,6 +13,12 @@ CONFIG_FILE = '{}/appimage.yml'.format(CONFIG_PATH) CONFIG_DIR = '{}/appimage'.format(CONFIG_PATH) UPDATES_IGNORED_FILE = '{}/updates_ignored.txt'.format(CONFIG_DIR) SYMLINKS_DIR = '{}/.local/bin'.format(str(Path.home())) +URL_COMPRESSED_DATABASES = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/dbs.tar.gz' +DATABASES_DIR = '{}/appimage'.format(CACHE_PATH) +DATABASE_APPS_FILE = '{}/apps.db'.format(DATABASES_DIR) +DATABASE_RELEASES_FILE = '{}/releases.db'.format(DATABASES_DIR) +DATABASES_TS_FILE = '{}/dbs.ts'.format(DATABASES_DIR) +DESKTOP_ENTRIES_PATH = '{}/.local/share/applications'.format(str(Path.home())) def get_icon_path() -> str: diff --git a/bauh/gems/appimage/config.py b/bauh/gems/appimage/config.py index 8ceefa84..6c80328e 100644 --- a/bauh/gems/appimage/config.py +++ b/bauh/gems/appimage/config.py @@ -4,9 +4,8 @@ from bauh.gems.appimage import CONFIG_FILE def read_config(update_file: bool = False) -> dict: default = { - 'db_updater': { - 'interval': 60 * 20, - 'enabled': True + 'database': { + 'expiration': 60 } } return read(CONFIG_FILE, default, update_file=update_file) diff --git a/bauh/gems/appimage/controller.py b/bauh/gems/appimage/controller.py index d55d6571..bbe4241b 100644 --- a/bauh/gems/appimage/controller.py +++ b/bauh/gems/appimage/controller.py @@ -9,7 +9,6 @@ import traceback from datetime import datetime from math import floor from pathlib import Path -from threading import Lock from typing import Set, Type, List, Tuple, Optional from colorama import Fore @@ -29,17 +28,13 @@ from bauh.commons.config import save_config from bauh.commons.html import bold from bauh.commons.system import SystemProcess, new_subprocess, ProcessHandler, run_cmd, SimpleProcess from bauh.gems.appimage import query, INSTALLATION_PATH, LOCAL_PATH, SUGGESTIONS_FILE, CONFIG_FILE, ROOT_DIR, \ - CONFIG_DIR, UPDATES_IGNORED_FILE, util, get_default_manual_installation_file_dir + CONFIG_DIR, UPDATES_IGNORED_FILE, util, get_default_manual_installation_file_dir, DATABASE_APPS_FILE, \ + DATABASE_RELEASES_FILE, DESKTOP_ENTRIES_PATH, DATABASES_DIR from bauh.gems.appimage.config import read_config from bauh.gems.appimage.model import AppImage from bauh.gems.appimage.util import replace_desktop_entry_exec_command from bauh.gems.appimage.worker import DatabaseUpdater, SymlinksVerifier -DB_APPS_PATH = '{}/{}'.format(str(Path.home()), '.local/share/bauh/appimage/apps.db') -DB_RELEASES_PATH = '{}/{}'.format(str(Path.home()), '.local/share/bauh/appimage/releases.db') - -DESKTOP_ENTRIES_PATH = '{}/.local/share/applications'.format(str(Path.home())) - RE_DESKTOP_ICON = re.compile(r'Icon\s*=\s*.+\n') RE_ICON_ENDS_WITH = re.compile(r'.+\.(png|svg)$') RE_APPIMAGE_NAME = re.compile(r'(.+)\.appimage', flags=re.IGNORECASE) @@ -81,7 +76,6 @@ class AppImageManager(SoftwareManager): self.http_client = context.http_client self.logger = context.logger self.file_downloader = context.file_downloader - self.db_locks = {DB_APPS_PATH: Lock(), DB_RELEASES_PATH: Lock()} self.custom_actions = [CustomSoftwareAction(i18n_label_key='appimage.custom_action.install_file', i18n_status_key='appimage.custom_action.install_file.status', manager=self, @@ -177,60 +171,74 @@ class AppImageManager(SoftwareManager): def _get_db_connection(self, db_path: str) -> sqlite3.Connection: if os.path.exists(db_path): - self.db_locks[db_path].acquire() - return sqlite3.connect(db_path) + try: + return sqlite3.connect(db_path) + except: + self.logger.error("Could not connect to database file '{}'".format(db_path)) + traceback.print_exc() else: - self.logger.warning("Could not get a database connection. File '{}' not found".format(db_path)) - - def _close_connection(self, db_path: str, con: sqlite3.Connection): - con.close() - self.db_locks[db_path].release() + self.logger.warning("Could not get a connection for database '{}'".format(db_path)) def _gen_app_key(self, app: AppImage): return '{}{}'.format(app.name.lower(), app.github.lower() if app.github else '') def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1, is_url: bool = False) -> SearchResult: if is_url: - return SearchResult([], [], 0) + return SearchResult.empty() - res = SearchResult([], [], 0) - connection = self._get_db_connection(DB_APPS_PATH) + apps_conn = self._get_db_connection(DATABASE_APPS_FILE) - if connection: - try: - cursor = connection.cursor() - cursor.execute(query.SEARCH_APPS_BY_NAME_OR_DESCRIPTION.format(words, words)) + if not apps_conn: + return SearchResult.empty() - found_map = {} - idx = 0 - for r in cursor.fetchall(): - app = AppImage(*r, i18n=self.i18n, custom_actions=self.custom_app_actions) - res.new.append(app) - found_map[self._gen_app_key(app)] = {'app': app, 'idx': idx} - idx += 1 + not_installed, found_map = [], {} - finally: - self._close_connection(DB_APPS_PATH, connection) + try: + cursor = apps_conn.cursor() + cursor.execute(query.SEARCH_APPS_BY_NAME_OR_DESCRIPTION.format(words, words)) - if res.new: - installed = self.read_installed(disk_loader, limit, only_apps=False, pkg_types=None, internet_available=True).installed + idx = 0 + for r in cursor.fetchall(): + app = AppImage(*r, i18n=self.i18n, custom_actions=self.custom_app_actions) + not_installed.append(app) + found_map[self._gen_app_key(app)] = {'app': app, 'idx': idx} + idx += 1 + except: + self.logger.error("An exception happened while querying the 'apps' database") + traceback.print_exc() + apps_conn.close() + return SearchResult.empty() - if installed: - for iapp in installed: - key = self._gen_app_key(iapp) + installed_found = [] - new_found = found_map.get(key) + if not_installed: + installed = self.read_installed(disk_loader=disk_loader, limit=limit, + only_apps=False, + pkg_types=None, + connection=apps_conn, + internet_available=True).installed + if installed: + for appim in installed: + key = self._gen_app_key(appim) - if new_found: - del res.new[new_found['idx']] - res.installed.append(iapp) + new_found = found_map.get(key) - res.total = len(res.installed) + len(res.new) - return res + if new_found: + del not_installed[new_found['idx']] + installed_found.append(appim) - def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, - pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None, connection: sqlite3.Connection = None) -> SearchResult: - res = SearchResult([], [], 0) + try: + apps_conn.close() + except: + self.logger.error("An exception happened when trying to close the connection to database file '{}'".format(DATABASE_APPS_FILE)) + traceback.print_exc() + + return SearchResult(new=not_installed, installed=installed_found, total=len(not_installed) + len(installed_found)) + + def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int = -1, only_apps: bool = False, + pkg_types: Optional[Set[Type[SoftwarePackage]]] = None, internet_available: bool = None, connection: sqlite3.Connection = None) -> SearchResult: + installed_apps = [] + res = SearchResult(installed_apps, [], 0) if os.path.exists(INSTALLATION_PATH): installed = run_cmd('ls {}*/data.json'.format(INSTALLATION_PATH), print_error=False) @@ -243,19 +251,19 @@ class AppImageManager(SoftwareManager): app = AppImage(installed=True, i18n=self.i18n, custom_actions=self.custom_app_actions, **json.loads(f.read())) app.icon_url = app.icon_path - res.installed.append(app) + installed_apps.append(app) names.add("'{}'".format(app.name.lower())) - if res.installed: - con = self._get_db_connection(DB_APPS_PATH) if not connection else connection + if installed_apps: + apps_con = self._get_db_connection(DATABASE_APPS_FILE) if not connection else connection - if con: + if apps_con: try: - cursor = con.cursor() + cursor = apps_con.cursor() cursor.execute(query.FIND_APPS_BY_NAME.format(','.join(names))) for tup in cursor.fetchall(): - for app in res.installed: + for app in installed_apps: if app.name.lower() == tup[0].lower() and (not app.github or app.github.lower() == tup[1].lower()): continuous_version = app.version == 'continuous' continuous_update = tup[2] == 'continuous' @@ -276,15 +284,16 @@ class AppImageManager(SoftwareManager): break except: + self.logger.error("An exception happened while querying the database file {}".format(apps_con)) traceback.print_exc() finally: - if not connection: - self._close_connection(DB_APPS_PATH, con) + if not connection: # the connection can only be closed if it was opened within this method + apps_con.close() ignored_updates = self._read_ignored_updates() if ignored_updates: - for app in res.installed: + for app in installed_apps: if app.supports_ignored_updates() and app.name in ignored_updates: app.updates_ignored = True @@ -406,40 +415,53 @@ class AppImageManager(SoftwareManager): history = [] res = PackageHistory(pkg, history, -1) - connection = self._get_db_connection(DB_APPS_PATH) + app_con = self._get_db_connection(DATABASE_APPS_FILE) - if connection: - try: - cursor = connection.cursor() + if not app_con: + return res - cursor.execute(query.FIND_APP_ID_BY_NAME_AND_GITHUB.format(pkg.name.lower(), pkg.github.lower() if pkg.github else '')) - app_tuple = cursor.fetchone() + try: + cursor = app_con.cursor() - if not app_tuple: - self.logger.warning("Could not retrieve {} from the database {}".format(pkg, DB_APPS_PATH)) - return res - finally: - self._close_connection(DB_APPS_PATH, connection) + cursor.execute(query.FIND_APP_ID_BY_NAME_AND_GITHUB.format(pkg.name.lower(), pkg.github.lower() if pkg.github else '')) + app_tuple = cursor.fetchone() - connection = self._get_db_connection(DB_RELEASES_PATH) + if not app_tuple: + self.logger.warning("Could not retrieve {} from the database {}".format(pkg, DATABASE_APPS_FILE)) + return res + except: + self.logger.error("An exception happened while querying the database file '{}'".format(DATABASE_APPS_FILE)) + traceback.print_exc() + app_con.close() + return res - if connection: - try: - cursor = connection.cursor() + app_con.close() - releases = cursor.execute(query.FIND_RELEASES_BY_APP_ID.format(app_tuple[0])) + releases_con = self._get_db_connection(DATABASE_RELEASES_FILE) - if releases: - for idx, tup in enumerate(releases): - history.append({'0_version': tup[0], '1_published_at': datetime.strptime(tup[2], '%Y-%m-%dT%H:%M:%SZ') if tup[2] else '', '2_url_download': tup[1]}) + if not releases_con: + return res - if res.pkg_status_idx == -1 and pkg.version == tup[0]: - res.pkg_status_idx = idx + try: + cursor = releases_con.cursor() - finally: - self._close_connection(DB_RELEASES_PATH, connection) + releases = cursor.execute(query.FIND_RELEASES_BY_APP_ID.format(app_tuple[0])) - return res + if releases: + for idx, tup in enumerate(releases): + history.append({'0_version': tup[0], + '1_published_at': datetime.strptime(tup[2], '%Y-%m-%dT%H:%M:%SZ') if tup[ + 2] else '', '2_url_download': tup[1]}) + + if res.pkg_status_idx == -1 and pkg.version == tup[0]: + res.pkg_status_idx = idx + + return res + except: + self.logger.error("An exception happened while querying the database file '{}'".format(DATABASE_RELEASES_FILE)) + traceback.print_exc() + finally: + releases_con.close() def _find_desktop_file(self, folder: str) -> str: for r, d, files in os.walk(folder): @@ -452,7 +474,7 @@ class AppImageManager(SoftwareManager): if RE_ICON_ENDS_WITH.match(f): return f - def install(self, pkg: AppImage, root_password: str, disk_loader: DiskCacheLoader, watcher: ProcessWatcher) -> TransactionResult: + def install(self, pkg: AppImage, root_password: str, disk_loader: Optional[DiskCacheLoader], watcher: ProcessWatcher) -> TransactionResult: handler = ProcessHandler(watcher) out_dir = INSTALLATION_PATH + pkg.get_clean_name() @@ -587,22 +609,17 @@ class AppImageManager(SoftwareManager): return False def prepare(self, task_manager: TaskManager, root_password: str, internet_available: bool): - local_config = read_config(update_file=True) - interval = local_config['db_updater']['interval'] or 20 * 60 - - updater = DatabaseUpdater(task_man=task_manager, - i18n=self.context.i18n, - http_client=self.context.http_client, logger=self.context.logger, - db_locks=self.db_locks, interval=interval, - internet_checker=self.context.internet_checker) - if local_config['db_updater']['enabled']: - updater.start() - elif internet_available: - updater.download_databases() # only once - symlink_check = SymlinksVerifier(taskman=task_manager, i18n=self.i18n, logger=self.logger) symlink_check.start() + if internet_available: + updater = DatabaseUpdater(task_man=task_manager, i18n=self.context.i18n, + http_client=self.context.http_client, logger=self.context.logger) + + if updater.should_update(read_config()): + updater.register_task() + updater.start() + def list_updates(self, internet_available: bool) -> List[PackageUpdate]: res = self.read_installed(disk_loader=None, internet_available=internet_available) @@ -615,12 +632,15 @@ class AppImageManager(SoftwareManager): return updates def list_warnings(self, internet_available: bool) -> List[str]: - pass + dbfiles = glob.glob('{}/*.db'.format(DATABASES_DIR)) + + if not dbfiles or len({f for f in (DATABASE_APPS_FILE, DATABASE_RELEASES_FILE) if f in dbfiles}) != 2: + return [self.i18n['appimage.warning.missing_db_files'].format(appimage=bold('AppImage'))] def list_suggestions(self, limit: int, filter_installed: bool) -> List[PackageSuggestion]: res = [] - connection = self._get_db_connection(DB_APPS_PATH) + connection = self._get_db_connection(DATABASE_APPS_FILE) if connection: file = self.http_client.get(SUGGESTIONS_FILE) @@ -661,7 +681,7 @@ class AppImageManager(SoftwareManager): except: traceback.print_exc() finally: - self._close_connection(DB_APPS_PATH, connection) + connection.close() return res @@ -705,34 +725,22 @@ class AppImageManager(SoftwareManager): config = read_config() max_width = floor(screen_width * 0.15) - enabled_opts = [InputOption(label=self.i18n['yes'].capitalize(), value=True), - InputOption(label=self.i18n['no'].capitalize(), value=False)] - - updater_opts = [ - SingleSelectComponent(label=self.i18n['appimage.config.db_updates.activated'], - options=enabled_opts, - default_option=[o for o in enabled_opts if o.value == config['db_updater']['enabled']][0], - max_per_line=len(enabled_opts), - type_=SelectViewType.RADIO, - tooltip=self.i18n['appimage.config.db_updates.activated.tip'], - max_width=max_width, - id_='up_enabled'), - TextInputComponent(label=self.i18n['interval'], - value=str(config['db_updater']['interval']), - tooltip=self.i18n['appimage.config.db_updates.interval.tip'], + opts = [ + TextInputComponent(label=self.i18n['appimage.config.database.expiration'], + value=int(config['database']['expiration']) if isinstance(config['database']['expiration'], int) else '', + tooltip=self.i18n['appimage.config.database.expiration.tip'], only_int=True, max_width=max_width, - id_='up_int') + id_='appim_db_exp') ] - return PanelComponent([FormComponent(updater_opts, self.i18n['appimage.config.db_updates'])]) + return PanelComponent([FormComponent(opts, self.i18n['appimage.config.database'])]) def save_settings(self, component: PanelComponent) -> Tuple[bool, Optional[List[str]]]: config = read_config() panel = component.components[0] - config['db_updater']['enabled'] = panel.get_component('up_enabled').get_selected() - config['db_updater']['interval'] = panel.get_component('up_int').get_int_value() + config['database']['expiration'] = panel.get_component('appim_db_exp').get_int_value() try: save_config(config, CONFIG_FILE) diff --git a/bauh/gems/appimage/resources/locale/ca b/bauh/gems/appimage/resources/locale/ca index bfb71078..37147d06 100644 --- a/bauh/gems/appimage/resources/locale/ca +++ b/bauh/gems/appimage/resources/locale/ca @@ -1,7 +1,6 @@ -appimage.config.db_updates=Database update -appimage.config.db_updates.activated=activated -appimage.config.db_updates.activated.tip=It will be possible to check for updates related to the installed applications -appimage.config.db_updates.interval.tip=Update interval in SECONDS +appimage.config.database=Database +appimage.config.database.expiration=Expiration +appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it. appimage.custom_action.install_file=Install AppImage file appimage.custom_action.install_file.details=Installation details appimage.custom_action.install_file.invalid_file=You must define a valid AppImage file @@ -34,5 +33,6 @@ appimage.install.permission=S’està concedint el permís d’execució a {} appimage.task.db_update=Actualització de bases de dades appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=No s’ha pogut suprimir el directori d’instal·lació de l’aplicació {} +appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. gem.appimage.info=Aplicacions publicades a https://appimage.github.io/ gem.appimage.label=AppImage \ No newline at end of file diff --git a/bauh/gems/appimage/resources/locale/de b/bauh/gems/appimage/resources/locale/de index e7a36881..fc365c97 100644 --- a/bauh/gems/appimage/resources/locale/de +++ b/bauh/gems/appimage/resources/locale/de @@ -1,7 +1,6 @@ -appimage.config.db_updates=Database update -appimage.config.db_updates.activated=activated -appimage.config.db_updates.activated.tip=It will be possible to check for updates related to the installed applications -appimage.config.db_updates.interval.tip=Update interval in SECONDS +appimage.config.database=Database +appimage.config.database.expiration=Expiration +appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it. appimage.custom_action.install_file=Install AppImage file appimage.custom_action.install_file.details=Installation details appimage.custom_action.install_file.invalid_file=You must define a valid AppImage file @@ -34,5 +33,6 @@ appimage.install.permission=Ausführberechtigungen für {} appimage.task.db_update=Updating databases appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=Anwendungordner {} konnte nicht entfernt werden +appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. gem.appimage.info=Anwendungen veröffentlicht auf https://appimage.github.io/ gem.appimage.label=AppImage \ No newline at end of file diff --git a/bauh/gems/appimage/resources/locale/en b/bauh/gems/appimage/resources/locale/en index d69efe5a..142f777a 100644 --- a/bauh/gems/appimage/resources/locale/en +++ b/bauh/gems/appimage/resources/locale/en @@ -1,7 +1,6 @@ -appimage.config.db_updates=Database update -appimage.config.db_updates.activated=activated -appimage.config.db_updates.activated.tip=It will be possible to check for updates related to the installed applications -appimage.config.db_updates.interval.tip=Update interval in SECONDS +appimage.config.database=Database +appimage.config.database.expiration=Expiration +appimage.config.database.expiration.tip=appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date. Use 0 if you always want to update it. appimage.custom_action.install_file=Install AppImage file appimage.custom_action.install_file.details=Installation details appimage.custom_action.install_file.invalid_file=You must define a valid AppImage file @@ -34,5 +33,6 @@ appimage.install.permission=Giving execution permission to {} appimage.task.db_update=Updating databases appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=Could not remove the application installation directory {} +appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. gem.appimage.info=Applications published at https://appimage.github.io/ gem.appimage.label=AppImage \ No newline at end of file diff --git a/bauh/gems/appimage/resources/locale/es b/bauh/gems/appimage/resources/locale/es index 47adc1b8..a7b88024 100644 --- a/bauh/gems/appimage/resources/locale/es +++ b/bauh/gems/appimage/resources/locale/es @@ -1,7 +1,6 @@ -appimage.config.db_updates=Actualización de la base de datos -appimage.config.db_updates.activated=activada -appimage.config.db_updates.activated.tip=Será posible buscar actualizaciones relacionadas con las aplicaciones instaladas -appimage.config.db_updates.interval.tip=Intervalo de actualización en SEGUNDOS +appimage.config.database=Base de datos +appimage.config.database.expiration=Expiración +appimage.config.database.expiration.tip=Define el período (en minutos) en el que la base de datos será considerada actualizada durante el proceso de inicialización. Use 0 si siempre desea actualizarla. appimage.custom_action.install_file=Instalar archivo AppImage appimage.custom_action.install_file.details=Detalles de instalación appimage.custom_action.install_file.invalid_file=Debe definir un archivo AppImage válido @@ -34,5 +33,6 @@ appimage.install.permission=Concediendo permiso de ejecución a {} appimage.task.db_update=Actualizando base de datos appimage.task.symlink_check=Verificando links simbólicos appimage.uninstall.error.remove_folder=No se pudo eliminar el directorio de instalación de la aplicación {} +appimage.warning.missing_db_files=No se encontraron archivos de la base de datos {appimage}. Algunas acciones no funcionarán correctamente. gem.appimage.info=Aplicativos publicados en https://appimage.github.io/ gem.appimage.label=AppImage \ No newline at end of file diff --git a/bauh/gems/appimage/resources/locale/fr b/bauh/gems/appimage/resources/locale/fr index e263fb0c..484322d9 100644 --- a/bauh/gems/appimage/resources/locale/fr +++ b/bauh/gems/appimage/resources/locale/fr @@ -1,7 +1,6 @@ -appimage.config.db_updates=Mise à jour de la base de données -appimage.config.db_updates.activated=activé -appimage.config.db_updates.activated.tip=La recherche de mises à jour liées aux applications installées sera possible -appimage.config.db_updates.interval.tip=Intervalle de mises à jour en SECONDES +appimage.config.database=Database +appimage.config.database.expiration=Expiration +appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date. Use 0 if you always want to update it. appimage.custom_action.install_file=Installer fichier AppImage appimage.custom_action.install_file.details=Détails d'installation appimage.custom_action.install_file.invalid_file=Vous devez définir un fichier AppImage valide @@ -34,5 +33,6 @@ appimage.install.permission={} est maintenant exécutable appimage.task.db_update=Mise à jour des bases de données appimage.task.symlink_check=Verification des symlinks appimage.uninstall.error.remove_folder=Impossible de supprimer le répertoire d'installation {} +appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. gem.appimage.info=Applications publiées à https://appimage.github.io/ gem.appimage.label=AppImage diff --git a/bauh/gems/appimage/resources/locale/it b/bauh/gems/appimage/resources/locale/it index df393fef..76bf6a6c 100644 --- a/bauh/gems/appimage/resources/locale/it +++ b/bauh/gems/appimage/resources/locale/it @@ -1,7 +1,6 @@ -appimage.config.db_updates=Database update -appimage.config.db_updates.activated=activated -appimage.config.db_updates.activated.tip=It will be possible to check for updates related to the installed applications -appimage.config.db_updates.interval.tip=Update interval in SECONDS +appimage.config.database=Database +appimage.config.database.expiration=Expiration +appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it. appimage.custom_action.install_file=Install AppImage file appimage.custom_action.install_file.details=Installation details appimage.custom_action.install_file.invalid_file=You must define a valid AppImage file @@ -34,5 +33,6 @@ appimage.install.permission=Gdare il permesso di esecuzione a {} appimage.task.db_update=Aggiornamento dei database appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=Impossibile rimuovere la directory di installazione dell'applicazione {} +appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. gem.appimage.info=Applicazioni pubblicate su https://appimage.github.io/ gem.appimage.label=AppImage \ No newline at end of file diff --git a/bauh/gems/appimage/resources/locale/pt b/bauh/gems/appimage/resources/locale/pt index be125a45..27dfbe43 100644 --- a/bauh/gems/appimage/resources/locale/pt +++ b/bauh/gems/appimage/resources/locale/pt @@ -1,7 +1,6 @@ -appimage.config.db_updates=Atualização da base de dados -appimage.config.db_updates.activated=ativada -appimage.config.db_updates.activated.tip=Será possível verificar se há atualizações para os aplicativos instalados -appimage.config.db_updates.interval.tip=Intervalo de atualização em SEGUNDOS +appimage.config.database=Banco de dados +appimage.config.database.expiration=Expiração +appimage.config.database.expiration.tip=Define o período (em minutos) no qual o banco de dados será considerado atualizado durante o processo de inicialização. Use 0 se você quiser que ele sempre seja atualizado. appimage.custom_action.install_file=Instalar arquivo AppImage appimage.custom_action.install_file.details=Detalhes de instalação appimage.custom_action.install_file.invalid_file=Você precisa definir um arquivo AppImage válido @@ -34,5 +33,6 @@ appimage.install.permission=Concedendo permissão de execução para {} appimage.task.db_update=Atualizando base de dados appimage.task.symlink_check=Verificando links simbólicos appimage.uninstall.error.remove_folder=Não foi possível remover o diretóri ode instalação do aplicativo {} +appimage.warning.missing_db_files=Arquivos do banco de dados {appimage} não foram encontrados. Algumas ações não funcionarão corretamente. gem.appimage.info=Aplicativos publicados em https://appimage.github.io/ gem.appimage.label=AppImage \ No newline at end of file diff --git a/bauh/gems/appimage/resources/locale/ru b/bauh/gems/appimage/resources/locale/ru index eaaac1f5..2a072713 100644 --- a/bauh/gems/appimage/resources/locale/ru +++ b/bauh/gems/appimage/resources/locale/ru @@ -1,7 +1,6 @@ -appimage.config.db_updates=Обновление базы данных -appimage.config.db_updates.activated=Активировать -appimage.config.db_updates.activated.tip=Позволит проверять обновления установленных приложений -appimage.config.db_updates.interval.tip=Интервал обновления в секундах +appimage.config.database=Database +appimage.config.database.expiration=Expiration +appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it. appimage.custom_action.install_file=Установить файл AppImage appimage.custom_action.install_file.details=Детали установки appimage.custom_action.install_file.invalid_file=Вы должны определить допустимый файл AppImage @@ -34,5 +33,6 @@ appimage.install.permission=Установить права на выполне appimage.task.db_update=Обновление базы данных appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder=Не удалось удалить каталог приложения {} +appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. gem.appimage.info=Приложения, опубликованные на https://appimage.github.io/ gem.appimage.label=AppImage diff --git a/bauh/gems/appimage/resources/locale/tr b/bauh/gems/appimage/resources/locale/tr index 6dd8e0cb..232ff11e 100644 --- a/bauh/gems/appimage/resources/locale/tr +++ b/bauh/gems/appimage/resources/locale/tr @@ -1,7 +1,6 @@ -appimage.config.db_updates=Veritabanı güncelle -appimage.config.db_updates.activated=Aktifleştirildi -appimage.config.db_updates.activated.tip=Yüklü uygulamalarla ilgili güncellemeleri kontrol etmek mümkün olacak -appimage.config.db_updates.interval.tip=SANİYE cinsinden güncelleme aralığı +appimage.config.database=Database +appimage.config.database.expiration=Expiration +appimage.config.database.expiration.tip=It defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it. appimage.custom_action.install_file=AppImage dosyasını yükle appimage.custom_action.install_file.details=Kurulum detayları appimage.custom_action.install_file.invalid_file=Geçerli bir AppImage dosyası tanımlamanız gerekir @@ -34,5 +33,6 @@ appimage.install.permission={} için yürütme izni veriliyor appimage.task.db_update=Veritabanları güncelleniyor appimage.task.symlink_check=Checking symlinks appimage.uninstall.error.remove_folder={} uygulama kurulum dizini kaldırılamadı +appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly. gem.appimage.info=https://appimage.github.io/ adresinde yayınlanan uygulamalar gem.appimage.label=AppImage diff --git a/bauh/gems/appimage/worker.py b/bauh/gems/appimage/worker.py index ab3673e2..678942c7 100644 --- a/bauh/gems/appimage/worker.py +++ b/bauh/gems/appimage/worker.py @@ -5,108 +5,151 @@ import os import tarfile import time import traceback +from datetime import datetime, timedelta from pathlib import Path from threading import Thread - -import requests +from typing import Optional from bauh.api.abstract.handler import TaskManager, ProcessWatcher from bauh.api.http import HttpClient -from bauh.commons.internet import InternetChecker -from bauh.gems.appimage import LOCAL_PATH, get_icon_path, INSTALLATION_PATH, SYMLINKS_DIR, util +from bauh.gems.appimage import get_icon_path, INSTALLATION_PATH, SYMLINKS_DIR, util, DATABASES_TS_FILE, \ + DATABASES_DIR, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES from bauh.gems.appimage.model import AppImage from bauh.view.util.translation import I18n class DatabaseUpdater(Thread): - URL_DB = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/dbs.tar.gz' - COMPRESS_FILE_PATH = LOCAL_PATH + '/db.tar.gz' + COMPRESS_FILE_PATH = '{}/db.tar.gz'.format(DATABASES_DIR) - def __init__(self, task_man: TaskManager, i18n: I18n, http_client: HttpClient, - logger: logging.Logger, db_locks: dict, interval: int, - internet_checker: InternetChecker): + def __init__(self, task_man: TaskManager, i18n: I18n, http_client: HttpClient, logger: logging.Logger): super(DatabaseUpdater, self).__init__(daemon=True) self.http_client = http_client self.logger = logger - self.db_locks = db_locks - self.sleep_time = interval self.i18n = i18n self.task_man = task_man self.task_id = 'appim_db' - self.internet_checker = internet_checker + + def should_update(self, appimage_config: dict) -> bool: + ti = time.time() + + try: + db_exp = int(appimage_config['database']['expiration']) + except ValueError: + self.logger.error("Could not parse settings property 'database.expiration': {}".format(appimage_config['database']['expiration'])) + return True + + if db_exp <= 0: + self.logger.info("No expiration time configured for the AppImage database") + return True + + files = {*glob.glob('{}/*'.format(DATABASES_DIR))} + + if not files: + self.logger.warning('No database files on {}'.format(DATABASES_DIR)) + return True + + if DATABASES_TS_FILE not in files: + self.logger.warning("No database timestamp file found ({})".format(DATABASES_TS_FILE)) + return True + + if DATABASE_APPS_FILE not in files: + self.logger.warning("Database file '{}' not found".format(DATABASE_APPS_FILE)) + return True + + if DATABASE_RELEASES_FILE not in files: + self.logger.warning("Database file '{}' not found".format(DATABASE_RELEASES_FILE)) + return True + + with open(DATABASES_TS_FILE) as f: + dbs_ts_str = f.read() + + try: + dbs_timestamp = datetime.fromtimestamp(float(dbs_ts_str)) + except: + self.logger.error('Could not parse the databases timestamp: {}'.format(dbs_ts_str)) + traceback.print_exc() + return True + + update = dbs_timestamp + timedelta(minutes=db_exp) <= datetime.utcnow() + self.logger.info('Finished. Took {0:.2f} seconds'.format(time.time() - ti)) + return update + + def register_task(self): + if self.task_man: + self.task_man.register_task(self.task_id, self.i18n['appimage.task.db_update'], get_icon_path()) def _finish_task(self): if self.task_man: self.task_man.update_progress(self.task_id, 100, None) self.task_man.finish_task(self.task_id) self.task_man = None + self.logger.info("Finished") + + def _update_task_progress(self, progress: float, substatus: Optional[str] = None): + if self.task_man: + self.task_man.update_progress(self.task_id, progress, substatus) def download_databases(self): - if self.task_man: - self.task_man.register_task(self.task_id, self.i18n['appimage.task.db_update'], get_icon_path()) - self.task_man.update_progress(self.task_id, 10, None) - - try: - if not self.internet_checker.is_available(): - self._finish_task() - return - except requests.exceptions.ConnectionError: - self.logger.warning('The internet connection seems to be off.') - self._finish_task() - return - + self._update_task_progress(1) # TODO add substatus self.logger.info('Retrieving AppImage databases') + database_timestamp = datetime.utcnow().timestamp() try: - res = self.http_client.get(self.URL_DB, session=False) + res = self.http_client.get(URL_COMPRESSED_DATABASES, session=False) except Exception as e: self.logger.error("An error ocurred while downloading the AppImage database: {}".format(e.__class__.__name__)) res = None - if res: - Path(LOCAL_PATH).mkdir(parents=True, exist_ok=True) - - with open(self.COMPRESS_FILE_PATH, 'wb+') as f: - f.write(res.content) - - self.logger.info("Database file saved at {}".format(self.COMPRESS_FILE_PATH)) - - old_db_files = glob.glob(LOCAL_PATH + '/*.db') - - if old_db_files: - self.logger.info('Deleting old database files') - for f in old_db_files: - self.db_locks[f].acquire() - try: - os.remove(f) - finally: - self.db_locks[f].release() - - self.logger.info('Old database files deleted') - - self.logger.info('Uncompressing {}'.format(self.COMPRESS_FILE_PATH)) - - try: - tf = tarfile.open(self.COMPRESS_FILE_PATH) - tf.extractall(LOCAL_PATH) - self.logger.info('Successfully uncompressed file {}'.format(self.COMPRESS_FILE_PATH)) - except: - self.logger.error('Could not extract file {}'.format(self.COMPRESS_FILE_PATH)) - traceback.print_exc() - finally: - self.logger.info('Deleting {}'.format(self.COMPRESS_FILE_PATH)) - os.remove(self.COMPRESS_FILE_PATH) - self.logger.info('Successfully removed {}'.format(self.COMPRESS_FILE_PATH)) - self._finish_task() - else: - self.logger.warning('Could not download the database file {}'.format(self.URL_DB)) + if not res: + self.logger.warning('Could not download the database file {}'.format(URL_COMPRESSED_DATABASES)) self._finish_task() + return + + self._update_task_progress(25) # TODO add substatus + Path(DATABASES_DIR).mkdir(parents=True, exist_ok=True) + + with open(self.COMPRESS_FILE_PATH, 'wb+') as f: + f.write(res.content) + + self.logger.info("Database file saved at {}".format(self.COMPRESS_FILE_PATH)) + + self._update_task_progress(50) # TODO add substatus + old_db_files = glob.glob(DATABASES_DIR + '/*.db') + + if old_db_files: + self.logger.info('Deleting old database files') + for f in old_db_files: + os.remove(f) + + self.logger.info('Old database files deleted') + + self._update_task_progress(75) # TODO add substatus + self.logger.info('Uncompressing {}'.format(self.COMPRESS_FILE_PATH)) + + try: + tf = tarfile.open(self.COMPRESS_FILE_PATH) + tf.extractall(DATABASES_DIR) + self.logger.info('Successfully uncompressed file {}'.format(self.COMPRESS_FILE_PATH)) + except: + self.logger.error('Could not extract file {}'.format(self.COMPRESS_FILE_PATH)) + traceback.print_exc() + finally: + self.logger.info('Deleting {}'.format(self.COMPRESS_FILE_PATH)) + os.remove(self.COMPRESS_FILE_PATH) + self.logger.info('File {} deleted'.format(self.COMPRESS_FILE_PATH)) + + self._update_task_progress(95) + self.logger.info("Saving database timestamp {}".format(database_timestamp)) + + with open(DATABASES_TS_FILE, 'w+') as f: + f.write(str(database_timestamp)) + + self.logger.info("Database timestamp saved") + + self._finish_task() def run(self): - while True: - self.download_databases() - self.logger.info('Sleeping') - time.sleep(self.sleep_time) + self.download_databases() class SymlinksVerifier(Thread): diff --git a/bauh/view/util/cache.py b/bauh/view/util/cache.py index b4ac1533..963edd0d 100644 --- a/bauh/view/util/cache.py +++ b/bauh/view/util/cache.py @@ -1,6 +1,7 @@ import datetime import time from threading import Lock, Thread +from typing import Optional from bauh.api.abstract.cache import MemoryCache, MemoryCacheFactory @@ -106,7 +107,7 @@ class DefaultMemoryCacheFactory(MemoryCacheFactory): self.expiration_time = expiration_time self.cleaner = cleaner - def new(self, expiration: int = None) -> MemoryCache: + def new(self, expiration: Optional[int] = None) -> MemoryCache: instance = DefaultMemoryCache(expiration if expiration is not None else self.expiration_time) if self.cleaner: