spalencsar/bearhub
1
0
Fork 0
mirror of https://github.com/spalencsar/bearhub.git synced 2026-07-07 00:04:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

[appimage] improvement -> caching suggestions to disk

Browse Source
This commit is contained in:
Vinicius Moreira
2021-02-01 17:09:30 -03:00
parent 8030e6cbe8
commit 7a3245ae5e
16 changed files with 191 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [0.9.14] 2021
### Improvements
- AppImage
- caching suggestions to disk. The cache expiration can be controlled through the new property settings `suggestions.expiration` (in hours. Default: 24).
- Web
- nativefier URL moved to [bauh-files](https://github.com/vinifmor/bauh-files/blob/master/web/env/v1/environment.yml)

2
README.md
View File

@@ -198,6 +198,8 @@ bauh is officially distributed through [PyPi](https://pypi.org/project/bauh) and
```
database:
expiration: 60 # defines the period (in minutes) in which the database will be considered up to date during the initialization process. Use 0 if you always want to update it.
suggestions:
expiration: 24 # defines the period (in hours) in which the cached suggestions will be considered up to date. Use 0 if you always want to update them.
```

10
bauh/gems/appimage/__init__.py
View File

@@ -14,11 +14,13 @@ CONFIG_DIR = '{}/appimage'.format(CONFIG_PATH)
UPDATES_IGNORED_FILE = '{}/updates_ignored.txt'.format(CONFIG_DIR)
SYMLINKS_DIR = '{}/.local/bin'.format(str(Path.home()))
URL_COMPRESSED_DATABASES = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/dbs.tar.gz'
DATABASES_DIR = '{}/appimage'.format(CACHE_PATH)
DATABASE_APPS_FILE = '{}/apps.db'.format(DATABASES_DIR)
DATABASE_RELEASES_FILE = '{}/releases.db'.format(DATABASES_DIR)
DATABASES_TS_FILE = '{}/dbs.ts'.format(DATABASES_DIR)
APPIMAGE_CACHE_PATH = '{}/appimage'.format(CACHE_PATH)
DATABASE_APPS_FILE = '{}/apps.db'.format(APPIMAGE_CACHE_PATH)
DATABASE_RELEASES_FILE = '{}/releases.db'.format(APPIMAGE_CACHE_PATH)
DATABASES_TS_FILE = '{}/dbs.ts'.format(APPIMAGE_CACHE_PATH)
DESKTOP_ENTRIES_PATH = '{}/.local/share/applications'.format(str(Path.home()))
SUGGESTIONS_CACHED_FILE = '{}/suggestions.txt'.format(APPIMAGE_CACHE_PATH)
SUGGESTIONS_CACHED_TS_FILE = '{}/suggestions.ts'.format(APPIMAGE_CACHE_PATH)
def get_icon_path() -> str:

3
bauh/gems/appimage/config.py
View File

@@ -11,5 +11,8 @@ class AppImageConfigManager(YAMLConfigManager):
return {
'database': {
'expiration': 60
},
'suggestions': {
'expiration': 24
}
}

24
bauh/gems/appimage/controller.py
View File

@@ -27,13 +27,13 @@ from bauh.commons import resource
from bauh.commons.boot import CreateConfigFile
from bauh.commons.html import bold
from bauh.commons.system import SystemProcess, new_subprocess, ProcessHandler, run_cmd, SimpleProcess
from bauh.gems.appimage import query, INSTALLATION_PATH, LOCAL_PATH, SUGGESTIONS_FILE, ROOT_DIR, \
from bauh.gems.appimage import query, INSTALLATION_PATH, LOCAL_PATH, ROOT_DIR, \
CONFIG_DIR, UPDATES_IGNORED_FILE, util, get_default_manual_installation_file_dir, DATABASE_APPS_FILE, \
DATABASE_RELEASES_FILE, DESKTOP_ENTRIES_PATH, DATABASES_DIR, get_icon_path
DATABASE_RELEASES_FILE, DESKTOP_ENTRIES_PATH, APPIMAGE_CACHE_PATH, get_icon_path
from bauh.gems.appimage.config import AppImageConfigManager
from bauh.gems.appimage.model import AppImage
from bauh.gems.appimage.util import replace_desktop_entry_exec_command
from bauh.gems.appimage.worker import DatabaseUpdater, SymlinksVerifier
from bauh.gems.appimage.worker import DatabaseUpdater, SymlinksVerifier, AppImageSuggestionsDownloader
RE_DESKTOP_ICON = re.compile(r'Icon\s*=\s*.+\n')
RE_ICON_ENDS_WITH = re.compile(r'.+\.(png|svg)$')
@@ -629,6 +629,10 @@ class AppImageManager(SoftwareManager):
create_config=create_config, http_client=self.context.http_client,
logger=self.context.logger).start()
AppImageSuggestionsDownloader(taskman=task_manager, i18n=self.context.i18n,
http_client=self.context.http_client, logger=self.context.logger,
create_config=create_config).start()
def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
res = self.read_installed(disk_loader=None, internet_available=internet_available)
@@ -641,7 +645,7 @@ class AppImageManager(SoftwareManager):
return updates
def list_warnings(self, internet_available: bool) -> List[str]:
dbfiles = glob.glob('{}/*.db'.format(DATABASES_DIR))
dbfiles = glob.glob('{}/*.db'.format(APPIMAGE_CACHE_PATH))
if not dbfiles or len({f for f in (DATABASE_APPS_FILE, DATABASE_RELEASES_FILE) if f in dbfiles}) != 2:
return [self.i18n['appimage.warning.missing_db_files'].format(appimage=bold('AppImage'))]
@@ -652,16 +656,16 @@ class AppImageManager(SoftwareManager):
connection = self._get_db_connection(DATABASE_APPS_FILE)
if connection:
file = self.http_client.get(SUGGESTIONS_FILE)
suggestions = AppImageSuggestionsDownloader(appimage_config=self.configman.get_config(), logger=self.logger,
i18n=self.i18n, http_client=self.http_client,
taskman=TaskManager()).read()
if not file or not file.text:
self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_FILE))
if not suggestions:
self.logger.warning("Could not read suggestions")
return res
else:
self.logger.info("Mapping suggestions")
try:
sugs = [l for l in file.text.split('\n') if l]
if filter_installed:
installed = {i.name.lower() for i in self.read_installed(disk_loader=None, connection=connection).installed}
else:
@@ -669,7 +673,7 @@ class AppImageManager(SoftwareManager):
sugs_map = {}
for s in sugs:
for s in suggestions:
lsplit = s.split('=')
name = lsplit[1].strip()

1
bauh/gems/appimage/resources/locale/ca
View File

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
appimage.update_database.uncompressing=Uncompressing files
appimage.task.db_update=Actualització de bases de dades
appimage.task.db_update.checking=Checking for updates
appimage.task.suggestions=Downloading suggestions
appimage.task.symlink_check=Checking symlinks
appimage.uninstall.error.remove_folder=No s’ha pogut suprimir el directori d’instal·lació de l’aplicació {}
appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

1
bauh/gems/appimage/resources/locale/de
View File

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
appimage.update_database.uncompressing=Uncompressing files
appimage.task.db_update=Updating databases
appimage.task.db_update.checking=Checking for updates
appimage.task.suggestions=Downloading suggestions
appimage.task.symlink_check=Checking symlinks
appimage.uninstall.error.remove_folder=Anwendungordner {} konnte nicht entfernt werden
appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

1
bauh/gems/appimage/resources/locale/en
View File

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
appimage.update_database.uncompressing=Uncompressing files
appimage.task.db_update=Updating databases
appimage.task.db_update.checking=Checking for updates
appimage.task.suggestions=Downloading suggestions
appimage.task.symlink_check=Checking symlinks
appimage.uninstall.error.remove_folder=Could not remove the application installation directory {}
appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

1
bauh/gems/appimage/resources/locale/es
View File

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Descargando archivos de la base de datos
appimage.update_database.uncompressing=Descomprindo archivos
appimage.task.db_update=Actualizando base de datos
appimage.task.db_update.checking=Buscando actualizaciones
appimage.task.suggestions=Descargando sugerencias
appimage.task.symlink_check=Verificando links simbólicos
appimage.uninstall.error.remove_folder=No se pudo eliminar el directorio de instalación de la aplicación {}
appimage.warning.missing_db_files=No se encontraron archivos de la base de datos {appimage}. Algunas acciones no funcionarán correctamente.

1
bauh/gems/appimage/resources/locale/fr
View File

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
appimage.update_database.uncompressing=Uncompressing files
appimage.task.db_update=Mise à jour des bases de données
appimage.task.db_update.checking=Checking for updates
appimage.task.suggestions=Downloading suggestions
appimage.task.symlink_check=Verification des symlinks
appimage.uninstall.error.remove_folder=Impossible de supprimer le répertoire d'installation {}
appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

1
bauh/gems/appimage/resources/locale/it
View File

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
appimage.update_database.uncompressing=Uncompressing files
appimage.task.db_update=Aggiornamento dei database
appimage.task.db_update.checking=Checking for updates
appimage.task.suggestions=Downloading suggestions
appimage.task.symlink_check=Checking symlinks
appimage.uninstall.error.remove_folder=Impossibile rimuovere la directory di installazione dell'applicazione {}
appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

1
bauh/gems/appimage/resources/locale/pt
View File

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Baixando arquivos do banco de dados
appimage.update_database.uncompressing=Descomprimindo arquivos
appimage.task.db_update=Atualizando base de dados
appimage.task.db_update.checking=Checando atualizações
appimage.task.suggestions=Baixando sugestões
appimage.task.symlink_check=Verificando links simbólicos
appimage.uninstall.error.remove_folder=Não foi possível remover o diretóri ode instalação do aplicativo {}
appimage.warning.missing_db_files=Arquivos do banco de dados {appimage} não foram encontrados. Algumas ações não funcionarão corretamente.

1
bauh/gems/appimage/resources/locale/ru
View File

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
appimage.update_database.uncompressing=Uncompressing files
appimage.task.db_update=Обновление базы данных
appimage.task.db_update.checking=Checking for updates
appimage.task.suggestions=Downloading suggestions
appimage.task.symlink_check=Checking symlinks
appimage.uninstall.error.remove_folder=Не удалось удалить каталог приложения {}
appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

1
bauh/gems/appimage/resources/locale/tr
View File

@@ -37,6 +37,7 @@ appimage.update_database.downloading=Downloading database files
appimage.update_database.uncompressing=Uncompressing files
appimage.task.db_update=Veritabanları güncelleniyor
appimage.task.db_update.checking=Checking for updates
appimage.task.suggestions=Downloading suggestions
appimage.task.symlink_check=Checking symlinks
appimage.uninstall.error.remove_folder={} uygulama kurulum dizini kaldırılamadı
appimage.warning.missing_db_files={appimage} database files were not found. Some actions will not work properly.

162
bauh/gems/appimage/worker.py
View File

@@ -8,20 +8,23 @@ import traceback
from datetime import datetime, timedelta
from pathlib import Path
from threading import Thread
from typing import Optional
from typing import Optional, List
import requests
from bauh.api.abstract.handler import TaskManager, ProcessWatcher
from bauh.api.http import HttpClient
from bauh.commons.boot import CreateConfigFile
from bauh.commons.html import bold
from bauh.gems.appimage import get_icon_path, INSTALLATION_PATH, SYMLINKS_DIR, util, DATABASES_TS_FILE, \
DATABASES_DIR, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES
APPIMAGE_CACHE_PATH, DATABASE_APPS_FILE, DATABASE_RELEASES_FILE, URL_COMPRESSED_DATABASES, SUGGESTIONS_FILE, \
SUGGESTIONS_CACHED_TS_FILE, SUGGESTIONS_CACHED_FILE
from bauh.gems.appimage.model import AppImage
from bauh.view.util.translation import I18n
class DatabaseUpdater(Thread):
COMPRESS_FILE_PATH = '{}/db.tar.gz'.format(DATABASES_DIR)
COMPRESS_FILE_PATH = '{}/db.tar.gz'.format(APPIMAGE_CACHE_PATH)
def __init__(self, i18n: I18n, http_client: HttpClient, logger: logging.Logger, taskman: TaskManager,
watcher: Optional[ProcessWatcher] = None, appimage_config: Optional[dict] = None, create_config: Optional[CreateConfigFile] = None):
@@ -49,10 +52,10 @@ class DatabaseUpdater(Thread):
self.logger.info("No expiration time configured for the AppImage database")
return True
files = {*glob.glob('{}/*'.format(DATABASES_DIR))}
files = {*glob.glob('{}/*'.format(APPIMAGE_CACHE_PATH))}
if not files:
self.logger.warning('No database files on {}'.format(DATABASES_DIR))
self.logger.warning('No database files on {}'.format(APPIMAGE_CACHE_PATH))
return True
if DATABASES_TS_FILE not in files:
@@ -102,7 +105,7 @@ class DatabaseUpdater(Thread):
self.logger.warning('Could not download the database file {}'.format(URL_COMPRESSED_DATABASES))
return False
Path(DATABASES_DIR).mkdir(parents=True, exist_ok=True)
Path(APPIMAGE_CACHE_PATH).mkdir(parents=True, exist_ok=True)
with open(self.COMPRESS_FILE_PATH, 'wb+') as f:
f.write(res.content)
@@ -110,7 +113,7 @@ class DatabaseUpdater(Thread):
self.logger.info("Database file saved at {}".format(self.COMPRESS_FILE_PATH))
self._update_task_progress(50, self.i18n['appimage.update_database.deleting_old'])
old_db_files = glob.glob(DATABASES_DIR + '/*.db')
old_db_files = glob.glob(APPIMAGE_CACHE_PATH + '/*.db')
if old_db_files:
self.logger.info('Deleting old database files')
@@ -124,7 +127,7 @@ class DatabaseUpdater(Thread):
try:
tf = tarfile.open(self.COMPRESS_FILE_PATH)
tf.extractall(DATABASES_DIR)
tf.extractall(APPIMAGE_CACHE_PATH)
self.logger.info('Successfully uncompressed file {}'.format(self.COMPRESS_FILE_PATH))
except:
self.logger.error('Could not extract file {}'.format(self.COMPRESS_FILE_PATH))
@@ -279,3 +282,146 @@ class SymlinksVerifier(Thread):
self.logger.info("No AppImage applications found. Aborting")
self.taskman.update_progress(self.task_id, 100, '')
self.taskman.finish_task(self.task_id)
class AppImageSuggestionsDownloader(Thread):
def __init__(self, logger: logging.Logger, http_client: HttpClient, i18n: I18n, taskman: TaskManager, create_config: Optional[CreateConfigFile] = None, appimage_config: Optional[dict] = None):
super(AppImageSuggestionsDownloader, self).__init__(daemon=True)
self.create_config = create_config
self.logger = logger
self.i18n = i18n
self.http_client = http_client
self.taskman = taskman
self.config = appimage_config
self.task_id = 'appim.suggestions'
self.taskman.register_task(id_=self.task_id, label=i18n['appimage.task.suggestions'], icon_path=get_icon_path())
def should_download(self, appimage_config: dict) -> bool:
try:
exp_hours = int(appimage_config['suggestions']['expiration'])
except:
self.logger.error("An exception happened while trying to parse 'suggestions.expiration'")
traceback.print_exc()
return True
if exp_hours <= 0:
self.logger.info("Suggestions cache is disabled")
return True
if not os.path.exists(SUGGESTIONS_CACHED_FILE):
self.logger.info("'{}' not found. It must be downloaded".format(SUGGESTIONS_CACHED_FILE))
return True
if not os.path.exists(SUGGESTIONS_CACHED_TS_FILE):
self.logger.info("'{}' not found. The suggestions file must be downloaded.")
return True
with open(SUGGESTIONS_CACHED_TS_FILE) as f:
timestamp_str = f.read()
try:
suggestions_timestamp = datetime.fromtimestamp(float(timestamp_str))
except:
self.logger.error('Could not parse the cached suggestions timestamp: {}'.format(timestamp_str))
traceback.print_exc()
return True
update = suggestions_timestamp + timedelta(hours=exp_hours) <= datetime.utcnow()
return update
def read(self) -> Optional[List[str]]:
self.logger.info("Checking if suggestions should be downloaded")
if self.should_download(self.config):
suggestions_timestamp = datetime.utcnow().timestamp()
suggestions_str = self.download()
Thread(target=self.cache_suggestions, args=(suggestions_str, suggestions_timestamp), daemon=True).start()
else:
self.logger.info("Reading cached suggestions from '{}'".format(SUGGESTIONS_CACHED_FILE))
with open(SUGGESTIONS_CACHED_FILE) as f:
suggestions_str = f.read()
return self.map_suggestions(suggestions_str) if suggestions_str else None
def cache_suggestions(self, text: str, timestamp: float):
self.logger.info("Caching suggestions to '{}'".format(SUGGESTIONS_FILE))
cache_dir = os.path.dirname(SUGGESTIONS_CACHED_FILE)
try:
Path(cache_dir).mkdir(parents=True, exist_ok=True)
cache_dir_ok = True
except OSError:
self.logger.error("Could not create cache directory '{}'".format(cache_dir))
traceback.print_exc()
cache_dir_ok = False
if cache_dir_ok:
try:
with open(SUGGESTIONS_CACHED_FILE, 'w+') as f:
f.write(text)
except:
self.logger.error("An exception happened while writing the file '{}'".format(SUGGESTIONS_FILE))
traceback.print_exc()
try:
with open(SUGGESTIONS_CACHED_TS_FILE, 'w+') as f:
f.write(str(timestamp))
except:
self.logger.error("An exception happened while writing the file '{}'".format(SUGGESTIONS_CACHED_TS_FILE))
traceback.print_exc()
def download(self) -> Optional[str]:
self.logger.info("Downloading suggestions from {}".format(SUGGESTIONS_FILE))
try:
res = self.http_client.get(SUGGESTIONS_FILE)
except requests.exceptions.ConnectionError:
self.logger.warning("Could not download suggestion from '{}'".format(SUGGESTIONS_FILE))
return
if not res:
self.logger.warning("Could not download suggestion from '{}'".format(SUGGESTIONS_FILE))
return
if not res.text:
self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_FILE))
return
return res.text
def map_suggestions(self, text: str) -> List[str]:
return [line for line in text.split('\n') if line]
def run(self):
if self.create_config:
self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name)))
self.create_config.join()
self.config = self.create_config.config
ti = time.time()
self.taskman.update_progress(self.task_id, 1, None)
self.logger.info("Checking if suggestions should be downloaded")
should_download = self.should_download(self.config)
self.taskman.update_progress(self.task_id, 30, None)
try:
if should_download:
suggestions_timestamp = datetime.utcnow().timestamp()
suggestions_str = self.download()
self.taskman.update_progress(self.task_id, 70, None)
if suggestions_str:
self.cache_suggestions(suggestions_str, suggestions_timestamp)
else:
self.logger.info("Cached suggestions are up-to-date")
except:
self.logger.error("An unexpected exception happened")
traceback.print_exc()
self.taskman.update_progress(self.task_id, 100, None)
self.taskman.finish_task(self.task_id)
tf = time.time()
self.logger.info("Took {0:.9f} seconds to download suggestions".format(tf - ti))

2
bauh/view/core/controller.py
View File

@@ -477,7 +477,7 @@ class GenericSoftwareManager(SoftwareManager):
mti = time.time()
man_sugs = man.list_suggestions(limit=limit, filter_installed=filter_installed)
mtf = time.time()
self.logger.info(man.__class__.__name__ + ' took {0:.2f} seconds'.format(mtf - mti))
self.logger.info(man.__class__.__name__ + ' took {0:.5f} seconds'.format(mtf - mti))
if man_sugs:
if 0 < limit < len(man_sugs):