spalencsar/bearhub
1
0
Fork 0
mirror of https://github.com/spalencsar/bearhub.git synced 2026-07-06 22:54:16 +02:00
Code Issues Packages Projects Releases Wiki Activity

[appimage] downloading compressed database files from the cloud

Browse Source
This commit is contained in:
Vinicius Moreira
2019-10-11 13:54:14 -03:00
parent da12bc2086
commit 6db819ca9c
2 changed files with 41 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
bauh/gems/appimage/worker.py
View File

@@ -1,19 +1,20 @@
import logging
import os
import tarfile
import time
import traceback
from multiprocessing import Process
from pathlib import Path
from threading import Thread
from bauh.api.constants import HOME_PATH
from bauh.api.http import HttpClient
from bauh.gems.appimage import BASE_PATH
class DatabaseUpdater(Thread if bool(int(os.getenv('BAUH_DEBUG', 0))) else Process):
URL_APPS = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/apps.db'
URL_RELEASES = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/releases.db'
APPS_PATH = '{}/.local/share/bauh/appimage/apps.db'.format(HOME_PATH)
RELEASES_PATH = '{}/.local/share/bauh/appimage/releases.db'.format(HOME_PATH)
URL_DB = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/dbs.tar.gz'
COMPRESS_FILE_PATH = BASE_PATH + '/db.tar.gz'
def __init__(self, http_client: HttpClient, logger: logging.Logger):
super(DatabaseUpdater, self).__init__(daemon=True)
@@ -22,31 +23,48 @@ class DatabaseUpdater(Thread if bool(int(os.getenv('BAUH_DEBUG', 0))) else Proce
self.enabled = bool(int(os.getenv('BAUH_APPIMAGE_DB_UPDATER', 1)))
self.sleep = 60 * 20
def _download_file(self, url: str, output: str):
res = self.http_client.get(url, headers={'Authorization': 'token {}'.format(os.getenv('GITHUB_TOKEN'))})
def _download_databases(self):
self.logger.info('Retrieving AppImage databases')
res = self.http_client.get(self.URL_DB, headers={'Authorization': 'token {}'.format(os.getenv('GITHUB_TOKEN'))})
if res:
with open(output, 'wb+') as f:
Path(BASE_PATH).mkdir(parents=True, exist_ok=True)
with open(self.COMPRESS_FILE_PATH, 'wb+') as f:
f.write(res.content)
self.logger.info("Database file saved at {}".format(output))
self.logger.info("Database file saved at {}".format(self.COMPRESS_FILE_PATH))
old_db_files = {file for r, d, f in os.walk(BASE_PATH) for file in f if file.endswith('.db')}
if old_db_files:
self.logger.info('Deleting old database files')
for f in old_db_files:
os.remove('{}/{}'.format(BASE_PATH, f))
self.logger.info('Old database files deleted')
self.logger.info('Uncompressing {}'.format(self.COMPRESS_FILE_PATH))
try:
tf = tarfile.open(self.COMPRESS_FILE_PATH)
tf.extractall(BASE_PATH)
self.logger.info('Successfully uncompressed file {}'.format(self.COMPRESS_FILE_PATH))
except:
self.logger.error('Could not extract file {}'.format(self.COMPRESS_FILE_PATH))
traceback.print_exc()
finally:
self.logger.info('Deleting {}'.format(self.COMPRESS_FILE_PATH))
os.remove(self.COMPRESS_FILE_PATH)
self.logger.info('Successfully removed {}'.format(self.COMPRESS_FILE_PATH))
else:
self.logger.warning('Could not download the database file {}'.format(url))
self.logger.warning('Could not download the database file {}'.format(self.URL_DB))
def run(self):
if self.enabled:
while True:
self.logger.info('Retrieving AppImage databases')
threads = [Thread(target=self._download_file, args=(self.URL_APPS, self.APPS_PATH)),
Thread(target=self._download_file, args=(self.URL_RELEASES, self.RELEASES_PATH))]
for t in threads:
t.start()
for t in threads:
t.join()
self._download_databases()
self.logger.info('Sleeping')
time.sleep(self.sleep)
else: