[appimage] downloading compressed database files from the cloud

bauh/gems/appimage/__init__.py

@@ -3,4 +3,5 @@ import os
 from bauh.api.constants import HOME_PATH
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
-INSTALLATION_PATH = '{}/.local/share/bauh/appimage/installed/'.format(HOME_PATH)
+BASE_PATH = '{}/.local/share/bauh/appimage'.format(HOME_PATH)
+INSTALLATION_PATH = BASE_PATH + '/installed/'

bauh/gems/appimage/worker.py

@@ -1,19 +1,20 @@
 import logging
 import os
+import tarfile
 import time
+import traceback
 from multiprocessing import Process
+from pathlib import Path
 from threading import Thread
 
-from bauh.api.constants import HOME_PATH
 from bauh.api.http import HttpClient
+from bauh.gems.appimage import BASE_PATH
 
 
 class DatabaseUpdater(Thread if bool(int(os.getenv('BAUH_DEBUG', 0))) else Process):
 
-    URL_APPS = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/apps.db'
+    URL_DB = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/dbs.tar.gz'
-    URL_RELEASES = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/appimage/releases.db'
+    COMPRESS_FILE_PATH = BASE_PATH + '/db.tar.gz'
-    APPS_PATH = '{}/.local/share/bauh/appimage/apps.db'.format(HOME_PATH)
-    RELEASES_PATH = '{}/.local/share/bauh/appimage/releases.db'.format(HOME_PATH)
 
     def __init__(self, http_client: HttpClient, logger: logging.Logger):
         super(DatabaseUpdater, self).__init__(daemon=True)
@@ -22,31 +23,48 @@ class DatabaseUpdater(Thread if bool(int(os.getenv('BAUH_DEBUG', 0))) else Proce
         self.enabled = bool(int(os.getenv('BAUH_APPIMAGE_DB_UPDATER', 1)))
         self.sleep = 60 * 20
 
-    def _download_file(self, url: str, output: str):
+    def _download_databases(self):
-        res = self.http_client.get(url, headers={'Authorization': 'token {}'.format(os.getenv('GITHUB_TOKEN'))})
+        self.logger.info('Retrieving AppImage databases')
+
+        res = self.http_client.get(self.URL_DB, headers={'Authorization': 'token {}'.format(os.getenv('GITHUB_TOKEN'))})
 
         if res:
-            with open(output, 'wb+') as f:
+            Path(BASE_PATH).mkdir(parents=True, exist_ok=True)
+
+            with open(self.COMPRESS_FILE_PATH, 'wb+') as f:
                 f.write(res.content)
 
-            self.logger.info("Database file saved at {}".format(output))
+            self.logger.info("Database file saved at {}".format(self.COMPRESS_FILE_PATH))
+
+            old_db_files = {file for r, d, f in os.walk(BASE_PATH) for file in f if file.endswith('.db')}
+
+            if old_db_files:
+                self.logger.info('Deleting old database files')
+                for f in old_db_files:
+                    os.remove('{}/{}'.format(BASE_PATH, f))
+                self.logger.info('Old database files deleted')
+
+            self.logger.info('Uncompressing {}'.format(self.COMPRESS_FILE_PATH))
+
+            try:
+                tf = tarfile.open(self.COMPRESS_FILE_PATH)
+                tf.extractall(BASE_PATH)
+                self.logger.info('Successfully uncompressed file {}'.format(self.COMPRESS_FILE_PATH))
+            except:
+                self.logger.error('Could not extract file {}'.format(self.COMPRESS_FILE_PATH))
+                traceback.print_exc()
+            finally:
+                self.logger.info('Deleting {}'.format(self.COMPRESS_FILE_PATH))
+                os.remove(self.COMPRESS_FILE_PATH)
+                self.logger.info('Successfully removed {}'.format(self.COMPRESS_FILE_PATH))
+
         else:
-            self.logger.warning('Could not download the database file {}'.format(url))
+            self.logger.warning('Could not download the database file {}'.format(self.URL_DB))
 
     def run(self):
         if self.enabled:
             while True:
-                self.logger.info('Retrieving AppImage databases')
+                self._download_databases()
-
-                threads = [Thread(target=self._download_file, args=(self.URL_APPS, self.APPS_PATH)),
-                           Thread(target=self._download_file, args=(self.URL_RELEASES, self.RELEASES_PATH))]
-
-                for t in threads:
-                    t.start()
-
-                for t in threads:
-                    t.join()
-
                 self.logger.info('Sleeping')
                 time.sleep(self.sleep)
         else: