Caching installed application data to disk

fpakman/core/controller.py

@@ -1,3 +1,5 @@
+import os
+import shutil
 from abc import ABC, abstractmethod
 from datetime import datetime
 from threading import Lock
@@ -5,20 +7,20 @@ from typing import List
 
 import requests
 
-from fpakman.core import flatpak
+from fpakman.core import flatpak, disk
+from fpakman.core.disk import DiskCacheLoader, DiskCacheLoaderFactory
 from fpakman.core.model import FlatpakApplication, ApplicationData, ApplicationStatus, Application
-from fpakman.core.worker import FlatpakAsyncDataLoaderManager
 from fpakman.util.cache import Cache
 
 
 class ApplicationManager(ABC):
 
     @abstractmethod
-    def search(self, word: str) -> List[Application]:
+    def search(self, word: str, disk_loader: DiskCacheLoader) -> List[Application]:
         pass
 
     @abstractmethod
-    def read_installed(self) -> List[Application]:
+    def read_installed(self, disk_loader: DiskCacheLoader, keep_workers: bool) -> List[Application]:
         pass
 
     @abstractmethod
@@ -61,20 +63,27 @@ class ApplicationManager(ABC):
     def is_enabled(self) -> bool:
         pass
 
+    @abstractmethod
+    def cache_to_disk(self, app: Application, icon_bytes: bytes, only_icon: bool):
+        pass
+
+from fpakman.core.worker import FlatpakAsyncDataLoaderManager
+
 
 class FlatpakManager(ApplicationManager):
 
-    def __init__(self, api_cache: Cache):
+    def __init__(self, api_cache: Cache, disk_cache: bool):
         self.api_cache = api_cache
         self.http_session = requests.Session()
         self.lock_read = Lock()
-        self.async_data_loader = FlatpakAsyncDataLoaderManager(api_cache=self.api_cache)
+        self.disk_cache = disk_cache
+        self.async_data_loader = FlatpakAsyncDataLoaderManager(api_cache=self.api_cache, manager=self)
         flatpak.set_default_remotes()
 
     def get_app_type(self):
         return FlatpakApplication
 
-    def _map_to_model(self, app: dict) -> FlatpakApplication:
+    def _map_to_model(self, app: dict, installed: bool, disk_loader: DiskCacheLoader) -> FlatpakApplication:
 
         model = FlatpakApplication(arch=app.get('arch'),
                                    branch=app.get('branch'),
@@ -86,6 +95,7 @@ class FlatpakManager(ApplicationManager):
                                                              name=app.get('name'),
                                                              version=app.get('version'),
                                                              latest_version=app.get('latest_version')))
+        model.installed = installed
 
         api_data = self.api_cache.get(app['id'])
 
@@ -93,25 +103,23 @@ class FlatpakManager(ApplicationManager):
 
         if not api_data or expired_data:
             if not app['runtime']:
+                disk_loader.add(model)  # preloading cached disk data
                 model.status = ApplicationStatus.LOADING_DATA
                 self.async_data_loader.load(model)
 
-        else:  # filling cached data
-            for attr, val in api_data.items():
-                if attr != 'expires_at' and val:
-                    setattr(model.base_data, attr, val)
+        else:
+            model.fill_cached_data(api_data)
 
         return model
 
-    def search(self, word: str) -> List[FlatpakApplication]:
+    def search(self, word: str, disk_loader: DiskCacheLoader) -> List[FlatpakApplication]:
 
         res = []
         apps_found = flatpak.search(word)
 
         if apps_found:
-
             already_read = set()
-            installed_apps = self.read_installed(keep_workers=True)
+            installed_apps = self.read_installed(disk_loader=disk_loader, keep_workers=True)
 
             if installed_apps:
                 for app_found in apps_found:
@@ -122,13 +130,15 @@ class FlatpakManager(ApplicationManager):
 
             for app_found in apps_found:
                 if app_found['id'] not in already_read:
-                    res.append(self._map_to_model(app_found))
+                    res.append(self._map_to_model(app_found, False, disk_loader))
 
+            disk_loader.stop = True
+            disk_loader.join()
             self.async_data_loader.stop_current_workers()
 
         return res
 
-    def read_installed(self, keep_workers: bool = False) -> List[FlatpakApplication]:
+    def read_installed(self, disk_loader: DiskCacheLoader, keep_workers: bool = False) -> List[FlatpakApplication]:
 
         self.lock_read.acquire()
 
@@ -143,8 +153,7 @@ class FlatpakManager(ApplicationManager):
                 models = []
 
                 for app in installed:
-                    model = self._map_to_model(app)
-                    model.installed = True
+                    model = self._map_to_model(app, True, disk_loader)
                     model.update = app['id'] in available_updates
                     models.append(model)
 
@@ -175,6 +184,9 @@ class FlatpakManager(ApplicationManager):
     def clean_cache_for(self, app: FlatpakApplication):
         self.api_cache.delete(app.base_data.id)
 
+        if app.supports_disk_cache() and os.path.exists(app.get_disk_cache_path()):
+            shutil.rmtree(app.get_disk_cache_path())
+
     def update_and_stream(self, app: FlatpakApplication):
         return flatpak.update_and_stream(app.ref)
 
@@ -197,26 +209,43 @@ class FlatpakManager(ApplicationManager):
     def is_enabled(self):
         return flatpak.is_installed()
 
+    def cache_to_disk(self, app: FlatpakApplication, icon_bytes: bytes, only_icon: bool):
+        if self.disk_cache and app.supports_disk_cache():
+            disk.save(app, icon_bytes, only_icon)
+
 
 class GenericApplicationManager(ApplicationManager):
 
-    def __init__(self, managers: List[ApplicationManager]):
+    def __init__(self, managers: List[ApplicationManager], disk_loader_factory: DiskCacheLoaderFactory):
         self.managers = managers
         self.map = {m.get_app_type(): m for m in self.managers}
+        self.disk_loader_factory = disk_loader_factory
 
-    def search(self, word: str) -> List[Application]:
+    def search(self, word: str, disk_loader: DiskCacheLoader = None) -> List[Application]:
         apps = []
+        disk_loader = self.disk_loader_factory.new()
+        disk_loader.start()
+
         for man in self.managers:
             if man.is_enabled():
-                apps.extend(man.search(word))
+                apps.extend(man.search(word, disk_loader))
 
+        disk_loader.stop = True
+        disk_loader.join()
         return apps
 
-    def read_installed(self) -> List[Application]:
+    def read_installed(self, disk_loader: DiskCacheLoader = None, keep_workers: bool = False) -> List[Application]:
         installed = []
+
+        disk_loader = self.disk_loader_factory.new()
+        disk_loader.start()
+
         for man in self.managers:
             if man.is_enabled():
-                installed.extend(man.read_installed())
+                installed.extend(man.read_installed(disk_loader=disk_loader, keep_workers=keep_workers))
+
+        disk_loader.stop = True
+        disk_loader.join()
 
         return installed
 
@@ -273,6 +302,13 @@ class GenericApplicationManager(ApplicationManager):
     def is_enabled(self):
         return True
 
-    def _get_manager_for(self, app: Application):
+    def _get_manager_for(self, app: Application) -> ApplicationManager:
         man = self.map[app.__class__]
         return man if man and man.is_enabled() else None
+
+    def cache_to_disk(self, app: Application, icon_bytes: bytes, only_icon: bool):
+        if self.disk_loader_factory.disk_cache and app.supports_disk_cache():
+            man = self._get_manager_for(app)
+
+            if man:
+                return man.cache_to_disk(app, icon_bytes=icon_bytes, only_icon=only_icon)

fpakman/core/disk.py

@@ -0,0 +1,79 @@
+import json
+import os
+from pathlib import Path
+from threading import Thread, Lock
+from typing import List
+
+from fpakman.core.model import Application, FlatpakApplication
+from fpakman.util.cache import Cache
+
+
+class DiskCacheLoader(Thread):
+
+    def __init__(self, enabled: bool, flatpak_api_cache: Cache, apps: List[Application] = []):
+        super(DiskCacheLoader, self).__init__(daemon=True)
+        self.apps = apps
+        self.stop = False
+        self.lock = Lock()
+        self.flatpak_api_cache = flatpak_api_cache
+        self.enabled = enabled
+
+    def run(self):
+
+        if self.enabled:
+            while True:
+                if self.apps:
+                    self.lock.acquire()
+                    app = self.apps[0]
+                    del self.apps[0]
+                    self.lock.release()
+                    self.fill_cached_data(app)
+                elif self.stop:
+                    break
+
+    def add(self, app: Application):
+        if self.enabled:
+            if app and app.supports_disk_cache():
+                self.lock.acquire()
+                self.apps.append(app)
+                self.lock.release()
+
+    def fill_cached_data(self, app: Application):
+        if self.enabled:
+            if os.path.exists(app.get_disk_data_path()):
+                with open(app.get_disk_data_path()) as f:
+                    cached_data = json.loads(f.read())
+                    app.fill_cached_data(cached_data)
+
+                    if isinstance(app, FlatpakApplication):
+                        self.flatpak_api_cache.add_non_existing(app.base_data.id, cached_data)
+
+
+class DiskCacheLoaderFactory:
+
+    def __init__(self, disk_cache: bool, flatpak_api_cache: Cache):
+        self.disk_cache = disk_cache
+        self.flatpak_api_cache = flatpak_api_cache
+
+    def new(self):
+        return DiskCacheLoader(enabled=self.disk_cache, flatpak_api_cache=self.flatpak_api_cache)
+
+
+def save(app: Application, icon_bytes: bytes = None, only_icon: bool = False):
+
+    if app.supports_disk_cache():
+
+        if not only_icon:
+            Path(app.get_disk_cache_path()).mkdir(parents=True, exist_ok=True)
+
+            if isinstance(app, FlatpakApplication):
+                data = app.get_data_to_cache()
+
+                with open(app.get_disk_data_path(), 'w+') as f:
+                    f.write(json.dumps(data))
+
+        if icon_bytes:
+            Path(app.get_disk_cache_path()).mkdir(parents=True, exist_ok=True)
+
+            with open(app.get_disk_icon_path(), 'wb+') as f:
+                f.write(icon_bytes)

fpakman/core/flatpak.py

@@ -107,7 +107,7 @@ def uninstall_and_stream(app_ref: str):
     :param app_ref:
     :return:
     """
-    return system.stream_cmd([BASE_CMD, 'uninstall', app_ref, '-y'])
+    return system.cmd_to_subprocess([BASE_CMD, 'uninstall', app_ref, '-y'])
 
 
 def list_updates_as_str():
@@ -211,7 +211,7 @@ def search(word: str) -> List[dict]:
 
 
 def install_and_stream(app_id: str, origin: str):
-    return system.stream_cmd([BASE_CMD, 'install', origin, app_id, '-y'])
+    return system.cmd_to_subprocess([BASE_CMD, 'install', origin, app_id, '-y'])
 
 
 def set_default_remotes():

fpakman/core/model.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 from enum import Enum
 
 from fpakman.core import resource
+from fpakman.core.structure import flatpak_cache_path
 
 
 class ApplicationStatus(Enum):
@@ -52,13 +53,37 @@ class Application(ABC):
     def get_type(self):
         pass
 
-    def get_default_logo_path(self):
+    def get_default_icon_path(self):
         return resource.get_path('img/logo.svg')
 
     @abstractmethod
     def is_library(self):
         pass
 
+    @abstractmethod
+    def supports_disk_cache(self):
+        pass
+
+    @abstractmethod
+    def get_disk_cache_path(self):
+        pass
+
+    @abstractmethod
+    def get_disk_icon_path(self):
+        pass
+
+    @abstractmethod
+    def get_disk_data_path(self):
+        pass
+
+    @abstractmethod
+    def get_data_to_cache(self):
+        pass
+
+    @abstractmethod
+    def fill_cached_data(self, data: dict):
+        pass
+
 
 class FlatpakApplication(Application):
 
@@ -92,8 +117,34 @@ class FlatpakApplication(Application):
     def get_type(self):
         return 'flatpak'
 
-    def get_default_logo_path(self):
+    def get_default_icon_path(self):
         return resource.get_path('img/flathub.svg')
 
     def is_library(self):
         return self.runtime
+
+    def supports_disk_cache(self):
+        return self.installed and not self.is_library()
+
+    def get_disk_cache_path(self):
+        return '{}/{}'.format(flatpak_cache_path, self.base_data.id)
+
+    def get_disk_icon_path(self):
+        return '{}/icon.png'.format(self.get_disk_cache_path())
+
+    def get_disk_data_path(self):
+        return '{}/data.json'.format(self.get_disk_cache_path())
+
+    def get_data_to_cache(self):
+        return {
+            'description': self.base_data.description,
+            'icon_url': self.base_data.icon_url,
+            'latest_version': self.base_data.latest_version,
+            'version': self.base_data.version,
+            'name': self.base_data.name
+        }
+
+    def fill_cached_data(self, data: dict):
+        for attr in self.get_data_to_cache().keys():
+            if not getattr(self.base_data, attr):
+                setattr(self.base_data, attr, data[attr])

fpakman/core/structure.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+from fpakman import __app_name__
+
+home_path = Path.home()
+cache_path = '{}/.cache/{}'.format(home_path, __app_name__)
+flatpak_cache_path = '{}/flatpak/installed'.format(cache_path)
+
+
+def prepare_folder_structure(disk_cache: bool):
+    if disk_cache:
+        Path(flatpak_cache_path).mkdir(parents=True, exist_ok=True)

fpakman/core/system.py

@@ -20,7 +20,11 @@ def run_cmd(cmd: str, expected_code: int = 0, ignore_return_code: bool = False,
 
 
 def stream_cmd(cmd: List[str]):
-    return subprocess.Popen(cmd, stdout=subprocess.PIPE, env={'LANG': 'en'}).stdout
+    return cmd_to_subprocess(cmd).stdout
+
+
+def cmd_to_subprocess(cmd: List[str]):
+    return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={'LANG': 'en'})
 
 
 def notify_user(msg: str, icon_path: str = resource.get_path('img/logo.svg')):

fpakman/core/worker.py

@@ -7,20 +7,23 @@ import requests
 from colorama import Fore
 
 from fpakman.core.constants import FLATHUB_API_URL, FLATHUB_URL
+from fpakman.core.controller import ApplicationManager
 from fpakman.core.model import FlatpakApplication, ApplicationStatus
 from fpakman.util.cache import Cache
 
 
 class FlatpakAsyncDataLoader(Thread):
 
-    def __init__(self, http_session, api_cache: Cache, attempts: int = 3):
+    def __init__(self, manager: ApplicationManager, http_session, api_cache: Cache, attempts: int = 3, apps: List[FlatpakApplication] = []):
         super(FlatpakAsyncDataLoader, self).__init__(daemon=True)
-        self.apps = []
+        self.apps = apps
         self.http_session = http_session
         self.attempts = attempts
         self.api_cache = api_cache
         self.id_ = '{}#{}'.format(self.__class__.__name__, id(self))
         self.stop = False
+        self.to_persist = {}  # stores all data loaded by the instance
+        self.manager = manager
 
     def log_msg(self, msg: str, color: int = None):
         final_msg = StringIO()
@@ -41,61 +44,80 @@ class FlatpakAsyncDataLoader(Thread):
 
     def run(self):
         while True:
-            if not self.apps and self.stop:
-                break  # stop working
-            else:
+            if self.apps:
+                app = self.apps[0]
+                app.status = ApplicationStatus.LOADING_DATA
+
+                for _ in range(0, self.attempts):
+                    try:
+                        res = self.http_session.get('{}/apps/{}'.format(FLATHUB_API_URL, app.base_data.id), timeout=30)
+
+                        if res.status_code == 200 and res.text:
+                            data = res.json()
+
+                            if not app.base_data.version:
+                                app.base_data.version = data.get('version')
+
+                            if not app.base_data.name:
+                                app.base_data.name = data.get('name')
+
+                            app.base_data.description = data.get('description', data.get('summary', None))
+                            app.base_data.icon_url = data.get('iconMobileUrl', None)
+                            app.base_data.latest_version = data.get('currentReleaseVersion', app.base_data.version)
+
+                            if not app.base_data.version and app.base_data.latest_version:
+                                app.base_data.version = app.base_data.latest_version
+
+                            if app.base_data.icon_url and app.base_data.icon_url.startswith('/'):
+                                app.base_data.icon_url = FLATHUB_URL + app.base_data.icon_url
+
+                            loaded_data = app.get_data_to_cache()
+
+                            self.api_cache.add(app.base_data.id, loaded_data)
+                            app.status = ApplicationStatus.READY
+
+                            if app.supports_disk_cache():
+                                self.to_persist[app.base_data.id] = app
+
+                            break
+                        else:
+                            self.log_msg(
+                                "Could not retrieve app data for id '{}'. Server response: {}. Body: {}".format(
+                                    app.base_data.id, res.status_code, res.content.decode()), Fore.RED)
+                    except:
+                        self.log_msg("Could not retrieve app data for id '{}'".format(app.base_data.id), Fore.YELLOW)
+                        traceback.print_exc()
+
                 if self.apps:
-                    app = self.apps[0]
-                    app.status = ApplicationStatus.LOADING_DATA
-
-                    for _ in range(0, self.attempts):
-                        try:
-                            res = self.http_session.get('{}/apps/{}'.format(FLATHUB_API_URL, app.base_data.id), timeout=30)
-
-                            if res.status_code == 200 and res.text:
-                                data = res.json()
-
-                                if not app.base_data.version:
-                                    app.base_data.version = data.get('version')
-
-                                app.base_data.description = data.get('description', data.get('summary', None))
-                                app.base_data.icon_url = data.get('iconMobileUrl', None)
-                                app.base_data.latest_version = data.get('currentReleaseVersion', app.base_data.version)
-
-                                if app.base_data.icon_url and app.base_data.icon_url.startswith('/'):
-                                    app.base_data.icon_url = FLATHUB_URL + app.base_data.icon_url
-
-                                self.api_cache.add(app.base_data.id, {
-                                    'description': app.base_data.description,
-                                    'icon_url': app.base_data.icon_url,
-                                    'latest_version': app.base_data.latest_version,
-                                    'version': app.base_data.version
-                                })
-
-                                app.status = ApplicationStatus.READY
-                                break
-                            else:
-                                self.log_msg("Could not retrieve app data for id '{}'. Server response: {}. Body: {}".format(app.base_data.id, res.status_code, res.content.decode()), Fore.RED)
-                        except:
-                            self.log_msg("Could not retrieve app data for id '{}'".format(app.base_data.id), Fore.YELLOW)
-                            traceback.print_exc()
-
                     del self.apps[0]
 
+            elif self.stop:
+                self.cache_to_disk()
+                break  # stop working
+
     def add(self, app: FlatpakApplication):
         self.apps.append(app)
 
     def current_load(self):
         return len(self.apps)
 
+    def cache_to_disk(self):
+
+        if self.to_persist:
+            for app in self.to_persist.values():
+                self.manager.cache_to_disk(app=app, icon_bytes=None, only_icon=False)
+
+            self.to_persist = {}
+
 
 class FlatpakAsyncDataLoaderManager:
 
-    def __init__(self, api_cache: Cache, worker_load: int = 1, workers: List[FlatpakAsyncDataLoader] = []):
+    def __init__(self, manager: ApplicationManager, api_cache: Cache, worker_load: int = 1, workers: List[FlatpakAsyncDataLoader] = []):
         self.worker_load = worker_load
         self.current_workers = workers
         self.http_session = requests.Session()
         self.api_cache = api_cache
+        self.manager = manager
 
     def load(self, app: FlatpakApplication):
 
@@ -105,7 +127,8 @@ class FlatpakAsyncDataLoaderManager:
                 worker = available_workers[0]
             else:  # new worker
                 worker = FlatpakAsyncDataLoader(http_session=self.http_session,
-                                                api_cache=self.api_cache)
+                                                api_cache=self.api_cache,
+                                                manager=self.manager)
                 worker.start()
                 self.current_workers.append(worker)