Caching installed application data to disk

CHANGELOG.md

@@ -6,12 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [0.3.1]
 ### Improvements
-- Console output now is optional and now shown by default.
+- Console output now is optional and not shown by default.
+- Cleaning search bar when 'Refresh' is clicked.
 - Full Flatpak database is not loaded during initialization (management panel is quickly available for the user)
-- Applications data that must be retrieved from Flathub API are now retrieved on demand and cached
+- Applications data not available offline are now retrieved from Flathub API on demand and cached in memory and disk (only installed)
-- Cache cleaners (for icons and API data) to improve memory usage.
+- In-memory cached data have an expiration time and are cleaned overtime to reduce memory usage.
-- New environment variable and argument: FPAKMAN_ICON_EXPIRATION ('--icon-exp')
+- Code was refactored to support other types of packaging in the future (e.g: snap)
-- Code was refactored to support other types of packaging
+
+### Fixes:
+- flatpak 1.0.X: search is now retrieving the application name
 
 ## [0.3.0] - 2019-07-02
 ### Features

README.md

@@ -47,6 +47,7 @@ You can change some application settings via environment variables or arguments
 - **FPAKMAN_LOCALE**: define a custom app translation for a given locale key (e.g: 'pt', 'en', 'es', ...). Default: system locale.
 - **FPAKMAN_CACHE_EXPIRATION**: define a custom expiration time in SECONDS for cached API data. Default: 3600 (1 hour).
 - **FPAKMAN_ICON_EXPIRATION**: define a custom expiration time in SECONDS for cached icons. Default: 300 (5 minutes).
+- **FPAKMAN_DISK_CACHE**: enables / disables disk cache. When disk cache is enabled, the installed applications data are loaded faster. Use **0** (disable) or **1** (enable, default).
 
 
 ### Roadmap

fpakman/__init__.py

@@ -1 +1,2 @@
 __version__ = '0.3.1'
+__app_name__ = 'fpakman'

fpakman/app.py

@@ -8,6 +8,8 @@ from colorama import Fore
 
 from fpakman import __version__
 from fpakman.core import resource
+from fpakman.core.disk import DiskCacheLoaderFactory
+from fpakman.core.structure import prepare_folder_structure
 from fpakman.util import util
 from fpakman.core.controller import FlatpakManager, GenericApplicationManager
 from fpakman.util.cache import Cache
@@ -31,7 +33,8 @@ parser.add_argument('-e', '--cache-exp', action="store", default=int(os.getenv('
 parser.add_argument('-ie', '--icon-exp', action="store", default=int(os.getenv('FPAKMAN_ICON_EXPIRATION', 60 * 5)), type=int, help='cached icons expiration time in SECONDS. Default: %(default)s')
 parser.add_argument('-l', '--locale', action="store", default=os.getenv('FPAKMAN_LOCALE', 'en'), help='Translation key. Default: %(default)s')
 parser.add_argument('-i', '--check-interval', action="store", default=int(os.getenv('FPAKMAN_CHECK_INTERVAL', 60)), type=int, help='Updates check interval in SECONDS. Default: %(default)s')
-parser.add_argument('-n', '--update-notification', action="store", default=os.getenv('FPAKMAN_UPDATE_NOTIFICATION', 1), type=int, help='Enable/disable system notifications for new updates. Default: %(default)s')
+parser.add_argument('-n', '--update-notification', action="store", default=os.getenv('FPAKMAN_UPDATE_NOTIFICATION', 1), type=int, help='Enables / disables system notifications for new updates. Default: %(default)s')
+parser.add_argument('-dc', '--disk-cache', action="store", default=os.getenv('FPAKMAN_DISK_CACHE', 1), type=int, help='Enables / disables disk cache. When disk cache is enabled, the installed applications data are loaded faster. Default: %(default)s')
 args = parser.parse_args()
 
 if args.cache_exp < 0:
@@ -53,6 +56,8 @@ if args.update_notification == 0:
 
 locale_keys = util.get_locale_keys(args.locale)
 
+prepare_folder_structure(disk_cache=args.disk_cache)
+
 caches = []
 flatpak_api_cache = Cache(expiration_time=args.cache_exp)
 caches.append(flatpak_api_cache)
@@ -63,13 +68,15 @@ caches.append(icon_cache)
 app = QApplication(sys.argv)
 app.setWindowIcon(QIcon(resource.get_path('img/logo.svg')))
 
-manager = GenericApplicationManager([FlatpakManager(flatpak_api_cache)])
+disk_loader_factory = DiskCacheLoaderFactory(disk_cache=args.disk_cache, flatpak_api_cache=flatpak_api_cache)
+manager = GenericApplicationManager([FlatpakManager(flatpak_api_cache, disk_cache=args.disk_cache)], disk_loader_factory=disk_loader_factory)
 
 
 trayIcon = TrayIcon(locale_keys=locale_keys,
                     manager=manager,
                     check_interval=args.check_interval,
                     icon_cache=icon_cache,
+                    disk_cache=args.disk_cache,
                     update_notification=bool(args.update_notification))
 trayIcon.show()
 

fpakman/core/controller.py

@@ -1,3 +1,5 @@
+import os
+import shutil
 from abc import ABC, abstractmethod
 from datetime import datetime
 from threading import Lock
@@ -5,20 +7,20 @@ from typing import List
 
 import requests
 
-from fpakman.core import flatpak
+from fpakman.core import flatpak, disk
+from fpakman.core.disk import DiskCacheLoader, DiskCacheLoaderFactory
 from fpakman.core.model import FlatpakApplication, ApplicationData, ApplicationStatus, Application
-from fpakman.core.worker import FlatpakAsyncDataLoaderManager
 from fpakman.util.cache import Cache
 
 
 class ApplicationManager(ABC):
 
     @abstractmethod
-    def search(self, word: str) -> List[Application]:
+    def search(self, word: str, disk_loader: DiskCacheLoader) -> List[Application]:
         pass
 
     @abstractmethod
-    def read_installed(self) -> List[Application]:
+    def read_installed(self, disk_loader: DiskCacheLoader, keep_workers: bool) -> List[Application]:
         pass
 
     @abstractmethod
@@ -61,20 +63,27 @@ class ApplicationManager(ABC):
     def is_enabled(self) -> bool:
         pass
 
+    @abstractmethod
+    def cache_to_disk(self, app: Application, icon_bytes: bytes, only_icon: bool):
+        pass
+
+from fpakman.core.worker import FlatpakAsyncDataLoaderManager
+
 
 class FlatpakManager(ApplicationManager):
 
-    def __init__(self, api_cache: Cache):
+    def __init__(self, api_cache: Cache, disk_cache: bool):
         self.api_cache = api_cache
         self.http_session = requests.Session()
         self.lock_read = Lock()
-        self.async_data_loader = FlatpakAsyncDataLoaderManager(api_cache=self.api_cache)
+        self.disk_cache = disk_cache
+        self.async_data_loader = FlatpakAsyncDataLoaderManager(api_cache=self.api_cache, manager=self)
         flatpak.set_default_remotes()
 
     def get_app_type(self):
         return FlatpakApplication
 
-    def _map_to_model(self, app: dict) -> FlatpakApplication:
+    def _map_to_model(self, app: dict, installed: bool, disk_loader: DiskCacheLoader) -> FlatpakApplication:
 
         model = FlatpakApplication(arch=app.get('arch'),
                                    branch=app.get('branch'),
@@ -86,6 +95,7 @@ class FlatpakManager(ApplicationManager):
                                                              name=app.get('name'),
                                                              version=app.get('version'),
                                                              latest_version=app.get('latest_version')))
+        model.installed = installed
 
         api_data = self.api_cache.get(app['id'])
 
@@ -93,25 +103,23 @@ class FlatpakManager(ApplicationManager):
 
         if not api_data or expired_data:
             if not app['runtime']:
+                disk_loader.add(model)  # preloading cached disk data
                 model.status = ApplicationStatus.LOADING_DATA
                 self.async_data_loader.load(model)
 
-        else:  # filling cached data
+        else:
-            for attr, val in api_data.items():
+            model.fill_cached_data(api_data)
-                if attr != 'expires_at' and val:
-                    setattr(model.base_data, attr, val)
 
         return model
 
-    def search(self, word: str) -> List[FlatpakApplication]:
+    def search(self, word: str, disk_loader: DiskCacheLoader) -> List[FlatpakApplication]:
 
         res = []
         apps_found = flatpak.search(word)
 
         if apps_found:
-
             already_read = set()
-            installed_apps = self.read_installed(keep_workers=True)
+            installed_apps = self.read_installed(disk_loader=disk_loader, keep_workers=True)
 
             if installed_apps:
                 for app_found in apps_found:
@@ -122,13 +130,15 @@ class FlatpakManager(ApplicationManager):
 
             for app_found in apps_found:
                 if app_found['id'] not in already_read:
-                    res.append(self._map_to_model(app_found))
+                    res.append(self._map_to_model(app_found, False, disk_loader))
 
+            disk_loader.stop = True
+            disk_loader.join()
             self.async_data_loader.stop_current_workers()
 
         return res
 
-    def read_installed(self, keep_workers: bool = False) -> List[FlatpakApplication]:
+    def read_installed(self, disk_loader: DiskCacheLoader, keep_workers: bool = False) -> List[FlatpakApplication]:
 
         self.lock_read.acquire()
 
@@ -143,8 +153,7 @@ class FlatpakManager(ApplicationManager):
                 models = []
 
                 for app in installed:
-                    model = self._map_to_model(app)
+                    model = self._map_to_model(app, True, disk_loader)
-                    model.installed = True
                     model.update = app['id'] in available_updates
                     models.append(model)
 
@@ -175,6 +184,9 @@ class FlatpakManager(ApplicationManager):
     def clean_cache_for(self, app: FlatpakApplication):
         self.api_cache.delete(app.base_data.id)
 
+        if app.supports_disk_cache() and os.path.exists(app.get_disk_cache_path()):
+            shutil.rmtree(app.get_disk_cache_path())
+
     def update_and_stream(self, app: FlatpakApplication):
         return flatpak.update_and_stream(app.ref)
 
@@ -197,26 +209,43 @@ class FlatpakManager(ApplicationManager):
     def is_enabled(self):
         return flatpak.is_installed()
 
+    def cache_to_disk(self, app: FlatpakApplication, icon_bytes: bytes, only_icon: bool):
+        if self.disk_cache and app.supports_disk_cache():
+            disk.save(app, icon_bytes, only_icon)
+
 
 class GenericApplicationManager(ApplicationManager):
 
-    def __init__(self, managers: List[ApplicationManager]):
+    def __init__(self, managers: List[ApplicationManager], disk_loader_factory: DiskCacheLoaderFactory):
         self.managers = managers
         self.map = {m.get_app_type(): m for m in self.managers}
+        self.disk_loader_factory = disk_loader_factory
 
-    def search(self, word: str) -> List[Application]:
+    def search(self, word: str, disk_loader: DiskCacheLoader = None) -> List[Application]:
         apps = []
+        disk_loader = self.disk_loader_factory.new()
+        disk_loader.start()
+
         for man in self.managers:
             if man.is_enabled():
-                apps.extend(man.search(word))
+                apps.extend(man.search(word, disk_loader))
 
+        disk_loader.stop = True
+        disk_loader.join()
         return apps
 
-    def read_installed(self) -> List[Application]:
+    def read_installed(self, disk_loader: DiskCacheLoader = None, keep_workers: bool = False) -> List[Application]:
         installed = []
+
+        disk_loader = self.disk_loader_factory.new()
+        disk_loader.start()
+
         for man in self.managers:
             if man.is_enabled():
-                installed.extend(man.read_installed())
+                installed.extend(man.read_installed(disk_loader=disk_loader, keep_workers=keep_workers))
+
+        disk_loader.stop = True
+        disk_loader.join()
 
         return installed
 
@@ -273,6 +302,13 @@ class GenericApplicationManager(ApplicationManager):
     def is_enabled(self):
         return True
 
-    def _get_manager_for(self, app: Application):
+    def _get_manager_for(self, app: Application) -> ApplicationManager:
         man = self.map[app.__class__]
         return man if man and man.is_enabled() else None
+
+    def cache_to_disk(self, app: Application, icon_bytes: bytes, only_icon: bool):
+        if self.disk_loader_factory.disk_cache and app.supports_disk_cache():
+            man = self._get_manager_for(app)
+
+            if man:
+                return man.cache_to_disk(app, icon_bytes=icon_bytes, only_icon=only_icon)

fpakman/core/disk.py

@@ -0,0 +1,79 @@
+import json
+import os
+from pathlib import Path
+from threading import Thread, Lock
+from typing import List
+
+from fpakman.core.model import Application, FlatpakApplication
+from fpakman.util.cache import Cache
+
+
+class DiskCacheLoader(Thread):
+
+    def __init__(self, enabled: bool, flatpak_api_cache: Cache, apps: List[Application] = []):
+        super(DiskCacheLoader, self).__init__(daemon=True)
+        self.apps = apps
+        self.stop = False
+        self.lock = Lock()
+        self.flatpak_api_cache = flatpak_api_cache
+        self.enabled = enabled
+
+    def run(self):
+
+        if self.enabled:
+            while True:
+                if self.apps:
+                    self.lock.acquire()
+                    app = self.apps[0]
+                    del self.apps[0]
+                    self.lock.release()
+                    self.fill_cached_data(app)
+                elif self.stop:
+                    break
+
+    def add(self, app: Application):
+        if self.enabled:
+            if app and app.supports_disk_cache():
+                self.lock.acquire()
+                self.apps.append(app)
+                self.lock.release()
+
+    def fill_cached_data(self, app: Application):
+        if self.enabled:
+            if os.path.exists(app.get_disk_data_path()):
+                with open(app.get_disk_data_path()) as f:
+                    cached_data = json.loads(f.read())
+                    app.fill_cached_data(cached_data)
+
+                    if isinstance(app, FlatpakApplication):
+                        self.flatpak_api_cache.add_non_existing(app.base_data.id, cached_data)
+
+
+class DiskCacheLoaderFactory:
+
+    def __init__(self, disk_cache: bool, flatpak_api_cache: Cache):
+        self.disk_cache = disk_cache
+        self.flatpak_api_cache = flatpak_api_cache
+
+    def new(self):
+        return DiskCacheLoader(enabled=self.disk_cache, flatpak_api_cache=self.flatpak_api_cache)
+
+
+def save(app: Application, icon_bytes: bytes = None, only_icon: bool = False):
+
+    if app.supports_disk_cache():
+
+        if not only_icon:
+            Path(app.get_disk_cache_path()).mkdir(parents=True, exist_ok=True)
+
+            if isinstance(app, FlatpakApplication):
+                data = app.get_data_to_cache()
+
+                with open(app.get_disk_data_path(), 'w+') as f:
+                    f.write(json.dumps(data))
+
+        if icon_bytes:
+            Path(app.get_disk_cache_path()).mkdir(parents=True, exist_ok=True)
+
+            with open(app.get_disk_icon_path(), 'wb+') as f:
+                f.write(icon_bytes)

fpakman/core/flatpak.py

@@ -107,7 +107,7 @@ def uninstall_and_stream(app_ref: str):
     :param app_ref:
     :return:
     """
-    return system.stream_cmd([BASE_CMD, 'uninstall', app_ref, '-y'])
+    return system.cmd_to_subprocess([BASE_CMD, 'uninstall', app_ref, '-y'])
 
 
 def list_updates_as_str():
@@ -211,7 +211,7 @@ def search(word: str) -> List[dict]:
 
 
 def install_and_stream(app_id: str, origin: str):
-    return system.stream_cmd([BASE_CMD, 'install', origin, app_id, '-y'])
+    return system.cmd_to_subprocess([BASE_CMD, 'install', origin, app_id, '-y'])
 
 
 def set_default_remotes():

fpakman/core/model.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 from enum import Enum
 
 from fpakman.core import resource
+from fpakman.core.structure import flatpak_cache_path
 
 
 class ApplicationStatus(Enum):
@@ -52,13 +53,37 @@ class Application(ABC):
     def get_type(self):
         pass
 
-    def get_default_logo_path(self):
+    def get_default_icon_path(self):
         return resource.get_path('img/logo.svg')
 
     @abstractmethod
     def is_library(self):
         pass
 
+    @abstractmethod
+    def supports_disk_cache(self):
+        pass
+
+    @abstractmethod
+    def get_disk_cache_path(self):
+        pass
+
+    @abstractmethod
+    def get_disk_icon_path(self):
+        pass
+
+    @abstractmethod
+    def get_disk_data_path(self):
+        pass
+
+    @abstractmethod
+    def get_data_to_cache(self):
+        pass
+
+    @abstractmethod
+    def fill_cached_data(self, data: dict):
+        pass
+
 
 class FlatpakApplication(Application):
 
@@ -92,8 +117,34 @@ class FlatpakApplication(Application):
     def get_type(self):
         return 'flatpak'
 
-    def get_default_logo_path(self):
+    def get_default_icon_path(self):
         return resource.get_path('img/flathub.svg')
 
     def is_library(self):
         return self.runtime
+
+    def supports_disk_cache(self):
+        return self.installed and not self.is_library()
+
+    def get_disk_cache_path(self):
+        return '{}/{}'.format(flatpak_cache_path, self.base_data.id)
+
+    def get_disk_icon_path(self):
+        return '{}/icon.png'.format(self.get_disk_cache_path())
+
+    def get_disk_data_path(self):
+        return '{}/data.json'.format(self.get_disk_cache_path())
+
+    def get_data_to_cache(self):
+        return {
+            'description': self.base_data.description,
+            'icon_url': self.base_data.icon_url,
+            'latest_version': self.base_data.latest_version,
+            'version': self.base_data.version,
+            'name': self.base_data.name
+        }
+
+    def fill_cached_data(self, data: dict):
+        for attr in self.get_data_to_cache().keys():
+            if not getattr(self.base_data, attr):
+                setattr(self.base_data, attr, data[attr])

fpakman/core/structure.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+from fpakman import __app_name__
+
+home_path = Path.home()
+cache_path = '{}/.cache/{}'.format(home_path, __app_name__)
+flatpak_cache_path = '{}/flatpak/installed'.format(cache_path)
+
+
+def prepare_folder_structure(disk_cache: bool):
+    if disk_cache:
+        Path(flatpak_cache_path).mkdir(parents=True, exist_ok=True)

fpakman/core/system.py

@@ -20,7 +20,11 @@ def run_cmd(cmd: str, expected_code: int = 0, ignore_return_code: bool = False,
 
 
 def stream_cmd(cmd: List[str]):
-    return subprocess.Popen(cmd, stdout=subprocess.PIPE, env={'LANG': 'en'}).stdout
+    return cmd_to_subprocess(cmd).stdout
+
+
+def cmd_to_subprocess(cmd: List[str]):
+    return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={'LANG': 'en'})
 
 
 def notify_user(msg: str, icon_path: str = resource.get_path('img/logo.svg')):

fpakman/core/worker.py

@@ -7,20 +7,23 @@ import requests
 from colorama import Fore
 
 from fpakman.core.constants import FLATHUB_API_URL, FLATHUB_URL
+from fpakman.core.controller import ApplicationManager
 from fpakman.core.model import FlatpakApplication, ApplicationStatus
 from fpakman.util.cache import Cache
 
 
 class FlatpakAsyncDataLoader(Thread):
 
-    def __init__(self, http_session, api_cache: Cache, attempts: int = 3):
+    def __init__(self, manager: ApplicationManager, http_session, api_cache: Cache, attempts: int = 3, apps: List[FlatpakApplication] = []):
         super(FlatpakAsyncDataLoader, self).__init__(daemon=True)
-        self.apps = []
+        self.apps = apps
         self.http_session = http_session
         self.attempts = attempts
         self.api_cache = api_cache
         self.id_ = '{}#{}'.format(self.__class__.__name__, id(self))
         self.stop = False
+        self.to_persist = {}  # stores all data loaded by the instance
+        self.manager = manager
 
     def log_msg(self, msg: str, color: int = None):
         final_msg = StringIO()
@@ -41,61 +44,80 @@ class FlatpakAsyncDataLoader(Thread):
 
     def run(self):
         while True:
-            if not self.apps and self.stop:
+            if self.apps:
-                break  # stop working
+                app = self.apps[0]
-            else:
+                app.status = ApplicationStatus.LOADING_DATA
+
+                for _ in range(0, self.attempts):
+                    try:
+                        res = self.http_session.get('{}/apps/{}'.format(FLATHUB_API_URL, app.base_data.id), timeout=30)
+
+                        if res.status_code == 200 and res.text:
+                            data = res.json()
+
+                            if not app.base_data.version:
+                                app.base_data.version = data.get('version')
+
+                            if not app.base_data.name:
+                                app.base_data.name = data.get('name')
+
+                            app.base_data.description = data.get('description', data.get('summary', None))
+                            app.base_data.icon_url = data.get('iconMobileUrl', None)
+                            app.base_data.latest_version = data.get('currentReleaseVersion', app.base_data.version)
+
+                            if not app.base_data.version and app.base_data.latest_version:
+                                app.base_data.version = app.base_data.latest_version
+
+                            if app.base_data.icon_url and app.base_data.icon_url.startswith('/'):
+                                app.base_data.icon_url = FLATHUB_URL + app.base_data.icon_url
+
+                            loaded_data = app.get_data_to_cache()
+
+                            self.api_cache.add(app.base_data.id, loaded_data)
+                            app.status = ApplicationStatus.READY
+
+                            if app.supports_disk_cache():
+                                self.to_persist[app.base_data.id] = app
+
+                            break
+                        else:
+                            self.log_msg(
+                                "Could not retrieve app data for id '{}'. Server response: {}. Body: {}".format(
+                                    app.base_data.id, res.status_code, res.content.decode()), Fore.RED)
+                    except:
+                        self.log_msg("Could not retrieve app data for id '{}'".format(app.base_data.id), Fore.YELLOW)
+                        traceback.print_exc()
+
                 if self.apps:
-                    app = self.apps[0]
-                    app.status = ApplicationStatus.LOADING_DATA
-
-                    for _ in range(0, self.attempts):
-                        try:
-                            res = self.http_session.get('{}/apps/{}'.format(FLATHUB_API_URL, app.base_data.id), timeout=30)
-
-                            if res.status_code == 200 and res.text:
-                                data = res.json()
-
-                                if not app.base_data.version:
-                                    app.base_data.version = data.get('version')
-
-                                app.base_data.description = data.get('description', data.get('summary', None))
-                                app.base_data.icon_url = data.get('iconMobileUrl', None)
-                                app.base_data.latest_version = data.get('currentReleaseVersion', app.base_data.version)
-
-                                if app.base_data.icon_url and app.base_data.icon_url.startswith('/'):
-                                    app.base_data.icon_url = FLATHUB_URL + app.base_data.icon_url
-
-                                self.api_cache.add(app.base_data.id, {
-                                    'description': app.base_data.description,
-                                    'icon_url': app.base_data.icon_url,
-                                    'latest_version': app.base_data.latest_version,
-                                    'version': app.base_data.version
-                                })
-
-                                app.status = ApplicationStatus.READY
-                                break
-                            else:
-                                self.log_msg("Could not retrieve app data for id '{}'. Server response: {}. Body: {}".format(app.base_data.id, res.status_code, res.content.decode()), Fore.RED)
-                        except:
-                            self.log_msg("Could not retrieve app data for id '{}'".format(app.base_data.id), Fore.YELLOW)
-                            traceback.print_exc()
-
                     del self.apps[0]
 
+            elif self.stop:
+                self.cache_to_disk()
+                break  # stop working
+
     def add(self, app: FlatpakApplication):
         self.apps.append(app)
 
     def current_load(self):
         return len(self.apps)
 
+    def cache_to_disk(self):
+
+        if self.to_persist:
+            for app in self.to_persist.values():
+                self.manager.cache_to_disk(app=app, icon_bytes=None, only_icon=False)
+
+            self.to_persist = {}
+
 
 class FlatpakAsyncDataLoaderManager:
 
-    def __init__(self, api_cache: Cache, worker_load: int = 1, workers: List[FlatpakAsyncDataLoader] = []):
+    def __init__(self, manager: ApplicationManager, api_cache: Cache, worker_load: int = 1, workers: List[FlatpakAsyncDataLoader] = []):
         self.worker_load = worker_load
         self.current_workers = workers
         self.http_session = requests.Session()
         self.api_cache = api_cache
+        self.manager = manager
 
     def load(self, app: FlatpakApplication):
 
@@ -105,7 +127,8 @@ class FlatpakAsyncDataLoaderManager:
                 worker = available_workers[0]
             else:  # new worker
                 worker = FlatpakAsyncDataLoader(http_session=self.http_session,
-                                                api_cache=self.api_cache)
+                                                api_cache=self.api_cache,
+                                                manager=self.manager)
                 worker.start()
                 self.current_workers.append(worker)
 

fpakman/util/cache.py

@@ -16,7 +16,21 @@ class Cache:
 
         if self.is_enabled():
             self.lock.acquire()
-            self._cache[key] = {'val': val, 'expires_at': datetime.utcnow() + timedelta(seconds=self.expiration_time) if self.expiration_time > 0 else None}
+            self._add(key, val)
+            self.lock.release()
+
+    def _add(self, key: str, val: object):
+        self._cache[key] = {'val': val, 'expires_at': datetime.utcnow() + timedelta(seconds=self.expiration_time) if self.expiration_time > 0 else None}
+
+    def add_non_existing(self, key: str, val: object):
+
+        if self. is_enabled():
+            self.lock.acquire()
+            cur_val = self.get(key)
+
+            if cur_val is None:
+                self._add(key, val)
+
             self.lock.release()
 
     def get(self, key: str):

fpakman/view/qt/apps_table.py

@@ -1,3 +1,4 @@
+import os
 from threading import Lock
 from typing import List
 
@@ -8,7 +9,7 @@ from PyQt5.QtWidgets import QTableWidget, QTableView, QMenu, QAction, QTableWidg
     QHeaderView, QLabel
 
 from fpakman.core import resource
-from fpakman.core.model import FlatpakApplication, ApplicationStatus
+from fpakman.core.model import ApplicationStatus
 from fpakman.util.cache import Cache
 from fpakman.view.qt import dialog
 from fpakman.view.qt.view_model import ApplicationView, ApplicationViewStatus
@@ -39,10 +40,11 @@ class UpdateToggleButton(QToolButton):
 
 class AppsTable(QTableWidget):
 
-    def __init__(self, parent: QWidget, icon_cache: Cache):
+    def __init__(self, parent: QWidget, icon_cache: Cache, disk_cache: bool):
         super(AppsTable, self).__init__()
         self.setParent(parent)
         self.window = parent
+        self.disk_cache = disk_cache
         self.column_names = [parent.locale_keys[key].capitalize() for key in ['name',
                                                                               'version',
                                                                               'latest_version',
@@ -61,7 +63,7 @@ class AppsTable(QTableWidget):
         self.icon_logo = QIcon(resource.get_path('img/logo.svg'))
 
         self.network_man = QNetworkAccessManager()
-        self.network_man.finished.connect(self._load_icon)
+        self.network_man.finished.connect(self._load_icon_and_cache)
 
         self.icon_cache = icon_cache
         self.lock_async_data = Lock()
@@ -114,23 +116,21 @@ class AppsTable(QTableWidget):
 
         if self.window.apps:
 
-            visible, ready = 0, 0
             for idx, app_v in enumerate(self.window.apps):
 
-                if app_v.visible:
+                if app_v.visible and app_v.status == ApplicationViewStatus.LOADING and app_v.model.status == ApplicationStatus.READY:
-                    visible += 1
+                    self.network_man.get(QNetworkRequest(QUrl(app_v.model.base_data.icon_url)))
 
-                    if app_v.status == ApplicationViewStatus.LOADING and app_v.model.status == ApplicationStatus.READY:
+                    app_name = self.item(idx, 0).text()
-                        self.network_man.get(QNetworkRequest(QUrl(app_v.model.base_data.icon_url)))
-                        self.item(idx, 2).setText(app_v.model.base_data.latest_version)
-                        self._set_col_description(self.item(idx, 3), app_v)
-                        app_v.status = ApplicationViewStatus.READY
 
-                    if app_v.status == ApplicationViewStatus.READY:
+                    if not app_name or app_name == '...':
-                        ready += 1
+                        self.item(idx, 0).setText(app_v.model.base_data.name)
 
-            if ready == visible:
+                    self.item(idx, 2).setText(app_v.model.base_data.latest_version)
-                self.window.resize_and_center()
+                    self._set_col_description(self.item(idx, 3), app_v)
+                    app_v.status = ApplicationViewStatus.READY
+
+            self.window.resize_and_center()
 
         self.lock_async_data.release()
 
@@ -165,19 +165,29 @@ class AppsTable(QTableWidget):
     def _install_app(self):
         self.window.install_app(self.get_selected_app())
 
-    def _load_icon(self, http_response):
+    def _load_icon_and_cache(self, http_response):
         icon_url = http_response.url().toString()
 
-        if not self.icon_cache.get(icon_url):
+        icon_data = self.icon_cache.get(icon_url)
-            pixmap = QPixmap()
+        icon_was_cached = True
-            pixmap.loadFromData(http_response.readAll())
-            icon = QIcon(pixmap)
-            self.icon_cache.add(icon_url, icon)
 
-            for idx, app in enumerate(self.window.apps):
+        if not icon_data:
-                if app.model.base_data.icon_url == icon_url:
+            icon_was_cached = False
-                    self.item(idx, 0).setIcon(icon)
+            pixmap = QPixmap()
-                    break
+            icon_bytes = http_response.readAll()
+            pixmap.loadFromData(icon_bytes)
+            icon = QIcon(pixmap)
+            icon_data = {'icon': icon, 'bytes': icon_bytes}
+            self.icon_cache.add(icon_url, icon_data)
+
+        for idx, app in enumerate(self.window.apps):
+            if app.model.base_data.icon_url == icon_url:
+                col_name = self.item(idx, 0)
+                col_name.setIcon(icon_data['icon'])
+
+                if self.disk_cache and app.model.supports_disk_cache():
+                    if not icon_was_cached or not os.path.exists(app.model.get_disk_icon_path()):
+                        self.window.manager.cache_to_disk(app=app.model, icon_bytes=icon_data['bytes'], only_icon=True)
 
     def update_apps(self, app_views: List[ApplicationView]):
         self.setEnabled(True)
@@ -185,21 +195,7 @@ class AppsTable(QTableWidget):
 
         if app_views:
             for idx, app_v in enumerate(app_views):
-
+                col_name = self._gen_col_name(app_v)
-                col_name = QTableWidgetItem()
-                col_name.setText(app_v.model.base_data.name)
-                col_name.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
-
-                if not app_v.model.base_data.icon_url:
-                    col_name.setIcon(QIcon(app_v.model.get_default_logo_path()))
-                else:
-                    cached_icon = self.icon_cache.get(app_v.model.base_data.icon_url)
-
-                    if cached_icon:
-                        col_name.setIcon(cached_icon)
-                    else:
-                        col_name.setIcon(QIcon(app_v.model.get_default_logo_path()))
-
                 self.setItem(idx, 0, col_name)
 
                 col_version = QTableWidgetItem()
@@ -235,6 +231,22 @@ class AppsTable(QTableWidget):
                 col_update = UpdateToggleButton(app_v, self.window, self.window.locale_keys, app_v.model.update) if app_v.model.update else None
                 self.setCellWidget(idx, 6, col_update)
 
+    def _gen_col_name(self, app_v: ApplicationView):
+        col = QTableWidgetItem()
+        col.setText(app_v.model.base_data.name if app_v.model.base_data.name else '...')
+        col.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
+
+        if app_v.model.supports_disk_cache() and os.path.exists(app_v.model.get_disk_icon_path()):
+            icon = QIcon(app_v.model.get_disk_icon_path())
+        elif not app_v.model.base_data.icon_url:
+            icon = QIcon(app_v.model.get_default_icon_path())
+        else:
+            icon_data = self.icon_cache.get(app_v.model.base_data.icon_url)
+            icon = icon_data['icon'] if icon_data else QIcon(app_v.model.get_default_icon_path())
+
+        col.setIcon(icon)
+        return col
+
     def _set_col_description(self, col: QTableWidgetItem, app_v: ApplicationView):
         desc = app_v.get_async_attr('description', strip_html=True)
 

fpakman/view/qt/systray.py

@@ -39,11 +39,12 @@ class UpdateCheck(QThread):
 
 class TrayIcon(QSystemTrayIcon):
 
-    def __init__(self, locale_keys: dict, manager: ApplicationManager, icon_cache: Cache, check_interval: int = 60, update_notification: bool = True):
+    def __init__(self, locale_keys: dict, manager: ApplicationManager, icon_cache: Cache, disk_cache: bool, check_interval: int = 60, update_notification: bool = True):
         super(TrayIcon, self).__init__()
         self.locale_keys = locale_keys
         self.manager = manager
         self.icon_cache = icon_cache
+        self.disk_cache = disk_cache
 
         self.icon_default = QIcon(resource.get_path('img/logo.png'))
         self.icon_update = QIcon(resource.get_path('img/logo_update.png'))
@@ -108,7 +109,8 @@ class TrayIcon(QSystemTrayIcon):
             self.manage_window = ManageWindow(locale_keys=self.locale_keys,
                                               manager=self.manager,
                                               icon_cache=self.icon_cache,
-                                              tray_icon=self)
+                                              tray_icon=self,
+                                              disk_cache=self.disk_cache)
 
         if self.manage_window.isMinimized():
             self.manage_window.setWindowState(Qt.WindowNoState)

fpakman/view/qt/thread.py

@@ -56,13 +56,25 @@ class UninstallApp(QThread):
 
     def run(self):
         if self.app:
-            for output in self.manager.uninstall_and_stream(self.app.model):
+            subproc = self.manager.uninstall_and_stream(self.app.model)
+
+            for output in subproc.stdout:
                 line = output.decode().strip()
                 if line:
                     self.signal_output.emit(line)
 
-            self.icon_cache.delete(self.app.model.base_data.icon_url)
+            error = False
-            self.manager.clean_cache_for(self.app.model)
+
+            for output in subproc.stderr:
+                line = output.decode().strip()
+                if line:
+                    error = True
+                    self.signal_output.emit(line)
+
+            if not error:
+                self.icon_cache.delete(self.app.model.base_data.icon_url)
+                self.manager.clean_cache_for(self.app.model)
+
             self.signal_finished.emit()
 
 
@@ -147,19 +159,39 @@ class InstallApp(QThread):
     signal_finished = pyqtSignal()
     signal_output = pyqtSignal(str)
 
-    def __init__(self, manager: ApplicationManager, app: ApplicationView = None):
+    def __init__(self, manager: ApplicationManager, disk_cache: bool, icon_cache: Cache, app: ApplicationView = None):
         super(InstallApp, self).__init__()
         self.app = app
         self.manager = manager
+        self.icon_cache = icon_cache
+        self.disk_cache = disk_cache
 
     def run(self):
 
         if self.app:
-            for output in self.manager.install_and_stream(self.app.model):
+            subproc = self.manager.install_and_stream(self.app.model)
+
+            for output in subproc.stdout:
                 line = output.decode().strip()
                 if line:
                     self.signal_output.emit(line)
 
+            error = False
+
+            for output in subproc.stderr:
+                line = output.decode().strip()
+                if line:
+                    error = True
+                    self.signal_output.emit(line)
+
+            if not error and self.disk_cache:
+                self.app.model.installed = True
+
+                if self.app.model.supports_disk_cache():
+                    self.manager.cache_to_disk(app=self.app.model,
+                                               icon_bytes=self.icon_cache.get(self.app.model.base_data.icon_url)['bytes'],
+                                               only_icon=False)
+
         self.app = None
         self.signal_finished.emit()
 
@@ -224,3 +256,5 @@ class VerifyModels(QThread):
 
                 if stop_at <= datetime.utcnow():
                     break
+
+        self.apps = None

fpakman/view/qt/window.py

@@ -26,7 +26,7 @@ DARK_ORANGE = '#FF4500'
 class ManageWindow(QWidget):
     __BASE_HEIGHT__ = 400
 
-    def __init__(self, locale_keys: dict, icon_cache: Cache, manager: ApplicationManager, tray_icon=None):
+    def __init__(self, locale_keys: dict, icon_cache: Cache, manager: ApplicationManager, disk_cache: bool,  tray_icon=None):
         super(ManageWindow, self).__init__()
         self.locale_keys = locale_keys
         self.manager = manager
@@ -36,6 +36,7 @@ class ManageWindow(QWidget):
         self.apps = []
         self.label_flatpak = None
         self.icon_cache = icon_cache
+        self.disk_cache = disk_cache
 
         self.icon_flathub = QIcon(resource.get_path('img/logo.svg'))
         self.resize(ManageWindow.__BASE_HEIGHT__, ManageWindow.__BASE_HEIGHT__)
@@ -103,7 +104,7 @@ class ManageWindow(QWidget):
 
         self.layout.addWidget(toolbar)
 
-        self.table_apps = AppsTable(self, self.icon_cache)
+        self.table_apps = AppsTable(self, self.icon_cache, disk_cache=self.disk_cache)
         self.table_apps.change_headers_policy()
 
         self.layout.addWidget(self.table_apps)
@@ -150,7 +151,7 @@ class ManageWindow(QWidget):
         self.thread_search = SearchApps(self.manager)
         self.thread_search.signal_finished.connect(self._finish_search)
 
-        self.thread_install = InstallApp(self.manager)
+        self.thread_install = InstallApp(manager=self.manager, disk_cache=self.disk_cache, icon_cache=self.icon_cache)
         self.thread_install.signal_output.connect(self._update_action_output)
         self.thread_install.signal_finished.connect(self._finish_install)
 
@@ -236,6 +237,7 @@ class ManageWindow(QWidget):
     def refresh(self):
 
         if self._acquire_lock():
+            self.input_search.clear()
             self._begin_action(self.locale_keys['manage_window.status.refreshing'])
 
             self.thread_refresh.start()