refactoring memory and disk cache abstractions

bauh/app.py

@@ -3,32 +3,36 @@ import sys
 from PyQt5.QtGui import QIcon
 from PyQt5.QtWidgets import QApplication
 from bauh_api.abstract.controller import ApplicationContext
-from bauh_api.util.cache import Cache
-from bauh_api.util.disk import DiskCacheLoaderFactory
 from bauh_api.util.http import HttpClient
 
 from bauh import __version__, __app_name__, app_args, ROOT_DIR
 from bauh.core import resource, extensions
 from bauh.core.controller import GenericSoftwareManager
 from bauh.util import util
-from bauh.util.memory import CacheCleaner
+from bauh.util.cache import DefaultMemoryCacheFactory, CacheCleaner
+from bauh.util.disk import DefaultDiskCacheLoaderFactory
 from bauh.view.qt.systray import TrayIcon
 from bauh.view.qt.window import ManageWindow
 
 args = app_args.read()
 
 i18n = util.get_locale_keys(args.locale)
-caches, cache_map = [], {}
 
-context = ApplicationContext(i18n=i18n, http_client=HttpClient(), args=args, app_root_dir=ROOT_DIR)
-managers = extensions.load_managers(caches=caches, cache_map=cache_map, context=context)
+cache_cleaner = CacheCleaner()
+cache_factory = DefaultMemoryCacheFactory(expiration_time=args.cache_exp, cleaner=cache_cleaner)
+icon_cache = cache_factory.new(args.icon_exp)
 
-icon_cache = Cache(expiration_time=args.icon_exp)
-caches.append(icon_cache)
+context = ApplicationContext(i18n=i18n,
+                             http_client=HttpClient(),
+                             args=args,
+                             app_root_dir=ROOT_DIR,
+                             cache_factory=cache_factory,
+                             disk_loader_factory=DefaultDiskCacheLoaderFactory(disk_cache_enabled=args.disk_cache))
 
-disk_loader_factory = DiskCacheLoaderFactory(disk_cache=args.disk_cache, cache_map=cache_map)
+managers = extensions.load_managers(context=context)
 
-manager = GenericSoftwareManager(managers, disk_loader_factory=disk_loader_factory, context=context)
+
+manager = GenericSoftwareManager(managers, context=context)
 manager.prepare()
 
 app = QApplication(sys.argv)
@@ -56,5 +60,5 @@ else:
     manage_window.refresh_apps()
     manage_window.show()
 
-CacheCleaner(caches).start()
+cache_cleaner.start()
 sys.exit(app.exec_())

bauh/app_args.py

@@ -20,7 +20,7 @@ def read() -> Namespace:
     parser.add_argument('-v', '--version', action='version', version='%(prog)s {}'.format(__version__))
     parser.add_argument('-e', '--cache-exp', action="store",
                         default=int(os.getenv('BAUH_CACHE_EXPIRATION', 60 * 60)), type=int,
-                        help='cached API data expiration time in SECONDS. Default: %(default)s')
+                        help='default memory caches expiration time in SECONDS. Default: %(default)s')
     parser.add_argument('-ie', '--icon-exp', action="store", default=int(os.getenv('BAUH_ICON_EXPIRATION', 60 * 5)),
                         type=int, help='cached icons expiration time in SECONDS. Default: %(default)s')
     parser.add_argument('-l', '--locale', action="store", default=os.getenv('BAUH_LOCALE', 'en'),

bauh/core/controller.py

@@ -2,24 +2,23 @@ from threading import Thread
 from typing import List, Set, Type
 
 from bauh_api.abstract.controller import SoftwareManager, SearchResult, ApplicationContext
+from bauh_api.abstract.disk import DiskCacheLoader
 from bauh_api.abstract.handler import ProcessWatcher
 from bauh_api.abstract.model import SoftwarePackage, PackageUpdate, PackageHistory
-from bauh_api.util.disk import DiskCacheLoader
-from bauh_api.util.disk import DiskCacheLoaderFactory
 
 SUGGESTIONS_LIMIT = 6
 
 
 class GenericSoftwareManager(SoftwareManager):
 
-    def __init__(self, managers: List[SoftwareManager], context: ApplicationContext, disk_loader_factory: DiskCacheLoaderFactory):
-        super(GenericSoftwareManager, self).__init__(context=context, app_cache=None)
+    def __init__(self, managers: List[SoftwareManager], context: ApplicationContext):
+        super(GenericSoftwareManager, self).__init__(context=context)
         self.managers = managers
         self.map = {t: m for m in self.managers for t in m.get_managed_types()}
-        self.disk_loader_factory = disk_loader_factory
         self._enabled_map = {} if context.args.check_packaging_once else None
         self.thread_prepare = None
         self.i18n = context.i18n
+        self.disk_loader_factory = context.disk_loader_factory
 
     def _sort(self, apps: List[SoftwarePackage], word: str) -> List[SoftwarePackage]:
 
@@ -188,7 +187,7 @@ class GenericSoftwareManager(SoftwareManager):
         return man if man and self._is_enabled(man) else None
 
     def cache_to_disk(self, app: SoftwarePackage, icon_bytes: bytes, only_icon: bool):
-        if self.disk_loader_factory.disk_cache and app.supports_disk_cache():
+        if self.context.args.disk_cache and app.supports_disk_cache():
             man = self._get_manager_for(app)
 
             if man:

bauh/core/extensions.py

@@ -1,10 +1,9 @@
 import inspect
 import os
 import pkgutil
-from typing import List, Dict
+from typing import List
 
 from bauh_api.abstract.controller import SoftwareManager, ApplicationContext
-from bauh_api.util.cache import Cache
 
 from bauh import __app_name__
 from bauh.util import util
@@ -24,7 +23,7 @@ def find_manager(member):
                     return manager_found
 
 
-def load_managers(caches: List[Cache], cache_map: Dict[type, Cache], context: ApplicationContext) -> List[SoftwareManager]:
+def load_managers(context: ApplicationContext) -> List[SoftwareManager]:
     managers = []
 
     for m in pkgutil.iter_modules():
@@ -40,13 +39,6 @@ def load_managers(caches: List[Cache], cache_map: Dict[type, Cache], context: Ap
                     if os.path.exists(locale_path):
                         context.i18n.update(util.get_locale_keys(context.args.locale, locale_path))
 
-                    app_cache = Cache(expiration_time=context.args.cache_exp)
-                    man = manager_class(context=context, app_cache=app_cache)
-
-                    for t in man.get_managed_types():
-                        cache_map[t] = app_cache
-
-                    caches.append(app_cache)
-                    managers.append(man)
+                    managers.append(manager_class(context=context))
 
     return managers

bauh/util/cache.py

@@ -0,0 +1,108 @@
+import datetime
+import time
+from threading import Lock, Thread
+from typing import List
+
+from bauh_api.abstract.cache import MemoryCache, MemoryCacheFactory
+
+
+class DefaultMemoryCache(MemoryCache):
+    """
+    A synchronized cache implementation
+    """
+
+    def __init__(self, expiration_time: int):
+        super(DefaultMemoryCache, self).__init__()
+        self.expiration_time = expiration_time
+        self._cache = {}
+        self.lock = Lock()
+
+    def is_enabled(self):
+        return self.expiration_time < 0 or self.expiration_time > 0
+
+    def add(self, key: str, val: object):
+        if key and self.is_enabled():
+            self.lock.acquire()
+            self._add(key, val)
+            self.lock.release()
+
+    def _add(self, key: str, val: object):
+        if key:
+            self._cache[key] = {'val': val, 'expires_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.expiration_time) if self.expiration_time > 0 else None}
+
+    def add_non_existing(self, key: str, val: object):
+        if key and self. is_enabled():
+            self.lock.acquire()
+            cur_val = self.get(key)
+
+            if cur_val is None:
+                self._add(key, val)
+
+            self.lock.release()
+
+    def get(self, key: str):
+        if key and self.is_enabled():
+            val = self._cache.get(key)
+
+            if val:
+                expiration = val.get('expires_at')
+
+                if expiration and expiration <= datetime.datetime.utcnow():
+                    self.lock.acquire()
+                    del self._cache[key]
+                    self.lock.release()
+                    return None
+
+                return val['val']
+
+    def delete(self, key):
+        if key and self.is_enabled():
+            if key in self._cache:
+                self.lock.acquire()
+                del self._cache[key]
+                self.lock.release()
+
+    def keys(self):
+        return set(self._cache.keys()) if self.is_enabled() else set()
+
+    def clean_expired(self):
+        if self.is_enabled():
+            for key in self.keys():
+                self.get(key)
+
+
+class CacheCleaner(Thread):
+
+    def __init__(self, check_interval: int = 15):
+        super(CacheCleaner, self).__init__(daemon=True)
+        self.caches = []
+        self.check_interval = check_interval
+
+    def register(self, cache: MemoryCache):
+        if cache.is_enabled():
+            self.caches.append(cache)
+
+    def run(self):
+        if self.caches:
+            while True:
+                for cache in self.caches:
+                    cache.clean_expired()
+
+                time.sleep(self.check_interval)
+
+
+class DefaultMemoryCacheFactory(MemoryCacheFactory):
+
+    def __init__(self, expiration_time: int, cleaner: CacheCleaner):
+        """
+        :param expiration_time: default expiration time for all instantiated caches
+        :param cleaner
+        """
+        super(DefaultMemoryCacheFactory, self).__init__()
+        self.expiration_time = expiration_time
+        self.cleaner = cleaner
+
+    def new(self, expiration: int = None) -> MemoryCache:
+        instance = DefaultMemoryCache(expiration if expiration is not None else self.expiration_time)
+        self.cleaner.register(instance)
+        return instance

bauh/util/disk.py

@@ -0,0 +1,72 @@
+import json
+import os
+from threading import Thread, Lock
+from typing import Type, Dict
+
+from bauh_api.abstract.cache import MemoryCache
+from bauh_api.abstract.disk import DiskCacheLoader, DiskCacheLoaderFactory
+from bauh_api.abstract.model import SoftwarePackage
+
+
+class AsyncDiskCacheLoader(Thread, DiskCacheLoader):
+
+    def __init__(self, enabled: bool, cache_map: Dict[Type[SoftwarePackage], MemoryCache]):
+        super(AsyncDiskCacheLoader, self).__init__(daemon=True)
+        self.apps = []
+        self.stop = False
+        self.lock = Lock()
+        self.cache_map = cache_map
+        self.enabled = enabled
+
+    def fill(self, pkg: SoftwarePackage):
+        """
+        Adds a package which data must be read from the disk to a queue.
+        :param pkg:
+        :return:
+        """
+        if self.enabled:
+            if pkg and pkg.supports_disk_cache():
+                self.lock.acquire()
+                self.apps.append(pkg)
+                self.lock.release()
+
+    def run(self):
+        if self.enabled:
+            while True:
+                if self.apps:
+                    self.lock.acquire()
+                    app = self.apps[0]
+                    del self.apps[0]
+                    self.lock.release()
+                    self._fill_cached_data(app)
+                elif self.stop:
+                    break
+
+    def _fill_cached_data(self, pkg: SoftwarePackage):
+        if self.enabled:
+            if os.path.exists(pkg.get_disk_data_path()):
+                with open(pkg.get_disk_data_path()) as f:
+                    cached_data = json.loads(f.read())
+                    pkg.fill_cached_data(cached_data)
+                    cache = self.cache_map.get(pkg.__class__)\
+
+                    if cache:
+                        cache.add_non_existing(pkg.base_data.id, cached_data)
+
+
+class DefaultDiskCacheLoaderFactory(DiskCacheLoaderFactory):
+
+    def __init__(self, disk_cache_enabled: bool):
+        super(DefaultDiskCacheLoaderFactory, self).__init__()
+        self.disk_cache_enabled = disk_cache_enabled
+        self.cache_map = {}
+
+    def map(self, pkg_type: Type[SoftwarePackage], cache: MemoryCache):
+        if pkg_type:
+            if pkg_type in self.cache_map:
+                raise Exception('{} is already mapped')
+
+            self.cache_map[pkg_type] = cache
+
+    def new(self) -> AsyncDiskCacheLoader:
+        return AsyncDiskCacheLoader(enabled=self.disk_cache_enabled, cache_map=self.cache_map)

bauh/util/memory.py

@@ -1,23 +0,0 @@
-import time
-from threading import Thread
-from typing import List
-
-from bauh_api.util.cache import Cache
-
-
-class CacheCleaner(Thread):
-
-    def __init__(self, caches: List[Cache], check_interval: int = 15):
-        super(CacheCleaner, self).__init__(daemon=True)
-        self.caches = [c for c in caches if c.is_enabled()]
-        self.check_interval = check_interval
-
-    def run(self):
-
-        if self.caches:
-            while True:
-                for cache in self.caches:
-                    cache.clean_expired()
-
-                time.sleep(self.check_interval)
-

bauh/view/qt/apps_table.py

@@ -7,8 +7,8 @@ from PyQt5.QtGui import QPixmap, QIcon, QCursor
 from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest
 from PyQt5.QtWidgets import QTableWidget, QTableView, QMenu, QAction, QTableWidgetItem, QToolButton, QWidget, \
     QHeaderView, QLabel, QHBoxLayout, QPushButton, QToolBar
+from bauh_api.abstract.cache import MemoryCache
 from bauh_api.abstract.model import PackageStatus
-from bauh_api.util.cache import Cache
 
 from bauh.core import resource
 from bauh.util import util
@@ -75,7 +75,7 @@ class UpdateToggleButton(QWidget):
 
 class AppsTable(QTableWidget):
 
-    def __init__(self, parent: QWidget, icon_cache: Cache, disk_cache: bool, download_icons: bool):
+    def __init__(self, parent: QWidget, icon_cache: MemoryCache, disk_cache: bool, download_icons: bool):
         super(AppsTable, self).__init__()
         self.setParent(parent)
         self.window = parent

bauh/view/qt/history.py

@@ -4,13 +4,13 @@ from functools import reduce
 from PyQt5.QtCore import Qt
 from PyQt5.QtGui import QColor
 from PyQt5.QtWidgets import QDialog, QVBoxLayout, QTableWidget, QTableWidgetItem, QHeaderView
+from bauh_api.abstract.cache import MemoryCache
 from bauh_api.abstract.model import PackageHistory
-from bauh_api.util.cache import Cache
 
 
 class HistoryDialog(QDialog):
 
-    def __init__(self, history: PackageHistory, icon_cache: Cache, locale_keys: dict):
+    def __init__(self, history: PackageHistory, icon_cache: MemoryCache, locale_keys: dict):
         super(HistoryDialog, self).__init__()
 
         self.setWindowTitle('{} - {} '.format(locale_keys['popup.history.title'], history.pkg.base_data.name))

bauh/view/qt/info.py

@@ -2,7 +2,7 @@ from PyQt5.QtCore import QSize
 from PyQt5.QtGui import QIcon
 from PyQt5.QtWidgets import QDialog, QVBoxLayout, QGroupBox, \
     QLineEdit, QLabel, QGridLayout, QPushButton, QPlainTextEdit, QToolBar
-from bauh_api.util.cache import Cache
+from bauh_api.abstract.cache import MemoryCache
 
 from bauh.util import util
 
@@ -11,7 +11,7 @@ IGNORED_ATTRS = {'name', '__app__'}
 
 class InfoDialog(QDialog):
 
-    def __init__(self, app: dict, icon_cache: Cache, locale_keys: dict, screen_size: QSize()):
+    def __init__(self, app: dict, icon_cache: MemoryCache, locale_keys: dict, screen_size: QSize()):
         super(InfoDialog, self).__init__()
         self.setWindowTitle(app['__app__'].model.base_data.name)
         self.screen_size = screen_size

bauh/view/qt/thread.py

@@ -5,12 +5,12 @@ from typing import List, Type, Set
 
 import requests
 from PyQt5.QtCore import QThread, pyqtSignal
+from bauh_api.abstract.cache import MemoryCache
 from bauh_api.abstract.controller import SoftwareManager
 from bauh_api.abstract.handler import ProcessWatcher
 from bauh_api.abstract.model import PackageStatus, SoftwarePackage
 from bauh_api.abstract.view import InputViewComponent, MessageType
 from bauh_api.exception import NoInternetException
-from bauh_api.util.cache import Cache
 
 from bauh.view.qt.view_model import PackageView
 
@@ -127,7 +127,7 @@ class RefreshApps(AsyncAction):
 
 class UninstallApp(AsyncAction):
 
-    def __init__(self, manager: SoftwareManager, icon_cache: Cache, app: PackageView = None):
+    def __init__(self, manager: SoftwareManager, icon_cache: MemoryCache, app: PackageView = None):
         super(UninstallApp, self).__init__()
         self.app = app
         self.manager = manager
@@ -223,7 +223,7 @@ class SearchApps(AsyncAction):
 
 class InstallApp(AsyncAction):
 
-    def __init__(self, manager: SoftwareManager, disk_cache: bool, icon_cache: Cache, locale_keys: dict, pkg: PackageView = None):
+    def __init__(self, manager: SoftwareManager, disk_cache: bool, icon_cache: MemoryCache, locale_keys: dict, pkg: PackageView = None):
         super(InstallApp, self).__init__()
         self.pkg = pkg
         self.manager = manager

bauh/view/qt/window.py

@@ -6,10 +6,10 @@ from PyQt5.QtCore import QEvent, Qt, QSize, pyqtSignal
 from PyQt5.QtGui import QIcon, QWindowStateChangeEvent, QPixmap
 from PyQt5.QtWidgets import QWidget, QVBoxLayout, QApplication, QCheckBox, QHeaderView, QToolButton, QToolBar, \
     QLabel, QPlainTextEdit, QLineEdit, QProgressBar, QPushButton, QComboBox
+from bauh_api.abstract.cache import MemoryCache
 from bauh_api.abstract.controller import SoftwareManager
 from bauh_api.abstract.model import SoftwarePackage
 from bauh_api.abstract.view import MessageType
-from bauh_api.util.cache import Cache
 
 from bauh.core import resource
 from bauh.util import util
@@ -38,7 +38,7 @@ class ManageWindow(QWidget):
     def _toolbar_button_style(self, bg: str):
         return 'QPushButton { color: white; font-weight: bold; background: ' + bg + '}'
 
-    def __init__(self, locale_keys: dict, icon_cache: Cache, manager: SoftwareManager, disk_cache: bool, download_icons: bool, screen_size, suggestions: bool, tray_icon=None):
+    def __init__(self, locale_keys: dict, icon_cache: MemoryCache, manager: SoftwareManager, disk_cache: bool, download_icons: bool, screen_size, suggestions: bool, tray_icon=None):
         super(ManageWindow, self).__init__()
         self.locale_keys = locale_keys
         self.manager = manager