From bd175947237af936998395dfe96223d28fe00463 Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Fri, 30 Aug 2019 17:26:35 -0300 Subject: [PATCH] refactoring memory and disk cache abstractions --- bauh/app.py | 26 +++++---- bauh/app_args.py | 2 +- bauh/core/controller.py | 11 ++-- bauh/core/extensions.py | 14 ++--- bauh/util/cache.py | 108 +++++++++++++++++++++++++++++++++++++ bauh/util/disk.py | 72 +++++++++++++++++++++++++ bauh/util/memory.py | 23 -------- bauh/view/qt/apps_table.py | 4 +- bauh/view/qt/history.py | 4 +- bauh/view/qt/info.py | 4 +- bauh/view/qt/thread.py | 6 +-- bauh/view/qt/window.py | 4 +- 12 files changed, 215 insertions(+), 63 deletions(-) create mode 100644 bauh/util/cache.py create mode 100644 bauh/util/disk.py delete mode 100644 bauh/util/memory.py diff --git a/bauh/app.py b/bauh/app.py index 4f324210..2456da61 100755 --- a/bauh/app.py +++ b/bauh/app.py @@ -3,32 +3,36 @@ import sys from PyQt5.QtGui import QIcon from PyQt5.QtWidgets import QApplication from bauh_api.abstract.controller import ApplicationContext -from bauh_api.util.cache import Cache -from bauh_api.util.disk import DiskCacheLoaderFactory from bauh_api.util.http import HttpClient from bauh import __version__, __app_name__, app_args, ROOT_DIR from bauh.core import resource, extensions from bauh.core.controller import GenericSoftwareManager from bauh.util import util -from bauh.util.memory import CacheCleaner +from bauh.util.cache import DefaultMemoryCacheFactory, CacheCleaner +from bauh.util.disk import DefaultDiskCacheLoaderFactory from bauh.view.qt.systray import TrayIcon from bauh.view.qt.window import ManageWindow args = app_args.read() i18n = util.get_locale_keys(args.locale) -caches, cache_map = [], {} -context = ApplicationContext(i18n=i18n, http_client=HttpClient(), args=args, app_root_dir=ROOT_DIR) -managers = extensions.load_managers(caches=caches, cache_map=cache_map, context=context) +cache_cleaner = CacheCleaner() +cache_factory = DefaultMemoryCacheFactory(expiration_time=args.cache_exp, cleaner=cache_cleaner) +icon_cache = cache_factory.new(args.icon_exp) -icon_cache = Cache(expiration_time=args.icon_exp) -caches.append(icon_cache) +context = ApplicationContext(i18n=i18n, + http_client=HttpClient(), + args=args, + app_root_dir=ROOT_DIR, + cache_factory=cache_factory, + disk_loader_factory=DefaultDiskCacheLoaderFactory(disk_cache_enabled=args.disk_cache)) -disk_loader_factory = DiskCacheLoaderFactory(disk_cache=args.disk_cache, cache_map=cache_map) +managers = extensions.load_managers(context=context) -manager = GenericSoftwareManager(managers, disk_loader_factory=disk_loader_factory, context=context) + +manager = GenericSoftwareManager(managers, context=context) manager.prepare() app = QApplication(sys.argv) @@ -56,5 +60,5 @@ else: manage_window.refresh_apps() manage_window.show() -CacheCleaner(caches).start() +cache_cleaner.start() sys.exit(app.exec_()) diff --git a/bauh/app_args.py b/bauh/app_args.py index 0290fa05..aaa8f38f 100644 --- a/bauh/app_args.py +++ b/bauh/app_args.py @@ -20,7 +20,7 @@ def read() -> Namespace: parser.add_argument('-v', '--version', action='version', version='%(prog)s {}'.format(__version__)) parser.add_argument('-e', '--cache-exp', action="store", default=int(os.getenv('BAUH_CACHE_EXPIRATION', 60 * 60)), type=int, - help='cached API data expiration time in SECONDS. Default: %(default)s') + help='default memory caches expiration time in SECONDS. Default: %(default)s') parser.add_argument('-ie', '--icon-exp', action="store", default=int(os.getenv('BAUH_ICON_EXPIRATION', 60 * 5)), type=int, help='cached icons expiration time in SECONDS. Default: %(default)s') parser.add_argument('-l', '--locale', action="store", default=os.getenv('BAUH_LOCALE', 'en'), diff --git a/bauh/core/controller.py b/bauh/core/controller.py index c3e0fcca..afffc9cb 100755 --- a/bauh/core/controller.py +++ b/bauh/core/controller.py @@ -2,24 +2,23 @@ from threading import Thread from typing import List, Set, Type from bauh_api.abstract.controller import SoftwareManager, SearchResult, ApplicationContext +from bauh_api.abstract.disk import DiskCacheLoader from bauh_api.abstract.handler import ProcessWatcher from bauh_api.abstract.model import SoftwarePackage, PackageUpdate, PackageHistory -from bauh_api.util.disk import DiskCacheLoader -from bauh_api.util.disk import DiskCacheLoaderFactory SUGGESTIONS_LIMIT = 6 class GenericSoftwareManager(SoftwareManager): - def __init__(self, managers: List[SoftwareManager], context: ApplicationContext, disk_loader_factory: DiskCacheLoaderFactory): - super(GenericSoftwareManager, self).__init__(context=context, app_cache=None) + def __init__(self, managers: List[SoftwareManager], context: ApplicationContext): + super(GenericSoftwareManager, self).__init__(context=context) self.managers = managers self.map = {t: m for m in self.managers for t in m.get_managed_types()} - self.disk_loader_factory = disk_loader_factory self._enabled_map = {} if context.args.check_packaging_once else None self.thread_prepare = None self.i18n = context.i18n + self.disk_loader_factory = context.disk_loader_factory def _sort(self, apps: List[SoftwarePackage], word: str) -> List[SoftwarePackage]: @@ -188,7 +187,7 @@ class GenericSoftwareManager(SoftwareManager): return man if man and self._is_enabled(man) else None def cache_to_disk(self, app: SoftwarePackage, icon_bytes: bytes, only_icon: bool): - if self.disk_loader_factory.disk_cache and app.supports_disk_cache(): + if self.context.args.disk_cache and app.supports_disk_cache(): man = self._get_manager_for(app) if man: diff --git a/bauh/core/extensions.py b/bauh/core/extensions.py index 4336c5e6..0450f52a 100644 --- a/bauh/core/extensions.py +++ b/bauh/core/extensions.py @@ -1,10 +1,9 @@ import inspect import os import pkgutil -from typing import List, Dict +from typing import List from bauh_api.abstract.controller import SoftwareManager, ApplicationContext -from bauh_api.util.cache import Cache from bauh import __app_name__ from bauh.util import util @@ -24,7 +23,7 @@ def find_manager(member): return manager_found -def load_managers(caches: List[Cache], cache_map: Dict[type, Cache], context: ApplicationContext) -> List[SoftwareManager]: +def load_managers(context: ApplicationContext) -> List[SoftwareManager]: managers = [] for m in pkgutil.iter_modules(): @@ -40,13 +39,6 @@ def load_managers(caches: List[Cache], cache_map: Dict[type, Cache], context: Ap if os.path.exists(locale_path): context.i18n.update(util.get_locale_keys(context.args.locale, locale_path)) - app_cache = Cache(expiration_time=context.args.cache_exp) - man = manager_class(context=context, app_cache=app_cache) - - for t in man.get_managed_types(): - cache_map[t] = app_cache - - caches.append(app_cache) - managers.append(man) + managers.append(manager_class(context=context)) return managers diff --git a/bauh/util/cache.py b/bauh/util/cache.py new file mode 100644 index 00000000..9e822257 --- /dev/null +++ b/bauh/util/cache.py @@ -0,0 +1,108 @@ +import datetime +import time +from threading import Lock, Thread +from typing import List + +from bauh_api.abstract.cache import MemoryCache, MemoryCacheFactory + + +class DefaultMemoryCache(MemoryCache): + """ + A synchronized cache implementation + """ + + def __init__(self, expiration_time: int): + super(DefaultMemoryCache, self).__init__() + self.expiration_time = expiration_time + self._cache = {} + self.lock = Lock() + + def is_enabled(self): + return self.expiration_time < 0 or self.expiration_time > 0 + + def add(self, key: str, val: object): + if key and self.is_enabled(): + self.lock.acquire() + self._add(key, val) + self.lock.release() + + def _add(self, key: str, val: object): + if key: + self._cache[key] = {'val': val, 'expires_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.expiration_time) if self.expiration_time > 0 else None} + + def add_non_existing(self, key: str, val: object): + if key and self. is_enabled(): + self.lock.acquire() + cur_val = self.get(key) + + if cur_val is None: + self._add(key, val) + + self.lock.release() + + def get(self, key: str): + if key and self.is_enabled(): + val = self._cache.get(key) + + if val: + expiration = val.get('expires_at') + + if expiration and expiration <= datetime.datetime.utcnow(): + self.lock.acquire() + del self._cache[key] + self.lock.release() + return None + + return val['val'] + + def delete(self, key): + if key and self.is_enabled(): + if key in self._cache: + self.lock.acquire() + del self._cache[key] + self.lock.release() + + def keys(self): + return set(self._cache.keys()) if self.is_enabled() else set() + + def clean_expired(self): + if self.is_enabled(): + for key in self.keys(): + self.get(key) + + +class CacheCleaner(Thread): + + def __init__(self, check_interval: int = 15): + super(CacheCleaner, self).__init__(daemon=True) + self.caches = [] + self.check_interval = check_interval + + def register(self, cache: MemoryCache): + if cache.is_enabled(): + self.caches.append(cache) + + def run(self): + if self.caches: + while True: + for cache in self.caches: + cache.clean_expired() + + time.sleep(self.check_interval) + + +class DefaultMemoryCacheFactory(MemoryCacheFactory): + + def __init__(self, expiration_time: int, cleaner: CacheCleaner): + """ + :param expiration_time: default expiration time for all instantiated caches + :param cleaner + """ + super(DefaultMemoryCacheFactory, self).__init__() + self.expiration_time = expiration_time + self.cleaner = cleaner + + def new(self, expiration: int = None) -> MemoryCache: + instance = DefaultMemoryCache(expiration if expiration is not None else self.expiration_time) + self.cleaner.register(instance) + return instance diff --git a/bauh/util/disk.py b/bauh/util/disk.py new file mode 100644 index 00000000..5e26f6eb --- /dev/null +++ b/bauh/util/disk.py @@ -0,0 +1,72 @@ +import json +import os +from threading import Thread, Lock +from typing import Type, Dict + +from bauh_api.abstract.cache import MemoryCache +from bauh_api.abstract.disk import DiskCacheLoader, DiskCacheLoaderFactory +from bauh_api.abstract.model import SoftwarePackage + + +class AsyncDiskCacheLoader(Thread, DiskCacheLoader): + + def __init__(self, enabled: bool, cache_map: Dict[Type[SoftwarePackage], MemoryCache]): + super(AsyncDiskCacheLoader, self).__init__(daemon=True) + self.apps = [] + self.stop = False + self.lock = Lock() + self.cache_map = cache_map + self.enabled = enabled + + def fill(self, pkg: SoftwarePackage): + """ + Adds a package which data must be read from the disk to a queue. + :param pkg: + :return: + """ + if self.enabled: + if pkg and pkg.supports_disk_cache(): + self.lock.acquire() + self.apps.append(pkg) + self.lock.release() + + def run(self): + if self.enabled: + while True: + if self.apps: + self.lock.acquire() + app = self.apps[0] + del self.apps[0] + self.lock.release() + self._fill_cached_data(app) + elif self.stop: + break + + def _fill_cached_data(self, pkg: SoftwarePackage): + if self.enabled: + if os.path.exists(pkg.get_disk_data_path()): + with open(pkg.get_disk_data_path()) as f: + cached_data = json.loads(f.read()) + pkg.fill_cached_data(cached_data) + cache = self.cache_map.get(pkg.__class__)\ + + if cache: + cache.add_non_existing(pkg.base_data.id, cached_data) + + +class DefaultDiskCacheLoaderFactory(DiskCacheLoaderFactory): + + def __init__(self, disk_cache_enabled: bool): + super(DefaultDiskCacheLoaderFactory, self).__init__() + self.disk_cache_enabled = disk_cache_enabled + self.cache_map = {} + + def map(self, pkg_type: Type[SoftwarePackage], cache: MemoryCache): + if pkg_type: + if pkg_type in self.cache_map: + raise Exception('{} is already mapped') + + self.cache_map[pkg_type] = cache + + def new(self) -> AsyncDiskCacheLoader: + return AsyncDiskCacheLoader(enabled=self.disk_cache_enabled, cache_map=self.cache_map) diff --git a/bauh/util/memory.py b/bauh/util/memory.py deleted file mode 100644 index 215abb0e..00000000 --- a/bauh/util/memory.py +++ /dev/null @@ -1,23 +0,0 @@ -import time -from threading import Thread -from typing import List - -from bauh_api.util.cache import Cache - - -class CacheCleaner(Thread): - - def __init__(self, caches: List[Cache], check_interval: int = 15): - super(CacheCleaner, self).__init__(daemon=True) - self.caches = [c for c in caches if c.is_enabled()] - self.check_interval = check_interval - - def run(self): - - if self.caches: - while True: - for cache in self.caches: - cache.clean_expired() - - time.sleep(self.check_interval) - diff --git a/bauh/view/qt/apps_table.py b/bauh/view/qt/apps_table.py index 1cde0056..a2679e72 100644 --- a/bauh/view/qt/apps_table.py +++ b/bauh/view/qt/apps_table.py @@ -7,8 +7,8 @@ from PyQt5.QtGui import QPixmap, QIcon, QCursor from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest from PyQt5.QtWidgets import QTableWidget, QTableView, QMenu, QAction, QTableWidgetItem, QToolButton, QWidget, \ QHeaderView, QLabel, QHBoxLayout, QPushButton, QToolBar +from bauh_api.abstract.cache import MemoryCache from bauh_api.abstract.model import PackageStatus -from bauh_api.util.cache import Cache from bauh.core import resource from bauh.util import util @@ -75,7 +75,7 @@ class UpdateToggleButton(QWidget): class AppsTable(QTableWidget): - def __init__(self, parent: QWidget, icon_cache: Cache, disk_cache: bool, download_icons: bool): + def __init__(self, parent: QWidget, icon_cache: MemoryCache, disk_cache: bool, download_icons: bool): super(AppsTable, self).__init__() self.setParent(parent) self.window = parent diff --git a/bauh/view/qt/history.py b/bauh/view/qt/history.py index e5e01116..de27c6a6 100644 --- a/bauh/view/qt/history.py +++ b/bauh/view/qt/history.py @@ -4,13 +4,13 @@ from functools import reduce from PyQt5.QtCore import Qt from PyQt5.QtGui import QColor from PyQt5.QtWidgets import QDialog, QVBoxLayout, QTableWidget, QTableWidgetItem, QHeaderView +from bauh_api.abstract.cache import MemoryCache from bauh_api.abstract.model import PackageHistory -from bauh_api.util.cache import Cache class HistoryDialog(QDialog): - def __init__(self, history: PackageHistory, icon_cache: Cache, locale_keys: dict): + def __init__(self, history: PackageHistory, icon_cache: MemoryCache, locale_keys: dict): super(HistoryDialog, self).__init__() self.setWindowTitle('{} - {} '.format(locale_keys['popup.history.title'], history.pkg.base_data.name)) diff --git a/bauh/view/qt/info.py b/bauh/view/qt/info.py index 38a68f84..1f4339dc 100644 --- a/bauh/view/qt/info.py +++ b/bauh/view/qt/info.py @@ -2,7 +2,7 @@ from PyQt5.QtCore import QSize from PyQt5.QtGui import QIcon from PyQt5.QtWidgets import QDialog, QVBoxLayout, QGroupBox, \ QLineEdit, QLabel, QGridLayout, QPushButton, QPlainTextEdit, QToolBar -from bauh_api.util.cache import Cache +from bauh_api.abstract.cache import MemoryCache from bauh.util import util @@ -11,7 +11,7 @@ IGNORED_ATTRS = {'name', '__app__'} class InfoDialog(QDialog): - def __init__(self, app: dict, icon_cache: Cache, locale_keys: dict, screen_size: QSize()): + def __init__(self, app: dict, icon_cache: MemoryCache, locale_keys: dict, screen_size: QSize()): super(InfoDialog, self).__init__() self.setWindowTitle(app['__app__'].model.base_data.name) self.screen_size = screen_size diff --git a/bauh/view/qt/thread.py b/bauh/view/qt/thread.py index 8519144a..cf40858b 100644 --- a/bauh/view/qt/thread.py +++ b/bauh/view/qt/thread.py @@ -5,12 +5,12 @@ from typing import List, Type, Set import requests from PyQt5.QtCore import QThread, pyqtSignal +from bauh_api.abstract.cache import MemoryCache from bauh_api.abstract.controller import SoftwareManager from bauh_api.abstract.handler import ProcessWatcher from bauh_api.abstract.model import PackageStatus, SoftwarePackage from bauh_api.abstract.view import InputViewComponent, MessageType from bauh_api.exception import NoInternetException -from bauh_api.util.cache import Cache from bauh.view.qt.view_model import PackageView @@ -127,7 +127,7 @@ class RefreshApps(AsyncAction): class UninstallApp(AsyncAction): - def __init__(self, manager: SoftwareManager, icon_cache: Cache, app: PackageView = None): + def __init__(self, manager: SoftwareManager, icon_cache: MemoryCache, app: PackageView = None): super(UninstallApp, self).__init__() self.app = app self.manager = manager @@ -223,7 +223,7 @@ class SearchApps(AsyncAction): class InstallApp(AsyncAction): - def __init__(self, manager: SoftwareManager, disk_cache: bool, icon_cache: Cache, locale_keys: dict, pkg: PackageView = None): + def __init__(self, manager: SoftwareManager, disk_cache: bool, icon_cache: MemoryCache, locale_keys: dict, pkg: PackageView = None): super(InstallApp, self).__init__() self.pkg = pkg self.manager = manager diff --git a/bauh/view/qt/window.py b/bauh/view/qt/window.py index 1126f63f..d5b5f5d3 100755 --- a/bauh/view/qt/window.py +++ b/bauh/view/qt/window.py @@ -6,10 +6,10 @@ from PyQt5.QtCore import QEvent, Qt, QSize, pyqtSignal from PyQt5.QtGui import QIcon, QWindowStateChangeEvent, QPixmap from PyQt5.QtWidgets import QWidget, QVBoxLayout, QApplication, QCheckBox, QHeaderView, QToolButton, QToolBar, \ QLabel, QPlainTextEdit, QLineEdit, QProgressBar, QPushButton, QComboBox +from bauh_api.abstract.cache import MemoryCache from bauh_api.abstract.controller import SoftwareManager from bauh_api.abstract.model import SoftwarePackage from bauh_api.abstract.view import MessageType -from bauh_api.util.cache import Cache from bauh.core import resource from bauh.util import util @@ -38,7 +38,7 @@ class ManageWindow(QWidget): def _toolbar_button_style(self, bg: str): return 'QPushButton { color: white; font-weight: bold; background: ' + bg + '}' - def __init__(self, locale_keys: dict, icon_cache: Cache, manager: SoftwareManager, disk_cache: bool, download_icons: bool, screen_size, suggestions: bool, tray_icon=None): + def __init__(self, locale_keys: dict, icon_cache: MemoryCache, manager: SoftwareManager, disk_cache: bool, download_icons: bool, screen_size, suggestions: bool, tray_icon=None): super(ManageWindow, self).__init__() self.locale_keys = locale_keys self.manager = manager