spalencsar/bearhub
1
0
Fork 0
mirror of https://github.com/spalencsar/bearhub.git synced 2026-07-07 00:04:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

[snap] caching categories to the disk | not locking the application boot while categories are being retrieved

Browse Source
This commit is contained in:
Vinicius Moreira
2019-10-22 16:36:10 -03:00
parent 930308834b
commit c358d077be
6 changed files with 82 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@@ -17,6 +17,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- showing a "user-friendly" popup when there are integrity issues with the source-files of a building package
- not waiting for the categories file to be retrieved from the cloud during application boot ( reduces boot time )
- caching cloud to categories to the disk so they can be used in scenarios when it is not possible to retrieve them ( e.g: internet is off )
- Snap:
- not waiting for the categories file to be retrieved from the cloud during application boot ( reduces boot time )
- caching cloud to categories to the disk so they can be used in scenarios when it is not possible to retrieve them ( e.g: internet is off )
- minor thread improvements
### Fixes

10
bauh/gems/arch/controller.py
View File

@@ -52,10 +52,10 @@ class ArchManager(SoftwareManager):
self.enabled = True
self.arch_distro = context.distro == 'arch'
self.categories_mapper = AURCategoriesMapper(context.http_client, context.logger, self, self.context.disk_cache)
self.categories_map = {}
self.categories = {}
def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader):
app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories_map)
app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories)
app.downgrade_enabled = downgrade_enabled
if app.installed:
@@ -117,7 +117,7 @@ class ArchManager(SoftwareManager):
if pkgsinfo:
for pkgdata in pkgsinfo:
pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories_map)
pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories)
pkg.downgrade_enabled = downgrade_enabled
if disk_loader:
@@ -136,7 +136,7 @@ class ArchManager(SoftwareManager):
latest_version=data.get('version'), description=data.get('description'),
installed=True, mirror='aur')
pkg.categories = self.categories_map.get(pkg.name)
pkg.categories = self.categories.get(pkg.name)
pkg.downgrade_enabled = downgrade_enabled
if disk_loader:
@@ -731,7 +731,7 @@ class ArchManager(SoftwareManager):
self.categories_mapper.join()
for pkg in api_res:
if pkg.get('Name') in sug_names:
res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}, self.categories_map), suggestions.ALL.get(pkg['Name'])))
res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}, self.categories), suggestions.ALL.get(pkg['Name'])))
return res

8
bauh/gems/arch/worker.py
View File

@@ -198,8 +198,7 @@ class AURCategoriesMapper(Thread):
self.logger.info('Loaded categories for {} AUR packages'.format(len(categories_map)))
if self.disk_cache and categories_map:
t = Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True)
t.start()
Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start()
return categories_map
except:
@@ -218,5 +217,6 @@ class AURCategoriesMapper(Thread):
if categories:
self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__))
self.manager.categories_map = categories
self.logger.info('Finished')
self.manager.categories = categories
self.logger.info('Finished')

4
bauh/gems/snap/__init__.py
View File

@@ -1,2 +1,6 @@
import os
from bauh.api.constants import CACHE_PATH
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
SNAP_CACHE_PATH = CACHE_PATH + '/snap'

11
bauh/gems/snap/controller.py
View File

@@ -19,6 +19,7 @@ from bauh.gems.snap.worker import SnapAsyncDataLoader, CategoriesDownloader
RE_AVAILABLE_CHANNELS = re.compile(re.compile(r'(\w+)\s+(snap install.+)'))
class SnapManager(SoftwareManager):
def __init__(self, context: ApplicationContext):
@@ -31,7 +32,7 @@ class SnapManager(SoftwareManager):
self.logger = context.logger
self.ubuntu_distro = context.distro == 'ubuntu'
self.categories = {}
self.categories_downloader = CategoriesDownloader(self.http_client, self.logger)
self.categories_downloader = CategoriesDownloader(self.http_client, self.logger, self, context.disk_cache)
def map_json(self, app_json: dict, installed: bool, disk_loader: DiskCacheLoader, internet: bool = True) -> SnapApplication:
app = SnapApplication(publisher=app_json.get('publisher'),
@@ -91,6 +92,7 @@ class SnapManager(SoftwareManager):
def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = None) -> SearchResult:
if snap.is_snapd_running():
self.categories_downloader.join()
installed = [self.map_json(app_json, installed=True, disk_loader=disk_loader, internet=internet_available) for app_json in snap.read_installed(self.ubuntu_distro)]
return SearchResult(installed, None, len(installed))
else:
@@ -173,10 +175,7 @@ class SnapManager(SoftwareManager):
return ProcessHandler(watcher).handle(SystemProcess(subproc=snap.refresh_and_stream(pkg.name, root_password)))
def prepare(self):
categories = self.categories_downloader.get_categories()
if categories:
self.categories = categories
self.categories_downloader.start()
def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
pass
@@ -207,6 +206,8 @@ class SnapManager(SoftwareManager):
sugs.sort(key=lambda t: t[1].value, reverse=True)
threads = []
self.categories_downloader.join()
for sug in sugs:
if limit <= 0 or len(res) < limit:

72
bauh/gems/snap/worker.py
View File

@@ -1,6 +1,7 @@
import logging
import os
import traceback
from pathlib import Path
from threading import Thread
from typing import Dict, List
@@ -11,7 +12,7 @@ from bauh.api.abstract.context import ApplicationContext
from bauh.api.abstract.controller import SoftwareManager
from bauh.api.abstract.model import PackageStatus
from bauh.api.http import HttpClient
from bauh.gems.snap import snap
from bauh.gems.snap import snap, SNAP_CACHE_PATH
from bauh.gems.snap.constants import SNAP_API_URL
from bauh.gems.snap.model import SnapApplication
@@ -76,13 +77,51 @@ class SnapAsyncDataLoader(Thread):
self.manager.cache_to_disk(pkg=self.app, icon_bytes=None, only_icon=False)
class CategoriesDownloader:
class CategoriesDownloader(Thread):
URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/categories.txt'
CATEGORIES_FILE_PATH = SNAP_CACHE_PATH + '/categories.txt'
def __init__(self, http_client: HttpClient, logger: logging.Logger):
def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool):
super(CategoriesDownloader, self).__init__(daemon=True)
self.http_client = http_client
self.logger = logger
self.manager = manager
self.disk_cache = disk_cache
def _read_categories_from_disk(self) -> dict:
if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH):
self.logger.info("Reading cached categories from the disk")
with open(self.CATEGORIES_FILE_PATH) as f:
categories = f.read()
return self._map_categories(categories)
return {}
def _map_categories(self, categories: str) -> dict:
categories_map = {}
for l in categories.split('\n'):
if l:
data = l.split('=')
categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
return categories_map
def _cache_categories_to_disk(self, categories: str):
self.logger.info('Caching Snap categories to the disk')
try:
Path(SNAP_CACHE_PATH).mkdir(parents=True, exist_ok=True)
with open(self.CATEGORIES_FILE_PATH, 'w+') as f:
f.write(categories)
self.logger.info("Snap categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
except:
self.logger.error("Could not cache Snap categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
traceback.print_exc()
def get_categories(self) -> Dict[str, List[str]]:
self.logger.info('Downloading Snap category definitions from {}'.format(self.URL_CATEGORIES_FILE))
@@ -92,20 +131,29 @@ class CategoriesDownloader:
if res:
try:
categories_map = {}
for l in res.text.split('\n'):
if l:
data = l.split('=')
categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
categories = self._map_categories(res.text)
self.logger.info('Loaded categories for {} Snap applications'.format(len(categories)))
self.logger.info('Loaded categories for {} Snap applications'.format(len(categories_map)))
return categories_map
if self.disk_cache and categories:
Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start()
return categories
except:
self.logger.error("Could not parse categories definitions")
traceback.print_exc()
return {}
else:
self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE))
except requests.exceptions.ConnectionError:
self.logger.warning('The internet connection seems to be off.')
return {}
return self._read_categories_from_disk()
def run(self):
categories = self.get_categories()
if categories:
self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__))
self.manager.categories = categories
self.logger.info('Finished')