spalencsar/bearhub
1
0
Fork 0
mirror of https://github.com/spalencsar/bearhub.git synced 2026-07-07 01:14:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

[aur] caching categories to the disk | not locking the application boot while the AUR categories are being retrieved

Browse Source
This commit is contained in:
Vinicius Moreira
2019-10-22 16:13:32 -03:00
parent 9b17407067
commit 930308834b
7 changed files with 87 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bauh/gems/arch/__init__.py
View File

@@ -1,4 +1,8 @@
import os
from bauh.api.constants import CACHE_PATH
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
BUILD_DIR = '/tmp/bauh/aur'
ARCH_CACHE_PATH = CACHE_PATH + '/arch'

24
bauh/gems/arch/controller.py
View File

@@ -22,7 +22,7 @@ from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, c
from bauh.gems.arch.aur import AURClient
from bauh.gems.arch.mapper import ArchDataMapper
from bauh.gems.arch.model import ArchPackage
from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, CategoriesDownloader
from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, AURCategoriesMapper
URL_GIT = 'https://aur.archlinux.org/{}.git'
URL_PKG_DOWNLOAD = 'https://aur.archlinux.org/cgit/aur.git/snapshot/{}.tar.gz'
@@ -51,11 +51,11 @@ class ArchManager(SoftwareManager):
self.logger = context.logger
self.enabled = True
self.arch_distro = context.distro == 'arch'
self.categories_downloader = CategoriesDownloader(context.http_client, context.logger)
self.categories_mapper = AURCategoriesMapper(context.http_client, context.logger, self, self.context.disk_cache)
self.categories_map = {}
def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader):
app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'])
app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed'], self.categories_map)
app.downgrade_enabled = downgrade_enabled
if app.installed:
@@ -66,7 +66,7 @@ class ArchManager(SoftwareManager):
else:
res.new.append(app)
Thread(target=self.mapper.fill_package_build, args=(app,)).start()
Thread(target=self.mapper.fill_package_build, args=(app,), daemon=True).start()
def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1) -> SearchResult:
self.comp_optimizer.join()
@@ -75,7 +75,7 @@ class ArchManager(SoftwareManager):
res = SearchResult([], [], 0)
installed = {}
read_installed = Thread(target=lambda: installed.update(pacman.list_and_map_installed()))
read_installed = Thread(target=lambda: installed.update(pacman.list_and_map_installed()), daemon=True)
read_installed.start()
api_res = self.aur_client.search(words)
@@ -117,9 +117,8 @@ class ArchManager(SoftwareManager):
if pkgsinfo:
for pkgdata in pkgsinfo:
pkg = self.mapper.map_api_data(pkgdata, not_signed)
pkg = self.mapper.map_api_data(pkgdata, not_signed, self.categories_map)
pkg.downgrade_enabled = downgrade_enabled
pkg.categories = self.categories_map.get(pkg.name)
if disk_loader:
disk_loader.fill(pkg)
@@ -161,6 +160,7 @@ class ArchManager(SoftwareManager):
apps = []
if installed and installed['not_signed']:
self.dcache_updater.join()
self.categories_mapper.join()
self._fill_aur_pkgs(installed['not_signed'], apps, disk_loader, internet_available)
@@ -684,7 +684,7 @@ class ArchManager(SoftwareManager):
return False
def cache_to_disk(self, pkg: ArchPackage, icon_bytes: bytes, only_icon: bool):
pass
pass
def requires_root(self, action: str, pkg: ArchPackage):
return action != 'search'
@@ -693,10 +693,7 @@ class ArchManager(SoftwareManager):
self.dcache_updater.start()
self.comp_optimizer.start()
self.aur_index_updater.start()
categories = self.categories_downloader.get_categories()
if categories:
self.categories_map = categories
self.categories_mapper.start()
def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
installed = self.read_installed(disk_loader=None, internet_available=internet_available).installed
@@ -731,9 +728,10 @@ class ArchManager(SoftwareManager):
api_res = self.aur_client.get_info(sug_names)
if api_res:
self.categories_mapper.join()
for pkg in api_res:
if pkg.get('Name') in sug_names:
res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}), suggestions.ALL.get(pkg['Name'])))
res.append(PackageSuggestion(self.mapper.map_api_data(pkg, {}, self.categories_map), suggestions.ALL.get(pkg['Name'])))
return res

6
bauh/gems/arch/mapper.py
View File

@@ -110,14 +110,18 @@ class ArchDataMapper:
if res and res.status_code == 200 and res.text:
pkg.pkgbuild = res.text
def map_api_data(self, apidata: dict, installed: dict) -> ArchPackage:
def map_api_data(self, apidata: dict, installed: dict, categories: dict) -> ArchPackage:
data = installed.get(apidata.get('Name'))
app = ArchPackage(name=apidata.get('Name'), installed=bool(data), mirror='aur')
app.status = PackageStatus.LOADING_DATA
if categories:
app.categories = categories.get(app.name)
if data:
app.version = data.get('version')
app.description = data.get('description')
self.fill_api_data(app, apidata, fill_version=not data)
return app

6
bauh/gems/arch/model.py
View File

@@ -2,10 +2,8 @@ import datetime
from typing import List
from bauh.api.abstract.model import SoftwarePackage
from bauh.api.constants import CACHE_PATH
from bauh.commons import resource
from bauh.gems.arch import ROOT_DIR
from bauh.gems.arch import ROOT_DIR, ARCH_CACHE_PATH
CACHED_ATTRS = {'command', 'icon_path', 'mirror', 'maintainer', 'desktop_entry'}
@@ -35,7 +33,7 @@ class ArchPackage(SoftwarePackage):
@staticmethod
def disk_cache_path(pkgname: str, mirror: str):
return CACHE_PATH + '/arch/installed/' + ('aur' if mirror == 'aur' else 'mirror') + '/' + pkgname
return ARCH_CACHE_PATH + '/installed/' + ('aur' if mirror == 'aur' else 'mirror') + '/' + pkgname
def get_pkg_build_url(self):
if self.package_base:

73
bauh/gems/arch/worker.py
View File

@@ -5,6 +5,7 @@ import time
import traceback
from math import ceil
from multiprocessing import Process
from pathlib import Path
from threading import Thread
from typing import Dict, List
@@ -12,9 +13,9 @@ import requests
from bauh.api.abstract.context import ApplicationContext
from bauh.api.abstract.controller import SoftwareManager
from bauh.api.constants import HOME_PATH
from bauh.api.constants import HOME_PATH, CACHE_PATH
from bauh.api.http import HttpClient
from bauh.gems.arch import pacman, disk
from bauh.gems.arch import pacman, disk, ARCH_CACHE_PATH
URL_INDEX = 'https://aur.archlinux.org/packages.gz'
URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg={}'
@@ -139,13 +140,51 @@ class ArchCompilationOptimizer(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else P
self.logger.warning("A custom 'makepkg.conf' is already defined at '{}'".format(HOME_PATH))
class CategoriesDownloader:
class AURCategoriesMapper(Thread):
URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt'
CATEGORIES_CACHE_DIR = ARCH_CACHE_PATH + '/categories'
CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt'
def __init__(self, http_client: HttpClient, logger: logging.Logger):
def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool):
super(AURCategoriesMapper, self).__init__(daemon=True)
self.http_client = http_client
self.logger = logger
self.manager = manager
self.disk_cache = disk_cache
def _read_categories_from_disk(self) -> dict:
if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH):
self.logger.info("Reading cached categories from the disk")
with open(self.CATEGORIES_FILE_PATH) as f:
categories = f.read()
return self._map_categories(categories)
return {}
def _map_categories(self, categories: str) -> dict:
categories_map = {}
for l in categories.split('\n'):
if l:
data = l.split('=')
categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
return categories_map
def _cache_categories_to_disk(self, categories: str):
self.logger.info('Caching AUR categories to the disk')
try:
Path(self.CATEGORIES_CACHE_DIR).mkdir(parents=True, exist_ok=True)
with open(self.CATEGORIES_FILE_PATH, 'w+') as f:
f.write(categories)
self.logger.info("AUR categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
except:
self.logger.error("Could not cache AUR categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
traceback.print_exc()
def get_categories(self) -> Dict[str, List[str]]:
self.logger.info('Downloading AUR category definitions from {}'.format(self.URL_CATEGORIES_FILE))
@@ -155,21 +194,29 @@ class CategoriesDownloader:
if res:
try:
categories_map = {}
for l in res.text.split('\n'):
if l:
data = l.split('=')
categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
categories_map = self._map_categories(res.text)
self.logger.info('Loaded categories for {} AUR packages'.format(len(categories_map)))
if self.disk_cache and categories_map:
t = Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True)
t.start()
return categories_map
except:
self.logger.error("Could not parse AUR categories definitions")
traceback.print_exc()
return {}
else:
self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE))
return {}
except requests.exceptions.ConnectionError:
self.logger.warning('The internet connection seems to be off.')
return {}
return self._read_categories_from_disk()
def run(self):
categories = self.get_categories()
if categories:
self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__))
self.manager.categories_map = categories
self.logger.info('Finished')