diff --git a/CHANGELOG.md b/CHANGELOG.md index 813158ac..dc6c62a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Improvements - AppImage: - cleaning the downloaded database files when **--reset** is passed as parameter +- AUR: + - The AUR indexer daemon is not running every 20 minutes anymore. It will only run during the boot, and will generate the optimized index + at **/tmp/bauh/arch/aur.txt**. This new behavior does not harm the current experience, and reduces memory usage. More information about this behavior in [README](https://github.com/vinifmor/bauh/blob/master/README.md). + +### Fixes +- AUR: + - an exception happens when retrieving matches from the cached AUR index ## [0.7.4] 2019-12-09 diff --git a/README.md b/README.md index 32b62b1a..c4bfd21e 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ will be pre-downloaded faster ( it does **NOT** modify your **pacman** settings Obs: this feature can be disabled through the environment variable **BAUH_ARCH_OPTIMIZE=0** ( For more information about these optimizations, have a look at [Makepkg](https://wiki.archlinux.org/index.php/Makepkg) ) -- Arch package memory-indexer running every 20 minutes. This memory index is used when AUR Api cannot handle the amount of results found for a given search. It can be disabled via the environment variable **BAUH_ARCH_AUR_INDEX_UPDATER=0**. +- During bauh initialization the a full AUR normalized index is saved at /tmp/bauh/arch/aur.txt, and it will only be used if the AUR Api cannot handle the number of matches for a given query. - If some of your installed packages are not categorized, send an e-mail to **bauh4linux@gmail.com** informing their names and categories in the following format: ```name=category1[,category2,category3,...]``` - Transitive dependencies checking can be disabled through the environment variable **BAUH_ARCH_CHECK_SUBDEPS=0**. The dependency checking process will be faster, but the application will ask for a confirmation every time a not installed dependency is detected. diff --git a/bauh/gems/arch/__init__.py b/bauh/gems/arch/__init__.py index 2a027234..e77b4c57 100644 --- a/bauh/gems/arch/__init__.py +++ b/bauh/gems/arch/__init__.py @@ -10,6 +10,7 @@ CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt' URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt' CONFIG_DIR = '{}/.config/bauh/arch'.format(HOME_PATH) CUSTOM_MAKEPKG_PATH = '{}/makepkg.conf'.format(CONFIG_DIR) +AUR_INDEX_FILE = '{}/aur.txt'.format(BUILD_DIR) def should_optimize_compilation() -> bool: diff --git a/bauh/gems/arch/aur.py b/bauh/gems/arch/aur.py index defb1e71..5ed024ca 100644 --- a/bauh/gems/arch/aur.py +++ b/bauh/gems/arch/aur.py @@ -1,10 +1,12 @@ +import logging +import os import re from typing import Set, List from bauh.api.http import HttpClient import urllib.parse -from bauh.gems.arch import pacman +from bauh.gems.arch import pacman, AUR_INDEX_FILE from bauh.gems.arch.exceptions import PackageNotFoundException URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&' @@ -22,9 +24,9 @@ def map_pkgbuild(pkgbuild: str) -> dict: class AURClient: - def __init__(self, http_client: HttpClient): + def __init__(self, http_client: HttpClient, logger: logging.Logger): self.http_client = http_client - self.names_index = set() + self.logger = logger def search(self, words: str) -> dict: return self.http_client.get_json(URL_SEARCH + words) @@ -64,3 +66,17 @@ class AURClient: def _map_names_as_queries(self, names) -> str: return '&'.join(['arg[{}]={}'.format(i, urllib.parse.quote(n)) for i, n in enumerate(names)]) + + def read_local_index(self) -> dict: + self.logger.info('Checking if the AUR index file exists') + if os.path.exists(AUR_INDEX_FILE): + self.logger.info('Reading AUR index file from {}'.format(AUR_INDEX_FILE)) + index = {} + with open(AUR_INDEX_FILE) as f: + for l in f.readlines(): + if l: + lsplit = l.split('=') + index[lsplit[0]] = lsplit[1].strip() + self.logger.info("AUR index file read") + return index + self.logger.warning('The AUR index file was not found') diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 7dfe6d1a..917da21a 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -52,9 +52,7 @@ class ArchManager(SoftwareManager): self.mapper = ArchDataMapper(http_client=context.http_client) self.i18n = context.i18n - self.aur_client = AURClient(context.http_client) - self.names_index = {} - self.aur_index_updater = AURIndexUpdater(context, self) + self.aur_client = AURClient(context.http_client, context.logger) self.dcache_updater = ArchDiskCacheUpdater(context.logger, context.disk_cache) self.comp_optimizer = ArchCompilationOptimizer(context.logger) self.logger = context.logger @@ -103,10 +101,11 @@ class ArchManager(SoftwareManager): self._upgrade_search_result(pkgdata, installed, downgrade_enabled, res, disk_loader) else: # if there are no results from the API (it could be because there were too many), tries the names index: - if self.names_index: - + aur_index = self.aur_client.read_local_index() + if aur_index: + self.logger.info("Querying through the local AUR index") to_query = set() - for norm_name, real_name in self.names_index.items(): + for norm_name, real_name in aur_index.items(): if words in norm_name: to_query.add(real_name) @@ -119,7 +118,7 @@ class ArchManager(SoftwareManager): read_installed.join() for pkgdata in pkgsinfo: - self._upgrade_search_result(pkgdata, installed, res) + self._upgrade_search_result(pkgdata, installed, downgrade_enabled, res, disk_loader) res.total = len(res.installed) + len(res.new) return res @@ -790,8 +789,8 @@ class ArchManager(SoftwareManager): def prepare(self): self.dcache_updater.start() self.comp_optimizer.start() - self.aur_index_updater.start() self.categories_mapper.start() + AURIndexUpdater(self.context).start() def list_updates(self, internet_available: bool) -> List[PackageUpdate]: installed = self.read_installed(disk_loader=None, internet_available=internet_available).installed diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index b135e627..792aaf73 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -1,8 +1,6 @@ import logging import os import re -import time -from math import ceil from multiprocessing import Process from pathlib import Path from threading import Thread @@ -10,8 +8,8 @@ from threading import Thread import requests from bauh.api.abstract.context import ApplicationContext -from bauh.api.abstract.controller import SoftwareManager -from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_PATH, CONFIG_DIR, should_optimize_compilation +from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_PATH, CONFIG_DIR, should_optimize_compilation, BUILD_DIR, \ + AUR_INDEX_FILE URL_INDEX = 'https://aur.archlinux.org/packages.gz' URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg={}' @@ -20,35 +18,36 @@ GLOBAL_MAKEPKG = '/etc/makepkg.conf' RE_MAKE_FLAGS = re.compile(r'#?\s*MAKEFLAGS\s*=\s*.+\s*') RE_COMPRESS_XZ = re.compile(r'#?\s*COMPRESSXZ\s*=\s*.+') +RE_CLEAR_REPLACE = re.compile(r'[\-_.]') class AURIndexUpdater(Thread): - def __init__(self, context: ApplicationContext, man: SoftwareManager): + def __init__(self, context: ApplicationContext): super(AURIndexUpdater, self).__init__(daemon=True) self.http_client = context.http_client self.logger = context.logger - self.man = man - self.enabled = bool(int(os.getenv('BAUH_ARCH_AUR_INDEX_UPDATER', 1))) def run(self): - if self.enabled: - while True: - self.logger.info('Pre-indexing AUR packages in memory') - try: - res = self.http_client.get(URL_INDEX) + self.logger.info('Pre-indexing AUR packages') + try: + res = self.http_client.get(URL_INDEX) - if res and res.text: - self.man.names_index = {n.replace('-', '').replace('_', '').replace('.', ''): n for n in res.text.split('\n') if n and not n.startswith('#')} - self.logger.info('Pre-indexed {} AUR package names in memory'.format(len(self.man.names_index))) - else: - self.logger.warning('No data returned from: {}'.format(URL_INDEX)) - except requests.exceptions.ConnectionError: - self.logger.warning('No internet connection: could not pre-index packages') + if res and res.text: + indexed = 0 + Path(BUILD_DIR).mkdir(parents=True, exist_ok=True) - time.sleep(60 * 20) # updates every 20 minutes - else: - self.logger.info("AUR index updater disabled") + with open(AUR_INDEX_FILE, 'w+') as f: + for n in res.text.split('\n'): + if n and not n.startswith('#'): + f.write('{}={}\n'.format(RE_CLEAR_REPLACE.sub('', n), n)) + indexed += 1 + + self.logger.info('Pre-indexed {} AUR package names at {}'.format(indexed, AUR_INDEX_FILE)) + else: + self.logger.warning('No data returned from: {}'.format(URL_INDEX)) + except requests.exceptions.ConnectionError: + self.logger.warning('No internet connection: could not pre-index packages') class ArchDiskCacheUpdater(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else Process):