[aur] AUR normalized index cached to disk instead of memory

CHANGELOG.md

@@ -14,6 +14,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Improvements
 - AppImage:
     - cleaning the downloaded database files when **--reset** is passed as parameter
+- AUR:
+    - The AUR indexer daemon is not running every 20 minutes anymore. It will only run during the boot, and will generate the optimized index
+    at **/tmp/bauh/arch/aur.txt**. This new behavior does not harm the current experience, and reduces memory usage. More information about this behavior in [README](https://github.com/vinifmor/bauh/blob/master/README.md).
+
+### Fixes
+- AUR:
+    - an exception happens when retrieving matches from the cached AUR index
 
 
 ## [0.7.4] 2019-12-09

README.md

@@ -128,7 +128,7 @@ will be pre-downloaded faster ( it does **NOT** modify your **pacman** settings
 
     Obs: this feature can be disabled through the environment variable **BAUH_ARCH_OPTIMIZE=0**
     ( For more information about these optimizations, have a look at [Makepkg](https://wiki.archlinux.org/index.php/Makepkg) )
-- Arch package memory-indexer running every 20 minutes. This memory index is used when AUR Api cannot handle the amount of results found for a given search. It can be disabled via the environment variable **BAUH_ARCH_AUR_INDEX_UPDATER=0**.
+- During bauh initialization the a full AUR normalized index is saved at /tmp/bauh/arch/aur.txt, and it will only be used if the AUR Api cannot handle the number of matches for a given query.
 - If some of your installed packages are not categorized, send an e-mail to **bauh4linux@gmail.com** informing their names and categories in the following format: ```name=category1[,category2,category3,...]```
 - Transitive dependencies checking can be disabled through the environment variable **BAUH_ARCH_CHECK_SUBDEPS=0**. The dependency checking process will be faster, but the application will ask for a confirmation every time a not installed dependency is detected.
 

bauh/gems/arch/__init__.py

@@ -10,6 +10,7 @@ CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt'
 URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt'
 CONFIG_DIR = '{}/.config/bauh/arch'.format(HOME_PATH)
 CUSTOM_MAKEPKG_PATH = '{}/makepkg.conf'.format(CONFIG_DIR)
+AUR_INDEX_FILE = '{}/aur.txt'.format(BUILD_DIR)
 
 
 def should_optimize_compilation() -> bool:

bauh/gems/arch/aur.py

@@ -1,10 +1,12 @@
+import logging
+import os
 import re
 from typing import Set, List
 
 from bauh.api.http import HttpClient
 import urllib.parse
 
-from bauh.gems.arch import pacman
+from bauh.gems.arch import pacman, AUR_INDEX_FILE
 from bauh.gems.arch.exceptions import PackageNotFoundException
 
 URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&'
@@ -22,9 +24,9 @@ def map_pkgbuild(pkgbuild: str) -> dict:
 
 class AURClient:
 
-    def __init__(self, http_client: HttpClient):
+    def __init__(self, http_client: HttpClient, logger: logging.Logger):
         self.http_client = http_client
-        self.names_index = set()
+        self.logger = logger
 
     def search(self, words: str) -> dict:
         return self.http_client.get_json(URL_SEARCH + words)
@@ -64,3 +66,17 @@ class AURClient:
 
     def _map_names_as_queries(self, names) -> str:
         return '&'.join(['arg[{}]={}'.format(i, urllib.parse.quote(n)) for i, n in enumerate(names)])
+
+    def read_local_index(self) -> dict:
+        self.logger.info('Checking if the AUR index file exists')
+        if os.path.exists(AUR_INDEX_FILE):
+            self.logger.info('Reading AUR index file from {}'.format(AUR_INDEX_FILE))
+            index = {}
+            with open(AUR_INDEX_FILE) as f:
+                for l in f.readlines():
+                    if l:
+                        lsplit = l.split('=')
+                        index[lsplit[0]] = lsplit[1].strip()
+            self.logger.info("AUR index file read")
+            return index
+        self.logger.warning('The AUR index file was not found')

bauh/gems/arch/controller.py

@@ -52,9 +52,7 @@ class ArchManager(SoftwareManager):
 
         self.mapper = ArchDataMapper(http_client=context.http_client)
         self.i18n = context.i18n
-        self.aur_client = AURClient(context.http_client)
+        self.aur_client = AURClient(context.http_client, context.logger)
-        self.names_index = {}
-        self.aur_index_updater = AURIndexUpdater(context, self)
         self.dcache_updater = ArchDiskCacheUpdater(context.logger, context.disk_cache)
         self.comp_optimizer = ArchCompilationOptimizer(context.logger)
         self.logger = context.logger
@@ -103,10 +101,11 @@ class ArchManager(SoftwareManager):
                 self._upgrade_search_result(pkgdata, installed, downgrade_enabled, res, disk_loader)
 
         else:  # if there are no results from the API (it could be because there were too many), tries the names index:
-            if self.names_index:
+            aur_index = self.aur_client.read_local_index()
-
+            if aur_index:
+                self.logger.info("Querying through the local AUR index")
                 to_query = set()
-                for norm_name, real_name in self.names_index.items():
+                for norm_name, real_name in aur_index.items():
                     if words in norm_name:
                         to_query.add(real_name)
 
@@ -119,7 +118,7 @@ class ArchManager(SoftwareManager):
                     read_installed.join()
 
                     for pkgdata in pkgsinfo:
-                        self._upgrade_search_result(pkgdata, installed, res)
+                        self._upgrade_search_result(pkgdata, installed, downgrade_enabled, res, disk_loader)
 
         res.total = len(res.installed) + len(res.new)
         return res
@@ -790,8 +789,8 @@ class ArchManager(SoftwareManager):
     def prepare(self):
         self.dcache_updater.start()
         self.comp_optimizer.start()
-        self.aur_index_updater.start()
         self.categories_mapper.start()
+        AURIndexUpdater(self.context).start()
 
     def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
         installed = self.read_installed(disk_loader=None, internet_available=internet_available).installed

bauh/gems/arch/worker.py

@@ -1,8 +1,6 @@
 import logging
 import os
 import re
-import time
-from math import ceil
 from multiprocessing import Process
 from pathlib import Path
 from threading import Thread
@@ -10,8 +8,8 @@ from threading import Thread
 import requests
 
 from bauh.api.abstract.context import ApplicationContext
-from bauh.api.abstract.controller import SoftwareManager
+from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_PATH, CONFIG_DIR, should_optimize_compilation, BUILD_DIR, \
-from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_PATH, CONFIG_DIR, should_optimize_compilation
+    AUR_INDEX_FILE
 
 URL_INDEX = 'https://aur.archlinux.org/packages.gz'
 URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg={}'
@@ -20,36 +18,37 @@ GLOBAL_MAKEPKG = '/etc/makepkg.conf'
 
 RE_MAKE_FLAGS = re.compile(r'#?\s*MAKEFLAGS\s*=\s*.+\s*')
 RE_COMPRESS_XZ = re.compile(r'#?\s*COMPRESSXZ\s*=\s*.+')
+RE_CLEAR_REPLACE = re.compile(r'[\-_.]')
 
 
 class AURIndexUpdater(Thread):
 
-    def __init__(self, context: ApplicationContext, man: SoftwareManager):
+    def __init__(self, context: ApplicationContext):
         super(AURIndexUpdater, self).__init__(daemon=True)
         self.http_client = context.http_client
         self.logger = context.logger
-        self.man = man
-        self.enabled = bool(int(os.getenv('BAUH_ARCH_AUR_INDEX_UPDATER', 1)))
 
     def run(self):
-        if self.enabled:
+        self.logger.info('Pre-indexing AUR packages')
-            while True:
-                self.logger.info('Pre-indexing AUR packages in memory')
         try:
             res = self.http_client.get(URL_INDEX)
 
             if res and res.text:
-                        self.man.names_index = {n.replace('-', '').replace('_', '').replace('.', ''): n for n in res.text.split('\n') if n and not n.startswith('#')}
+                indexed = 0
-                        self.logger.info('Pre-indexed {} AUR package names in memory'.format(len(self.man.names_index)))
+                Path(BUILD_DIR).mkdir(parents=True, exist_ok=True)
+
+                with open(AUR_INDEX_FILE, 'w+') as f:
+                    for n in res.text.split('\n'):
+                        if n and not n.startswith('#'):
+                            f.write('{}={}\n'.format(RE_CLEAR_REPLACE.sub('', n), n))
+                            indexed += 1
+
+                self.logger.info('Pre-indexed {} AUR package names at {}'.format(indexed, AUR_INDEX_FILE))
             else:
                 self.logger.warning('No data returned from: {}'.format(URL_INDEX))
         except requests.exceptions.ConnectionError:
             self.logger.warning('No internet connection: could not pre-index packages')
 
-                time.sleep(60 * 20)  # updates every 20 minutes
-        else:
-            self.logger.info("AUR index updater disabled")
-
 
 class ArchDiskCacheUpdater(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else Process):