improvement -> all suported types now display a 'Checking configuration file' task during the initialization process

2026-07-10 06:34:15 +02:00 · 2020-12-30 13:35:15 -03:00
parent 01bd532465
commit c91e982243
69 changed files with 1059 additions and 561 deletions
--- a/bauh/gems/arch/worker.py
+++ b/bauh/gems/arch/worker.py
@@ -13,11 +13,12 @@ import requests

 from bauh.api.abstract.context import ApplicationContext
 from bauh.api.abstract.handler import TaskManager
+from bauh.commons import system
+from bauh.commons.boot import CreateConfigFile
 from bauh.commons.html import bold
-from bauh.commons.system import run_cmd, new_root_subprocess, ProcessHandler
-from bauh.commons.util import datetime_as_milis
+from bauh.commons.system import new_root_subprocess, ProcessHandler
 from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_FILE, CONFIG_DIR, AUR_INDEX_FILE, get_icon_path, database, \
-    mirrors, ARCH_CACHE_PATH, BUILD_DIR, AUR_INDEX_TS_FILE
+    mirrors, ARCH_CACHE_PATH, AUR_INDEX_TS_FILE, aur
 from bauh.gems.arch.aur import URL_INDEX
 from bauh.view.util.translation import I18n

@@ -31,18 +32,20 @@ RE_CLEAR_REPLACE = re.compile(r'[\-_.]')

 class AURIndexUpdater(Thread):

-    def __init__(self, context: ApplicationContext, taskman: TaskManager):
+    def __init__(self, context: ApplicationContext, taskman: TaskManager, create_config: Optional[CreateConfigFile] = None, arch_config: Optional[dict] = None):
        super(AURIndexUpdater, self).__init__(daemon=True)
        self.http_client = context.http_client
        self.i18n = context.i18n
        self.logger = context.logger
        self.taskman = taskman
        self.task_id = 'index_aur'
+        self.create_config = create_config
+        self.config = arch_config
+        self.taskman.register_task(self.task_id, self.i18n['arch.task.aur.index.status'], get_icon_path())

-    @staticmethod
-    def should_update(arch_config: dict) -> bool:
+    def should_update(self) -> bool:
        try:
-            exp_minutes = int(arch_config['aur_idx_exp'])
+            exp_minutes = int(self.config['aur_idx_exp'])
        except:
            traceback.print_exc()
            return True
@@ -66,11 +69,9 @@ class AURIndexUpdater(Thread):
            traceback.print_exc()
            return True

-    def run(self):
-        ti = time.time()
+    def update_index(self):
        self.logger.info('Indexing AUR packages')
-        self.taskman.register_task(self.task_id, self.i18n['arch.task.aur.index.status'], get_icon_path())
-        self.taskman.update_progress(self.task_id, 1, self.i18n['arch.task.aur.index.substatus.download'])
+        self.taskman.update_progress(self.task_id, 5, self.i18n['arch.task.aur.index.substatus.download'])
        try:
            index_ts = datetime.utcnow().timestamp()
            res = self.http_client.get(URL_INDEX)
@@ -115,6 +116,22 @@ class AURIndexUpdater(Thread):
            self.logger.warning('No internet connection: could not pre-index packages')
            self.taskman.update_progress(self.task_id, 100, self.i18n['arch.task.aur.index.substatus.error.download'])

+    def run(self):
+        ti = time.time()
+
+        if self.create_config:
+            self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(self.create_config.task_name))
+            self.create_config.join()
+            self.config = self.create_config.config
+
+        self.taskman.update_progress(self.task_id, 1, self.i18n['arch.task.aur.index.substatus.checking'])
+
+        if self.should_update():
+            self.update_index()
+        else:
+            self.logger.info("AUR index is up to date. Aborting...")
+            self.taskman.update_progress(self.task_id, 100, None)
+
        tf = time.time()
        self.taskman.finish_task(self.task_id)
        self.logger.info("Finished. Took {0:.2f} seconds".format(tf - ti))
@@ -122,48 +139,56 @@ class AURIndexUpdater(Thread):

 class ArchDiskCacheUpdater(Thread):

-    def __init__(self, task_man: TaskManager, arch_config: dict, i18n: I18n, logger: logging.Logger, controller: "ArchManager", internet_available: bool,
-                 aur_supported: bool, aur_indexer: Thread):
+    def __init__(self, taskman: TaskManager, i18n: I18n, logger: logging.Logger,
+                 controller: "ArchManager", internet_available: bool, aur_indexer: Thread,
+                 create_config: CreateConfigFile):
        super(ArchDiskCacheUpdater, self).__init__(daemon=True)
        self.logger = logger
-        self.task_man = task_man
+        self.taskman = taskman
        self.task_id = 'arch_cache_up'
        self.i18n = i18n
        self.indexed = 0
        self.indexed_template = self.i18n['arch.task.disk_cache.indexed'] + ': {}/ {}'
        self.to_index = 0
        self.progress = 0  # progress is defined by the number of packages prepared and indexed
-        self.repositories = arch_config['repositories']
-        self.aur = aur_supported
        self.controller = controller
        self.internet_available = internet_available
        self.installed_hash_path = '{}/installed.sha1'.format(ARCH_CACHE_PATH)
        self.installed_cache_dir = '{}/installed'.format(ARCH_CACHE_PATH)
        self.aur_indexer = aur_indexer
+        self.create_config = create_config
+        self.taskman.register_task(self.task_id, self.i18n['arch.task.disk_cache'], get_icon_path())

    def update_indexed(self, pkgname: str):
        self.indexed += 1
        sub = self.indexed_template.format(self.indexed, self.to_index)
        progress = self.progress + (self.indexed / self.to_index) * 50
-        self.task_man.update_progress(self.task_id, progress, sub)
+        self.taskman.update_progress(self.task_id, progress, sub)

    def _update_progress(self, progress: float, msg: str):
        self.progress = progress
-        self.task_man.update_progress(self.task_id, self.progress, msg)
+        self.taskman.update_progress(self.task_id, self.progress, msg)

    def _notify_reading_files(self):
        self._update_progress(50, self.i18n['arch.task.disk_cache.indexing'])

    def run(self):
-        if not any([self.aur, self.repositories]):
+        ti = time.time()
+        self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(self.create_config.task_name))
+        self.create_config.join()
+
+        config = self.create_config.config
+        aur_supported, repositories = aur.is_supported(config), config['repositories']
+
+        self.taskman.update_progress(self.task_id, 1, None)
+        if not any([aur_supported, repositories]):
+            self.taskman.update_progress(self.task_id, 100, self.i18n['arch.task.disabled'])
+            self.taskman.finish_task(self.task_id)
            return

-        ti = time.time()
-        self.task_man.register_task(self.task_id, self.i18n['arch.task.disk_cache'], get_icon_path())
-
        self.logger.info("Checking already cached package data")
-
        self._update_progress(1, self.i18n['arch.task.disk_cache.checking'])
+
        cache_dirs = [fpath for fpath in glob.glob('{}/*'.format(self.installed_cache_dir)) if os.path.isdir(fpath)]

        not_cached_names = None
@@ -177,8 +202,8 @@ class ArchDiskCacheUpdater(Thread):
            self._update_progress(20, self.i18n['arch.task.disk_cache.checking'])

            if not not_cached_names:
-                self.task_man.update_progress(self.task_id, 100, '')
-                self.task_man.finish_task(self.task_id)
+                self.taskman.update_progress(self.task_id, 100, '')
+                self.taskman.finish_task(self.task_id)
                tf = time.time()
                time_msg = '{0:.2f} seconds'.format(tf - ti)
                self.logger.info('Finished: no package data to cache ({})'.format(time_msg))
@@ -186,8 +211,8 @@ class ArchDiskCacheUpdater(Thread):

        self.logger.info('Pre-caching installed Arch packages data to disk')

-        if self.aur and self.aur_indexer:
-            self.task_man.update_progress(self.task_id, 20, self.i18n['arch.task.disk_cache.waiting_aur_index'].format(bold(self.i18n['arch.task.aur.index.status'])))
+        if aur_supported and self.aur_indexer:
+            self.taskman.update_progress(self.task_id, 20, self.i18n['arch.task.disk_cache.waiting_aur_index'].format(bold(self.i18n['arch.task.aur.index.status'])))
            self.aur_indexer.join()

        self._update_progress(21, self.i18n['arch.task.disk_cache.checking'])
@@ -199,7 +224,7 @@ class ArchDiskCacheUpdater(Thread):

        saved = 0

-        pkgs = {p.name: p for p in installed if ((self.aur and p.repository == 'aur') or (self.repositories and p.repository != 'aur')) and not os.path.exists(p.get_disk_cache_path())}
+        pkgs = {p.name: p for p in installed if ((aur_supported and p.repository == 'aur') or (repositories and p.repository != 'aur')) and not os.path.exists(p.get_disk_cache_path())}
        self.to_index = len(pkgs)

        # overwrite == True because the verification already happened
@@ -207,8 +232,8 @@ class ArchDiskCacheUpdater(Thread):
        saved += disk.write_several(pkgs=pkgs,
                                    after_desktop_files=self._notify_reading_files,
                                    after_written=self.update_indexed, overwrite=True)
-        self.task_man.update_progress(self.task_id, 100, None)
-        self.task_man.finish_task(self.task_id)
+        self.taskman.update_progress(self.task_id, 100, None)
+        self.taskman.finish_task(self.task_id)

        tf = time.time()
        time_msg = '{0:.2f} seconds'.format(tf - ti)
@@ -217,7 +242,7 @@ class ArchDiskCacheUpdater(Thread):

 class ArchCompilationOptimizer(Thread):

-    def __init__(self, arch_config: dict, i18n: I18n, logger: logging.Logger, taskman: Optional[TaskManager] = None):
+    def __init__(self, i18n: I18n, logger: logging.Logger, taskman: TaskManager, create_config: Optional[CreateConfigFile] = None):
        super(ArchCompilationOptimizer, self).__init__(daemon=True)
        self.logger = logger
        self.i18n = i18n
@@ -227,17 +252,12 @@ class ArchCompilationOptimizer(Thread):
        self.re_ccache = re.compile(r'!?ccache')
        self.taskman = taskman
        self.task_id = 'arch_make_optm'
-        self.optimizations = bool(arch_config['optimize'])
+        self.create_config = create_config
+        self.taskman.register_task(self.task_id, self.i18n['arch.task.optimizing'].format(bold('makepkg.conf')), get_icon_path())

    def _is_ccache_installed(self) -> bool:
-        return bool(run_cmd('which ccache', print_error=False))
-
-    def _update_progress(self, progress: float, substatus: str = None):
-        if self.taskman:
-            self.taskman.update_progress(self.task_id, progress, substatus)
-
-            if progress == 100:
-                self.taskman.finish_task(self.task_id)
+        code, _ = system.execute(cmd='which ccache', output=False)
+        return code == 0

    def optimize(self):
        ti = time.time()
@@ -271,7 +291,7 @@ class ArchCompilationOptimizer(Thread):
                else:
                    optimizations.append('MAKEFLAGS="-j$(nproc)"')

-            self._update_progress(20)
+            self.taskman.update_progress(self.task_id, 20, None)

            compress_xz = self.re_compress_xz.findall(custom_makepkg or global_makepkg)

@@ -286,7 +306,7 @@ class ArchCompilationOptimizer(Thread):
            else:
                optimizations.append('COMPRESSXZ=(xz -c -z - --threads=0)')

-            self._update_progress(40)
+            self.taskman.update_progress(self.task_id, 40, None)

            compress_zst = self.re_compress_zst.findall(custom_makepkg or global_makepkg)

@@ -301,7 +321,7 @@ class ArchCompilationOptimizer(Thread):
            else:
                optimizations.append('COMPRESSZST=(zstd -c -z -q - --threads=0)')

-            self._update_progress(60)
+            self.taskman.update_progress(self.task_id, 60, None)

            build_envs = self.re_build_env.findall(custom_makepkg or global_makepkg)

@@ -331,7 +351,7 @@ class ArchCompilationOptimizer(Thread):
                    self.logger.info('Adding a BUILDENV declaration')
                    optimizations.append('BUILDENV=(ccache)')

-            self._update_progress(80)
+            self.taskman.update_progress(self.task_id, 80, None)

            if custom_makepkg and optimizations:
                generated_by = '# <generated by bauh>\n'
@@ -348,43 +368,91 @@ class ArchCompilationOptimizer(Thread):
                    self.logger.info("Removing old optimized 'makepkg.conf' at '{}'".format(CUSTOM_MAKEPKG_FILE))
                    os.remove(CUSTOM_MAKEPKG_FILE)

+            self.taskman.update_progress(self.task_id, 100, None)
            tf = time.time()
-            self._update_progress(100)
-            self.logger.info("Optimizations took {0:.2f} seconds".format(tf - ti))
-            self.logger.info('Finished')
+            self.logger.info('Finished. {0:.2f} seconds'.format(tf - ti))

    def run(self):
-        if not self.optimizations:
-            self.logger.info("Arch packages compilation optimizations are disabled")
+        ti = time.time()
+        if self.create_config:
+            self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name)))
+            self.create_config.join()

-            if os.path.exists(CUSTOM_MAKEPKG_FILE):
-                self.logger.info("Removing custom 'makepkg.conf' -> '{}'".format(CUSTOM_MAKEPKG_FILE))
-                os.remove(CUSTOM_MAKEPKG_FILE)
+            self.taskman.update_progress(self.task_id, 1, None)

-            self.logger.info('Finished')
-        else:
-            if self.taskman:
-                self.taskman.register_task(self.task_id, self.i18n['arch.task.optimizing'].format(bold('makepkg.conf')), get_icon_path())
+            if self.create_config.config['optimize'] and aur.is_supported(self.create_config.config):
+                try:
+                    self.optimize()
+                except:
+                    self.logger.error("Unexpected exception")
+                    traceback.print_exc()
+                    self.taskman.update_progress(self.task_id, 100, None)
+            else:
+                self.logger.info("AUR packages compilation optimizations are disabled")

-            self.optimize()
+                if os.path.exists(CUSTOM_MAKEPKG_FILE):
+                    try:
+                        self.logger.info("Removing custom 'makepkg.conf' -> '{}'".format(CUSTOM_MAKEPKG_FILE))
+                        os.remove(CUSTOM_MAKEPKG_FILE)
+                    except:
+                        self.logger.error("Unexpected exception")
+                        traceback.print_exc()
+
+                self.taskman.update_progress(self.task_id, 100, self.i18n['arch.task.disabled'])
+
+        tf = time.time()
+        self.taskman.finish_task(self.task_id)
+        self.logger.info('Finished. Took {0:.2f} seconds'.format(tf - ti))


 class RefreshMirrors(Thread):

-    def __init__(self, taskman: TaskManager, root_password: str, i18n: I18n, sort_limit: int, logger: logging.Logger):
+    def __init__(self, taskman: TaskManager, root_password: str, i18n: I18n, logger: logging.Logger,
+                 create_config: CreateConfigFile):
        super(RefreshMirrors, self).__init__(daemon=True)
        self.taskman = taskman
        self.i18n = i18n
        self.logger = logger
        self.root_password = root_password
        self.task_id = "arch_mirrors"
-        self.sort_limit = sort_limit
+        self.create_config = create_config
+        self.refreshed = False
+        self.task_name = self.i18n['arch.task.mirrors']
+        self.taskman.register_task(self.task_id, self.task_name, get_icon_path())

    def _notify_output(self, output: str):
        self.taskman.update_output(self.task_id, output)

+    @staticmethod
+    def is_enabled(arch_config: dict, aur_supported: bool) -> bool:
+        return (arch_config['repositories'] or aur_supported) \
+               and arch_config['refresh_mirrors_startup'] and pacman.is_mirrors_available()
+
+    @classmethod
+    def should_synchronize(cls, arch_config: dict, aur_supported: bool, logger: logging.Logger) -> bool:
+        return cls.is_enabled(arch_config, aur_supported) and mirrors.should_sync(logger)
+
    def run(self):
-        self.taskman.register_task(self.task_id, self.i18n['arch.task.mirrors'], get_icon_path())
+        ti = time.time()
+        self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name)))
+        self.create_config.join()
+
+        arch_config = self.create_config.config
+        aur_supported = aur.is_supported(arch_config)
+
+        self.taskman.update_progress(self.task_id, 1, self.i18n['arch.task.checking_settings'])
+
+        if not self.is_enabled(arch_config, aur_supported):
+            self.taskman.update_progress(self.task_id, 100, self.i18n['arch.task.disabled'])
+            self.taskman.finish_task(self.task_id)
+            return
+
+        if not mirrors.should_sync(self.logger):
+            self.taskman.update_progress(self.task_id, 100, self.i18n['arch.task.mirrors.cached'])
+            self.taskman.finish_task(self.task_id)
+            return
+
+        sort_limit = arch_config['mirrors_sort_limit']
        self.logger.info("Refreshing mirrors")

        handler = ProcessHandler()
@@ -394,15 +462,16 @@ class RefreshMirrors(Thread):

            if success:

-                if self.sort_limit is not None and self.sort_limit >= 0:
+                if sort_limit is not None and sort_limit >= 0:
                    self.taskman.update_progress(self.task_id, 50, self.i18n['arch.custom_action.refresh_mirrors.status.updating'])
                    try:
-                        handler.handle_simple(pacman.sort_fastest_mirrors(self.root_password, self.sort_limit), output_handler=self._notify_output)
+                        handler.handle_simple(pacman.sort_fastest_mirrors(self.root_password, sort_limit), output_handler=self._notify_output)
                    except:
                        self.logger.error("Could not sort mirrors by speed")
                        traceback.print_exc()

                mirrors.register_sync(self.logger)
+                self.refreshed = True
            else:
                self.logger.error("It was not possible to refresh mirrors")
        except:
@@ -411,12 +480,14 @@ class RefreshMirrors(Thread):

        self.taskman.update_progress(self.task_id, 100, None)
        self.taskman.finish_task(self.task_id)
-        self.logger.info("Finished")
+        tf = time.time()
+        self.logger.info("Finished. Took {0:.2f} seconds".format(tf - ti))


 class SyncDatabases(Thread):

-    def __init__(self, taskman: TaskManager, root_password: str, i18n: I18n, logger: logging.Logger, refresh_mirrors: RefreshMirrors = None):
+    def __init__(self, taskman: TaskManager, root_password: str, i18n: I18n, logger: logging.Logger,
+                 refresh_mirrors: RefreshMirrors, create_config: CreateConfigFile):
        super(SyncDatabases, self).__init__(daemon=True)
        self.task_man = taskman
        self.i18n = i18n
@@ -425,15 +496,46 @@ class SyncDatabases(Thread):
        self.root_password = root_password
        self.refresh_mirrors = refresh_mirrors
        self.logger = logger
+        self.create_config = create_config
+        self.task_name = self.i18n['arch.sync_databases.substatus']
+        self.taskman.register_task(self.task_id, self.task_name, get_icon_path())
+        self.synchronized = False
+
+    @staticmethod
+    def is_enabled(arch_config: dict, aur_supported: bool) -> bool:
+        return arch_config['sync_databases_startup'] and (aur_supported or arch_config['repositories'])
+
+    @classmethod
+    def should_sync(cls, mirrors_refreshed: bool, arch_config: dict, aur_supported: bool, logger: logging.Logger):
+        return mirrors_refreshed or (cls.is_enabled(arch_config, aur_supported) and database.should_sync(arch_config, aur_supported, None, logger))

    def run(self) -> None:
+        self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.create_config.task_name)))
+        self.create_config.join()
+
+        self.taskman.update_progress(self.task_id, 0, self.i18n['task.waiting_task'].format(bold(self.refresh_mirrors.task_name)))
+        self.refresh_mirrors.join()
+
+        self.taskman.update_progress(self.task_id, 1,  self.i18n['arch.task.checking_settings'])
+
+        arch_config = self.create_config.config
+        aur_supported = aur.is_supported(arch_config)
+        if not self.is_enabled(arch_config, aur_supported):
+            self.taskman.update_progress(self.task_id, 100, self.i18n['arch.task.disabled'])
+            self.taskman.finish_task(self.task_id)
+            return
+
+        shoud_sync = self.refresh_mirrors.refreshed or (database.should_sync(arch_config, aur_supported, None, self.logger))
+
+        if not shoud_sync:
+            self.taskman.update_progress(self.task_id, 100, self.i18n['arch.sync_databases.substatus.synchronized'])
+            self.taskman.finish_task(self.task_id)
+            self.synchronized = True
+            return
+
        self.logger.info("Synchronizing databases")
        self.taskman.register_task(self.task_id, self.i18n['arch.sync_databases.substatus'], get_icon_path())

-        if self.refresh_mirrors and self.refresh_mirrors.is_alive():
-            self.taskman.update_progress(self.task_id, 0, self.i18n['arch.task.sync_databases.waiting'].format('"{}"'.format(self.i18n['arch.task.mirrors'])))
-            self.refresh_mirrors.join()
-
        progress = 10
        dbs = pacman.get_databases()
        self.taskman.update_progress(self.task_id, progress, None)
@@ -472,6 +574,7 @@ class SyncDatabases(Thread):

                if p.returncode == 0:
                    database.register_sync(self.logger)
+                    self.synchronized = True
                else:
                    self.logger.error("Could not synchronize database")