arch: automatically optimizing arch packages compilation | aria2c stdout sleep

CHANGELOG.md

@@ -31,7 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - small UI improvements
 - installation logs are saved at **/tmp/bauh/logs/install**
 - Environment variable / parameter BAUH_UPDATE_NOTIFICATION renamed to BAUH_SYSTEM_NOTIFICATIONS and now works for any system notification
-- Environment variable / parameter BAUH_DOWNLOAD_MULTITHREAD (--download-mthread): if source files should be downloaded using multi-threads.
+- Environment variable / parameter BAUH_DOWNLOAD_MULTITHREAD (--download-mthread): if source files should be downloaded using multi-threads (not supported by all **gems**).
 
 ### UI Changes
 - **Upgrade selected** and **Refresh** buttons now have text labels and new colors
@@ -49,7 +49,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### AUR support (**arch gem**):
 - Search, install, uninstall, downgrade, retrieve history and launch packages
-- Faster source files download when (**axel**) [https://github.com/axel-download-accelerator/axel] is installed on your system
+- Faster source files download when (**aria2**) [https://github.com/aria2/aria2] is installed on your system ( see **README.md** for more information )
+- Automatically improves package compilations (see *README.md** for more information )
 
 ### Code
 - Code was internally modularized as: **api** (conceptual classes used to create custom software managers), **gems** (software managers), **commons** (common classes shared between the **view** and **gems**), **view** (UI code)

README.md

@@ -87,6 +87,22 @@ The application settings are stored in **/home/$USER/.config/bauh/config.json**
 - If you don't mind app suggestions, disable it (**bauh --sugs=0**)
 - Let the disk cache always enabled so **bauh** does not need to dynamically retrieve some data every time you launch it.
 
+### Flatpak support ( flatpak gem )
+- The user is able to search, install, uninstall, downgrade and retrieve the applications history
+
+### Snap support ( snap gem )
+- The user is able to search, install, uninstall and downgrade applications
+
+### AUR support ( arch gem )
+- The user is able to search, install, uninstall, downgrade and retrieve the packages history
+- It handles conflicts, and missing / optional packages installations
+- Automatically makes a simple package compilation improvement: if **MAKEFLAGS** is not set in **/etc/makepkg.conf** and **/home/$USER/makepkg.conf** does not exist,
+then a copy of **/etc/makepkg.conf** will be generated at **/home/$USER/makepkg.conf** defining MAKEFLAGS to work with
+the number of your machine processors multiplied by 1.5 rounded up ( this feature can be disabled through the environment variable **BAUH_ARCH_OPTIMIZE=0** )
+- If (**aria2**) [https://github.com/aria2/aria2] is installed on your system and multi-threaded downloads are enabled ( see**BAUH_DOWNLOAD_MULTITHREAD** ), the source packages
+will be downloaded faster.
+
+
 ### Logs
 - Installation logs are saved at **/tmp/bauh/logs/install**
 

bauh/api/abstract/download.py

@@ -19,3 +19,10 @@ class FileDownloader(ABC):
     @abstractmethod
     def is_multithreaded(self) -> bool:
         pass
+
+    @abstractmethod
+    def get_default_client_name(self) -> str:
+        """
+        :return: retrieve current downloader client name
+        """
+        pass

bauh/app_args.py

@@ -36,7 +36,7 @@ def read() -> Namespace:
     parser.add_argument('-md', '--max-displayed', action="store", default=os.getenv('BAUH_MAX_DISPLAYED', 50), type=int, help='Maximum number of displayed packages in the management panel table. Default: %(default)s')
     parser.add_argument('--logs', action="store", default=int(os.getenv('BAUH_LOGS', 0)), choices=[0, 1], type=int, help='If the application logs should be displayed. Default: %(default)s')
     parser.add_argument('--show-panel', action="store_true", help='Shows the management panel after the app icon is attached to the tray.')
-    parser.add_argument('-dm', '--download-mthread', action="store", default=os.getenv('BAUH_DOWNLOAD_MULTITHREAD', 1), choices=[0, 1], type=int, help='If installation files should be downloaded using multi-threads (only possible if axel is installed). Default: %(default)s')
+    parser.add_argument('-dmt', '--download-mthread', action="store", default=os.getenv('BAUH_DOWNLOAD_MULTITHREAD', 1), choices=[0, 1], type=int, help='If installation files should be downloaded using multi-threads (only possible if aria2c is installed). Not all gems support this feature. Check README.md. Default: %(default)s')
     return parser.parse_args()
     
     
@@ -71,7 +71,7 @@ def validate(args: Namespace, logger: logging.Logger):
     if args.logs == 1:
         logger.info("Logs are enabled")
 
-    if args.download_mthread == 0:
+    if args.download_mthread == 1:
-        logger.info("Multithreaded downloads disabled")
+        logger.info("Multi-threaded downloads enabled")
 
     return args

bauh/commons/system.py

@@ -1,6 +1,7 @@
 import os
 import subprocess
 import sys
+import time
 from subprocess import PIPE
 from typing import List
 
@@ -41,12 +42,14 @@ class SystemProcess:
     Represents a system process being executed.
     """
 
-    def __init__(self, subproc: subprocess.Popen, success_phrase: str = None, wrong_error_phrase: str = '[sudo] password for', check_error_output: bool = True, skip_stdout: bool = False):
+    def __init__(self, subproc: subprocess.Popen, success_phrase: str = None, wrong_error_phrase: str = '[sudo] password for',
+                 check_error_output: bool = True, skip_stdout: bool = False, output_delay: float = None):
         self.subproc = subproc
         self.success_phrase = success_phrase
         self.wrong_error_phrase = wrong_error_phrase
         self.check_error_output = check_error_output
         self.skip_stdout = skip_stdout
+        self.output_delay = output_delay
 
     def wait(self):
         self.subproc.wait()
@@ -79,6 +82,9 @@ class ProcessHandler:
                     if process.success_phrase and process.success_phrase in line:
                         already_succeeded = True
 
+                if not already_succeeded and process.output_delay:
+                    time.sleep(process.output_delay)
+
             if already_succeeded:
                 return True
 
@@ -96,6 +102,9 @@ class ProcessHandler:
                     elif process.skip_stdout and process.success_phrase and process.success_phrase in line:
                         already_succeeded = True
 
+                if not already_succeeded and process.output_delay:
+                    time.sleep(process.output_delay)
+
         if already_succeeded:
             return True
 

bauh/gems/arch/controller.py

@@ -18,7 +18,7 @@ from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, c
 from bauh.gems.arch.aur import AURClient
 from bauh.gems.arch.mapper import ArchDataMapper
 from bauh.gems.arch.model import ArchPackage
-from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater
+from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer
 
 URL_GIT = 'https://aur.archlinux.org/{}.git'
 URL_PKG_DOWNLOAD = 'https://aur.archlinux.org/cgit/aur.git/snapshot/{}.tar.gz'
@@ -43,6 +43,7 @@ class ArchManager(SoftwareManager):
         self.names_index = {}
         self.aur_index_updater = AURIndexUpdater(context, self)
         self.dcache_updater = ArchDiskCacheUpdater(context.logger, context.disk_cache)
+        self.comp_optimizer = ArchCompilationOptimizer(context.logger)
         self.logger = context.logger
         self.enabled = True
         self.arch_distro = self.context.linux_distro[0].lower() == 'arch'
@@ -62,6 +63,8 @@ class ArchManager(SoftwareManager):
         Thread(target=self.mapper.fill_package_build, args=(app,)).start()
 
     def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1) -> SearchResult:
+        self.comp_optimizer.join()
+
         downgrade_enabled = git.is_enabled()
         res = SearchResult([], [], 0)
 
@@ -362,28 +365,31 @@ class ArchManager(SoftwareManager):
                             if RE_PRE_DOWNLOADABLE_FILES.findall(f):
                                 pre_download_files.append(f)
 
-            for f in pre_download_files:
+            if pre_download_files:
-                fdata = f.split('::')
+                downloader = self.context.file_downloader.get_default_client_name()
 
-                args = {'watcher': watcher, 'cwd': project_dir}
+                for f in pre_download_files:
-                if len(fdata) > 1:
+                    fdata = f.split('::')
-                    args.update({'file_url': fdata[1], 'output_path': fdata[0]})
-                else:
-                    args.update({'file_url': fdata[0], 'output_path': None})
 
-                file_size = self.context.http_client.get_content_length(args['file_url'])
+                    args = {'watcher': watcher, 'cwd': project_dir}
-                file_size = int(file_size) / (1024 ** 2) if file_size else None
+                    if len(fdata) > 1:
+                        args.update({'file_url': fdata[1], 'output_path': fdata[0]})
+                    else:
+                        args.update({'file_url': fdata[0], 'output_path': None})
 
-                watcher.change_substatus(self.i18n['downloading'] + ' ' + bold(args['file_url'].split('/')[-1]) + ' ( {0:.2f} Mb )'.format(file_size) if file_size else '')
+                    file_size = self.context.http_client.get_content_length(args['file_url'])
-                if not self.context.file_downloader.download(**args):
+                    file_size = int(file_size) / (1024 ** 2) if file_size else None
-                    watcher.print('Could not download source file {}'.format(args['file_url']))
+
-                    return False
+                    watcher.change_substatus(bold('[{}] ').format(downloader) + self.i18n['downloading'] + ' ' + bold(args['file_url'].split('/')[-1]) + ' ( {0:.2f} Mb )'.format(file_size) if file_size else '')
+                    if not self.context.file_downloader.download(**args):
+                        watcher.print('Could not download source file {}'.format(args['file_url']))
+                        return False
 
         return True
 
     def _make_pkg(self, pkgname: str, root_password: str, handler: ProcessHandler, build_dir: str, project_dir: str, dependency: bool, skip_optdeps: bool = False, change_progress: bool = True) -> bool:
-        if not self._pre_download_source(pkgname, project_dir, handler.watcher):
+
-            return False
+        self._pre_download_source(pkgname, project_dir, handler.watcher)
 
         self._update_progress(handler.watcher, 50, change_progress)
         if not self._install_missings_deps_and_keys(pkgname, root_password, handler, project_dir):
@@ -391,7 +397,7 @@ class ArchManager(SoftwareManager):
 
         # building main package
         handler.watcher.change_substatus(self.i18n['arch.building.package'].format(bold(pkgname)))
-        pkgbuilt = handler.handle(SystemProcess(new_subprocess(['makepkg', '-ALcsmf'], cwd=project_dir, lang=None), check_error_output=False))
+        pkgbuilt = handler.handle(SystemProcess(new_subprocess(['makepkg', '-ALcsmf'], cwd=project_dir, lang=None), check_error_output=False, skip_stdout=True))
         self._update_progress(handler.watcher, 65, change_progress)
 
         if pkgbuilt:
@@ -642,6 +648,7 @@ class ArchManager(SoftwareManager):
 
     def prepare(self):
         self.dcache_updater.start()
+        self.comp_optimizer.start()
         self.aur_index_updater.start()
 
     def list_updates(self) -> List[PackageUpdate]:

bauh/gems/arch/worker.py

@@ -1,17 +1,24 @@
 import logging
 import os
+import re
 import time
+from math import ceil
 from multiprocessing import Process
 from threading import Thread
 
 from bauh.api.abstract.context import ApplicationContext
 from bauh.api.abstract.controller import SoftwareManager
-
+from bauh.api.constants import HOME_PATH
 from bauh.gems.arch import pacman, disk
 
 URL_INDEX = 'https://aur.archlinux.org/packages.gz'
 URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg={}'
 
+GLOBAL_MAKEPKG = '/etc/makepkg.conf'
+USER_MAKEPKG = '{}/makepkg.conf'.format(HOME_PATH)
+
+RE_MAKE_FLAGS = re.compile(r'#?\s*MAKEFLAGS\s*=\s*.+\s*')
+
 
 class AURIndexUpdater(Thread):
 
@@ -52,3 +59,50 @@ class ArchDiskCacheUpdater(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else Proce
                 saved = disk.save_several({app for app in installed['not_signed']}, 'aur', overwrite=False)
 
             self.logger.info('Pre-cached data of {} AUR packages to the disk'.format(saved))
+
+
+class ArchCompilationOptimizer(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else Process):
+
+    def __init__(self, logger: logging.Logger):
+        super(ArchCompilationOptimizer, self).__init__(daemon=True)
+        self.logger = logger
+        self.compilation_optimizations = bool(int(os.getenv('BAUH_ARCH_OPTIMIZE', 1)))
+
+    def run(self):
+
+        if not self.compilation_optimizations:
+            self.logger.info("Arch packages compilation optimization is disabled. Aborting...")
+        else:
+            try:
+                ncpus = ceil(os.cpu_count() * 1.5)
+            except:
+                self.logger.error('Could not determine the number of processors. Aborting...')
+                return
+
+            if os.path.exists(GLOBAL_MAKEPKG) and not os.path.exists(USER_MAKEPKG):
+                self.logger.info("Verifying if it is possible to optimize Arch packages compilation")
+
+                with open(GLOBAL_MAKEPKG) as f:
+                    global_makepkg = f.read()
+
+                makeflags = RE_MAKE_FLAGS.findall(global_makepkg)
+                user_makepkg = None
+
+                if makeflags:
+                    not_commented = [f for f in makeflags if not makeflags[0].startswith('#')]
+
+                    if not not_commented:
+                        user_makepkg = RE_MAKE_FLAGS.sub('', global_makepkg)
+                    else:
+                        self.logger.warning("It seems '{}' compilation flags are already customized".format(GLOBAL_MAKEPKG))
+                else:
+                    user_makepkg = global_makepkg
+
+                if user_makepkg:
+                    user_makepkg = '# <generated by bauh>\n' + user_makepkg + '\nMAKEFLAGS="-j{}"'.format(ncpus)
+                    with open(USER_MAKEPKG, 'w+') as f:
+                        f.write(user_makepkg)
+
+                    self.logger.info("A custom optimized 'makepkg.conf' was generated at '{}'".format(HOME_PATH))
+            else:
+                self.logger.warning("A custom 'makepkg.conf' is already defined at '{}'".format(HOME_PATH))

bauh/view/core/controller.py

@@ -82,7 +82,7 @@ class GenericSoftwareManager(SoftwareManager):
             mti = time.time()
             apps_found = man.search(words=word, disk_loader=disk_loader)
             mtf = time.time()
-            self.logger.info(man.__class__.__name__ +  " took {0:.2f} seconds".format(mtf - mti))
+            self.logger.info(man.__class__.__name__ + " took {0:.2f} seconds".format(mtf - mti))
 
             res.installed.extend(apps_found.installed)
             res.new.extend(apps_found.new)

bauh/view/core/downloader.py

@@ -2,7 +2,6 @@ import logging
 import os
 import time
 import traceback
-from multiprocessing import cpu_count
 
 from bauh.api.abstract.download import FileDownloader
 from bauh.api.abstract.handler import ProcessWatcher
@@ -15,29 +14,24 @@ class AdaptableFileDownloader(FileDownloader):
         self.logger = logger
         self.multithread_enabled = multithread_enabled
 
-        if multithread_enabled:
-            try:
-                self.download_threads = int(cpu_count() * 2)
-            except:
-                self.download_threads = 4
-
-            self.download_threads = 16 if self.download_threads > 16 else self.download_threads
-        else:
-            self.download_threads = 1
-
-    def is_axel_available(self) -> bool:
-        return bool(run_cmd('which axel'))
-
     def is_aria2c_available(self) -> bool:
         return bool(run_cmd('which aria2c'))
 
     def _get_aria2c_process(self, url: str, output_path: str, cwd: str) -> SystemProcess:
-        cmd = ['aria2c', '-x{}'.format(self.download_threads), url,
+        cmd = ['aria2c', url,
+               '--no-conf',
+               '--max-connection-per-server=16',
+               '--split=16',
                '--enable-color=false',
                '--stderr=true',
                '--summary-interval=0',
                '--disable-ipv6',
-               '--min-split-size=1M']
+               '--min-split-size=1M',
+               '--allow-overwrite=true',
+               '--continue=true',
+               '--timeout=5',
+               '--max-file-not-found=3',
+               '--remote-time=true']
 
         if output_path:
             output_split = output_path.split('/')
@@ -47,16 +41,8 @@ class AdaptableFileDownloader(FileDownloader):
         return SystemProcess(new_subprocess(cmd=cmd, cwd=cwd),
                              skip_stdout=True,
                              check_error_output=False,
-                             success_phrase='download completed')
+                             success_phrase='download completed',
-
+                             output_delay=0.001)
-    def _get_axel_cmd(self, url: str, output_path: str, cwd: str) -> SystemProcess:
-        cmd = ['axel', '-k', '-n', str(self.download_threads), url]
-
-        if output_path:
-            cmd.append('-o')
-            cmd.append(output_path)
-
-        return SystemProcess(new_subprocess(cmd, cwd=cwd))
 
     def _get_wget_process(self, url: str, output_path: str, cwd: str) -> SystemProcess:
         cmd = ['wget', url]
@@ -87,7 +73,6 @@ class AdaptableFileDownloader(FileDownloader):
             if self.is_multithreaded():
                 ti = time.time()
                 process = self._get_aria2c_process(file_url, output_path, final_cwd)
-                # process = self._get_axel_cmd(file_url, output_path, final_cwd)
             else:
                 ti = time.time()
                 process = self._get_wget_process(file_url, output_path, final_cwd)
@@ -109,3 +94,6 @@ class AdaptableFileDownloader(FileDownloader):
     def is_multithreaded(self) -> bool:
         return self.multithread_enabled and self.is_aria2c_available()
 
+    def get_default_client_name(self) -> str:
+        return 'aria2c' if self. is_multithreaded() else 'wget'
+