diff --git a/CHANGELOG.md b/CHANGELOG.md index 82afd749..4422cd81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - small UI improvements - installation logs are saved at **/tmp/bauh/logs/install** - Environment variable / parameter BAUH_UPDATE_NOTIFICATION renamed to BAUH_SYSTEM_NOTIFICATIONS and now works for any system notification -- Environment variable / parameter BAUH_DOWNLOAD_MULTITHREAD (--download-mthread): if source files should be downloaded using multi-threads. +- Environment variable / parameter BAUH_DOWNLOAD_MULTITHREAD (--download-mthread): if source files should be downloaded using multi-threads (not supported by all **gems**). ### UI Changes - **Upgrade selected** and **Refresh** buttons now have text labels and new colors @@ -49,7 +49,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### AUR support (**arch gem**): - Search, install, uninstall, downgrade, retrieve history and launch packages -- Faster source files download when (**axel**) [https://github.com/axel-download-accelerator/axel] is installed on your system +- Faster source files download when (**aria2**) [https://github.com/aria2/aria2] is installed on your system ( see **README.md** for more information ) +- Automatically improves package compilations (see *README.md** for more information ) ### Code - Code was internally modularized as: **api** (conceptual classes used to create custom software managers), **gems** (software managers), **commons** (common classes shared between the **view** and **gems**), **view** (UI code) diff --git a/README.md b/README.md index f856e83e..711b1130 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,22 @@ The application settings are stored in **/home/$USER/.config/bauh/config.json** - If you don't mind app suggestions, disable it (**bauh --sugs=0**) - Let the disk cache always enabled so **bauh** does not need to dynamically retrieve some data every time you launch it. +### Flatpak support ( flatpak gem ) +- The user is able to search, install, uninstall, downgrade and retrieve the applications history + +### Snap support ( snap gem ) +- The user is able to search, install, uninstall and downgrade applications + +### AUR support ( arch gem ) +- The user is able to search, install, uninstall, downgrade and retrieve the packages history +- It handles conflicts, and missing / optional packages installations +- Automatically makes a simple package compilation improvement: if **MAKEFLAGS** is not set in **/etc/makepkg.conf** and **/home/$USER/makepkg.conf** does not exist, +then a copy of **/etc/makepkg.conf** will be generated at **/home/$USER/makepkg.conf** defining MAKEFLAGS to work with +the number of your machine processors multiplied by 1.5 rounded up ( this feature can be disabled through the environment variable **BAUH_ARCH_OPTIMIZE=0** ) +- If (**aria2**) [https://github.com/aria2/aria2] is installed on your system and multi-threaded downloads are enabled ( see**BAUH_DOWNLOAD_MULTITHREAD** ), the source packages +will be downloaded faster. + + ### Logs - Installation logs are saved at **/tmp/bauh/logs/install** diff --git a/bauh/api/abstract/download.py b/bauh/api/abstract/download.py index 97833216..8d91e49d 100644 --- a/bauh/api/abstract/download.py +++ b/bauh/api/abstract/download.py @@ -19,3 +19,10 @@ class FileDownloader(ABC): @abstractmethod def is_multithreaded(self) -> bool: pass + + @abstractmethod + def get_default_client_name(self) -> str: + """ + :return: retrieve current downloader client name + """ + pass diff --git a/bauh/app_args.py b/bauh/app_args.py index 1d6cc5eb..5460a7de 100644 --- a/bauh/app_args.py +++ b/bauh/app_args.py @@ -36,7 +36,7 @@ def read() -> Namespace: parser.add_argument('-md', '--max-displayed', action="store", default=os.getenv('BAUH_MAX_DISPLAYED', 50), type=int, help='Maximum number of displayed packages in the management panel table. Default: %(default)s') parser.add_argument('--logs', action="store", default=int(os.getenv('BAUH_LOGS', 0)), choices=[0, 1], type=int, help='If the application logs should be displayed. Default: %(default)s') parser.add_argument('--show-panel', action="store_true", help='Shows the management panel after the app icon is attached to the tray.') - parser.add_argument('-dm', '--download-mthread', action="store", default=os.getenv('BAUH_DOWNLOAD_MULTITHREAD', 1), choices=[0, 1], type=int, help='If installation files should be downloaded using multi-threads (only possible if axel is installed). Default: %(default)s') + parser.add_argument('-dmt', '--download-mthread', action="store", default=os.getenv('BAUH_DOWNLOAD_MULTITHREAD', 1), choices=[0, 1], type=int, help='If installation files should be downloaded using multi-threads (only possible if aria2c is installed). Not all gems support this feature. Check README.md. Default: %(default)s') return parser.parse_args() @@ -71,7 +71,7 @@ def validate(args: Namespace, logger: logging.Logger): if args.logs == 1: logger.info("Logs are enabled") - if args.download_mthread == 0: - logger.info("Multithreaded downloads disabled") + if args.download_mthread == 1: + logger.info("Multi-threaded downloads enabled") return args diff --git a/bauh/commons/system.py b/bauh/commons/system.py index 70ec1171..cddaba99 100644 --- a/bauh/commons/system.py +++ b/bauh/commons/system.py @@ -1,6 +1,7 @@ import os import subprocess import sys +import time from subprocess import PIPE from typing import List @@ -41,12 +42,14 @@ class SystemProcess: Represents a system process being executed. """ - def __init__(self, subproc: subprocess.Popen, success_phrase: str = None, wrong_error_phrase: str = '[sudo] password for', check_error_output: bool = True, skip_stdout: bool = False): + def __init__(self, subproc: subprocess.Popen, success_phrase: str = None, wrong_error_phrase: str = '[sudo] password for', + check_error_output: bool = True, skip_stdout: bool = False, output_delay: float = None): self.subproc = subproc self.success_phrase = success_phrase self.wrong_error_phrase = wrong_error_phrase self.check_error_output = check_error_output self.skip_stdout = skip_stdout + self.output_delay = output_delay def wait(self): self.subproc.wait() @@ -79,6 +82,9 @@ class ProcessHandler: if process.success_phrase and process.success_phrase in line: already_succeeded = True + if not already_succeeded and process.output_delay: + time.sleep(process.output_delay) + if already_succeeded: return True @@ -96,6 +102,9 @@ class ProcessHandler: elif process.skip_stdout and process.success_phrase and process.success_phrase in line: already_succeeded = True + if not already_succeeded and process.output_delay: + time.sleep(process.output_delay) + if already_succeeded: return True diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 10efea31..6c884f26 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -18,7 +18,7 @@ from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, c from bauh.gems.arch.aur import AURClient from bauh.gems.arch.mapper import ArchDataMapper from bauh.gems.arch.model import ArchPackage -from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater +from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer URL_GIT = 'https://aur.archlinux.org/{}.git' URL_PKG_DOWNLOAD = 'https://aur.archlinux.org/cgit/aur.git/snapshot/{}.tar.gz' @@ -43,6 +43,7 @@ class ArchManager(SoftwareManager): self.names_index = {} self.aur_index_updater = AURIndexUpdater(context, self) self.dcache_updater = ArchDiskCacheUpdater(context.logger, context.disk_cache) + self.comp_optimizer = ArchCompilationOptimizer(context.logger) self.logger = context.logger self.enabled = True self.arch_distro = self.context.linux_distro[0].lower() == 'arch' @@ -62,6 +63,8 @@ class ArchManager(SoftwareManager): Thread(target=self.mapper.fill_package_build, args=(app,)).start() def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1) -> SearchResult: + self.comp_optimizer.join() + downgrade_enabled = git.is_enabled() res = SearchResult([], [], 0) @@ -362,28 +365,31 @@ class ArchManager(SoftwareManager): if RE_PRE_DOWNLOADABLE_FILES.findall(f): pre_download_files.append(f) - for f in pre_download_files: - fdata = f.split('::') + if pre_download_files: + downloader = self.context.file_downloader.get_default_client_name() - args = {'watcher': watcher, 'cwd': project_dir} - if len(fdata) > 1: - args.update({'file_url': fdata[1], 'output_path': fdata[0]}) - else: - args.update({'file_url': fdata[0], 'output_path': None}) + for f in pre_download_files: + fdata = f.split('::') - file_size = self.context.http_client.get_content_length(args['file_url']) - file_size = int(file_size) / (1024 ** 2) if file_size else None + args = {'watcher': watcher, 'cwd': project_dir} + if len(fdata) > 1: + args.update({'file_url': fdata[1], 'output_path': fdata[0]}) + else: + args.update({'file_url': fdata[0], 'output_path': None}) - watcher.change_substatus(self.i18n['downloading'] + ' ' + bold(args['file_url'].split('/')[-1]) + ' ( {0:.2f} Mb )'.format(file_size) if file_size else '') - if not self.context.file_downloader.download(**args): - watcher.print('Could not download source file {}'.format(args['file_url'])) - return False + file_size = self.context.http_client.get_content_length(args['file_url']) + file_size = int(file_size) / (1024 ** 2) if file_size else None + + watcher.change_substatus(bold('[{}] ').format(downloader) + self.i18n['downloading'] + ' ' + bold(args['file_url'].split('/')[-1]) + ' ( {0:.2f} Mb )'.format(file_size) if file_size else '') + if not self.context.file_downloader.download(**args): + watcher.print('Could not download source file {}'.format(args['file_url'])) + return False return True def _make_pkg(self, pkgname: str, root_password: str, handler: ProcessHandler, build_dir: str, project_dir: str, dependency: bool, skip_optdeps: bool = False, change_progress: bool = True) -> bool: - if not self._pre_download_source(pkgname, project_dir, handler.watcher): - return False + + self._pre_download_source(pkgname, project_dir, handler.watcher) self._update_progress(handler.watcher, 50, change_progress) if not self._install_missings_deps_and_keys(pkgname, root_password, handler, project_dir): @@ -391,7 +397,7 @@ class ArchManager(SoftwareManager): # building main package handler.watcher.change_substatus(self.i18n['arch.building.package'].format(bold(pkgname))) - pkgbuilt = handler.handle(SystemProcess(new_subprocess(['makepkg', '-ALcsmf'], cwd=project_dir, lang=None), check_error_output=False)) + pkgbuilt = handler.handle(SystemProcess(new_subprocess(['makepkg', '-ALcsmf'], cwd=project_dir, lang=None), check_error_output=False, skip_stdout=True)) self._update_progress(handler.watcher, 65, change_progress) if pkgbuilt: @@ -642,6 +648,7 @@ class ArchManager(SoftwareManager): def prepare(self): self.dcache_updater.start() + self.comp_optimizer.start() self.aur_index_updater.start() def list_updates(self) -> List[PackageUpdate]: diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index 1b6e5d80..cd2a412b 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -1,17 +1,24 @@ import logging import os +import re import time +from math import ceil from multiprocessing import Process from threading import Thread from bauh.api.abstract.context import ApplicationContext from bauh.api.abstract.controller import SoftwareManager - +from bauh.api.constants import HOME_PATH from bauh.gems.arch import pacman, disk URL_INDEX = 'https://aur.archlinux.org/packages.gz' URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg={}' +GLOBAL_MAKEPKG = '/etc/makepkg.conf' +USER_MAKEPKG = '{}/makepkg.conf'.format(HOME_PATH) + +RE_MAKE_FLAGS = re.compile(r'#?\s*MAKEFLAGS\s*=\s*.+\s*') + class AURIndexUpdater(Thread): @@ -52,3 +59,50 @@ class ArchDiskCacheUpdater(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else Proce saved = disk.save_several({app for app in installed['not_signed']}, 'aur', overwrite=False) self.logger.info('Pre-cached data of {} AUR packages to the disk'.format(saved)) + + +class ArchCompilationOptimizer(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else Process): + + def __init__(self, logger: logging.Logger): + super(ArchCompilationOptimizer, self).__init__(daemon=True) + self.logger = logger + self.compilation_optimizations = bool(int(os.getenv('BAUH_ARCH_OPTIMIZE', 1))) + + def run(self): + + if not self.compilation_optimizations: + self.logger.info("Arch packages compilation optimization is disabled. Aborting...") + else: + try: + ncpus = ceil(os.cpu_count() * 1.5) + except: + self.logger.error('Could not determine the number of processors. Aborting...') + return + + if os.path.exists(GLOBAL_MAKEPKG) and not os.path.exists(USER_MAKEPKG): + self.logger.info("Verifying if it is possible to optimize Arch packages compilation") + + with open(GLOBAL_MAKEPKG) as f: + global_makepkg = f.read() + + makeflags = RE_MAKE_FLAGS.findall(global_makepkg) + user_makepkg = None + + if makeflags: + not_commented = [f for f in makeflags if not makeflags[0].startswith('#')] + + if not not_commented: + user_makepkg = RE_MAKE_FLAGS.sub('', global_makepkg) + else: + self.logger.warning("It seems '{}' compilation flags are already customized".format(GLOBAL_MAKEPKG)) + else: + user_makepkg = global_makepkg + + if user_makepkg: + user_makepkg = '# \n' + user_makepkg + '\nMAKEFLAGS="-j{}"'.format(ncpus) + with open(USER_MAKEPKG, 'w+') as f: + f.write(user_makepkg) + + self.logger.info("A custom optimized 'makepkg.conf' was generated at '{}'".format(HOME_PATH)) + else: + self.logger.warning("A custom 'makepkg.conf' is already defined at '{}'".format(HOME_PATH)) diff --git a/bauh/view/core/controller.py b/bauh/view/core/controller.py index 95985a34..80226e70 100755 --- a/bauh/view/core/controller.py +++ b/bauh/view/core/controller.py @@ -82,7 +82,7 @@ class GenericSoftwareManager(SoftwareManager): mti = time.time() apps_found = man.search(words=word, disk_loader=disk_loader) mtf = time.time() - self.logger.info(man.__class__.__name__ + " took {0:.2f} seconds".format(mtf - mti)) + self.logger.info(man.__class__.__name__ + " took {0:.2f} seconds".format(mtf - mti)) res.installed.extend(apps_found.installed) res.new.extend(apps_found.new) diff --git a/bauh/view/core/downloader.py b/bauh/view/core/downloader.py index 7888d2e5..4451caef 100644 --- a/bauh/view/core/downloader.py +++ b/bauh/view/core/downloader.py @@ -2,7 +2,6 @@ import logging import os import time import traceback -from multiprocessing import cpu_count from bauh.api.abstract.download import FileDownloader from bauh.api.abstract.handler import ProcessWatcher @@ -15,29 +14,24 @@ class AdaptableFileDownloader(FileDownloader): self.logger = logger self.multithread_enabled = multithread_enabled - if multithread_enabled: - try: - self.download_threads = int(cpu_count() * 2) - except: - self.download_threads = 4 - - self.download_threads = 16 if self.download_threads > 16 else self.download_threads - else: - self.download_threads = 1 - - def is_axel_available(self) -> bool: - return bool(run_cmd('which axel')) - def is_aria2c_available(self) -> bool: return bool(run_cmd('which aria2c')) def _get_aria2c_process(self, url: str, output_path: str, cwd: str) -> SystemProcess: - cmd = ['aria2c', '-x{}'.format(self.download_threads), url, + cmd = ['aria2c', url, + '--no-conf', + '--max-connection-per-server=16', + '--split=16', '--enable-color=false', '--stderr=true', '--summary-interval=0', '--disable-ipv6', - '--min-split-size=1M'] + '--min-split-size=1M', + '--allow-overwrite=true', + '--continue=true', + '--timeout=5', + '--max-file-not-found=3', + '--remote-time=true'] if output_path: output_split = output_path.split('/') @@ -47,16 +41,8 @@ class AdaptableFileDownloader(FileDownloader): return SystemProcess(new_subprocess(cmd=cmd, cwd=cwd), skip_stdout=True, check_error_output=False, - success_phrase='download completed') - - def _get_axel_cmd(self, url: str, output_path: str, cwd: str) -> SystemProcess: - cmd = ['axel', '-k', '-n', str(self.download_threads), url] - - if output_path: - cmd.append('-o') - cmd.append(output_path) - - return SystemProcess(new_subprocess(cmd, cwd=cwd)) + success_phrase='download completed', + output_delay=0.001) def _get_wget_process(self, url: str, output_path: str, cwd: str) -> SystemProcess: cmd = ['wget', url] @@ -87,7 +73,6 @@ class AdaptableFileDownloader(FileDownloader): if self.is_multithreaded(): ti = time.time() process = self._get_aria2c_process(file_url, output_path, final_cwd) - # process = self._get_axel_cmd(file_url, output_path, final_cwd) else: ti = time.time() process = self._get_wget_process(file_url, output_path, final_cwd) @@ -109,3 +94,6 @@ class AdaptableFileDownloader(FileDownloader): def is_multithreaded(self) -> bool: return self.multithread_enabled and self.is_aria2c_available() + def get_default_client_name(self) -> str: + return 'aria2c' if self. is_multithreaded() else 'wget' +