diff --git a/CHANGELOG.md b/CHANGELOG.md index e7963e1b..82afd749 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Progress bar status can now be controlled by the software manager while an operation is being executed - Flatpak: showing runtime branches as versions when they are not available - small UI improvements +- installation logs are saved at **/tmp/bauh/logs/install** - Environment variable / parameter BAUH_UPDATE_NOTIFICATION renamed to BAUH_SYSTEM_NOTIFICATIONS and now works for any system notification +- Environment variable / parameter BAUH_DOWNLOAD_MULTITHREAD (--download-mthread): if source files should be downloaded using multi-threads. ### UI Changes - **Upgrade selected** and **Refresh** buttons now have text labels and new colors @@ -45,6 +47,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Ubuntu root password check - [Ubuntu 19.04 pip3 install issue] (https://github.com/vinifmor/bauh/issues/3) +### AUR support (**arch gem**): +- Search, install, uninstall, downgrade, retrieve history and launch packages +- Faster source files download when (**axel**) [https://github.com/axel-download-accelerator/axel] is installed on your system + ### Code - Code was internally modularized as: **api** (conceptual classes used to create custom software managers), **gems** (software managers), **commons** (common classes shared between the **view** and **gems**), **view** (UI code) - **api** allows custom operations, so the **gems** can provide actions that the current GUI does not support (Snap "refresh" was refactored as a custom operation) diff --git a/README.md b/README.md index f7c6731a..f856e83e 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ You can change some application settings via environment variables or arguments - **BAUH_SUGGESTIONS**: If application suggestions should be displayed if no packaged considered as an application is installed (runtimes / libraries do not count as applications). Use **0** (disable) or **1** (enable, default). - **BAUH_MAX_DISPLAYED**: Maximum number of displayed packages in the management panel table. Default: 50. - **BAUH_LOGS**: enable **bauh** logs (for debugging purposes). Use: **0** (disable, default) or **1** (enable) - +- **BAUH_DOWNLOAD_MULTITHREAD**: enable multi-threaded download for installation files ( only possible if **axel** is installed ). Use **0** (disable) or **1** (enabled, default). The application settings are stored in **/home/$USER/.config/bauh/config.json** @@ -87,6 +87,8 @@ The application settings are stored in **/home/$USER/.config/bauh/config.json** - If you don't mind app suggestions, disable it (**bauh --sugs=0**) - Let the disk cache always enabled so **bauh** does not need to dynamically retrieve some data every time you launch it. +### Logs +- Installation logs are saved at **/tmp/bauh/logs/install** ### Code structure #### Modules diff --git a/bauh/api/abstract/context.py b/bauh/api/abstract/context.py index 4ad92f11..131f35c6 100644 --- a/bauh/api/abstract/context.py +++ b/bauh/api/abstract/context.py @@ -1,8 +1,10 @@ import logging import platform +import sys from bauh.api.abstract.cache import MemoryCacheFactory from bauh.api.abstract.disk import DiskCacheLoaderFactory +from bauh.api.abstract.download import FileDownloader from bauh.api.http import HttpClient @@ -10,7 +12,7 @@ class ApplicationContext: def __init__(self, disk_cache: bool, download_icons: bool, http_client: HttpClient, app_root_dir: str, i18n: dict, cache_factory: MemoryCacheFactory, disk_loader_factory: DiskCacheLoaderFactory, - logger: logging.Logger): + logger: logging.Logger, file_downloader: FileDownloader): """ :param disk_cache: if package data should be cached to disk :param download_icons: if packages icons should be downloaded @@ -20,6 +22,7 @@ class ApplicationContext: :param cache_factory: :param disk_loader_factory: :param logger: a logger instance + :param file_downloader: """ self.disk_cache = disk_cache self.download_icons = download_icons @@ -30,3 +33,8 @@ class ApplicationContext: self.disk_loader_factory = disk_loader_factory self.logger = logger self.linux_distro = platform.linux_distribution() + self.file_downloader = file_downloader + self.arch_x86_64 = sys.maxsize > 2**32 + + def is_system_x86_64(self): + return self.arch_x86_64 diff --git a/bauh/api/abstract/download.py b/bauh/api/abstract/download.py new file mode 100644 index 00000000..97833216 --- /dev/null +++ b/bauh/api/abstract/download.py @@ -0,0 +1,21 @@ +from abc import ABC, abstractmethod + +from bauh.api.abstract.handler import ProcessWatcher + + +class FileDownloader(ABC): + + @abstractmethod + def download(self, file_url: str, watcher: ProcessWatcher, output_path: str, cwd: str) -> bool: + """ + :param file_url: + :param watcher: + :param output_path: the downloaded file output path. Leave None for the current directory and the same file name + :param cwd: current working directory. Leave None if does not matter. + :return: success / failure + """ + pass + + @abstractmethod + def is_multithreaded(self) -> bool: + pass diff --git a/bauh/api/http.py b/bauh/api/http.py index d01d9cc7..97bbdc1d 100644 --- a/bauh/api/http.py +++ b/bauh/api/http.py @@ -38,3 +38,13 @@ class HttpClient: def get_json(self, url: str): res = self.get(url) return res.json() if res else None + + def get_content_length(self, url: str) -> int: + """ + :param url: + :return: + """ + res = self.session.head(url) + + if res.status_code == 200: + return res.headers['content-length'] diff --git a/bauh/app.py b/bauh/app.py index 2c49ebd2..a8bce3c3 100755 --- a/bauh/app.py +++ b/bauh/app.py @@ -8,6 +8,7 @@ from bauh.api.abstract.controller import ApplicationContext from bauh.api.http import HttpClient from bauh.view.core import gems, config from bauh.view.core.controller import GenericSoftwareManager +from bauh.view.core.downloader import AdaptableFileDownloader from bauh.view.util import util, logs, resource from bauh.view.qt.systray import TrayIcon from bauh.view.qt.window import ManageWindow @@ -33,7 +34,8 @@ def main(): app_root_dir=ROOT_DIR, cache_factory=cache_factory, disk_loader_factory=DefaultDiskCacheLoaderFactory(disk_cache_enabled=args.disk_cache, logger=logger), - logger=logger) + logger=logger, + file_downloader=AdaptableFileDownloader(logger, bool(args.download_mthread))) user_config = config.read() app = QApplication(sys.argv) diff --git a/bauh/app_args.py b/bauh/app_args.py index a7a596a7..1d6cc5eb 100644 --- a/bauh/app_args.py +++ b/bauh/app_args.py @@ -36,6 +36,7 @@ def read() -> Namespace: parser.add_argument('-md', '--max-displayed', action="store", default=os.getenv('BAUH_MAX_DISPLAYED', 50), type=int, help='Maximum number of displayed packages in the management panel table. Default: %(default)s') parser.add_argument('--logs', action="store", default=int(os.getenv('BAUH_LOGS', 0)), choices=[0, 1], type=int, help='If the application logs should be displayed. Default: %(default)s') parser.add_argument('--show-panel', action="store_true", help='Shows the management panel after the app icon is attached to the tray.') + parser.add_argument('-dm', '--download-mthread', action="store", default=os.getenv('BAUH_DOWNLOAD_MULTITHREAD', 1), choices=[0, 1], type=int, help='If installation files should be downloaded using multi-threads (only possible if axel is installed). Default: %(default)s') return parser.parse_args() @@ -70,4 +71,7 @@ def validate(args: Namespace, logger: logging.Logger): if args.logs == 1: logger.info("Logs are enabled") + if args.download_mthread == 0: + logger.info("Multithreaded downloads disabled") + return args diff --git a/bauh/gems/arch/aur.py b/bauh/gems/arch/aur.py index d2afd8a5..ce01364f 100644 --- a/bauh/gems/arch/aur.py +++ b/bauh/gems/arch/aur.py @@ -11,6 +11,8 @@ URL_SEARCH = 'https://aur.archlinux.org/rpc/?v=5&type=search&arg=' RE_SRCINFO_KEYS = re.compile(r'(\w+)\s+=\s+(.+)\n') +KNOWN_LIST_FIELDS = ('validpgpkeys', 'depends', 'optdepends', 'sha512sums', 'sha512sums_x86_64', 'source', 'source_x86_64') + def map_pkgbuild(pkgbuild: str) -> dict: return {attr: val.replace('"', '').replace("'", '').replace('(', '').replace(')', '') for attr, val in re.findall(r'\n(\w+)=(.+)', pkgbuild)} @@ -36,16 +38,13 @@ class AURClient: info = {} for field in RE_SRCINFO_KEYS.findall(res.text): if field[0] not in info: - info[field[0]] = field[1] + info[field[0]] = [field[1]] if field[0] in KNOWN_LIST_FIELDS else field[1] else: if not isinstance(info[field[0]], list): info[field[0]] = [info[field[0]]] info[field[0]].append(field[1]) - if info.get('validpgpkeys') and isinstance(info['validpgpkeys'], str): - info['validpgpkeys'] = [info['validpgpkeys']] - return info def _map_names_as_queries(self, names) -> str: diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 5b96a004..34d57655 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -26,6 +26,9 @@ URL_SRC_INFO = 'https://aur.archlinux.org/cgit/aur.git/plain/.SRCINFO?h=' RE_SPLIT_VERSION = re.compile(r'(=|>|<)') +SOURCE_FIELDS = ('source', 'source_x86_64') +RE_PRE_DOWNLOADABLE_FILES = re.compile(r'(https?|ftp)://.+\.\w+[^gpg]$') + class ArchManager(SoftwareManager): @@ -251,7 +254,7 @@ class ArchManager(SoftwareManager): '07_maintainer': pkg.maintainer, '08_first_submitted': pkg.first_submitted, '09_last_modified': pkg.last_modified, - '10_url': pkg.url_download, + '10_url': pkg.url_download } srcinfo = self.aur_client.get_src_info(pkg.name) @@ -344,7 +347,43 @@ class ArchManager(SoftwareManager): return pkg_mirrors + def _pre_download_source(self, pkgname: str, project_dir: str, watcher: ProcessWatcher) -> bool: + if self.context.file_downloader.is_multithreaded(): + srcinfo = self.aur_client.get_src_info(pkgname) + + pre_download_files = [] + + for attr in SOURCE_FIELDS: + if srcinfo.get(attr): + if attr == 'source_x86_x64' and not self.context.is_system_x86_64(): + continue + else: + for f in srcinfo[attr]: + if RE_PRE_DOWNLOADABLE_FILES.findall(f): + pre_download_files.append(f) + + for f in pre_download_files: + fdata = f.split('::') + + args = {'watcher': watcher, 'cwd': project_dir} + if len(fdata) > 1: + args.update({'file_url': fdata[1], 'output_path': fdata[0]}) + else: + args.update({'file_url': fdata[0], 'output_path': None}) + + file_size = self.context.http_client.get_content_length(args['file_url']) + file_size = int(file_size) / (1024 ** 2) if file_size else None + + watcher.change_substatus(self.i18n['downloading'] + ' ' + bold(args['file_url'].split('/')[-1]) + ' ( {0:.2f} Mb )'.format(file_size) if file_size else '') + if not self.context.file_downloader.download(**args): + watcher.print('Could not download source file {}'.format(args['file_url'])) + return False + + return True + def _make_pkg(self, pkgname: str, root_password: str, handler: ProcessHandler, build_dir: str, project_dir: str, dependency: bool, skip_optdeps: bool = False, change_progress: bool = True) -> bool: + if not self._pre_download_source(pkgname, project_dir, handler.watcher): + return False self._update_progress(handler.watcher, 50, change_progress) if not self._install_missings_deps_and_keys(pkgname, root_password, handler, project_dir): @@ -378,7 +417,7 @@ class ArchManager(SoftwareManager): def _install_missings_deps_and_keys(self, pkgname: str, root_password: str, handler: ProcessHandler, pkgdir: str) -> bool: handler.watcher.change_substatus(self.i18n['arch.checking.deps'].format(bold(pkgname))) - check_res = makepkg.check_missing_deps(pkgdir, handler.watcher) + check_res = makepkg.check(pkgdir, handler.watcher) if check_res: if check_res.get('gpg_key'): diff --git a/bauh/gems/arch/makepkg.py b/bauh/gems/arch/makepkg.py index 843aac78..d4e7917c 100644 --- a/bauh/gems/arch/makepkg.py +++ b/bauh/gems/arch/makepkg.py @@ -7,35 +7,39 @@ RE_DEPS_PATTERN = re.compile(r'\n?\s+->\s(.+)\n') RE_UNKNOWN_GPG_KEY = re.compile(r'\(unknown public key (\w+)\)') -def check_missing_deps(pkgdir: str, watcher: ProcessWatcher) -> dict: - depcheck = new_subprocess(['makepkg', '-L', '--check'], cwd=pkgdir) +def check(pkgdir: str, watcher: ProcessWatcher) -> dict: + depcheck = new_subprocess(['makepkg', '-ALcf', '--check', '--noarchive'], cwd=pkgdir) res = {} + + output_lines = [] + for o in depcheck.stdout: if o: line = o.decode().strip() if line: + output_lines.append(line) watcher.print(line) - error_lines = [] for s in depcheck.stderr: if s: line = s.decode() if line: - error_lines.append(line) + output_lines.append(line) line_strip = line.strip() if line_strip: - watcher.print(line) + watcher.print(line_strip) - if error_lines: - error_str = ''.join(error_lines) + if output_lines: + error_str = ''.join(output_lines) if 'Missing dependencies' in error_str: res['missing_deps'] = RE_DEPS_PATTERN.findall(error_str) gpg_keys = RE_UNKNOWN_GPG_KEY.findall(error_str) + if gpg_keys: res['gpg_key'] = gpg_keys[0] diff --git a/bauh/gems/arch/pacman.py b/bauh/gems/arch/pacman.py index 97ef6408..f420fcee 100644 --- a/bauh/gems/arch/pacman.py +++ b/bauh/gems/arch/pacman.py @@ -61,6 +61,8 @@ def get_info_dict(pkg_name: str) -> dict: if attr == 'optional deps' and info_dict[attr]: info_dict[attr] = RE_DEPS.findall(info_dict[attr]) + elif attr == 'depends on' and info_dict[attr]: + info_dict[attr] = [d.strip() for d in info_dict[attr].split(' ') if d] return info_dict diff --git a/bauh/view/core/controller.py b/bauh/view/core/controller.py index e0ac0ba4..95985a34 100755 --- a/bauh/view/core/controller.py +++ b/bauh/view/core/controller.py @@ -218,7 +218,7 @@ class GenericSoftwareManager(SoftwareManager): return False finally: tf = time.time() - self.logger.info('Installation of {} took {} seconds'.format(app, tf - ti)) + self.logger.info('Installation of {}'.format(app) + 'took {0:.2f} minutes'.format((tf - ti)/60)) def get_info(self, app: SoftwarePackage): man = self._get_manager_for(app) diff --git a/bauh/view/core/downloader.py b/bauh/view/core/downloader.py new file mode 100644 index 00000000..fa33c8a5 --- /dev/null +++ b/bauh/view/core/downloader.py @@ -0,0 +1,53 @@ +import logging +import time +from multiprocessing import cpu_count +from typing import List + +from bauh.api.abstract.download import FileDownloader +from bauh.api.abstract.handler import ProcessWatcher +from bauh.commons.system import run_cmd, new_subprocess, ProcessHandler, SystemProcess + + +class AdaptableFileDownloader(FileDownloader): + + def __init__(self, logger: logging.Logger, multithread_enabled: bool): + self.download_threads = cpu_count() * 2 + self.logger = logger + self.multithread_enabled = multithread_enabled + + def is_axel_available(self) -> bool: + return bool(run_cmd('which axel')) + + def _get_download_command(self, url: str, output_path: str) -> List: + if self.is_multithreaded(): + cmd = ['axel', '-k', '-n', str(self.download_threads), url] + + if output_path: + cmd.append('-o') + cmd.append(output_path) + + return cmd + + cmd = ['wget', url] + + if output_path: + cmd.append('-O') + cmd.append(output_path) + + return cmd + + def download(self, file_url: str, watcher: ProcessWatcher, output_path: str, cwd: str): + handler = ProcessHandler(watcher) + cmd = self._get_download_command(file_url, output_path) + self.logger.info('Downloading {}'.format(file_url)) + watcher.print("[{}] downloading {}{}".format(cmd[0], file_url, " as {}".format(output_path) if output_path else '')) + + ti = time.time() + res = handler.handle(SystemProcess(new_subprocess(cmd=cmd, cwd=cwd if cwd else '.'))) + tf = time.time() + self.logger.info(file_url.split('/')[-1] + ' download took {0:.2f} minutes'.format((tf - ti) / 60)) + return res + + def is_multithreaded(self) -> bool: + return self.multithread_enabled and self.is_axel_available() + diff --git a/bauh/view/qt/info.py b/bauh/view/qt/info.py index 13893064..f4cdf3ea 100644 --- a/bauh/view/qt/info.py +++ b/bauh/view/qt/info.py @@ -12,7 +12,7 @@ class InfoDialog(QDialog): def __init__(self, app: dict, icon_cache: MemoryCache, locale_keys: dict, screen_size: QSize()): super(InfoDialog, self).__init__() - self.setWindowTitle(app['__app__'].model.name) + self.setWindowTitle(str(app['__app__'])) self.screen_size = screen_size self.i18n = locale_keys layout = QVBoxLayout() @@ -50,7 +50,7 @@ class InfoDialog(QDialog): i18n_key = app['__app__'].model.get_type() + '.info.' + attr.lower() if isinstance(app[attr], list): - val = '\n'.join([str(e) for e in app[attr]]) + val = '\n'.join(['* ' + str(e.strip()) for e in app[attr] if e]) else: val = str(app[attr]).strip() diff --git a/bauh/view/qt/window.py b/bauh/view/qt/window.py index 12e7234f..fb269157 100755 --- a/bauh/view/qt/window.py +++ b/bauh/view/qt/window.py @@ -1,5 +1,7 @@ import operator +import time from functools import reduce +from pathlib import Path from typing import List, Type, Set from PyQt5.QtCore import QEvent, Qt, QSize, pyqtSignal @@ -867,6 +869,18 @@ class ManageWindow(QWidget): self.thread_install.start() def _finish_install(self, pkgv: PackageView): + console_output = self.textarea_output.toPlainText() + + if console_output: + log_path = '/tmp/bauh/logs/install/{}/{}'.format(pkgv.model.get_type(), pkgv.model.name) + try: + Path(log_path).mkdir(parents=True, exist_ok=True) + + with open(log_path + '/{}.log'.format(int(time.time())), 'w+') as f: + f.write(console_output) + except: + self.textarea_output.appendPlainText("[warning] Could not write install log file to '{}'".format(log_path)) + self.input_search.setText('') self.finish_action() diff --git a/bauh/view/resources/locale/en b/bauh/view/resources/locale/en index 11af0fe5..83244c5f 100644 --- a/bauh/view/resources/locale/en +++ b/bauh/view/resources/locale/en @@ -115,4 +115,5 @@ manage_window.settings.gems=Application types style=style style.change.title=Style change style.change.question=To change the current style is necessary to restart {}. Proceed ? -manage_window.bt_settings.tooltip=Click here to open extra actions \ No newline at end of file +manage_window.bt_settings.tooltip=Click here to open extra actions +downloading=Downloading \ No newline at end of file diff --git a/bauh/view/resources/locale/es b/bauh/view/resources/locale/es index 859f3f8c..89a0da1e 100644 --- a/bauh/view/resources/locale/es +++ b/bauh/view/resources/locale/es @@ -117,4 +117,5 @@ manage_window.settings.gems=Tipos de aplicativos style=estilo style.change.title=Cambio de estilo style.change.question=Para cambiar el estilo actual es necesario reiniciar {}. ¿Proceder? -manage_window.bt_settings.tooltip=Haga clic aquí para abrir acciones adicionales \ No newline at end of file +manage_window.bt_settings.tooltip=Haga clic aquí para abrir acciones adicionales +downloading=Descargando \ No newline at end of file diff --git a/bauh/view/resources/locale/pt b/bauh/view/resources/locale/pt index 50e478af..ddd035ed 100644 --- a/bauh/view/resources/locale/pt +++ b/bauh/view/resources/locale/pt @@ -117,4 +117,5 @@ manage_window.settings.gems=Tipos de aplicativos style=estilo style.change.title=Mudança de estilo style.change.question=Para alterar o estilo atual é necessário reiniciar o {}. Continuar ? -manage_window.bt_settings.tooltip=Clique aqui para abrir ações adicionais \ No newline at end of file +manage_window.bt_settings.tooltip=Clique aqui para abrir ações adicionais +downloading=Baixando \ No newline at end of file