+ 
+
{}
'.format(i18n['arch.missing_deps.body'].format(name=bold(pkgname) if pkgname else '', deps=bold(str(len(deps))))) opts = [] @@ -52,6 +52,9 @@ def request_install_missing_deps(pkgname: str, deps: List[Tuple[str, str]], watc repo_deps = [d[0] for d in deps if d[1] != 'aur'] sizes = pacman.get_update_size(repo_deps) if repo_deps else {} + if context and sizes: + context.pkg_sizes.update(sizes) + for dep in deps: size = sizes.get(dep[0]) op = InputOption('{} ({}: {}) - {}: {}'.format(dep[0], diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 86dcb1f0..7b6bb9ca 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -35,6 +35,7 @@ from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, message, confirmatio from bauh.gems.arch.aur import AURClient from bauh.gems.arch.config import read_config from bauh.gems.arch.dependencies import DependenciesAnalyser +from bauh.gems.arch.download import MultithreadedDownloadService, ArchDownloadException from bauh.gems.arch.exceptions import PackageNotFoundException from bauh.gems.arch.mapper import ArchDataMapper from bauh.gems.arch.model import ArchPackage @@ -62,7 +63,7 @@ class TransactionContext: install_file: str = None, repository: str = None, pkg: ArchPackage = None, remote_repo_map: Dict[str, str] = None, provided_map: Dict[str, Set[str]] = None, remote_provided_map: Dict[str, Set[str]] = None, aur_idx: Set[str] = None, - missing_deps: List[Tuple[str, str]] = None): + missing_deps: List[Tuple[str, str]] = None, pkg_sizes: Dict[str, int] = {}): self.name = name self.base = base self.maintainer = maintainer @@ -83,6 +84,7 @@ class TransactionContext: self.remote_provided_map = remote_provided_map self.aur_idx = aur_idx self.missing_deps = missing_deps + self.pkg_sizes = pkg_sizes @classmethod def gen_context_from(cls, pkg: ArchPackage, arch_config: dict, root_password: str, handler: ProcessHandler) -> "TransactionContext": @@ -747,13 +749,23 @@ class ArchManager(SoftwareManager): return files - def _upgrade_repo_pkgs(self, pkgs: List[str], handler: ProcessHandler, root_password: str, overwrite_files: bool = False, - status_handler: TransactionStatusHandler = None) -> bool: + def _upgrade_repo_pkgs(self, pkgs: List[str], handler: ProcessHandler, root_password: str, arch_config: dict, overwrite_files: bool = False, + status_handler: TransactionStatusHandler = None, already_downloaded: bool = False, sizes: Dict[str, int] = None) -> bool: + + downloaded = 0 + + if not already_downloaded: + if self._should_download_packages(arch_config): + try: + downloaded = self._download_packages(pkgs, handler, root_password, sizes) + except ArchDownloadException: + return False + try: if status_handler: output_handler = status_handler else: - output_handler = TransactionStatusHandler(handler.watcher, self.i18n, len(pkgs), self.logger) + output_handler = TransactionStatusHandler(handler.watcher, self.i18n, len(pkgs), self.logger, downloading=downloaded) output_handler.start() success, upgrade_output = handler.handle_simple(pacman.upgrade_several(pkgnames=pkgs, @@ -791,7 +803,10 @@ class ArchManager(SoftwareManager): handler=handler, root_password=root_password, overwrite_files=True, - status_handler=output_handler) + status_handler=output_handler, + already_downloaded=True, + arch_config=arch_config, + sizes=sizes) else: output_handler.stop_working() output_handler.join() @@ -819,7 +834,7 @@ class ArchManager(SoftwareManager): watcher.change_substatus('') return False - aur_pkgs, repo_pkgs = [], [] + aur_pkgs, repo_pkgs, pkg_sizes = [], [], {} for req in (*requirements.to_install, *requirements.to_upgrade): if req.pkg.repository == 'aur': @@ -827,6 +842,8 @@ class ArchManager(SoftwareManager): else: repo_pkgs.append(req.pkg) + pkg_sizes[req.pkg.name] = req.required_size + if aur_pkgs and not self._check_action_allowed(aur_pkgs[0], watcher): return False @@ -852,7 +869,11 @@ class ArchManager(SoftwareManager): self.logger.info("Upgrading {} repository packages: {}".format(len(repo_pkgs_names), ', '.join(repo_pkgs_names))) - if not self._upgrade_repo_pkgs(pkgs=repo_pkgs_names, handler=handler, root_password=root_password): + if not self._upgrade_repo_pkgs(pkgs=repo_pkgs_names, + handler=handler, + root_password=root_password, + arch_config=arch_config, + sizes=pkg_sizes): return False if aur_pkgs: @@ -1263,7 +1284,7 @@ class ArchManager(SoftwareManager): return True - def _install_deps(self, context: TransactionContext, deps: List[Tuple[str, str]]) -> Iterable[str]: + def _install_deps(self, context: TransactionContext, deps: List[Tuple[str, str]]) -> Iterable[str]: """ :param pkgs_repos: :param root_password: @@ -1305,7 +1326,15 @@ class ArchManager(SoftwareManager): if not self._request_conflict_resolution(dep, conflict_pkg , context): return {dep} - status_handler = TransactionStatusHandler(context.watcher, self.i18n, len(repo_dep_names), self.logger, percentage=len(repo_deps) > 1) + downloaded = 0 + if self._should_download_packages(context.config): + try: + downloaded = self._download_packages(repo_dep_names, context.handler, context.root_password, context.pkg_sizes) + except ArchDownloadException: + return False + + status_handler = TransactionStatusHandler(watcher=context.watcher, i18n=self.i18n, npkgs=len(repo_dep_names), + logger=self.logger, percentage=len(repo_deps) > 1, downloading=downloaded) status_handler.start() installed, _ = context.handler.handle_simple(pacman.install_as_process(pkgpaths=repo_dep_names, root_password=context.root_password, @@ -1426,7 +1455,7 @@ class ArchManager(SoftwareManager): def _ask_and_install_missing_deps(self, context: TransactionContext, missing_deps: List[Tuple[str, str]]) -> bool: context.watcher.change_substatus(self.i18n['arch.missing_deps_found'].format(bold(context.name))) - if not confirmation.request_install_missing_deps(context.name, missing_deps, context.watcher, self.i18n): + if not confirmation.request_install_missing_deps(context.name, missing_deps, context.watcher, self.i18n, context): context.watcher.print(self.i18n['action.cancelled']) return False @@ -1583,7 +1612,7 @@ class ArchManager(SoftwareManager): sorted_deps = sorting.sort(to_sort, {**deps_data, **subdeps_data}, provided_map) - if display_deps_dialog and not confirmation.request_install_missing_deps(None, sorted_deps, context.watcher, self.i18n): + if display_deps_dialog and not confirmation.request_install_missing_deps(None, sorted_deps, context.watcher, self.i18n, context): context.watcher.print(self.i18n['action.cancelled']) return True # because the main package installation was successful @@ -1597,7 +1626,21 @@ class ArchManager(SoftwareManager): return True - def _install(self, context: TransactionContext): + def _should_download_packages(self, arch_config: dict) -> bool: + return bool(arch_config['repositories_mthread_download']) and self.context.file_downloader.is_multithreaded() + + def _download_packages(self, pkgnames: List[str], handler: ProcessHandler, root_password: str, sizes: Dict[str, int] = None) -> int: + download_service = MultithreadedDownloadService(file_downloader=self.context.file_downloader, + http_client=self.http_client, + logger=self.context.logger, + i18n=self.i18n) + self.logger.info("Initializing multi-threaded download for {} package(s)".format(len(pkgnames))) + return download_service.download_packages(pkgs=pkgnames, + handler=handler, + sizes=sizes, + root_password=root_password) + + def _install(self, context: TransactionContext) -> bool: check_install_output = [] pkgpath = context.get_package_path() @@ -1635,7 +1678,6 @@ class ArchManager(SoftwareManager): else: self.logger.info("No conflict detected for '{}'".format(context.name)) - context.watcher.change_substatus(self.i18n['arch.installing.package'].format(bold(context.name))) self._update_progress(context, 80) to_install = [] @@ -1645,12 +1687,25 @@ class ArchManager(SoftwareManager): to_install.append(pkgpath) + downloaded = 0 + if self._should_download_packages(context.config): + to_download = [p for p in to_install if not p.startswith('/')] + + if to_download: + try: + downloaded = self._download_packages(to_download, context.handler, context.root_password, context.pkg_sizes) + except ArchDownloadException: + return False + if not context.dependency: - status_handler = TransactionStatusHandler(context.watcher, self.i18n, len(to_install), self.logger, percentage=len(to_install) > 1) if not context.dependency else None + status_handler = TransactionStatusHandler(context.watcher, self.i18n, len(to_install), self.logger, + percentage=len(to_install) > 1, + downloading=downloaded) if not context.dependency else None status_handler.start() else: status_handler = None + context.watcher.change_substatus(self.i18n['arch.installing.package'].format(bold(context.name))) installed, _ = context.handler.handle_simple(pacman.install_as_process(pkgpaths=to_install, root_password=context.root_password, file=context.has_install_file(), @@ -1848,7 +1903,7 @@ class ArchManager(SoftwareManager): context.missing_deps = missing_deps context.watcher.change_substatus(self.i18n['arch.missing_deps_found'].format(bold(context.name))) - if not confirmation.request_install_missing_deps(context.name, missing_deps, context.watcher, self.i18n): + if not confirmation.request_install_missing_deps(context.name, missing_deps, context.watcher, self.i18n, context): context.watcher.print(self.i18n['action.cancelled']) return False @@ -2043,6 +2098,12 @@ class ArchManager(SoftwareManager): tooltip_key='arch.config.optimize.tip', value=bool(local_config['optimize']), max_width=max_width), + self._gen_bool_selector(id_='mthread_download', + label_key='arch.config.pacman_mthread_download', + tooltip_key='arch.config.pacman_mthread_download.tip', + value=local_config['repositories_mthread_download'], + max_width=max_width, + capitalize_label=True), self._gen_bool_selector(id_='sync_dbs', label_key='arch.config.sync_dbs', tooltip_key='arch.config.sync_dbs.tip', @@ -2081,6 +2142,7 @@ class ArchManager(SoftwareManager): config['clean_cached'] = form_install.get_component('clean_cached').get_selected() config['refresh_mirrors_startup'] = form_install.get_component('ref_mirs').get_selected() config['mirrors_sort_limit'] = form_install.get_component('mirrors_sort_limit').get_int_value() + config['repositories_mthread_download'] = form_install.get_component('mthread_download').get_selected() try: save_config(config, CONFIG_FILE) diff --git a/bauh/gems/arch/download.py b/bauh/gems/arch/download.py new file mode 100644 index 00000000..889f9b2f --- /dev/null +++ b/bauh/gems/arch/download.py @@ -0,0 +1,155 @@ +import glob +import logging +import os +import time +import traceback +from math import floor +from typing import List, Iterable, Dict + +from bauh.api.abstract.download import FileDownloader +from bauh.api.abstract.handler import ProcessWatcher +from bauh.api.abstract.view import MessageType +from bauh.api.http import HttpClient +from bauh.commons.html import bold +from bauh.commons.system import ProcessHandler, SimpleProcess +from bauh.gems.arch import pacman +from bauh.view.util.translation import I18n + + +class ArchDownloadException(Exception): + pass + + +class CacheDirCreationException(ArchDownloadException): + pass + + +class MultiThreadedDownloader: + + def __init__(self, file_downloader: FileDownloader, http_client: HttpClient, mirrors_available: Iterable[str], + mirrors_branch: str, cache_dir: str, logger: logging.Logger): + self.downloader = file_downloader + self.http_client = http_client + self.mirrors = mirrors_available + self.branch = mirrors_branch + self.extensions = ['.tar.zst', '.tar.xz'] + self.cache_dir = cache_dir + self.logger = logger + + def download_package(self, pkg: Dict[str, str], root_password: str, substatus_prefix: str, watcher: ProcessWatcher, size: int) -> bool: + if self.mirrors and self.branch: + pkgname = '{}-{}{}.pkg'.format(pkg['n'], pkg['v'], ('-{}'.format(pkg['a']) if pkg['a'] else '')) + + if {f for f in glob.glob(self.cache_dir + '/*') if f.split('/')[-1].startswith(pkgname)}: + watcher.print("{} ({}) file found o cache dir {}. Skipping download.".format(pkg['n'], pkg['v'], self.cache_dir)) + return True + + arch = pkg['a'] if pkg.get('a') and pkg['a'] != 'any' else 'x86_64' + + url_base = '{}/{}/{}/{}'.format(self.branch, pkg['r'], arch, pkgname) + base_output_path = '{}/{}'.format(self.cache_dir, pkgname) + + for mirror in self.mirrors: + for ext in self.extensions: + url = '{}{}{}'.format(mirror, url_base, ext) + output_path = base_output_path + ext + + watcher.print("Downloading '{}' from mirror '{}'".format(pkgname, mirror)) + + pkg_downloaded = self.downloader.download(file_url=url, watcher=watcher, output_path=output_path, + cwd='.', root_password=root_password, display_file_size=True, + substatus_prefix=substatus_prefix, + known_size=size) + if not pkg_downloaded: + watcher.print("Could not download '{}' from mirror '{}'".format(pkgname, mirror)) + else: + self.logger.info("Package '{}' successfully downloaded".format(pkg['n'])) + self.logger.info("Downloading package '{}' signature".format(pkg['n'])) + + sig_downloaded = self.downloader.download(file_url=url + '.sig', watcher=watcher, + output_path=output_path + '.sig', + cwd='.', root_password=root_password, + display_file_size=False, + substatus_prefix=substatus_prefix, + max_threads=1) + + if not sig_downloaded: + self.logger.warning("Could not download package '{}' signature".format(pkg['n'])) + else: + self.logger.info("Package '{}' signature successfully downloaded".format(pkg['n'])) + return True + return False + + +class MultithreadedDownloadService: + + def __init__(self, file_downloader: FileDownloader, http_client: HttpClient, logger: logging.Logger, i18n: I18n): + self.file_downloader = file_downloader + self.http_client = http_client + self.logger = logger + self.i18n = i18n + + def download_packages(self, pkgs: List[str], handler: ProcessHandler, root_password: str, sizes: Dict[str, int] = None) -> int: + ti = time.time() + watcher = handler.watcher + mirrors = pacman.list_available_mirrors() + + if not mirrors: + self.logger.warning('repository mirrors seem to be not reachable') + watcher.print('[warning] repository mirrors seem to be not reachable') + watcher.print('[warning] multi-threaded download cancelled') + return 0 + + branch = pacman.get_mirrors_branch() + + if not branch: + self.logger.warning('no default repository branch found') + watcher.print('[warning] no default repository branch found') + watcher.print('[warning] multi-threaded download cancelled') + return 0 + + cache_dir = pacman.get_cache_dir() + + if not os.path.exists(cache_dir): + success, _ = handler.handle_simple(SimpleProcess(['mkdir', '-p', cache_dir], root_password=root_password)) + + if not success: + msg = "could not create cache dir '{}'".format(cache_dir) + self.logger.warning(msg) + watcher.print("[warning] {}".format(cache_dir)) + watcher.show_message(title=self.i18n['warning'].capitalize(), + body=self.i18n['arch.mthread_downloaded.error.cache_dir'].format(bold(cache_dir)), + type_=MessageType.WARNING) + raise CacheDirCreationException() + + downloader = MultiThreadedDownloader(file_downloader=self.file_downloader, + mirrors_available=mirrors, + mirrors_branch=branch, + http_client=self.http_client, + logger=self.logger, + cache_dir=cache_dir) + + downloaded = 0 + pkgs_data = pacman.list_download_data(pkgs) + + for pkg in pkgs_data: + self.logger.info('Preparing to download package: {} ({})'.format(pkg['n'], pkg['v'])) + try: + perc = '({0:.2f}%)'.format((downloaded / (2 * len(pkgs))) * 100) + status_prefix = '{} [{}/{}]'.format(perc, downloaded + 1, len(pkgs)) + + if downloader.download_package(pkg=pkg, + root_password=root_password, + watcher=handler.watcher, + substatus_prefix=status_prefix, + size=sizes.get(pkg['n']) if sizes else None): + downloaded += 1 + except: + traceback.print_exc() + watcher.show_message(title=self.i18n['error'].capitalize(), + body=self.i18n['arch.mthread_downloaded.error.cancelled'], + type_=MessageType.ERROR) + raise ArchDownloadException() + tf = time.time() + self.logger.info("Download time: {0:.2f} seconds".format(tf - ti)) + return downloaded diff --git a/bauh/gems/arch/model.py b/bauh/gems/arch/model.py index 410950c2..5e4a941d 100644 --- a/bauh/gems/arch/model.py +++ b/bauh/gems/arch/model.py @@ -16,7 +16,7 @@ class ArchPackage(SoftwarePackage): first_submitted: datetime.datetime = None, last_modified: datetime.datetime = None, maintainer: str = None, url_download: str = None, pkgbuild: str = None, repository: str = None, desktop_entry: str = None, installed: bool = False, srcinfo: dict = None, dependencies: Set[str] = None, - categories: List[str] = None, i18n: I18n = None, update_ignored: bool = False): + categories: List[str] = None, i18n: I18n = None, update_ignored: bool = False, arch: str = None): super(ArchPackage, self).__init__(name=name, version=version, latest_version=latest_version, description=description, installed=installed, categories=categories) @@ -35,6 +35,7 @@ class ArchPackage(SoftwarePackage): self.desktop_entry = desktop_entry self.src_info = srcinfo self.dependencies = dependencies + self.arch = arch self.i18n = i18n self.update_ignored = update_ignored diff --git a/bauh/gems/arch/output.py b/bauh/gems/arch/output.py index e52d3a44..6df509ad 100644 --- a/bauh/gems/arch/output.py +++ b/bauh/gems/arch/output.py @@ -8,12 +8,13 @@ from bauh.view.util.translation import I18n class TransactionStatusHandler(Thread): - def __init__(self, watcher: ProcessWatcher, i18n: I18n, npkgs: int, logger: logging.Logger, percentage: bool = True): + def __init__(self, watcher: ProcessWatcher, i18n: I18n, npkgs: int, logger: logging.Logger, + percentage: bool = True, downloading: int = 0): super(TransactionStatusHandler, self).__init__(daemon=True) self.watcher = watcher self.i18n = i18n self.npkgs = npkgs - self.downloading = 0 + self.downloading = downloading self.upgrading = 0 self.installing = 0 self.outputs = [] diff --git a/bauh/gems/arch/pacman.py b/bauh/gems/arch/pacman.py index 7d11e4d3..1eaf2d22 100644 --- a/bauh/gems/arch/pacman.py +++ b/bauh/gems/arch/pacman.py @@ -3,6 +3,7 @@ import re from threading import Thread from typing import List, Set, Tuple, Dict, Iterable +from bauh.commons import system from bauh.commons.system import run_cmd, new_subprocess, new_root_subprocess, SystemProcess, SimpleProcess, \ ProcessHandler from bauh.commons.util import size_to_byte @@ -16,6 +17,7 @@ RE_INSTALLED_FIELDS = re.compile(r'(Name|Description|Version|Validated By)\s*:\s RE_INSTALLED_SIZE = re.compile(r'Installed Size\s*:\s*([0-9,\.]+)\s(\w+)\n?', re.IGNORECASE) RE_UPDATE_REQUIRED_FIELDS = re.compile(r'(\bProvides\b|\bInstalled Size\b|\bConflicts With\b)\s*:\s(.+)\n') RE_REMOVE_TRANSITIVE_DEPS = re.compile(r'removing\s([\w\-_]+)\s.+required\sby\s([\w\-_]+)\n?') +RE_AVAILABLE_MIRRORS = re.compile(r'.+\s+OK\s+.+\s+(\d+:\d+)\s+.+(http.+)') def is_available() -> bool: @@ -599,6 +601,36 @@ def map_provided(remote: bool = False, pkgs: Iterable[str] = None) -> Dict[str, return provided_map +def list_download_data(pkgs: Iterable[str]) -> List[Dict[str, str]]: + _, output = system.run(['pacman', '-Si', *pkgs]) + + if output: + res = [] + data = {'a': None, 'v': None, 'r': None, 'n': None} + + for l in output.split('\n'): + if l: + if l[0] != ' ': + line = l.strip() + field_sep_idx = line.index(':') + field = line[0:field_sep_idx].strip() + val = line[field_sep_idx + 1:].strip() + + if field == 'Repository': + data['r'] = val + elif field == 'Name': + data['n'] = val + elif field == 'Version': + data['v'] = val.split('=')[0] + elif field == 'Architecture': + data['a'] = val + elif data.get('a'): + res.append(data) + data = {'a': None, 'v': None, 'r': None, 'n': None} + + return res + + def map_updates_data(pkgs: Iterable[str], files: bool = False) -> dict: if files: output = run_cmd('pacman -Qi -p {}'.format(' '.join(pkgs))) @@ -1021,3 +1053,19 @@ def list_installed_names() -> Set[str]: output = run_cmd('pacman -Qq', print_error=False) return {name.strip() for name in output.split('\n') if name} if output else set() + +def list_available_mirrors() -> List[str]: + _, output = system.run(['pacman-mirrors', '--status', '--no-color']) + + if output: + mirrors = RE_AVAILABLE_MIRRORS.findall(output) + + if mirrors: + mirrors.sort(key=lambda o: o[0]) + return [m[1] for m in mirrors] + + +def get_mirrors_branch() -> str: + _, output = system.run(['pacman-mirrors', '-G']) + return output.strip() + diff --git a/bauh/gems/arch/resources/locale/ca b/bauh/gems/arch/resources/locale/ca index 5cd03adb..3fc38bdd 100644 --- a/bauh/gems/arch/resources/locale/ca +++ b/bauh/gems/arch/resources/locale/ca @@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Mirrors sort limit arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting. arch.config.optimize=optimize arch.config.optimize.tip=La configuració optimitzada s'utilitzarà per fer més ràpida la instal·lació, actualització i reversió dels paquets, en cas contrari s'utilitzarà la configuració del sistema. +arch.config.pacman_mthread_download=Multithreaded download (repositories) +arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed arch.config.refresh_mirrors=Refresh mirrors on startup arch.config.refresh_mirrors.tip=Refresh the package mirrors once a day on startup ( or after a device reboot ) arch.config.repos=Repositories packages @@ -153,6 +155,8 @@ arch.makepkg.optimizing=Optimitzant la recopilació arch.missing_deps.body=S’han d’instal·lar les {deps} dependències següents abans de continuar amb la instal·lació de {name} arch.missing_deps.title=Dependències mancants arch.missing_deps_found=Dependències mancants per a {} +arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {} +arch.mthread_downloaded.error.cancelled=Operation cancelled arch.optdeps.checking=S’estan comprovant les dependències opcionals de {} arch.providers=providers arch.substatus.conflicts=Checking for conflicts diff --git a/bauh/gems/arch/resources/locale/de b/bauh/gems/arch/resources/locale/de index 84b5f629..9f7ccfa7 100644 --- a/bauh/gems/arch/resources/locale/de +++ b/bauh/gems/arch/resources/locale/de @@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Mirrors sort limit arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting. arch.config.optimize=optimize arch.config.optimize.tip=Optimized settings will be used in order to make the packages installation, upgrading and downgrading faster, otherwise the system settings will be used +arch.config.pacman_mthread_download=Multithreaded download (repositories) +arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed arch.config.refresh_mirrors=Refresh mirrors on startup arch.config.refresh_mirrors.tip=Refresh the package mirrors once a day on startup ( or after a device reboot ) arch.config.repos=Repositories packages @@ -153,6 +155,8 @@ arch.makepkg.optimizing=Optimiert die Zusammenstellung arch.missing_deps.body=Die folgenden {deps} Abhängigkeiten müssten installiert sein, bevor mit der {name} Installation fortgefahren werden kann arch.missing_deps.title=Fehlende Abhängigkeiten arch.missing_deps_found=Fehlende Abhängigkeiten für {} +arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {} +arch.mthread_downloaded.error.cancelled=Operation cancelled arch.optdeps.checking={} optionale Abhängigkeiten überprüfen arch.providers=providers arch.substatus.conflicts=Checking for conflicts diff --git a/bauh/gems/arch/resources/locale/en b/bauh/gems/arch/resources/locale/en index 6ea75a36..2f6abf95 100644 --- a/bauh/gems/arch/resources/locale/en +++ b/bauh/gems/arch/resources/locale/en @@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Mirrors sort limit arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting. arch.config.optimize=optimize arch.config.optimize.tip=Optimized settings will be used in order to make the packages installation, upgrading and downgrading faster, otherwise the system settings will be used +arch.config.pacman_mthread_download=Multi-threaded download (repositories) +arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed arch.config.refresh_mirrors=Refresh mirrors on startup arch.config.refresh_mirrors.tip=Refresh the package mirrors once a day on startup ( or after a device reboot ) arch.config.repos=Repositories packages @@ -153,6 +155,8 @@ arch.makepkg.optimizing=Optimizing the compilation arch.missing_deps.body=The following {deps} dependencies must be installed so the {name} installation can continue arch.missing_deps.title=Missing dependencies arch.missing_deps_found=Missing dependencies for {} +arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {} +arch.mthread_downloaded.error.cancelled=Operation cancelled arch.optdeps.checking=Checking {} optional dependencies arch.providers=providers arch.substatus.conflicts=Checking for conflicts diff --git a/bauh/gems/arch/resources/locale/es b/bauh/gems/arch/resources/locale/es index c496269c..155f82a3 100644 --- a/bauh/gems/arch/resources/locale/es +++ b/bauh/gems/arch/resources/locale/es @@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Límite de ordenación de espejos arch.config.mirrors_sort_limit.tip=Define el número máximo de espejos que se utilizarán para la ordenación por velocidad. Use 0 para no limitar o déjelo en blanco para deshabilitar la clasificación. arch.config.optimize=optimizar arch.config.optimize.tip=Se usará la configuración optimizada para que la instalación, actualización y reversión de los paquetes sean más rápidas, de lo contrario se usará la configuración del sistema +arch.config.pacman_mthread_download=Descarga segmentada (repositorios) +arch.config.pacman_mthread_download.tip=Si los paquetes de los repositorios deben descargarse con una herramienta que usa segmentación/threads (puede ser más rápido). Por el momento, esta configuración solo funcionará si el paquete aria2 esté instalado arch.config.refresh_mirrors=Actualizar espejos al iniciar arch.config.refresh_mirrors.tip=Actualiza los espejos de paquetes una vez al día al iniciar ( o después de reiniciar el dispositivo ) arch.config.repos=Paquetes de repositorios @@ -153,6 +155,8 @@ arch.makepkg.optimizing=Optimizing the compilation arch.missing_deps.body=Deben instalarse las siguientes {deps} dependencias para que la instalación de {name} pueda continuar arch.missing_deps.title=Dependencias faltantes arch.missing_deps_found=Dependencias faltantes para {} +arch.mthread_downloaded.error.cache_dir=No fue posible crear el directorio de caché {} +arch.mthread_downloaded.error.cancelled=Operación cancelada arch.optdeps.checking=Verificando las dependencias opcionales de {} arch.providers=proveedores arch.substatus.conflicts=Verificando conflictos diff --git a/bauh/gems/arch/resources/locale/it b/bauh/gems/arch/resources/locale/it index 4a2b4bf4..95c3ae33 100644 --- a/bauh/gems/arch/resources/locale/it +++ b/bauh/gems/arch/resources/locale/it @@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Mirrors sort limit arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting. arch.config.optimize=optimize arch.config.optimize.tip=Verranno utilizzate le impostazioni ottimizzate per velocizzare l'installazione, l'aggiornamento e l'inversione dei pacchetti, altrimenti verranno utilizzate le impostazioni di sistema +arch.config.pacman_mthread_download=Multithreaded download (repositories) +arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed arch.config.refresh_mirrors=Refresh mirrors on startup arch.config.refresh_mirrors.tip=Refresh the package mirrors once a day on startup ( or after a device reboot ) arch.config.repos=Repositories packages @@ -153,6 +155,8 @@ arch.makepkg.optimizing=Ottimizzando la compilazione arch.missing_deps.body=Le seguenti {deps} dipendenze devono essere installate prima che l'installazione di {name} continui arch.missing_deps.title=Dipendenze mancanti arch.missing_deps_found=Dipendenze mancanti per {} +arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {} +arch.mthread_downloaded.error.cancelled=Operation cancelled arch.optdeps.checking=Verifica di {} dipendenze opzionali arch.providers=providers arch.substatus.conflicts=Checking for conflicts diff --git a/bauh/gems/arch/resources/locale/pt b/bauh/gems/arch/resources/locale/pt index 072f6db7..1d7b1824 100644 --- a/bauh/gems/arch/resources/locale/pt +++ b/bauh/gems/arch/resources/locale/pt @@ -27,7 +27,8 @@ arch.config.clean_cache.tip=Se versões antigas de um pacote armazenadas em disc arch.config.mirrors_sort_limit=Limite de ordenação de espelhos arch.config.mirrors_sort_limit.tip=Define o número máximo de espelhos que serão utilizados para a ordenação por velocidade. Use 0 para não limitar ou deixe em branco para desabilitar a ordenação. arch.config.optimize=Otimizar -arch.config.optimize.tip=Utilizará configurações otimizadas para que a instalação, atualização e reversão de pacotes sejam mais rápida, caso contrário utilizará a do sistema +arch.config.pacman_mthread_download=Download segmentado (repositórios) +arch.config.pacman_mthread_download.tip=Se os pacotes dos repositórios devem baixados através de uma ferramenta que trabalha com segmentação/threads (pode ser mais rápido). No momento esta propriedade somente funcionará se o pacote aria2 estiver instalado. arch.config.refresh_mirrors=Atualizar espelhos ao iniciar arch.config.refresh_mirrors.tip=Atualiza os espelhos de pacotes uma vez ao dia na inicialização arch.config.repos=Pacotes de repositórios @@ -153,6 +154,8 @@ arch.makepkg.optimizing=Otimizando a compilação arch.missing_deps.body=As seguintes {deps} dependências devem ser instaladas para que a instalação de {name} continue arch.missing_deps.title=Dependências ausentes arch.missing_deps_found=Dependencias ausentes para {} +arch.mthread_downloaded.error.cache_dir=Não foi possível criar o diretório para cache {} +arch.mthread_downloaded.error.cancelled=Operação cancelada arch.optdeps.checking=Verificando as dependências opcionais de {} arch.providers=provedores arch.substatus.conflicts=Verificando conflitos diff --git a/bauh/gems/arch/resources/locale/ru b/bauh/gems/arch/resources/locale/ru index 902ce41d..8c9f7ae1 100644 --- a/bauh/gems/arch/resources/locale/ru +++ b/bauh/gems/arch/resources/locale/ru @@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Ограничение сортировки зе arch.config.mirrors_sort_limit.tip=Определяет максимальное количество зеркал, которые будут использоваться для сортировки по скорости. Используйте 0 для No limit или оставьте его пустым, чтобы отключить сортировку. arch.config.optimize=Оптимизация arch.config.optimize.tip=Оптимизированные настройки будут использоваться для ускорения установки пакетов, в противном случае будут использоваться системные настройки +arch.config.pacman_mthread_download=Multithreaded download (repositories) +arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed arch.config.refresh_mirrors=Обновить зеркала при запуске arch.config.refresh_mirrors.tip=Обновляйте зеркала пакета один раз в день при запуске (или после перезагрузки устройства) arch.config.repos=Пакеты репозиториев @@ -153,6 +155,8 @@ arch.makepkg.optimizing=Оптимизация компиляции arch.missing_deps.body=Необходимо установить следующие зависимости , чтобы продолжить установку {name} arch.missing_deps.title=Отсутствующие зависимости arch.missing_deps_found=Отсутствуют зависимости для {} +arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {} +arch.mthread_downloaded.error.cancelled=Operation cancelled arch.optdeps.checking=Проверка необязательных обязательных зависимостей {} arch.providers=providers arch.substatus.conflicts=Checking for conflicts diff --git a/bauh/gems/arch/resources/locale/tr b/bauh/gems/arch/resources/locale/tr index e13a8fba..7139d3f3 100644 --- a/bauh/gems/arch/resources/locale/tr +++ b/bauh/gems/arch/resources/locale/tr @@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Yansı sıralama sınırı arch.config.mirrors_sort_limit.tip=Hız sıralama için kullanılacak maksimum yansı sayısını tanımlar. Sınırsız olması için 0 kullanın veya sıralamayı devre dışı bırakmak için boş bırakın. arch.config.optimize=optimize arch.config.optimize.tip=Paketlerin kurulumunu, yükseltilmesini ve indirilmesini hızlandırmak için optimize edilmiş ayarlar kullanılacak, aksi takdirde sistem ayarları kullanılacak +arch.config.pacman_mthread_download=Multithreaded download (repositories) +arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed arch.config.refresh_mirrors=Başlangıçta yansıları yenile arch.config.refresh_mirrors.tip=Paket yansılarını başlangıçta günde bir kez (veya cihaz yeniden başlatıldıktan sonra) yenileyin arch.config.repos=Depo paketleri @@ -153,6 +155,8 @@ arch.makepkg.optimizing=Derlemeyi optimize et arch.missing_deps.body={name} kurulumunun devam edebilmesi için aşağıdaki {deps} bağımlılık kurulmalıdır arch.missing_deps.title=Eksik bağımlılıklar arch.missing_deps_found={} için eksik bağımlılıklar +arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {} +arch.mthread_downloaded.error.cancelled=Operation cancelled arch.optdeps.checking={} İsteğe bağlı bağımlılıkları kontrol et arch.providers=sağlayıcılar arch.substatus.conflicts=Checking for conflicts diff --git a/bauh/view/core/downloader.py b/bauh/view/core/downloader.py index ed31827d..1dfbf7b0 100644 --- a/bauh/view/core/downloader.py +++ b/bauh/view/core/downloader.py @@ -3,12 +3,14 @@ import os import re import time import traceback +from math import floor +from threading import Thread from bauh.api.abstract.download import FileDownloader from bauh.api.abstract.handler import ProcessWatcher from bauh.api.http import HttpClient from bauh.commons.html import bold -from bauh.commons.system import run_cmd, new_subprocess, ProcessHandler, SystemProcess, SimpleProcess +from bauh.commons.system import run_cmd, ProcessHandler, SimpleProcess, get_human_size_str from bauh.view.util.translation import I18n RE_HAS_EXTENSION = re.compile(r'.+\.\w+$') @@ -25,11 +27,21 @@ class AdaptableFileDownloader(FileDownloader): def is_aria2c_available(self) -> bool: return bool(run_cmd('which aria2c', print_error=False)) - def _get_aria2c_process(self, url: str, output_path: str, cwd: str) -> SystemProcess: + def _get_aria2c_process(self, url: str, output_path: str, cwd: str, root_password: str, max_threads: int, known_size: int) -> SimpleProcess: + + if max_threads and max_threads > 0: + threads = max_threads + elif known_size: + threads = 16 if known_size >= 16000000 else floor(known_size / 1000000) + + if threads <= 0: + threads = 1 + else: + threads = 16 + cmd = ['aria2c', url, '--no-conf', - '--max-connection-per-server=16', - '--split=16', + '--max-connection-per-server={}'.format(threads), '--enable-color=false', '--stderr=true', '--summary-interval=0', @@ -39,36 +51,46 @@ class AdaptableFileDownloader(FileDownloader): '--continue=true', '--timeout=5', '--max-file-not-found=3', + '--file-allocation=falloc', '--remote-time=true'] + if threads > 1: + cmd.append('--split={}'.format(threads)) + if output_path: output_split = output_path.split('/') cmd.append('--dir=' + '/'.join(output_split[:-1])) cmd.append('--out=' + output_split[-1]) - return SystemProcess(new_subprocess(cmd=cmd, cwd=cwd), - skip_stdout=True, - check_error_output=False, - success_phrases=['download completed'], - output_delay=0.001) + return SimpleProcess(cmd=cmd, root_password=root_password, cwd=cwd) - def _get_wget_process(self, url: str, output_path: str, cwd: str) -> SimpleProcess: + def _get_wget_process(self, url: str, output_path: str, cwd: str, root_password: str) -> SimpleProcess: cmd = ['wget', url, '--continue', '--retry-connrefused', '--tries=10', '--no-config'] if output_path: cmd.append('-O') cmd.append(output_path) - return SimpleProcess(cmd=cmd, cwd=cwd) + return SimpleProcess(cmd=cmd, cwd=cwd, root_password=root_password) - def _rm_bad_file(self, file_name: str, output_path: str, cwd): + def _rm_bad_file(self, file_name: str, output_path: str, cwd, handler: ProcessHandler, root_password: str): to_delete = output_path if output_path else '{}/{}'.format(cwd, file_name) if to_delete and os.path.exists(to_delete): self.logger.info('Removing downloaded file {}'.format(to_delete)) - os.remove(to_delete) + success, _ = handler.handle_simple(SimpleProcess(['rm', '-rf',to_delete], root_password=root_password)) + return success - def download(self, file_url: str, watcher: ProcessWatcher, output_path: str = None, cwd: str = None) -> bool: + def _display_file_size(self, file_url: str, base_substatus, watcher: ProcessWatcher): + try: + size = self.http_client.get_content_length(file_url) + + if size: + watcher.change_substatus(base_substatus + ' ( {} )'.format(size)) + except: + pass + + def download(self, file_url: str, watcher: ProcessWatcher, output_path: str = None, cwd: str = None, root_password: str = None, substatus_prefix: str = None, display_file_size: bool = True, max_threads: int = None, known_size: int = None) -> bool: self.logger.info('Downloading {}'.format(file_url)) handler = ProcessHandler(watcher) file_name = file_url.split('/')[-1] @@ -85,39 +107,45 @@ class AdaptableFileDownloader(FileDownloader): if self.is_multithreaded(): ti = time.time() - process = self._get_aria2c_process(file_url, output_path, final_cwd) - downloader = 'aria2c' + process = self._get_aria2c_process(file_url, output_path, final_cwd, root_password, max_threads, known_size) + downloader = 'aria2' else: ti = time.time() - process = self._get_wget_process(file_url, output_path, final_cwd) + process = self._get_wget_process(file_url, output_path, final_cwd, root_password) downloader = 'wget' - file_size = self.http_client.get_content_length(file_url) - name = file_url.split('/')[-1] if output_path and not RE_HAS_EXTENSION.match(name) and RE_HAS_EXTENSION.match(output_path): name = output_path.split('/')[-1] - msg = bold('[{}] ').format(downloader) + self.i18n['downloading'] + ' ' + bold(name) + (' ( {} )'.format(file_size) if file_size else '') + if substatus_prefix: + msg = substatus_prefix + ' ' + else: + msg = '' + + msg += bold('[{}] ').format(downloader) + self.i18n['downloading'] + ' ' + bold(name) if watcher: watcher.change_substatus(msg) - if isinstance(process, SimpleProcess): - success = handler.handle_simple(process) - else: - success = handler.handle(process) + if display_file_size: + if known_size: + watcher.change_substatus(msg + ' ( {} )'.format(get_human_size_str(known_size))) + else: + Thread(target=self._display_file_size, args=(file_url, msg, watcher)).start() + + success, _ = handler.handle_simple(process) except: traceback.print_exc() - self._rm_bad_file(file_name, output_path, final_cwd) + self._rm_bad_file(file_name, output_path, final_cwd, handler, root_password) tf = time.time() self.logger.info(file_name + ' download took {0:.2f} minutes'.format((tf - ti) / 60)) if not success: self.logger.error("Could not download '{}'".format(file_name)) - self._rm_bad_file(file_name, output_path, final_cwd) + self._rm_bad_file(file_name, output_path, final_cwd, handler, root_password) return success diff --git a/bauh/view/resources/locale/en b/bauh/view/resources/locale/en index d96ecd7c..36ab76f8 100644 --- a/bauh/view/resources/locale/en +++ b/bauh/view/resources/locale/en @@ -191,7 +191,7 @@ core.config.backup=Enabled core.config.download.icons.tip=If the application icons should be downloaded and displayed on the table core.config.download.icons=Download icons core.config.download.multithreaded.tip=Whether applications, packages and files should be downloaded with a tool that works with threads ( faster ). At the moment this setting will only work if the aria2 package is installed -core.config.download.multithreaded=Multithreaded download +core.config.download.multithreaded=Multi-threaded download core.config.downloads=downloads core.config.locale.label=language core.config.mem_cache.data_exp.tip=Defines the in-memory data lifetime ( in SECONDS ) diff --git a/pictures/releases/0.9.4/arch_repo_mthread.png b/pictures/releases/0.9.4/arch_repo_mthread.png new file mode 100644 index 00000000..503211ff Binary files /dev/null and b/pictures/releases/0.9.4/arch_repo_mthread.png differ