[feature][arch] supporting multi-threaded download for repository packages

CHANGELOG.md

@@ -22,6 +22,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     <p align="center">
         <img src="https://raw.githubusercontent.com/vinifmor/bauh/staging/pictures/releases/0.9.4/updates_ignored_category.png">
     </p>
+    
+- Arch
+	- supporting multi-threaded download for repository packages (enabled by default)
+    <p align="center">
+        <img src="https://raw.githubusercontent.com/vinifmor/bauh/staging/pictures/releases/0.9.4/arch_repo_mthread.png">
+    </p>
 
 
 ### Improvements
@@ -36,6 +42,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     - defining a minimum width and height based on the screen size
     - info window now has a height limit, a lower bar with Back and Close buttons, and is scrollable
     - minor changes
+- Downloads
+    - retrieving the downloading file sizes asynchronously
 
 ### Fixes
 - Flatpak

README.md

@@ -182,6 +182,7 @@ refresh_mirrors_startup: false # if the package mirrors should be refreshed duri
 mirrors_sort_limit: 5  # defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting. 
 aur:  true  # allows to manage AUR packages
 repositories: true  # allows to manage packages from the configured repositories
+repositories_mthread_download: true  # enable multi-threaded download for repository packages if aria2 is installed
 ``` 
 - Required dependencies:
     - **pacman**

bauh/api/abstract/download.py

@@ -6,12 +6,17 @@ from bauh.api.abstract.handler import ProcessWatcher
 class FileDownloader(ABC):
 
     @abstractmethod
-    def download(self, file_url: str, watcher: ProcessWatcher, output_path: str, cwd: str) -> bool:
+    def download(self, file_url: str, watcher: ProcessWatcher, output_path: str, cwd: str, root_password: str = None, substatus_prefix: str = None, display_file_size: bool = True, max_threads: int = None, known_size: int = None) -> bool:
         """
         :param file_url:
         :param watcher:
         :param output_path: the downloaded file output path. Leave None for the current directory and the same file name
         :param cwd: current working directory. Leave None if does not matter.
+        :param root_password: (if the output directory is protected)
+        :param substatus_prefix: custom substatus prefix ('prefix downloading xpto')
+        :param display_file_size: if the file size should be displayed on the substatus
+        :param max_threads: maximum number of threads (only available for multi-threaded download)
+        :param known_size: known file size
         :return: success / failure
         """
         pass

bauh/api/http.py

@@ -89,11 +89,12 @@ class HttpClient:
         if size:
             return system.get_human_size_str(size)
 
-    def exists(self, url: str, session: bool = True) -> bool:
+    def exists(self, url: str, session: bool = True, timeout: int = 5) -> bool:
-        params = {'url': url, 'allow_redirects': True, 'verify': False, 'timeout': 5}
+        params = {'url': url, 'allow_redirects': True, 'verify': False, 'timeout': timeout}
         if session:
             res = self.session.head(**params)
         else:
             res = self.session.get(**params)
 
         return res.status_code in (200, 403)
+        return False

bauh/commons/system.py

@@ -272,3 +272,8 @@ def get_human_size_str(size) -> str:
         if len(size_str.split('.')[0]) < 4:
             return '{0:.2f}'.format(float(size_str)) + ' ' + m[1]
     return str(int_size)
+
+
+def run(cmd: List[str], success_code: int = 0) -> Tuple[bool, str]:
+    p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    return p.returncode == success_code, p.stdout.decode()

bauh/gems/arch/__init__.py

@@ -6,6 +6,7 @@ from bauh.commons import resource
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 BUILD_DIR = '{}/arch'.format(TEMP_DIR)
+PACKAGE_CACHE_DIR = '{}/pkg_cache'.format(BUILD_DIR)
 ARCH_CACHE_PATH = CACHE_PATH + '/arch'
 CATEGORIES_FILE_PATH = ARCH_CACHE_PATH + '/categories.txt'
 URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/arch/categories.txt'

bauh/gems/arch/config.py

@@ -10,5 +10,6 @@ def read_config(update_file: bool = False) -> dict:
                 'repositories': True,
                 "refresh_mirrors_startup": False,
                 "sync_databases_startup": True,
-                'mirrors_sort_limit': 5}
+                'mirrors_sort_limit': 5,
+                'repositories_mthread_download': True}
     return read(CONFIG_FILE, template, update_file=update_file)

bauh/gems/arch/confirmation.py

@@ -44,7 +44,7 @@ def request_optional_deps(pkgname: str, pkg_repos: dict, watcher: ProcessWatcher
         return {o.value for o in view_opts.values}
 
 
-def request_install_missing_deps(pkgname: str, deps: List[Tuple[str, str]], watcher: ProcessWatcher, i18n: I18n) -> bool:
+def request_install_missing_deps(pkgname: str, deps: List[Tuple[str, str]], watcher: ProcessWatcher, i18n: I18n, context: "TransactionContext") -> bool:
     msg = '<p>{}</p>'.format(i18n['arch.missing_deps.body'].format(name=bold(pkgname) if pkgname else '', deps=bold(str(len(deps)))))
 
     opts = []
@@ -52,6 +52,9 @@ def request_install_missing_deps(pkgname: str, deps: List[Tuple[str, str]], watc
     repo_deps = [d[0] for d in deps if d[1] != 'aur']
     sizes = pacman.get_update_size(repo_deps) if repo_deps else {}
 
+    if context and sizes:
+        context.pkg_sizes.update(sizes)
+
     for dep in deps:
         size = sizes.get(dep[0])
         op = InputOption('{} ({}: {}) - {}: {}'.format(dep[0],

bauh/gems/arch/controller.py

@@ -35,6 +35,7 @@ from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, message, confirmatio
 from bauh.gems.arch.aur import AURClient
 from bauh.gems.arch.config import read_config
 from bauh.gems.arch.dependencies import DependenciesAnalyser
+from bauh.gems.arch.download import MultithreadedDownloadService, ArchDownloadException
 from bauh.gems.arch.exceptions import PackageNotFoundException
 from bauh.gems.arch.mapper import ArchDataMapper
 from bauh.gems.arch.model import ArchPackage
@@ -62,7 +63,7 @@ class TransactionContext:
                  install_file: str = None, repository: str = None, pkg: ArchPackage = None,
                  remote_repo_map: Dict[str, str] = None, provided_map: Dict[str, Set[str]] = None,
                  remote_provided_map: Dict[str, Set[str]] = None, aur_idx: Set[str] = None,
-                 missing_deps: List[Tuple[str, str]] = None):
+                 missing_deps: List[Tuple[str, str]] = None, pkg_sizes: Dict[str, int] = {}):
         self.name = name
         self.base = base
         self.maintainer = maintainer
@@ -83,6 +84,7 @@ class TransactionContext:
         self.remote_provided_map = remote_provided_map
         self.aur_idx = aur_idx
         self.missing_deps = missing_deps
+        self.pkg_sizes = pkg_sizes
 
     @classmethod
     def gen_context_from(cls, pkg: ArchPackage, arch_config: dict, root_password: str, handler: ProcessHandler) -> "TransactionContext":
@@ -747,13 +749,23 @@ class ArchManager(SoftwareManager):
 
         return files
 
-    def _upgrade_repo_pkgs(self, pkgs: List[str], handler: ProcessHandler, root_password: str, overwrite_files: bool = False,
+    def _upgrade_repo_pkgs(self, pkgs: List[str], handler: ProcessHandler, root_password: str, arch_config: dict, overwrite_files: bool = False,
-                           status_handler: TransactionStatusHandler = None) -> bool:
+                           status_handler: TransactionStatusHandler = None, already_downloaded: bool = False, sizes: Dict[str, int] = None) -> bool:
+
+        downloaded = 0
+
+        if not already_downloaded:
+            if self._should_download_packages(arch_config):
+                try:
+                    downloaded = self._download_packages(pkgs, handler, root_password, sizes)
+                except ArchDownloadException:
+                    return False
+
         try:
             if status_handler:
                 output_handler = status_handler
             else:
-                output_handler = TransactionStatusHandler(handler.watcher, self.i18n, len(pkgs), self.logger)
+                output_handler = TransactionStatusHandler(handler.watcher, self.i18n, len(pkgs), self.logger, downloading=downloaded)
                 output_handler.start()
 
             success, upgrade_output = handler.handle_simple(pacman.upgrade_several(pkgnames=pkgs,
@@ -791,7 +803,10 @@ class ArchManager(SoftwareManager):
                                                    handler=handler,
                                                    root_password=root_password,
                                                    overwrite_files=True,
-                                                   status_handler=output_handler)
+                                                   status_handler=output_handler,
+                                                   already_downloaded=True,
+                                                   arch_config=arch_config,
+                                                   sizes=sizes)
                 else:
                     output_handler.stop_working()
                     output_handler.join()
@@ -819,7 +834,7 @@ class ArchManager(SoftwareManager):
             watcher.change_substatus('')
             return False
 
-        aur_pkgs, repo_pkgs = [], []
+        aur_pkgs, repo_pkgs, pkg_sizes = [], [], {}
 
         for req in (*requirements.to_install, *requirements.to_upgrade):
             if req.pkg.repository == 'aur':
@@ -827,6 +842,8 @@ class ArchManager(SoftwareManager):
             else:
                 repo_pkgs.append(req.pkg)
 
+            pkg_sizes[req.pkg.name] = req.required_size
+
         if aur_pkgs and not self._check_action_allowed(aur_pkgs[0], watcher):
             return False
 
@@ -852,7 +869,11 @@ class ArchManager(SoftwareManager):
             self.logger.info("Upgrading {} repository packages: {}".format(len(repo_pkgs_names),
                                                                            ', '.join(repo_pkgs_names)))
 
-            if not self._upgrade_repo_pkgs(pkgs=repo_pkgs_names, handler=handler, root_password=root_password):
+            if not self._upgrade_repo_pkgs(pkgs=repo_pkgs_names,
+                                           handler=handler,
+                                           root_password=root_password,
+                                           arch_config=arch_config,
+                                           sizes=pkg_sizes):
                 return False
 
         if aur_pkgs:
@@ -1263,7 +1284,7 @@ class ArchManager(SoftwareManager):
 
             return True
 
-    def _install_deps(self, context: TransactionContext, deps: List[Tuple[str, str]]) -> Iterable[str]:
+    def  _install_deps(self, context: TransactionContext, deps: List[Tuple[str, str]]) -> Iterable[str]:
         """
         :param pkgs_repos:
         :param root_password:
@@ -1305,7 +1326,15 @@ class ArchManager(SoftwareManager):
                                 if not self._request_conflict_resolution(dep, conflict_pkg , context):
                                     return {dep}
 
-            status_handler = TransactionStatusHandler(context.watcher, self.i18n, len(repo_dep_names), self.logger, percentage=len(repo_deps) > 1)
+            downloaded = 0
+            if self._should_download_packages(context.config):
+                try:
+                    downloaded = self._download_packages(repo_dep_names, context.handler, context.root_password, context.pkg_sizes)
+                except ArchDownloadException:
+                    return False
+
+            status_handler = TransactionStatusHandler(watcher=context.watcher, i18n=self.i18n, npkgs=len(repo_dep_names),
+                                                      logger=self.logger, percentage=len(repo_deps) > 1, downloading=downloaded)
             status_handler.start()
             installed, _ = context.handler.handle_simple(pacman.install_as_process(pkgpaths=repo_dep_names,
                                                                                    root_password=context.root_password,
@@ -1426,7 +1455,7 @@ class ArchManager(SoftwareManager):
     def _ask_and_install_missing_deps(self, context: TransactionContext,  missing_deps: List[Tuple[str, str]]) -> bool:
         context.watcher.change_substatus(self.i18n['arch.missing_deps_found'].format(bold(context.name)))
 
-        if not confirmation.request_install_missing_deps(context.name, missing_deps, context.watcher, self.i18n):
+        if not confirmation.request_install_missing_deps(context.name, missing_deps, context.watcher, self.i18n, context):
             context.watcher.print(self.i18n['action.cancelled'])
             return False
 
@@ -1583,7 +1612,7 @@ class ArchManager(SoftwareManager):
 
                 sorted_deps = sorting.sort(to_sort, {**deps_data, **subdeps_data}, provided_map)
 
-                if display_deps_dialog and not confirmation.request_install_missing_deps(None, sorted_deps, context.watcher, self.i18n):
+                if display_deps_dialog and not confirmation.request_install_missing_deps(None, sorted_deps, context.watcher, self.i18n, context):
                     context.watcher.print(self.i18n['action.cancelled'])
                     return True  # because the main package installation was successful
 
@@ -1597,7 +1626,21 @@ class ArchManager(SoftwareManager):
 
         return True
 
-    def _install(self, context: TransactionContext):
+    def _should_download_packages(self, arch_config: dict) -> bool:
+        return bool(arch_config['repositories_mthread_download']) and self.context.file_downloader.is_multithreaded()
+
+    def _download_packages(self, pkgnames: List[str], handler: ProcessHandler, root_password: str, sizes: Dict[str, int] = None) -> int:
+        download_service = MultithreadedDownloadService(file_downloader=self.context.file_downloader,
+                                                        http_client=self.http_client,
+                                                        logger=self.context.logger,
+                                                        i18n=self.i18n)
+        self.logger.info("Initializing multi-threaded download for {} package(s)".format(len(pkgnames)))
+        return download_service.download_packages(pkgs=pkgnames,
+                                                  handler=handler,
+                                                  sizes=sizes,
+                                                  root_password=root_password)
+
+    def _install(self, context: TransactionContext) -> bool:
         check_install_output = []
         pkgpath = context.get_package_path()
 
@@ -1635,7 +1678,6 @@ class ArchManager(SoftwareManager):
         else:
             self.logger.info("No conflict detected for '{}'".format(context.name))
 
-        context.watcher.change_substatus(self.i18n['arch.installing.package'].format(bold(context.name)))
         self._update_progress(context, 80)
 
         to_install = []
@@ -1645,12 +1687,25 @@ class ArchManager(SoftwareManager):
 
         to_install.append(pkgpath)
 
+        downloaded = 0
+        if self._should_download_packages(context.config):
+            to_download = [p for p in to_install if not p.startswith('/')]
+
+            if to_download:
+                try:
+                    downloaded = self._download_packages(to_download, context.handler, context.root_password, context.pkg_sizes)
+                except ArchDownloadException:
+                    return False
+
         if not context.dependency:
-            status_handler = TransactionStatusHandler(context.watcher, self.i18n, len(to_install), self.logger, percentage=len(to_install) > 1) if not context.dependency else None
+            status_handler = TransactionStatusHandler(context.watcher, self.i18n, len(to_install), self.logger,
+                                                      percentage=len(to_install) > 1,
+                                                      downloading=downloaded) if not context.dependency else None
             status_handler.start()
         else:
             status_handler = None
 
+        context.watcher.change_substatus(self.i18n['arch.installing.package'].format(bold(context.name)))
         installed, _ = context.handler.handle_simple(pacman.install_as_process(pkgpaths=to_install,
                                                                                root_password=context.root_password,
                                                                                file=context.has_install_file(),
@@ -1848,7 +1903,7 @@ class ArchManager(SoftwareManager):
             context.missing_deps = missing_deps
             context.watcher.change_substatus(self.i18n['arch.missing_deps_found'].format(bold(context.name)))
 
-            if not confirmation.request_install_missing_deps(context.name, missing_deps, context.watcher, self.i18n):
+            if not confirmation.request_install_missing_deps(context.name, missing_deps, context.watcher, self.i18n, context):
                 context.watcher.print(self.i18n['action.cancelled'])
                 return False
 
@@ -2043,6 +2098,12 @@ class ArchManager(SoftwareManager):
                                     tooltip_key='arch.config.optimize.tip',
                                     value=bool(local_config['optimize']),
                                     max_width=max_width),
+            self._gen_bool_selector(id_='mthread_download',
+                                    label_key='arch.config.pacman_mthread_download',
+                                    tooltip_key='arch.config.pacman_mthread_download.tip',
+                                    value=local_config['repositories_mthread_download'],
+                                    max_width=max_width,
+                                    capitalize_label=True),
             self._gen_bool_selector(id_='sync_dbs',
                                     label_key='arch.config.sync_dbs',
                                     tooltip_key='arch.config.sync_dbs.tip',
@@ -2081,6 +2142,7 @@ class ArchManager(SoftwareManager):
         config['clean_cached'] = form_install.get_component('clean_cached').get_selected()
         config['refresh_mirrors_startup'] = form_install.get_component('ref_mirs').get_selected()
         config['mirrors_sort_limit'] = form_install.get_component('mirrors_sort_limit').get_int_value()
+        config['repositories_mthread_download'] = form_install.get_component('mthread_download').get_selected()
 
         try:
             save_config(config, CONFIG_FILE)

bauh/gems/arch/download.py

@@ -0,0 +1,155 @@
+import glob
+import logging
+import os
+import time
+import traceback
+from math import floor
+from typing import List, Iterable, Dict
+
+from bauh.api.abstract.download import FileDownloader
+from bauh.api.abstract.handler import ProcessWatcher
+from bauh.api.abstract.view import MessageType
+from bauh.api.http import HttpClient
+from bauh.commons.html import bold
+from bauh.commons.system import ProcessHandler, SimpleProcess
+from bauh.gems.arch import pacman
+from bauh.view.util.translation import I18n
+
+
+class ArchDownloadException(Exception):
+    pass
+
+
+class CacheDirCreationException(ArchDownloadException):
+    pass
+
+
+class MultiThreadedDownloader:
+
+    def __init__(self, file_downloader: FileDownloader, http_client: HttpClient, mirrors_available: Iterable[str],
+                 mirrors_branch: str, cache_dir: str, logger: logging.Logger):
+        self.downloader = file_downloader
+        self.http_client = http_client
+        self.mirrors = mirrors_available
+        self.branch = mirrors_branch
+        self.extensions = ['.tar.zst', '.tar.xz']
+        self.cache_dir = cache_dir
+        self.logger = logger
+
+    def download_package(self, pkg: Dict[str, str], root_password: str, substatus_prefix: str, watcher: ProcessWatcher, size: int) -> bool:
+        if self.mirrors and self.branch:
+            pkgname = '{}-{}{}.pkg'.format(pkg['n'], pkg['v'], ('-{}'.format(pkg['a']) if pkg['a'] else ''))
+
+            if {f for f in glob.glob(self.cache_dir + '/*') if f.split('/')[-1].startswith(pkgname)}:
+                watcher.print("{} ({}) file found o cache dir {}. Skipping download.".format(pkg['n'], pkg['v'], self.cache_dir))
+                return True
+
+            arch = pkg['a'] if pkg.get('a') and pkg['a'] != 'any' else 'x86_64'
+
+            url_base = '{}/{}/{}/{}'.format(self.branch, pkg['r'], arch, pkgname)
+            base_output_path = '{}/{}'.format(self.cache_dir, pkgname)
+
+            for mirror in self.mirrors:
+                for ext in self.extensions:
+                    url = '{}{}{}'.format(mirror, url_base, ext)
+                    output_path = base_output_path + ext
+
+                    watcher.print("Downloading '{}' from mirror '{}'".format(pkgname, mirror))
+
+                    pkg_downloaded = self.downloader.download(file_url=url, watcher=watcher, output_path=output_path,
+                                                              cwd='.', root_password=root_password, display_file_size=True,
+                                                              substatus_prefix=substatus_prefix,
+                                                              known_size=size)
+                    if not pkg_downloaded:
+                        watcher.print("Could not download '{}' from mirror '{}'".format(pkgname, mirror))
+                    else:
+                        self.logger.info("Package '{}' successfully downloaded".format(pkg['n']))
+                        self.logger.info("Downloading package '{}' signature".format(pkg['n']))
+
+                        sig_downloaded = self.downloader.download(file_url=url + '.sig', watcher=watcher,
+                                                                  output_path=output_path + '.sig',
+                                                                  cwd='.', root_password=root_password,
+                                                                  display_file_size=False,
+                                                                  substatus_prefix=substatus_prefix,
+                                                                  max_threads=1)
+
+                        if not sig_downloaded:
+                            self.logger.warning("Could not download package '{}' signature".format(pkg['n']))
+                        else:
+                            self.logger.info("Package '{}' signature successfully downloaded".format(pkg['n']))
+                            return True
+        return False
+
+
+class MultithreadedDownloadService:
+
+    def __init__(self, file_downloader: FileDownloader, http_client: HttpClient, logger: logging.Logger, i18n: I18n):
+        self.file_downloader = file_downloader
+        self.http_client = http_client
+        self.logger = logger
+        self.i18n = i18n
+
+    def download_packages(self, pkgs: List[str], handler: ProcessHandler, root_password: str, sizes: Dict[str, int] = None) -> int:
+        ti = time.time()
+        watcher = handler.watcher
+        mirrors = pacman.list_available_mirrors()
+
+        if not mirrors:
+            self.logger.warning('repository mirrors seem to be not reachable')
+            watcher.print('[warning] repository mirrors seem to be not reachable')
+            watcher.print('[warning] multi-threaded download cancelled')
+            return 0
+
+        branch = pacman.get_mirrors_branch()
+
+        if not branch:
+            self.logger.warning('no default repository branch found')
+            watcher.print('[warning] no default repository branch found')
+            watcher.print('[warning] multi-threaded download cancelled')
+            return 0
+
+        cache_dir = pacman.get_cache_dir()
+
+        if not os.path.exists(cache_dir):
+            success, _ = handler.handle_simple(SimpleProcess(['mkdir', '-p', cache_dir], root_password=root_password))
+
+            if not success:
+                msg = "could not create cache dir '{}'".format(cache_dir)
+                self.logger.warning(msg)
+                watcher.print("[warning] {}".format(cache_dir))
+                watcher.show_message(title=self.i18n['warning'].capitalize(),
+                                     body=self.i18n['arch.mthread_downloaded.error.cache_dir'].format(bold(cache_dir)),
+                                     type_=MessageType.WARNING)
+                raise CacheDirCreationException()
+
+        downloader = MultiThreadedDownloader(file_downloader=self.file_downloader,
+                                             mirrors_available=mirrors,
+                                             mirrors_branch=branch,
+                                             http_client=self.http_client,
+                                             logger=self.logger,
+                                             cache_dir=cache_dir)
+
+        downloaded = 0
+        pkgs_data = pacman.list_download_data(pkgs)
+
+        for pkg in pkgs_data:
+            self.logger.info('Preparing to download package: {} ({})'.format(pkg['n'], pkg['v']))
+            try:
+                perc = '({0:.2f}%)'.format((downloaded / (2 * len(pkgs))) * 100)
+                status_prefix = '{} [{}/{}]'.format(perc, downloaded + 1, len(pkgs))
+
+                if downloader.download_package(pkg=pkg,
+                                               root_password=root_password,
+                                               watcher=handler.watcher,
+                                               substatus_prefix=status_prefix,
+                                               size=sizes.get(pkg['n']) if sizes else None):
+                    downloaded += 1
+            except:
+                traceback.print_exc()
+                watcher.show_message(title=self.i18n['error'].capitalize(),
+                                     body=self.i18n['arch.mthread_downloaded.error.cancelled'],
+                                     type_=MessageType.ERROR)
+                raise ArchDownloadException()
+        tf = time.time()
+        self.logger.info("Download time: {0:.2f} seconds".format(tf - ti))
+        return downloaded

bauh/gems/arch/model.py

@@ -16,7 +16,7 @@ class ArchPackage(SoftwarePackage):
                  first_submitted: datetime.datetime = None, last_modified: datetime.datetime = None,
                  maintainer: str = None, url_download: str = None, pkgbuild: str = None, repository: str = None,
                  desktop_entry: str = None, installed: bool = False, srcinfo: dict = None, dependencies: Set[str] = None,
-                 categories: List[str] = None, i18n: I18n = None, update_ignored: bool = False):
+                 categories: List[str] = None, i18n: I18n = None, update_ignored: bool = False, arch: str = None):
 
         super(ArchPackage, self).__init__(name=name, version=version, latest_version=latest_version, description=description,
                                           installed=installed, categories=categories)
@@ -35,6 +35,7 @@ class ArchPackage(SoftwarePackage):
         self.desktop_entry = desktop_entry
         self.src_info = srcinfo
         self.dependencies = dependencies
+        self.arch = arch
         self.i18n = i18n
         self.update_ignored = update_ignored
 

bauh/gems/arch/output.py

@@ -8,12 +8,13 @@ from bauh.view.util.translation import I18n
 
 class TransactionStatusHandler(Thread):
 
-    def __init__(self, watcher: ProcessWatcher, i18n: I18n, npkgs: int, logger: logging.Logger, percentage: bool = True):
+    def __init__(self, watcher: ProcessWatcher, i18n: I18n, npkgs: int, logger: logging.Logger,
+                 percentage: bool = True, downloading: int = 0):
         super(TransactionStatusHandler, self).__init__(daemon=True)
         self.watcher = watcher
         self.i18n = i18n
         self.npkgs = npkgs
-        self.downloading = 0
+        self.downloading = downloading
         self.upgrading = 0
         self.installing = 0
         self.outputs = []

bauh/gems/arch/pacman.py

@@ -3,6 +3,7 @@ import re
 from threading import Thread
 from typing import List, Set, Tuple, Dict, Iterable
 
+from bauh.commons import system
 from bauh.commons.system import run_cmd, new_subprocess, new_root_subprocess, SystemProcess, SimpleProcess, \
     ProcessHandler
 from bauh.commons.util import size_to_byte
@@ -16,6 +17,7 @@ RE_INSTALLED_FIELDS = re.compile(r'(Name|Description|Version|Validated By)\s*:\s
 RE_INSTALLED_SIZE = re.compile(r'Installed Size\s*:\s*([0-9,\.]+)\s(\w+)\n?', re.IGNORECASE)
 RE_UPDATE_REQUIRED_FIELDS = re.compile(r'(\bProvides\b|\bInstalled Size\b|\bConflicts With\b)\s*:\s(.+)\n')
 RE_REMOVE_TRANSITIVE_DEPS = re.compile(r'removing\s([\w\-_]+)\s.+required\sby\s([\w\-_]+)\n?')
+RE_AVAILABLE_MIRRORS = re.compile(r'.+\s+OK\s+.+\s+(\d+:\d+)\s+.+(http.+)')
 
 
 def is_available() -> bool:
@@ -599,6 +601,36 @@ def map_provided(remote: bool = False, pkgs: Iterable[str] = None) -> Dict[str,
         return provided_map
 
 
+def list_download_data(pkgs: Iterable[str]) -> List[Dict[str, str]]:
+    _, output = system.run(['pacman', '-Si', *pkgs])
+
+    if output:
+        res = []
+        data = {'a': None, 'v': None, 'r': None, 'n': None}
+
+        for l in output.split('\n'):
+            if l:
+                if l[0] != ' ':
+                    line = l.strip()
+                    field_sep_idx = line.index(':')
+                    field = line[0:field_sep_idx].strip()
+                    val = line[field_sep_idx + 1:].strip()
+
+                    if field == 'Repository':
+                        data['r'] = val
+                    elif field == 'Name':
+                        data['n'] = val
+                    elif field == 'Version':
+                        data['v'] = val.split('=')[0]
+                    elif field == 'Architecture':
+                        data['a'] = val
+                    elif data.get('a'):
+                        res.append(data)
+                        data = {'a': None, 'v': None, 'r': None, 'n': None}
+
+        return res
+
+
 def map_updates_data(pkgs: Iterable[str], files: bool = False) -> dict:
     if files:
         output = run_cmd('pacman -Qi -p {}'.format(' '.join(pkgs)))
@@ -1021,3 +1053,19 @@ def list_installed_names() -> Set[str]:
     output = run_cmd('pacman -Qq', print_error=False)
     return {name.strip() for name in output.split('\n') if name} if output else set()
 
+
+def list_available_mirrors() -> List[str]:
+    _, output = system.run(['pacman-mirrors', '--status', '--no-color'])
+
+    if output:
+        mirrors = RE_AVAILABLE_MIRRORS.findall(output)
+
+        if mirrors:
+            mirrors.sort(key=lambda o: o[0])
+            return [m[1] for m in mirrors]
+
+
+def get_mirrors_branch() -> str:
+    _, output = system.run(['pacman-mirrors', '-G'])
+    return output.strip()
+

bauh/gems/arch/resources/locale/ca

@@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Mirrors sort limit
 arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting.
 arch.config.optimize=optimize
 arch.config.optimize.tip=La configuració optimitzada s'utilitzarà per fer més ràpida la instal·lació, actualització i reversió dels paquets, en cas contrari s'utilitzarà la configuració del sistema.
+arch.config.pacman_mthread_download=Multithreaded download (repositories)
+arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed
 arch.config.refresh_mirrors=Refresh mirrors on startup
 arch.config.refresh_mirrors.tip=Refresh the package mirrors once a day on startup ( or after a device reboot )
 arch.config.repos=Repositories packages
@@ -153,6 +155,8 @@ arch.makepkg.optimizing=Optimitzant la recopilació
 arch.missing_deps.body=S’han d’instal·lar les {deps} dependències següents abans de continuar amb la instal·lació de {name}
 arch.missing_deps.title=Dependències mancants
 arch.missing_deps_found=Dependències mancants per a {}
+arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {}
+arch.mthread_downloaded.error.cancelled=Operation cancelled
 arch.optdeps.checking=S’estan comprovant les dependències opcionals de {}
 arch.providers=providers
 arch.substatus.conflicts=Checking for conflicts

bauh/gems/arch/resources/locale/de

@@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Mirrors sort limit
 arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting.
 arch.config.optimize=optimize
 arch.config.optimize.tip=Optimized settings will be used in order to make the packages installation, upgrading and downgrading faster, otherwise the system settings will be used
+arch.config.pacman_mthread_download=Multithreaded download (repositories)
+arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed
 arch.config.refresh_mirrors=Refresh mirrors on startup
 arch.config.refresh_mirrors.tip=Refresh the package mirrors once a day on startup ( or after a device reboot )
 arch.config.repos=Repositories packages
@@ -153,6 +155,8 @@ arch.makepkg.optimizing=Optimiert die Zusammenstellung
 arch.missing_deps.body=Die folgenden {deps} Abhängigkeiten müssten installiert sein, bevor mit der {name} Installation fortgefahren werden kann
 arch.missing_deps.title=Fehlende Abhängigkeiten
 arch.missing_deps_found=Fehlende Abhängigkeiten für {}
+arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {}
+arch.mthread_downloaded.error.cancelled=Operation cancelled
 arch.optdeps.checking={} optionale Abhängigkeiten überprüfen
 arch.providers=providers
 arch.substatus.conflicts=Checking for conflicts

bauh/gems/arch/resources/locale/en

@@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Mirrors sort limit
 arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting.
 arch.config.optimize=optimize
 arch.config.optimize.tip=Optimized settings will be used in order to make the packages installation, upgrading and downgrading faster, otherwise the system settings will be used
+arch.config.pacman_mthread_download=Multi-threaded download (repositories)
+arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed
 arch.config.refresh_mirrors=Refresh mirrors on startup
 arch.config.refresh_mirrors.tip=Refresh the package mirrors once a day on startup ( or after a device reboot )
 arch.config.repos=Repositories packages
@@ -153,6 +155,8 @@ arch.makepkg.optimizing=Optimizing the compilation
 arch.missing_deps.body=The following {deps} dependencies must be installed so the {name} installation can continue
 arch.missing_deps.title=Missing dependencies
 arch.missing_deps_found=Missing dependencies for {}
+arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {}
+arch.mthread_downloaded.error.cancelled=Operation cancelled
 arch.optdeps.checking=Checking {} optional dependencies
 arch.providers=providers
 arch.substatus.conflicts=Checking for conflicts

bauh/gems/arch/resources/locale/es

@@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Límite de ordenación de espejos
 arch.config.mirrors_sort_limit.tip=Define el número máximo de espejos que se utilizarán para la ordenación por velocidad. Use 0 para no limitar o déjelo en blanco para deshabilitar la clasificación.
 arch.config.optimize=optimizar
 arch.config.optimize.tip=Se usará la configuración optimizada para que la instalación, actualización y reversión de los paquetes sean más rápidas, de lo contrario se usará la configuración del sistema
+arch.config.pacman_mthread_download=Descarga segmentada (repositorios)
+arch.config.pacman_mthread_download.tip=Si los paquetes de los repositorios deben descargarse con una herramienta que usa segmentación/threads (puede ser más rápido). Por el momento, esta configuración solo funcionará si el paquete aria2 esté instalado
 arch.config.refresh_mirrors=Actualizar espejos al iniciar
 arch.config.refresh_mirrors.tip=Actualiza los espejos de paquetes una vez al día al iniciar ( o después de reiniciar el dispositivo )
 arch.config.repos=Paquetes de repositorios
@@ -153,6 +155,8 @@ arch.makepkg.optimizing=Optimizing the compilation
 arch.missing_deps.body=Deben instalarse las siguientes {deps} dependencias para que la instalación de {name} pueda continuar
 arch.missing_deps.title=Dependencias faltantes
 arch.missing_deps_found=Dependencias faltantes para {}
+arch.mthread_downloaded.error.cache_dir=No fue posible crear el directorio de caché {}
+arch.mthread_downloaded.error.cancelled=Operación cancelada
 arch.optdeps.checking=Verificando las dependencias opcionales de {}
 arch.providers=proveedores
 arch.substatus.conflicts=Verificando conflictos

bauh/gems/arch/resources/locale/it

@@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Mirrors sort limit
 arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting.
 arch.config.optimize=optimize
 arch.config.optimize.tip=Verranno utilizzate le impostazioni ottimizzate per velocizzare l'installazione, l'aggiornamento e l'inversione dei pacchetti, altrimenti verranno utilizzate le impostazioni di sistema
+arch.config.pacman_mthread_download=Multithreaded download (repositories)
+arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed
 arch.config.refresh_mirrors=Refresh mirrors on startup
 arch.config.refresh_mirrors.tip=Refresh the package mirrors once a day on startup ( or after a device reboot )
 arch.config.repos=Repositories packages
@@ -153,6 +155,8 @@ arch.makepkg.optimizing=Ottimizzando la compilazione
 arch.missing_deps.body=Le seguenti {deps} dipendenze devono essere installate prima che l'installazione di {name} continui
 arch.missing_deps.title=Dipendenze mancanti
 arch.missing_deps_found=Dipendenze mancanti per {}
+arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {}
+arch.mthread_downloaded.error.cancelled=Operation cancelled
 arch.optdeps.checking=Verifica di {} dipendenze opzionali
 arch.providers=providers
 arch.substatus.conflicts=Checking for conflicts

bauh/gems/arch/resources/locale/pt

@@ -27,7 +27,8 @@ arch.config.clean_cache.tip=Se versões antigas de um pacote armazenadas em disc
 arch.config.mirrors_sort_limit=Limite de ordenação de espelhos
 arch.config.mirrors_sort_limit.tip=Define o número máximo de espelhos que serão utilizados para a ordenação por velocidade. Use 0 para não limitar ou deixe em branco para desabilitar a ordenação.
 arch.config.optimize=Otimizar
-arch.config.optimize.tip=Utilizará configurações otimizadas para que a instalação, atualização e reversão de pacotes sejam mais rápida, caso contrário utilizará a do sistema
+arch.config.pacman_mthread_download=Download segmentado (repositórios)
+arch.config.pacman_mthread_download.tip=Se os pacotes dos repositórios devem baixados através de uma ferramenta que trabalha com segmentação/threads (pode ser mais rápido). No momento esta propriedade somente funcionará se o pacote aria2 estiver instalado.
 arch.config.refresh_mirrors=Atualizar espelhos ao iniciar
 arch.config.refresh_mirrors.tip=Atualiza os espelhos de pacotes uma vez ao dia na inicialização
 arch.config.repos=Pacotes de repositórios
@@ -153,6 +154,8 @@ arch.makepkg.optimizing=Otimizando a compilação
 arch.missing_deps.body=As seguintes {deps} dependências devem ser instaladas para que a instalação de {name} continue
 arch.missing_deps.title=Dependências ausentes
 arch.missing_deps_found=Dependencias ausentes para {}
+arch.mthread_downloaded.error.cache_dir=Não foi possível criar o diretório para cache {}
+arch.mthread_downloaded.error.cancelled=Operação cancelada
 arch.optdeps.checking=Verificando as dependências opcionais de {}
 arch.providers=provedores
 arch.substatus.conflicts=Verificando conflitos

bauh/gems/arch/resources/locale/ru

@@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Ограничение сортировки зе
 arch.config.mirrors_sort_limit.tip=Определяет максимальное количество зеркал, которые будут использоваться для сортировки по скорости. Используйте 0 для No limit или оставьте его пустым, чтобы отключить сортировку.
 arch.config.optimize=Оптимизация
 arch.config.optimize.tip=Оптимизированные настройки будут использоваться для ускорения установки пакетов, в противном случае будут использоваться системные настройки
+arch.config.pacman_mthread_download=Multithreaded download (repositories)
+arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed
 arch.config.refresh_mirrors=Обновить зеркала при запуске
 arch.config.refresh_mirrors.tip=Обновляйте зеркала пакета один раз в день при запуске (или после перезагрузки устройства)
 arch.config.repos=Пакеты репозиториев
@@ -153,6 +155,8 @@ arch.makepkg.optimizing=Оптимизация компиляции
 arch.missing_deps.body=Необходимо установить следующие зависимости , чтобы продолжить установку {name}
 arch.missing_deps.title=Отсутствующие зависимости
 arch.missing_deps_found=Отсутствуют зависимости для {}
+arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {}
+arch.mthread_downloaded.error.cancelled=Operation cancelled
 arch.optdeps.checking=Проверка необязательных обязательных зависимостей {}
 arch.providers=providers
 arch.substatus.conflicts=Checking for conflicts

bauh/gems/arch/resources/locale/tr

@@ -28,6 +28,8 @@ arch.config.mirrors_sort_limit=Yansı sıralama sınırı
 arch.config.mirrors_sort_limit.tip=Hız sıralama için kullanılacak maksimum yansı sayısını tanımlar. Sınırsız olması için 0 kullanın veya sıralamayı devre dışı bırakmak için boş bırakın.
 arch.config.optimize=optimize
 arch.config.optimize.tip=Paketlerin kurulumunu, yükseltilmesini ve indirilmesini hızlandırmak için optimize edilmiş ayarlar kullanılacak, aksi takdirde sistem ayarları kullanılacak
+arch.config.pacman_mthread_download=Multithreaded download (repositories)
+arch.config.pacman_mthread_download.tip=Whether the repository packages should be downloaded with a tool that works with threads (it can be faster). At the moment this setting will only work if the aria2 package is installed
 arch.config.refresh_mirrors=Başlangıçta yansıları yenile
 arch.config.refresh_mirrors.tip=Paket yansılarını başlangıçta günde bir kez (veya cihaz yeniden başlatıldıktan sonra) yenileyin
 arch.config.repos=Depo paketleri
@@ -153,6 +155,8 @@ arch.makepkg.optimizing=Derlemeyi optimize et
 arch.missing_deps.body={name} kurulumunun devam edebilmesi için aşağıdaki {deps} bağımlılık kurulmalıdır
 arch.missing_deps.title=Eksik bağımlılıklar
 arch.missing_deps_found={} için eksik bağımlılıklar
+arch.mthread_downloaded.error.cache_dir=It was not possible to create the cache directory {}
+arch.mthread_downloaded.error.cancelled=Operation cancelled
 arch.optdeps.checking={} İsteğe bağlı bağımlılıkları kontrol et
 arch.providers=sağlayıcılar
 arch.substatus.conflicts=Checking for conflicts

bauh/view/core/downloader.py

@@ -3,12 +3,14 @@ import os
 import re
 import time
 import traceback
+from math import floor
+from threading import Thread
 
 from bauh.api.abstract.download import FileDownloader
 from bauh.api.abstract.handler import ProcessWatcher
 from bauh.api.http import HttpClient
 from bauh.commons.html import bold
-from bauh.commons.system import run_cmd, new_subprocess, ProcessHandler, SystemProcess, SimpleProcess
+from bauh.commons.system import run_cmd, ProcessHandler, SimpleProcess, get_human_size_str
 from bauh.view.util.translation import I18n
 
 RE_HAS_EXTENSION = re.compile(r'.+\.\w+$')
@@ -25,11 +27,21 @@ class AdaptableFileDownloader(FileDownloader):
     def is_aria2c_available(self) -> bool:
         return bool(run_cmd('which aria2c', print_error=False))
 
-    def _get_aria2c_process(self, url: str, output_path: str, cwd: str) -> SystemProcess:
+    def _get_aria2c_process(self, url: str, output_path: str, cwd: str, root_password: str, max_threads: int, known_size: int) -> SimpleProcess:
+
+        if max_threads and max_threads > 0:
+            threads = max_threads
+        elif known_size:
+            threads = 16 if known_size >= 16000000 else floor(known_size / 1000000)
+
+            if threads <= 0:
+                threads = 1
+        else:
+            threads = 16
+
         cmd = ['aria2c', url,
                '--no-conf',
-               '--max-connection-per-server=16',
+               '--max-connection-per-server={}'.format(threads),
-               '--split=16',
                '--enable-color=false',
                '--stderr=true',
                '--summary-interval=0',
@@ -39,36 +51,46 @@ class AdaptableFileDownloader(FileDownloader):
                '--continue=true',
                '--timeout=5',
                '--max-file-not-found=3',
+               '--file-allocation=falloc',
                '--remote-time=true']
 
+        if threads > 1:
+            cmd.append('--split={}'.format(threads))
+
         if output_path:
             output_split = output_path.split('/')
             cmd.append('--dir=' + '/'.join(output_split[:-1]))
             cmd.append('--out=' + output_split[-1])
 
-        return SystemProcess(new_subprocess(cmd=cmd, cwd=cwd),
+        return SimpleProcess(cmd=cmd, root_password=root_password, cwd=cwd)
-                             skip_stdout=True,
-                             check_error_output=False,
-                             success_phrases=['download completed'],
-                             output_delay=0.001)
 
-    def _get_wget_process(self, url: str, output_path: str, cwd: str) -> SimpleProcess:
+    def _get_wget_process(self, url: str, output_path: str, cwd: str, root_password: str) -> SimpleProcess:
         cmd = ['wget', url, '--continue', '--retry-connrefused', '--tries=10', '--no-config']
 
         if output_path:
             cmd.append('-O')
             cmd.append(output_path)
 
-        return SimpleProcess(cmd=cmd, cwd=cwd)
+        return SimpleProcess(cmd=cmd, cwd=cwd, root_password=root_password)
 
-    def _rm_bad_file(self, file_name: str, output_path: str, cwd):
+    def _rm_bad_file(self, file_name: str, output_path: str, cwd, handler: ProcessHandler, root_password: str):
         to_delete = output_path if output_path else '{}/{}'.format(cwd, file_name)
 
         if to_delete and os.path.exists(to_delete):
             self.logger.info('Removing downloaded file {}'.format(to_delete))
-            os.remove(to_delete)
+            success, _ = handler.handle_simple(SimpleProcess(['rm', '-rf',to_delete], root_password=root_password))
+            return success
 
-    def download(self, file_url: str, watcher: ProcessWatcher, output_path: str = None, cwd: str = None) -> bool:
+    def _display_file_size(self, file_url: str, base_substatus, watcher: ProcessWatcher):
+        try:
+            size = self.http_client.get_content_length(file_url)
+
+            if size:
+                watcher.change_substatus(base_substatus + ' ( {} )'.format(size))
+        except:
+            pass
+
+    def download(self, file_url: str, watcher: ProcessWatcher, output_path: str = None, cwd: str = None, root_password: str = None, substatus_prefix: str = None, display_file_size: bool = True, max_threads: int = None, known_size: int = None) -> bool:
         self.logger.info('Downloading {}'.format(file_url))
         handler = ProcessHandler(watcher)
         file_name = file_url.split('/')[-1]
@@ -85,39 +107,45 @@ class AdaptableFileDownloader(FileDownloader):
 
             if self.is_multithreaded():
                 ti = time.time()
-                process = self._get_aria2c_process(file_url, output_path, final_cwd)
+                process = self._get_aria2c_process(file_url, output_path, final_cwd, root_password, max_threads, known_size)
-                downloader = 'aria2c'
+                downloader = 'aria2'
             else:
                 ti = time.time()
-                process = self._get_wget_process(file_url, output_path, final_cwd)
+                process = self._get_wget_process(file_url, output_path, final_cwd, root_password)
                 downloader = 'wget'
 
-            file_size = self.http_client.get_content_length(file_url)
-
             name = file_url.split('/')[-1]
 
             if output_path and not RE_HAS_EXTENSION.match(name) and RE_HAS_EXTENSION.match(output_path):
                 name = output_path.split('/')[-1]
 
-            msg = bold('[{}] ').format(downloader) + self.i18n['downloading'] + ' ' + bold(name) + (' ( {} )'.format(file_size) if file_size else '')
+            if substatus_prefix:
+                msg = substatus_prefix + ' '
+            else:
+                msg = ''
+
+            msg += bold('[{}] ').format(downloader) + self.i18n['downloading'] + ' ' + bold(name)
 
             if watcher:
                 watcher.change_substatus(msg)
 
-            if isinstance(process, SimpleProcess):
+                if display_file_size:
-                success = handler.handle_simple(process)
+                    if known_size:
-            else:
+                        watcher.change_substatus(msg + ' ( {} )'.format(get_human_size_str(known_size)))
-                success = handler.handle(process)
+                    else:
+                        Thread(target=self._display_file_size, args=(file_url, msg, watcher)).start()
+
+            success, _ = handler.handle_simple(process)
         except:
             traceback.print_exc()
-            self._rm_bad_file(file_name, output_path, final_cwd)
+            self._rm_bad_file(file_name, output_path, final_cwd, handler, root_password)
 
         tf = time.time()
         self.logger.info(file_name + ' download took {0:.2f} minutes'.format((tf - ti) / 60))
 
         if not success:
             self.logger.error("Could not download '{}'".format(file_name))
-            self._rm_bad_file(file_name, output_path, final_cwd)
+            self._rm_bad_file(file_name, output_path, final_cwd, handler, root_password)
 
         return success
 

bauh/view/resources/locale/en

@@ -191,7 +191,7 @@ core.config.backup=Enabled
 core.config.download.icons.tip=If the application icons should be downloaded and displayed on the table
 core.config.download.icons=Download icons
 core.config.download.multithreaded.tip=Whether applications, packages and files should be downloaded with a tool that works with threads ( faster ). At the moment this setting will only work if the aria2 package is installed
-core.config.download.multithreaded=Multithreaded download
+core.config.download.multithreaded=Multi-threaded download
 core.config.downloads=downloads
 core.config.locale.label=language
 core.config.mem_cache.data_exp.tip=Defines the in-memory data lifetime ( in SECONDS )