[feature] axel support for multithreaded downloads

CHANGELOG.md

@@ -29,6 +29,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
         <img src="https://raw.githubusercontent.com/vinifmor/bauh/staging/pictures/releases/0.9.4/arch_repo_mthread.png">
     </p>
 
+- Settings
+    - [axel](https://github.com/axel-download-accelerator/axel) added as an alternative multi-threaded download tool. The tool can be defined through the new field **Multi-threaded download tool** on the settings window **Advanced** tab
+    <p align="center">
+        <img src="https://raw.githubusercontent.com/vinifmor/bauh/staging/pictures/releases/0.9.4/mthread_tool.png">
+    </p>
+
+
 
 ### Improvements
 - Arch

README.md

@@ -257,7 +257,8 @@ You can change some application settings via environment variables or arguments
 ```
 download:
   icons: true # allows bauh to download the applications icons when they are not saved on the disk
-  multithreaded: true  # allows bauh to use a multithreaded download client installed on the system to download applications source files faster ( current only **aria2** is supported )
+  multithreaded: true  # allows bauh to use a multithreaded download client installed on the system to download applications source files faster
+  multithreaded_client: null  # defines the multi-threaded download tool to be used. If null, the default installed tool will be used (priority: aria2 > axel). Possible tools/values: aria2, axel
 gems: null  # defines the enabled applications types managed by bauh ( a null value means all available ) 
 locale: null  # defines a different translation for bauh ( a null value will retrieve the system's default locale )
 store_root_password: true  # if the root password should be asked only once

bauh/api/abstract/download.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from typing import Iterable
 
 from bauh.api.abstract.handler import ProcessWatcher
 
@@ -26,8 +27,9 @@ class FileDownloader(ABC):
         pass
 
     @abstractmethod
-    def get_default_client_name(self) -> str:
+    def can_work(self) -> bool:
-        """
+        pass
-        :return: retrieve current downloader client name
+
-        """
+    @abstractmethod
+    def get_supported_multithreaded_clients(self) -> Iterable[str]:
         pass

bauh/gems/appimage/controller.py

@@ -517,16 +517,8 @@ class AppImageManager(SoftwareManager):
         res = run_cmd('which sqlite3')
         return res and not res.strip().startswith('which ')
 
-    def _is_wget_available(self):
-        res = run_cmd('which wget')
-        return res and not res.strip().startswith('which ')
-
-    def _is_aria2_available(self):
-        res = run_cmd('which aria2c')
-        return res and not res.strip().startswith('which ')
-
     def can_work(self) -> bool:
-        return self._is_sqlite3_available() and (self._is_wget_available() or self._is_aria2_available())
+        return self._is_sqlite3_available() and self.file_downloader.can_work()
 
     def requires_root(self, action: str, pkg: AppImage):
         return False

bauh/manage.py

@@ -38,7 +38,7 @@ def new_manage_panel(app_args: Namespace, app_config: dict, logger: logging.Logg
                                  logger=logger,
                                  distro=util.get_distro(),
                                  file_downloader=AdaptableFileDownloader(logger, bool(app_config['download']['multithreaded']),
-                                                                         i18n, http_client),
+                                                                         i18n, http_client, app_config['download']['multithreaded_client']),
                                  app_name=__app_name__)
 
     managers = gems.load_managers(context=context, locale=i18n.current_key, config=app_config, default_locale=DEFAULT_I18N_KEY)

bauh/view/core/config.py

@@ -44,6 +44,7 @@ def read_config(update_file: bool = False) -> dict:
         },
         'download': {
             'multithreaded': True,
+            'multithreaded_client': None,
             'icons': True
         },
         'store_root_password': True,

bauh/view/core/controller.py

@@ -432,7 +432,8 @@ class GenericSoftwareManager(SoftwareManager):
             self.settings_manager = GenericSettingsManager(managers=self.managers,
                                                            working_managers=self.working_managers,
                                                            logger=self.logger,
-                                                           i18n=self.i18n)
+                                                           i18n=self.i18n,
+                                                           file_downloader=self.context.file_downloader)
         else:
             self.settings_manager.managers = self.managers
             self.settings_manager.working_managers = self.working_managers

bauh/view/core/downloader.py

@@ -5,6 +5,7 @@ import time
 import traceback
 from math import floor
 from threading import Thread
+from typing import Iterable
 
 from bauh.api.abstract.download import FileDownloader
 from bauh.api.abstract.handler import ProcessWatcher
@@ -18,27 +19,27 @@ RE_HAS_EXTENSION = re.compile(r'.+\.\w+$')
 
 class AdaptableFileDownloader(FileDownloader):
 
-    def __init__(self, logger: logging.Logger, multithread_enabled: bool, i18n: I18n, http_client: HttpClient):
+    def __init__(self, logger: logging.Logger, multithread_enabled: bool, i18n: I18n, http_client: HttpClient, multithread_client: str):
         self.logger = logger
         self.multithread_enabled = multithread_enabled
         self.i18n = i18n
         self.http_client = http_client
+        self.supported_multithread_clients = {'aria2', 'axel'}
+        self.multithread_client = multithread_client
 
-    def is_aria2c_available(self) -> bool:
+    @staticmethod
+    def is_aria2c_available() -> bool:
         return bool(run_cmd('which aria2c', print_error=False))
 
-    def _get_aria2c_process(self, url: str, output_path: str, cwd: str, root_password: str, max_threads: int, known_size: int) -> SimpleProcess:
+    @staticmethod
+    def is_axel_available() -> bool:
+        return bool(run_cmd('which axel', print_error=False))
 
-        if max_threads and max_threads > 0:
+    @staticmethod
-            threads = max_threads
+    def is_wget_available() -> bool:
-        elif known_size:
+        return bool(run_cmd('which wget', print_error=False))
-            threads = 16 if known_size >= 16000000 else floor(known_size / 1000000)
-
-            if threads <= 0:
-                threads = 1
-        else:
-            threads = 16
 
+    def _get_aria2c_process(self, url: str, output_path: str, cwd: str, root_password: str, threads: int) -> SimpleProcess:
         cmd = ['aria2c', url,
                '--no-conf',
                '--max-connection-per-server={}'.format(threads),
@@ -64,6 +65,18 @@ class AdaptableFileDownloader(FileDownloader):
 
         return SimpleProcess(cmd=cmd, root_password=root_password, cwd=cwd)
 
+    def _get_axel_process(self, url: str, output_path: str, cwd: str, root_password: str, threads: int) -> SimpleProcess:
+        cmd = ['axel', url,
+               '--num-connections={}'.format(threads),
+               '--ipv4',
+               '--no-clobber',
+               '--timeout=5']
+
+        if output_path:
+            cmd.append('--output={}'.format(output_path))
+
+        return SimpleProcess(cmd=cmd, cwd=cwd, root_password=root_password)
+
     def _get_wget_process(self, url: str, output_path: str, cwd: str, root_password: str) -> SimpleProcess:
         cmd = ['wget', url, '--continue', '--retry-connrefused', '--tries=10', '--no-config']
 
@@ -90,6 +103,19 @@ class AdaptableFileDownloader(FileDownloader):
         except:
             pass
 
+    def _get_appropriate_threads_number(self, max_threads: int, known_size: int) -> int:
+        if max_threads and max_threads > 0:
+            threads = max_threads
+        elif known_size:
+            threads = 16 if known_size >= 16000000 else floor(known_size / 1000000)
+
+            if threads <= 0:
+                threads = 1
+        else:
+            threads = 16
+
+        return threads
+
     def download(self, file_url: str, watcher: ProcessWatcher, output_path: str = None, cwd: str = None, root_password: str = None, substatus_prefix: str = None, display_file_size: bool = True, max_threads: int = None, known_size: int = None) -> bool:
         self.logger.info('Downloading {}'.format(file_url))
         handler = ProcessHandler(watcher)
@@ -105,10 +131,18 @@ class AdaptableFileDownloader(FileDownloader):
                 os.remove(output_path)
                 self.logger.info("Old file {} removed".format(output_path))
 
-            if self.is_multithreaded():
+            client = self.get_available_multithreaded_tool()
+            if client:
+                threads = self._get_appropriate_threads_number(max_threads, known_size)
+
+                if client == 'aria2':
                     ti = time.time()
-                process = self._get_aria2c_process(file_url, output_path, final_cwd, root_password, max_threads, known_size)
+                    process = self._get_aria2c_process(file_url, output_path, final_cwd, root_password, threads)
                     downloader = 'aria2'
+                else:
+                    ti = time.time()
+                    process = self._get_axel_process(file_url, output_path, final_cwd, root_password, threads)
+                    downloader = 'axel'
             else:
                 ti = time.time()
                 process = self._get_wget_process(file_url, output_path, final_cwd, root_password)
@@ -150,7 +184,22 @@ class AdaptableFileDownloader(FileDownloader):
         return success
 
     def is_multithreaded(self) -> bool:
-        return self.multithread_enabled and self.is_aria2c_available()
+        return self.multithread_enabled and (self.is_aria2c_available() or self.is_axel_available())
 
-    def get_default_client_name(self) -> str:
+    def get_available_multithreaded_tool(self) -> str:
-        return 'aria2c' if self. is_multithreaded() else 'wget'
+        if self.multithread_enabled:
+            if self.multithread_client is None:
+                if self.is_aria2c_available():
+                    return 'aria2'
+                elif self.is_axel_available():
+                    return 'axel'
+            elif (self.multithread_client == 'aria2' or self.multithread_client not in self.get_supported_multithreaded_clients()) and self.is_aria2c_available():
+                return 'aria2'
+            elif self.is_axel_available():
+                return 'axel'
+
+    def can_work(self) -> bool:
+        return self.is_wget_available() or self.is_multithreaded()
+
+    def get_supported_multithreaded_clients(self) -> Iterable[str]:
+        return self.supported_multithread_clients

bauh/view/core/settings.py

@@ -8,11 +8,13 @@ from PyQt5.QtWidgets import QApplication, QStyleFactory
 
 from bauh import ROOT_DIR
 from bauh.api.abstract.controller import SoftwareManager
+from bauh.api.abstract.download import FileDownloader
 from bauh.api.abstract.view import ViewComponent, TabComponent, InputOption, TextComponent, MultipleSelectComponent, \
     PanelComponent, FormComponent, TabGroupComponent, SingleSelectComponent, SelectViewType, TextInputComponent, \
     FileChooserComponent
 from bauh.view.core import config, timeshift
 from bauh.view.core.config import read_config
+from bauh.view.core.downloader import AdaptableFileDownloader
 from bauh.view.util import translation
 from bauh.view.util.translation import I18n
 
@@ -20,11 +22,12 @@ from bauh.view.util.translation import I18n
 class GenericSettingsManager:
 
     def __init__(self, managers: List[SoftwareManager], working_managers: List[SoftwareManager],
-                 logger: logging.Logger, i18n: I18n):
+                 logger: logging.Logger, i18n: I18n, file_downloader: FileDownloader):
         self.i18n = i18n
         self.managers = managers
         self.working_managers = working_managers
         self.logger = logger
+        self.file_downloader = file_downloader
 
     def get_settings(self, screen_width: int, screen_height: int) -> ViewComponent:
         tabs = list()
@@ -118,7 +121,22 @@ class GenericSettingsManager:
                                                    max_width=default_width,
                                                    value=core_config['download']['multithreaded'])
 
-        sub_comps = [FormComponent([select_dmthread, select_trim_up, select_dep_check, input_data_exp, input_icon_exp], spaces=False)]
+        supported_mthread_clients = self.file_downloader.get_supported_multithreaded_clients()
+        mthread_client_opts = [(self.i18n['default'].capitalize(), None, None)]
+        mthread_client_opts.extend(((d, d, None) for d in supported_mthread_clients))
+        current_mthread_client = core_config['download']['multithreaded_client']
+
+        if current_mthread_client not in supported_mthread_clients:
+            current_mthread_client = None
+
+        select_mthread_client = self._gen_select(label=self.i18n['core.config.download.multithreaded_client'],
+                                                 tip=self.i18n['core.config.download.multithreaded_client.tip'],
+                                                 id_="mthread_client",
+                                                 max_width=default_width,
+                                                 opts=mthread_client_opts,
+                                                 value=current_mthread_client)
+
+        sub_comps = [FormComponent([select_dmthread, select_mthread_client, select_trim_up, select_dep_check, input_data_exp, input_icon_exp], spaces=False)]
         return TabComponent(self.i18n['core.config.tab.advanced'].capitalize(), PanelComponent(sub_comps), None, 'core.adv')
 
     def _gen_tray_settings(self, core_config: dict, screen_width: int, screen_height: int) -> TabComponent:
@@ -307,6 +325,13 @@ class GenericSettingsManager:
         download_mthreaded = adv_form.get_component('down_mthread').get_selected()
         core_config['download']['multithreaded'] = download_mthreaded
 
+        mthread_client = adv_form.get_component('mthread_client').get_selected()
+        core_config['download']['multithreaded_client'] = mthread_client
+
+        if isinstance(self.file_downloader, AdaptableFileDownloader):
+            self.file_downloader.multithread_client = mthread_client
+            self.file_downloader.multithread_enabled = download_mthreaded
+
         single_dep_check = adv_form.get_component('dep_check').get_selected()
         core_config['system']['single_dependency_checking'] = single_dep_check
 

bauh/view/resources/locale/ca

@@ -192,7 +192,9 @@ core.config.backup.upgrade=Before upgrading
 core.config.download.icons=Download icons
 core.config.download.icons.tip=If the application icons should be downloaded and displayed on the table
 core.config.download.multithreaded=Multithreaded download
-core.config.download.multithreaded.tip=Whether applications, packages and files should be downloaded with a tool that works with threads ( faster ). At the moment this setting will only work if the aria2 package is installed
+core.config.download.multithreaded.tip=Whether applications, packages and files should be downloaded with a tool that works with threads (faster).
+core.config.download.multithreaded_client=Multi-threaded download tool
+core.config.download.multithreaded_client.tip=The download tool that should be used for multi-threaded downloads. It must be installed on the system for this configuration to work.
 core.config.downloads=downloads
 core.config.locale.label=language
 core.config.mem_cache=Memory storage

bauh/view/resources/locale/de

@@ -191,7 +191,9 @@ core.config.backup.upgrade=Before upgrading
 core.config.download.icons=Download Icons
 core.config.download.icons.tip=Falls aktiviert werden die Anwendungs-Icons in der Tabelle angezeigt
 core.config.download.multithreaded=Paralleler Download
-core.config.download.multithreaded.tip=Falls aktiviert werden Downloads parallel (mit mehreren Threads) heruntergeladen. Dazu muss aria installiert sein.
+core.config.download.multithreaded.tip=Falls aktiviert werden Downloads parallel (mit mehreren Threads) heruntergeladen.
+core.config.download.multithreaded_client=Multi-threaded download tool
+core.config.download.multithreaded_client.tip=The download tool that should be used for multi-threaded downloads. It must be installed on the system for this configuration to work.
 core.config.downloads=Downloads
 core.config.locale.label=Sprache
 core.config.mem_cache=RAM Cache

bauh/view/resources/locale/en

@@ -190,8 +190,10 @@ core.config.backup.upgrade=Before upgrading
 core.config.backup=Enabled
 core.config.download.icons.tip=If the application icons should be downloaded and displayed on the table
 core.config.download.icons=Download icons
-core.config.download.multithreaded.tip=Whether applications, packages and files should be downloaded with a tool that works with threads ( faster ). At the moment this setting will only work if the aria2 package is installed
 core.config.download.multithreaded=Multi-threaded download
+core.config.download.multithreaded.tip=Whether applications, packages and files should be downloaded with a tool that works with threads (faster).
+core.config.download.multithreaded_client=Multi-threaded download tool
+core.config.download.multithreaded_client.tip=The download tool that should be used for multi-threaded downloads. It must be installed on the system for this configuration to work.
 core.config.downloads=downloads
 core.config.locale.label=language
 core.config.mem_cache.data_exp.tip=Defines the in-memory data lifetime ( in SECONDS )

bauh/view/resources/locale/es

@@ -192,7 +192,9 @@ core.config.backup.upgrade=Antes de actualizar
 core.config.download.icons=Descargar iconos
 core.config.download.icons.tip=Si los íconos de las aplicaciones se deben descargar y mostrar en la tabla
 core.config.download.multithreaded=Descarga segmentada
-core.config.download.multithreaded.tip=Si las aplicaciones, paquetes y archivos deben descargarse con una herramienta que usa segmentación / threads ( más rápido ). Por el momento, esta configuración solo funcionará si el paquete aria2 esté instalado
+core.config.download.multithreaded.tip=Si las aplicaciones, paquetes y archivos deben descargarse con una herramienta que usa segmentación / threads (más rápido).
+core.config.download.multithreaded_client=Herramienta para descarga segmentada
+core.config.download.multithreaded_client.tip=La herramienta que debe usarse para descargas segmentadas. Ella debe estar instalada en el sistema para que esta configuración funcione.
 core.config.downloads=downloads
 core.config.locale.label=idioma
 core.config.mem_cache=Almacenamiento de memoria

bauh/view/resources/locale/it

@@ -191,7 +191,9 @@ core.config.backup.upgrade=Before upgrading
 core.config.download.icons=Download icons
 core.config.download.icons.tip=If the application icons should be downloaded and displayed on the table
 core.config.download.multithreaded=Multithreaded download
-core.config.download.multithreaded.tip=Whether applications, packages and files should be downloaded with a tool that works with threads ( faster ). At the moment this setting will only work if the aria2 package is installed
+core.config.download.multithreaded.tip=Whether applications, packages and files should be downloaded with a tool that works with threads (faster).
+core.config.download.multithreaded_client=Multi-threaded download tool
+core.config.download.multithreaded_client.tip=The download tool that should be used for multi-threaded downloads. It must be installed on the system for this configuration to work.
 core.config.downloads=downloads
 core.config.locale.label=language
 core.config.mem_cache=Memory storage

bauh/view/resources/locale/pt

@@ -190,8 +190,10 @@ core.config.backup.upgrade=Antes de atualizar
 core.config.backup=Habilitada
 core.config.download.icons.tip=Se os ícones dos aplicativos devem ser baixados e exibidos na tabela
 core.config.download.icons=Baixar ícones
-core.config.download.multithreaded.tip=Se os aplicativos, pacotes e arquivos devem ser baixados através de uma ferramenta que trabalha com segmentação / threads ( mais rápido ). No momento esta propriedade somente funcionará se o pacote aria2 estiver instalado.
+core.config.download.multithreaded.tip=Se os aplicativos, pacotes e arquivos devem ser baixados através de uma ferramenta que trabalha com segmentação / threads (mais rápido).
 core.config.download.multithreaded=Download segmentado
+core.config.download.multithreaded_client=Ferramenta para download segmentado
+core.config.download.multithreaded_client.tip=A ferramenta que deve ser utilizada para downloads segmentados. Ela precisa estar instalada no sistema para que essa configuração funcione.
 core.config.downloads=downloads
 core.config.locale.label=idioma
 core.config.mem_cache.data_exp.tip=Define o tempo de vida dos dados em memória ( em SEGUNDOS )

bauh/view/resources/locale/ru

@@ -191,7 +191,9 @@ core.config.backup.upgrade=Before upgrading
 core.config.download.icons=Скачать иконки
 core.config.download.icons.tip=Загружать иконки приложения для отображаения на рабочем столе
 core.config.download.multithreaded=Многопоточная загрузка
-core.config.download.multithreaded.tip=Следует ли загружать приложения, пакеты и файлы с помощью инструмента, который работает с потоками ( быстрее ). На данный момент эта настройка будет работать только в том случае, если установлен пакет aria2
+core.config.download.multithreaded.tip=Следует ли загружать приложения, пакеты и файлы с помощью инструмента, который работает с потоками (быстрее).
+core.config.download.multithreaded_client=Multi-threaded download tool
+core.config.download.multithreaded_client.tip=The download tool that should be used for multi-threaded downloads. It must be installed on the system for this configuration to work.
 core.config.downloads=downloads
 core.config.locale.label=Язык
 core.config.mem_cache=Запоминающее устройство

bauh/view/resources/locale/tr

@@ -190,8 +190,10 @@ core.config.backup.upgrade=Önce yükseltiliyor
 core.config.backup=Etkin
 core.config.download.icons.tip=Tabloda görüntüleniyorsa uygulama simgeleri indirilmeli
 core.config.download.icons=Simgeleri indir
-core.config.download.multithreaded.tip=Uygulamaların, paketlerin ve dosyaların iş parçacıklarıyla (daha hızlı) çalışan bir araçla indirilip indirilmeyeceği. Şu anda bu ayar yalnızca aria2 paketi kuruluysa çalışır
+core.config.download.multithreaded.tip=Uygulamaların, paketlerin ve dosyaların iş parçacıklarıyla (daha hızlı) çalışan bir araçla indirilip indirilmeyeceği.
 core.config.download.multithreaded=Çoklu parçalı indirme
+core.config.download.multithreaded_client=Multi-threaded download tool
+core.config.download.multithreaded_client.tip=The download tool that should be used for multi-threaded downloads. It must be installed on the system for this configuration to work.
 core.config.downloads=indirilenler
 core.config.locale.label=dil
 core.config.mem_cache.data_exp.tip=Bellek içi veri ömrünü tanımlar ( SANİYE )