enhancement: replacing wget by new download implementation

CHANGELOG.md

@@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   - parallelized search
   - adding a new project definition/setup (`pyproject.toml`) file to comply with the latest standards
   - faster exit (threaded calls won't be waited)
+  - new download implementation replaces `wget` (implemented in Python using the `requests` library)
 - UI
   - the "Skip" button on the initialization panel is now enabled after 10 seconds [#310](https://github.com/vinifmor/bauh/issues/310)
   - faster package icons download

README.md

@@ -77,7 +77,7 @@ Key features
 - `aria2`: multi-threaded downloads
 - `axel`: multi-threaded downloads alternative
 - `libappindicator3-1`: tray-mode
-- `wget`, `sqlite3`, `fuse`: AppImage support
+- `sqlite3`, `fuse`: AppImage support
 - `flatpak`: Flatpaks support
 - `snapd`: Snaps support
 - `python3-lxml`, `python3-bs4`: Web apps support

bauh/api/abstract/download.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Iterable, List, Optional
+from typing import Iterable, List, Optional, Tuple
 
 from bauh.api.abstract.handler import ProcessWatcher
 
@@ -31,7 +31,7 @@ class FileDownloader(ABC):
         pass
 
     @abstractmethod
-    def get_supported_multithreaded_clients(self) -> Iterable[str]:
+    def get_supported_multithreaded_clients(self) -> Tuple[str, ...]:
         pass
 
     @abstractmethod
@@ -39,9 +39,9 @@ class FileDownloader(ABC):
         pass
 
     @abstractmethod
-    def list_available_multithreaded_clients(self) -> List[str]:
+    def list_available_multithreaded_clients(self) -> Tuple[str, ...]:
         pass
 
     @abstractmethod
-    def get_supported_clients(self) -> tuple:
+    def get_supported_clients(self) -> Tuple[str, ...]:
         pass

bauh/gems/appimage/controller.py

@@ -739,10 +739,6 @@ class AppImageManager(SoftwareManager, SettingsController):
         if not self._is_sqlite3_available():
             return False, self.i18n['missing_dep'].format(dep=bold('sqlite3'))
 
-        if not self.file_downloader.can_work():
-            download_clients = ', '.join(self.file_downloader.get_supported_clients())
-            return False, self.i18n['appimage.missing_downloader'].format(clients=download_clients)
-
         return True, None
 
     def requires_root(self, action: SoftwareAction, pkg: AppImage) -> bool:

bauh/gems/appimage/resources/locale/ca

@@ -41,7 +41,6 @@ appimage.install.desktop_entry=S’està creant una drecera del menú
 appimage.install.extract=S’està extraient el contingut de {}
 appimage.install.imported.rename_error=It was not possible to move the file {} to {}
 appimage.install.permission=S’està concedint el permís d’execució a {}
-appimage.missing_downloader=No download client installed ({clients})
 appimage.update_database.deleting_old=Removing old files
 appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files

bauh/gems/appimage/resources/locale/de

@@ -41,7 +41,6 @@ appimage.install.desktop_entry=Ein Tastenkürzel für das Menü festlegen
 appimage.install.extract=Extrahieren des Inhalts von {}
 appimage.install.imported.rename_error=Es war nicht möglich, die Datei {} nach {} zu verschieben
 appimage.install.permission=Erteilung der Ausführungsberechtigung für {}
-appimage.missing_downloader=Kein Download-Client installiert ({clients})
 appimage.update_database.deleting_old=Entfernen von alten Dateien
 appimage.update_database.downloading=Herunterladen von Datenbankdateien
 appimage.update_database.uncompressing=Dekomprimierung von Dateien

bauh/gems/appimage/resources/locale/en

@@ -41,7 +41,6 @@ appimage.install.desktop_entry=Generating a menu shortcut
 appimage.install.extract=Extracting the content from {}
 appimage.install.imported.rename_error=It was not possible to move the file {} to {}
 appimage.install.permission=Giving execution permission to {}
-appimage.missing_downloader=No download client installed ({clients})
 appimage.update_database.deleting_old=Removing old files
 appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files

bauh/gems/appimage/resources/locale/es

@@ -41,7 +41,6 @@ appimage.install.desktop_entry=Creando un atajo en el menú
 appimage.install.extract=Extrayendo el contenido de {}
 appimage.install.imported.rename_error=No fue posible mover el archivo {} a {}
 appimage.install.permission=Concediendo permiso de ejecución a {}
-appimage.missing_downloader=No hay ningún cliente de descarga instalado ({clients})
 appimage.update_database.deleting_old=Eliminando archivos antiguos
 appimage.update_database.downloading=Descargando archivos de la base de datos
 appimage.update_database.uncompressing=Descomprindo archivos

bauh/gems/appimage/resources/locale/fr

@@ -42,7 +42,6 @@ appimage.install.download.error=Échec du téléchargement du fichier {}. Le ser
 appimage.install.extract=Extractiion du contenu de {}
 appimage.install.imported.rename_error=Impossible de déplacer {} vers {}
 appimage.install.permission={} est maintenant exécutable
-appimage.missing_downloader=No download client installed ({clients})
 appimage.update_database.deleting_old=Removing old files
 appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files

bauh/gems/appimage/resources/locale/it

@@ -42,7 +42,6 @@ appimage.install.download.error=Non è stato possibile scaricare il file {}. Il
 appimage.install.extract=Estrarre il contenuto da {}
 appimage.install.imported.rename_error=It was not possible to move the file {} to {}
 appimage.install.permission=Gdare il permesso di esecuzione a {}
-appimage.missing_downloader=No download client installed ({clients})
 appimage.update_database.deleting_old=Removing old files
 appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files

bauh/gems/appimage/resources/locale/pt

@@ -41,7 +41,6 @@ appimage.install.desktop_entry=Criando um atalho no menu
 appimage.install.extract=Extraindo o conteúdo de {}
 appimage.install.imported.rename_error=Não foi possível mover o arquivo {} para {}
 appimage.install.permission=Concedendo permissão de execução para {}
-appimage.missing_downloader=Nenhum cliente de para download está instalado ({clients})
 appimage.update_database.deleting_old=Removendo arquicos antigos
 appimage.update_database.downloading=Baixando arquivos do banco de dados
 appimage.update_database.uncompressing=Descomprimindo arquivos

bauh/gems/appimage/resources/locale/ru

@@ -41,7 +41,6 @@ appimage.install.desktop_entry=Создать ярлык в меню
 appimage.install.extract=Извлечь содержимое из {}
 appimage.install.imported.rename_error=Не удалось переместить файл {} в {}
 appimage.install.permission=Установить права на выполнение для {}
-appimage.missing_downloader=Не установлен клиент загрузки ({clients})
 appimage.update_database.deleting_old=Удаление старых файлов
 appimage.update_database.downloading=Загрузка файлов базы данных
 appimage.update_database.uncompressing=Распаковка файлов

bauh/gems/appimage/resources/locale/tr

@@ -41,7 +41,6 @@ appimage.install.desktop_entry=Bir menü kısayolu oluşturuluyor
 appimage.install.extract={} 'den içerik ayıklanıyor
 appimage.install.imported.rename_error={} dosyasını {} klasörüne taşımak mümkün değildi
 appimage.install.permission={} için yürütme izni veriliyor
-appimage.missing_downloader=No download client installed ({clients})
 appimage.update_database.deleting_old=Removing old files
 appimage.update_database.downloading=Downloading database files
 appimage.update_database.uncompressing=Uncompressing files

bauh/view/core/downloader.py

@@ -1,14 +1,14 @@
-import logging
 import os
 import re
 import shutil
 import time
 import traceback
-from io import StringIO
+from io import StringIO, BytesIO
+from logging import Logger
 from math import floor
 from pathlib import Path
 from threading import Thread
-from typing import Iterable, List, Optional
+from typing import Optional, Tuple
 
 from bauh.api.abstract.download import FileDownloader
 from bauh.api.abstract.handler import ProcessWatcher
@@ -21,17 +21,92 @@ from bauh.view.util.translation import I18n
 RE_HAS_EXTENSION = re.compile(r'.+\.\w+$')
 
 
+class SelfFileDownloader(FileDownloader):
+
+    def __init__(self, logger: Logger, i18n: I18n, http_client: HttpClient,
+                 check_ssl: bool):
+        self._logger = logger
+        self._i18n = i18n
+        self._client = http_client
+        self._ssl = check_ssl
+
+    def is_multithreaded(self) -> bool:
+        return False
+
+    def can_work(self) -> bool:
+        return True
+
+    def get_supported_multithreaded_clients(self) -> Tuple[str, ...]:
+        return tuple()
+
+    def is_multithreaded_client_available(self, name: str) -> bool:
+        return False
+
+    def list_available_multithreaded_clients(self) -> Tuple[str, ...]:
+        return tuple()
+
+    def get_supported_clients(self) -> Tuple[str, ...]:
+        return tuple()
+
+    def download(self, file_url: str, watcher: Optional[ProcessWatcher], output_path: str, cwd: str,
+                 root_password: Optional[str] = None, substatus_prefix: str = None, display_file_size: bool = True,
+                 max_threads: int = None, known_size: int = None) -> bool:
+        try:
+            res = self._client.get(url=file_url, ignore_ssl=not self._ssl, stream=True)
+        except Exception:
+            return False
+
+        try:
+            content_length = int(res.headers.get("content-length", 0))
+        except Exception:
+            content_length = 0
+            self._logger.warning(f"Could not retrieve the content-length for file '{file_url}'")
+
+        file_name = file_url.split("/")[-1]
+        msg = StringIO()
+        msg.write(f"{substatus_prefix} " if substatus_prefix else "")
+        msg.write(f"{self._i18n['downloading']} {bold(file_name)}")
+        base_msg = msg.getvalue()
+
+        byte_stream = BytesIO()
+        total_downloaded = 0
+        known_size = content_length and content_length > 0
+        total_size_str = get_human_size_str(content_length) if known_size > 0 else "?"
+
+        for data in res.iter_content(chunk_size=1024):
+            byte_stream.write(data)
+            total_downloaded += len(data)
+            perc = f"({(total_downloaded / content_length) * 100:.2f}%) " if known_size > 0 else ""
+            watcher.change_substatus(f"{perc}{base_msg} ({get_human_size_str(total_downloaded)} / {total_size_str})")
+
+        self._logger.info(f"Writing downloaded file content to disk: {output_path}")
+
+        try:
+            with open(output_path, "wb+") as f:
+                f.write(byte_stream.getvalue())
+        except Exception:
+            self._logger.error(f"Unexpected exception when saving downloaded content to disk: {output_path}")
+            traceback.print_exc()
+            return False
+
+        return True
+
+
 class AdaptableFileDownloader(FileDownloader):
 
-    def __init__(self, logger: logging.Logger, multithread_enabled: bool, i18n: I18n, http_client: HttpClient,
+    def __init__(self, logger: Logger, multithread_enabled: bool, i18n: I18n, http_client: HttpClient,
                  multithread_client: str, check_ssl: bool):
         self.logger = logger
         self.multithread_enabled = multithread_enabled
         self.i18n = i18n
         self.http_client = http_client
-        self.supported_multithread_clients = ['aria2', 'axel']
+        self.supported_multithread_clients = ("aria2", "axel")
         self.multithread_client = multithread_client
         self.check_ssl = check_ssl
+        self._self_downloader = SelfFileDownloader(logger=logger,
+                                                   i18n=i18n,
+                                                   http_client=http_client,
+                                                   check_ssl=check_ssl)
 
     @staticmethod
     def is_aria2c_available() -> bool:
@@ -41,10 +116,6 @@ class AdaptableFileDownloader(FileDownloader):
     def is_axel_available() -> bool:
         return bool(shutil.which('axel'))
 
-    @staticmethod
-    def is_wget_available() -> bool:
-        return bool(shutil.which('wget'))
-
     def _get_aria2c_process(self, url: str, output_path: str, cwd: str, root_password: Optional[str], threads: int) -> SimpleProcess:
         cmd = ['aria2c', url,
                '--no-conf',
@@ -85,18 +156,6 @@ class AdaptableFileDownloader(FileDownloader):
 
         return SimpleProcess(cmd=cmd, cwd=cwd, root_password=root_password)
 
-    def _get_wget_process(self, url: str, output_path: str, cwd: str, root_password: Optional[str]) -> SimpleProcess:
-        cmd = ['wget', url, '-c', '--retry-connrefused', '-t', '10', '-nc']
-
-        if not self.check_ssl:
-            cmd.append('--no-check-certificate')
-
-        if output_path:
-            cmd.append('-O')
-            cmd.append(output_path)
-
-        return SimpleProcess(cmd=cmd, cwd=cwd, root_password=root_password)
-
     def _rm_bad_file(self, file_name: str, output_path: str, cwd, handler: ProcessHandler, root_password: Optional[str]):
         to_delete = output_path if output_path else f'{cwd}/{file_name}'
 
@@ -130,6 +189,45 @@ class AdaptableFileDownloader(FileDownloader):
 
         return threads
 
+    def _download_with_threads(self, client: str, file_url: str, output_path: str, cwd: str,
+                               max_threads: int, known_size: int, display_file_size: bool, handler: ProcessHandler,
+                               root_password: Optional[str] = None, substatus_prefix: Optional[str] = None) \
+            -> Tuple[float, bool]:
+
+        threads = self._get_appropriate_threads_number(max_threads, known_size)
+
+        if client == 'aria2':
+            start_time = time.time()
+            process = self._get_aria2c_process(file_url, output_path, cwd, root_password, threads)
+            downloader = 'aria2'
+        else:
+            start_time = time.time()
+            process = self._get_axel_process(file_url, output_path, cwd, root_password, threads)
+            downloader = 'axel'
+
+        name = file_url.split('/')[-1]
+
+        if output_path and not RE_HAS_EXTENSION.match(name) and RE_HAS_EXTENSION.match(output_path):
+            name = output_path.split('/')[-1]
+
+        if handler.watcher:
+            msg = StringIO()
+            msg.write(f'{substatus_prefix} ' if substatus_prefix else '')
+            msg.write(f"{bold('[{}]'.format(downloader))} {self.i18n['downloading']} {bold(name)}")
+
+            if display_file_size:
+                if known_size:
+                    msg.write(f' ( {get_human_size_str(known_size)} )')
+                    handler.watcher.change_substatus(msg.getvalue())
+                else:
+                    Thread(target=self._concat_file_size, args=(file_url, msg, handler.watcher), daemon=True).start()
+            else:
+                msg.write(' ( ? Mb )')
+                handler.watcher.change_substatus(msg.getvalue())
+
+        success, _ = handler.handle_simple(process)
+        return start_time, success
+
     def download(self, file_url: str, watcher: ProcessWatcher, output_path: str = None, cwd: str = None, root_password: Optional[str] = None, substatus_prefix: str = None, display_file_size: bool = True, max_threads: int = None, known_size: int = None) -> bool:
         self.logger.info(f'Downloading {file_url}')
         handler = ProcessHandler(watcher)
@@ -138,7 +236,7 @@ class AdaptableFileDownloader(FileDownloader):
         final_cwd = cwd if cwd else '.'
 
         success = False
-        ti = time.time()
+        start_time = time.time()
         try:
             if output_path:
                 if os.path.exists(output_path):
@@ -155,50 +253,27 @@ class AdaptableFileDownloader(FileDownloader):
                         watcher.print(self.i18n['error.mkdir'].format(dir=output_dir))
                         return False
 
-            client = self.get_available_multithreaded_tool()
+            threaded_client = self.get_available_multithreaded_tool()
-            if client:
+            if threaded_client:
-                threads = self._get_appropriate_threads_number(max_threads, known_size)
+                start_time, success = self._download_with_threads(client=threaded_client, file_url=file_url,
-
+                                                                  output_path=output_path,
-                if client == 'aria2':
+                                                                  cwd=final_cwd, max_threads=max_threads,
-                    ti = time.time()
+                                                                  known_size=known_size, handler=handler,
-                    process = self._get_aria2c_process(file_url, output_path, final_cwd, root_password, threads)
+                                                                  display_file_size=display_file_size,
-                    downloader = 'aria2'
+                                                                  root_password=root_password)
-                else:
-                    ti = time.time()
-                    process = self._get_axel_process(file_url, output_path, final_cwd, root_password, threads)
-                    downloader = 'axel'
             else:
-                ti = time.time()
+                start_time = time.time()
-                process = self._get_wget_process(file_url, output_path, final_cwd, root_password)
+                success = self._self_downloader.download(file_url=file_url, watcher=watcher, output_path=output_path,
-                downloader = 'wget'
+                                                         cwd=cwd, root_password=root_password,
-
+                                                         substatus_prefix=substatus_prefix,
-            name = file_url.split('/')[-1]
+                                                         display_file_size=display_file_size, max_threads=max_threads,
-
+                                                         known_size=known_size)
-            if output_path and not RE_HAS_EXTENSION.match(name) and RE_HAS_EXTENSION.match(output_path):
-                name = output_path.split('/')[-1]
-
-            if watcher:
-                msg = StringIO()
-                msg.write(f'{substatus_prefix} ' if substatus_prefix else '')
-                msg.write(f"{bold('[{}]'.format(downloader))} {self.i18n['downloading']} {bold(name)}")
-
-                if display_file_size:
-                    if known_size:
-                        msg.write(f' ( {get_human_size_str(known_size)} )')
-                        watcher.change_substatus(msg.getvalue())
-                    else:
-                        Thread(target=self._concat_file_size, args=(file_url, msg, watcher), daemon=True).start()
-                else:
-                    msg.write(' ( ? Mb )')
-                    watcher.change_substatus(msg.getvalue())
-
-            success, _ = handler.handle_simple(process)
         except Exception:
             traceback.print_exc()
             self._rm_bad_file(file_name, output_path, final_cwd, handler, root_password)
 
-        tf = time.time()
+        final_time = time.time()
-        self.logger.info(f'{file_name} download took {(tf - ti) / 60:.2f} minutes')
+        self.logger.info(f'{file_name} download took {(final_time - start_time) / 60:.2f} minutes')
 
         if not success:
             self.logger.error(f"Could not download '{file_name}'")
@@ -228,9 +303,9 @@ class AdaptableFileDownloader(FileDownloader):
                             return client
 
     def can_work(self) -> bool:
-        return self.is_wget_available() or self.is_multithreaded()
+        return True
 
-    def get_supported_multithreaded_clients(self) -> Iterable[str]:
+    def get_supported_multithreaded_clients(self) -> Tuple[str, ...]:
         return self.supported_multithread_clients
 
     def is_multithreaded_client_available(self, name: str) -> bool:
@@ -241,8 +316,8 @@ class AdaptableFileDownloader(FileDownloader):
         else:
             return False
 
-    def list_available_multithreaded_clients(self) -> List[str]:
+    def list_available_multithreaded_clients(self) -> Tuple[str, ...]:
-        return [c for c in self.supported_multithread_clients if self.is_multithreaded_client_available(c)]
+        return tuple(c for c in self.supported_multithread_clients if self.is_multithreaded_client_available(c))
 
-    def get_supported_clients(self) -> tuple:
+    def get_supported_clients(self) -> Tuple[str, ...]:
-        return 'wget', 'aria2', 'axel'
+        return "self", "aria2", "axel"

bauh/view/core/settings.py

@@ -146,7 +146,7 @@ class GenericSettingsManager(SettingsController):
         return TabComponent(self.i18n['core.config.tab.advanced'].capitalize(), panel, None, 'core.adv')
 
     def _gen_multithread_client_select(self, core_config: dict) -> SingleSelectComponent:
-        available_mthread_clients = self.file_downloader.list_available_multithreaded_clients()
+        available_mthread_clients = [*self.file_downloader.list_available_multithreaded_clients()]
         available_mthread_clients.sort()
 
         default_i18n_key = 'default' if available_mthread_clients else 'core.config.download.multithreaded_client.none'

linux_dist/appimage/AppImageBuilder.yml

@@ -40,7 +40,6 @@ AppDir:
       - python3-lxml
       - python3-bs4
       - sqlite3
-      - wget
       # - libfreetype6
       # - libfontconfig1
     exclude: