[view.qt] enhacement: custom package icons downloader (faster)

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   - adding a new project definition/setup (`pyproject.toml`) file to comply with the latest standards
 - UI
   - the "Skip" button on the initialization panel is now enabled after 10 seconds [#310](https://github.com/vinifmor/bauh/issues/310)
+  - faster package icons download
 
 ### Fixes
 - AppImage

bauh/view/qt/apps_table.py

@@ -4,9 +4,9 @@ from functools import reduce
 from threading import Lock
 from typing import List, Optional
 
-from PyQt5.QtCore import Qt, QUrl, QSize
+from PyQt5.QtCore import Qt, QSize
 from PyQt5.QtGui import QPixmap, QIcon, QCursor
-from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest, QNetworkReply
+from PyQt5.QtNetwork import QNetworkReply
 from PyQt5.QtWidgets import QTableWidget, QTableView, QMenu, QToolButton, QWidget, \
     QHeaderView, QLabel, QHBoxLayout, QToolBar, QSizePolicy
 
@@ -16,6 +16,7 @@ from bauh.commons.html import strip_html, bold
 from bauh.view.qt.components import IconButton, QCustomMenuAction, QCustomToolbar
 from bauh.view.qt.dialog import ConfirmationDialog
 from bauh.view.qt.qt_utils import get_current_screen_geometry
+from bauh.view.qt.thread import URLFileDownloader
 from bauh.view.qt.view_model import PackageView
 from bauh.view.util.translation import I18n
 
@@ -88,8 +89,7 @@ class PackagesTable(QTableWidget):
         self.horizontalScrollBar().setCursor(QCursor(Qt.PointingHandCursor))
         self.verticalScrollBar().setCursor(QCursor(Qt.PointingHandCursor))
 
-        self.network_man = QNetworkAccessManager()
+        self.file_downloader: Optional[URLFileDownloader] = None
-        self.network_man.finished.connect(self._load_icon_and_cache)
 
         self.icon_cache = icon_cache
         self.lock_async_data = Lock()
@@ -183,10 +183,9 @@ class PackagesTable(QTableWidget):
         self._update_row(pkg, screen_width, update_check_enabled=False, change_update_col=False)
 
     def update_package(self, pkg: PackageView, screen_width: int, change_update_col: bool = False):
-        if self.download_icons and pkg.model.icon_url:
+        if self.download_icons and pkg.model.icon_url and pkg.model.icon_url.startswith("http"):
-            icon_request = QNetworkRequest(QUrl(pkg.model.icon_url))
+            self._setup_file_downloader(max_workers=1, max_downloads=1)
-            icon_request.setAttribute(QNetworkRequest.FollowRedirectsAttribute, True)
+            self.file_downloader.get(pkg.model.icon_url, pkg.table_index)
-            self.network_man.get(icon_request)
 
         self._update_row(pkg, screen_width, change_update_col=change_update_col)
 
@@ -219,14 +218,15 @@ class PackagesTable(QTableWidget):
                               i18n=self.i18n).ask():
             self.window.install(pkgv)
 
-    def _load_icon_and_cache(self, http_response: QNetworkReply):
+    def _update_pkg_icon(self, url_: str,  content: Optional[bytes], table_idx: int):
-        icon_url = http_response.request().url().toString()
+        if not content:
+            return content
 
-        icon_data = self.icon_cache.get(icon_url)
+        icon_data = self.icon_cache.get(url_)
         icon_was_cached = True
 
         if not icon_data:
-            icon_bytes = http_response.readAll()
+            icon_bytes = content
 
             if not icon_bytes:
                 return
@@ -238,16 +238,16 @@ class PackagesTable(QTableWidget):
             if not pixmap.isNull():
                 icon = QIcon(pixmap)
                 icon_data = {'icon': icon, 'bytes': icon_bytes}
-                self.icon_cache.add(icon_url, icon_data)
+                self.icon_cache.add(url_, icon_data)
 
         if icon_data:
-            for idx, app in enumerate(self.window.pkgs):
+            for pkg in self.window.pkgs:
-                if app.model.icon_url == icon_url:
+                if pkg.table_index == table_idx:
-                    self._update_icon(self.cellWidget(idx, 0), icon_data['icon'])
+                    self._update_icon(self.cellWidget(table_idx, 0), icon_data['icon'])
 
-                    if app.model.supports_disk_cache() and app.model.get_disk_icon_path() and icon_data['bytes']:
+                    if pkg.model.supports_disk_cache() and pkg.model.get_disk_icon_path() and icon_data['bytes']:
-                        if not icon_was_cached or not os.path.exists(app.model.get_disk_icon_path()):
+                        if not icon_was_cached or not os.path.exists(pkg.model.get_disk_icon_path()):
-                            self.window.manager.cache_to_disk(pkg=app.model, icon_bytes=icon_data['bytes'],
+                            self.window.manager.cache_to_disk(pkg=pkg.model, icon_bytes=icon_data['bytes'],
                                                               only_icon=True)
 
     def update_packages(self, pkgs: List[PackageView], update_check_enabled: bool = True):
@@ -259,13 +259,18 @@ class PackagesTable(QTableWidget):
             self.setColumnCount(self.COL_NUMBER if update_check_enabled else self.COL_NUMBER - 1)
             self.setRowCount(len(pkgs))
 
+            file_downloader_defined = False
+
             for idx, pkg in enumerate(pkgs):
                 pkg.table_index = idx
 
-                if self.download_icons and pkg.model.status == PackageStatus.READY and pkg.model.icon_url:
+                if self.download_icons and pkg.model.status == PackageStatus.READY and pkg.model.icon_url \
-                    icon_request = QNetworkRequest(QUrl(pkg.model.icon_url))
+                        and pkg.model.icon_url.startswith("http"):
-                    icon_request.setAttribute(QNetworkRequest.FollowRedirectsAttribute, True)
+                    if not file_downloader_defined:
-                    self.network_man.get(icon_request)
+                        self._setup_file_downloader()
+                        file_downloader_defined = True
+
+                    self.file_downloader.get(pkg.model.icon_url, idx)
 
                 self._update_row(pkg, screen_width, update_check_enabled)
 
@@ -593,3 +598,14 @@ class PackagesTable(QTableWidget):
 
     def get_width(self):
         return reduce(operator.add, [self.columnWidth(i) for i in range(self.columnCount())])
+
+    def _setup_file_downloader(self, max_workers: int = 50, max_downloads: int = -1) -> None:
+        self.file_downloader = URLFileDownloader(parent=self,
+                                                 max_workers=max_workers,
+                                                 max_downloads=max_downloads)
+        self.file_downloader.signal_downloaded.connect(self._update_pkg_icon)
+        self.file_downloader.start()
+
+    def stop_file_downloader(self) -> None:
+        if self.file_downloader:
+            self.file_downloader.stop()

bauh/view/qt/thread.py

@@ -1,10 +1,13 @@
 import os
 import re
+import sys
 import time
 import traceback
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timedelta
 from io import StringIO
 from pathlib import Path
+from queue import Queue
 from typing import List, Type, Set, Tuple, Optional
 
 import requests
@@ -1100,3 +1103,52 @@ class StartAsyncAction(QThread):
             self.msleep(self.delay)
 
         self.signal_start.emit()
+
+
+class URLFileDownloader(QThread):
+
+    signal_downloaded = pyqtSignal(str, bytes, object)
+
+    def __init__(self, max_workers: int = 50, request_timeout: int = 30, inactivity_timeout: int = 5,
+                 max_downloads: int = -1, parent: Optional[QWidget] = None):
+        super().__init__(parent)
+        self._queue = Queue()
+        self._max_workers = max_workers
+        self._request_timeout = request_timeout
+        self._inactivity_timeout = inactivity_timeout
+        self._max_downloads = max_downloads
+        self._stop = False
+
+    def _get(self, url_: str, id_: Optional[object]):
+        try:
+            res = requests.get(url=url_, timeout=self._request_timeout)
+            content = res.content if res.status_code == 200 else None
+            self.signal_downloaded.emit(url_, content, id_)
+        except Exception as e:
+            sys.stderr.write(f"[ERROR] could not download file from '{url_}': "
+                             f"{e.__class__.__name__}({str(e.args)})\n")
+
+    def run(self) -> None:
+        download_count = 0
+        with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
+            while not self._stop and (self._max_downloads <= 0 or download_count < self._max_downloads):
+                try:
+                    url_, id_ = self._queue.get(timeout=self._inactivity_timeout)
+                    executor.submit(self._get, url_, id_)
+                    download_count += 1
+                except Exception:
+                    self._stop = True
+
+            executor.shutdown(wait=False, cancel_futures=True)
+
+    def stop(self) -> None:
+        self._stop = True
+
+    def get(self, url: str, id_: Optional[object]):
+        final_url = url.strip() if url else None
+
+        if final_url:
+            if final_url.startswith("http"):
+                self._queue.put((final_url, id_))
+            else:
+                self.signal_downloaded.emit(final_url, None, id_)

bauh/view/qt/window.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from typing import List, Type, Set, Tuple, Optional
 
 from PyQt5.QtCore import QEvent, Qt, pyqtSignal, QRect
-from PyQt5.QtGui import QIcon, QWindowStateChangeEvent, QCursor, QMoveEvent
+from PyQt5.QtGui import QIcon, QWindowStateChangeEvent, QCursor
 from PyQt5.QtWidgets import QWidget, QVBoxLayout, QCheckBox, QHeaderView, QToolBar, \
     QLabel, QPlainTextEdit, QProgressBar, QPushButton, QComboBox, QApplication, QListView, QSizePolicy, \
     QMenu, QHBoxLayout
@@ -598,6 +598,7 @@ class ManageWindow(QWidget):
 
     def _begin_loading_installed(self):
         if self.installed_loaded:
+            self.table_apps.stop_file_downloader()
             self.search_bar.clear()
             self.input_name.set_text('')
             self._begin_action(self.i18n['manage_window.status.installed'])
@@ -725,6 +726,7 @@ class ManageWindow(QWidget):
         self.textarea_details.hide()
 
     def begin_refresh_packages(self, pkg_types: Optional[Set[Type[SoftwarePackage]]] = None):
+        self.table_apps.stop_file_downloader()
         self.search_bar.clear()
 
         self._begin_action(self.i18n['manage_window.status.refreshing'])
@@ -761,6 +763,7 @@ class ManageWindow(QWidget):
         self._finish_refresh_packages({'installed': None, 'types': None}, as_installed=False)
 
     def begin_load_suggestions(self, filter_installed: bool):
+        self.table_apps.stop_file_downloader()
         self.search_bar.clear()
         self._begin_action(self.i18n['manage_window.status.suggestions'])
         self._handle_console_option(False)
@@ -1265,8 +1268,9 @@ class ManageWindow(QWidget):
         if not proceed:
             return
 
-        self._begin_action(action_label='{} {}'.format(self.i18n['manage_window.status.downgrading'], pkg.model.name),
+        self.table_apps.stop_file_downloader()
-                           action_id=ACTION_DOWNGRADE)
+        label = f"{self.i18n['manage_window.status.downgrading']} {pkg.model.name}"
+        self._begin_action(action_label=label, action_id=ACTION_DOWNGRADE)
         self.comp_manager.set_components_visible(False)
         self._handle_console_option(True)
 
@@ -1360,16 +1364,15 @@ class ManageWindow(QWidget):
             dialog_history = HistoryDialog(res['history'], self.icon_cache, self.i18n)
             dialog_history.exec_()
 
-    def _begin_search(self, word, action_id: int = None):
-        self.filter_updates = False
-        self.filter_installed = False
-        self._begin_action('{} {}'.format(self.i18n['manage_window.status.searching'], word if word else ''), action_id=action_id)
-
     def search(self):
         word = self.search_bar.text().strip()
         if word:
+            self.table_apps.stop_file_downloader()
             self._handle_console(False)
-            self._begin_search(word, action_id=ACTION_SEARCH)
+            self.filter_updates = False
+            self.filter_installed = False
+            label = f"{self.i18n['manage_window.status.searching']} {word if word else ''}"
+            self._begin_action(action_label=label, action_id=ACTION_SEARCH)
             self.comp_manager.set_components_visible(False)
             self.thread_search.word = word
             self.thread_search.start()
@@ -1545,6 +1548,9 @@ class ManageWindow(QWidget):
             else:
                 action_label += f' {pkg.model.name}'
 
+        if action.refresh:
+            self.table_apps.stop_file_downloader()
+
         self._begin_action(action_label=action_label, action_id=ACTION_CUSTOM_ACTION)
         self.comp_manager.set_components_visible(False)
         self._handle_console_option(True)