From 6efa763236635afaaba87a8ad50fbe37026f4ded Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Sat, 25 Nov 2023 17:28:44 -0300 Subject: [PATCH] [view.qt] enhacement: custom package icons downloader (faster) --- CHANGELOG.md | 1 + bauh/view/qt/apps_table.py | 62 ++++++++++++++++++++++++-------------- bauh/view/qt/thread.py | 52 ++++++++++++++++++++++++++++++++ bauh/view/qt/window.py | 24 +++++++++------ 4 files changed, 107 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48d9d406..19f7cb5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - adding a new project definition/setup (`pyproject.toml`) file to comply with the latest standards - UI - the "Skip" button on the initialization panel is now enabled after 10 seconds [#310](https://github.com/vinifmor/bauh/issues/310) + - faster package icons download ### Fixes - AppImage diff --git a/bauh/view/qt/apps_table.py b/bauh/view/qt/apps_table.py index 29c2922d..f7b120db 100644 --- a/bauh/view/qt/apps_table.py +++ b/bauh/view/qt/apps_table.py @@ -4,9 +4,9 @@ from functools import reduce from threading import Lock from typing import List, Optional -from PyQt5.QtCore import Qt, QUrl, QSize +from PyQt5.QtCore import Qt, QSize from PyQt5.QtGui import QPixmap, QIcon, QCursor -from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest, QNetworkReply +from PyQt5.QtNetwork import QNetworkReply from PyQt5.QtWidgets import QTableWidget, QTableView, QMenu, QToolButton, QWidget, \ QHeaderView, QLabel, QHBoxLayout, QToolBar, QSizePolicy @@ -16,6 +16,7 @@ from bauh.commons.html import strip_html, bold from bauh.view.qt.components import IconButton, QCustomMenuAction, QCustomToolbar from bauh.view.qt.dialog import ConfirmationDialog from bauh.view.qt.qt_utils import get_current_screen_geometry +from bauh.view.qt.thread import URLFileDownloader from bauh.view.qt.view_model import PackageView from bauh.view.util.translation import I18n @@ -88,8 +89,7 @@ class PackagesTable(QTableWidget): self.horizontalScrollBar().setCursor(QCursor(Qt.PointingHandCursor)) self.verticalScrollBar().setCursor(QCursor(Qt.PointingHandCursor)) - self.network_man = QNetworkAccessManager() - self.network_man.finished.connect(self._load_icon_and_cache) + self.file_downloader: Optional[URLFileDownloader] = None self.icon_cache = icon_cache self.lock_async_data = Lock() @@ -183,10 +183,9 @@ class PackagesTable(QTableWidget): self._update_row(pkg, screen_width, update_check_enabled=False, change_update_col=False) def update_package(self, pkg: PackageView, screen_width: int, change_update_col: bool = False): - if self.download_icons and pkg.model.icon_url: - icon_request = QNetworkRequest(QUrl(pkg.model.icon_url)) - icon_request.setAttribute(QNetworkRequest.FollowRedirectsAttribute, True) - self.network_man.get(icon_request) + if self.download_icons and pkg.model.icon_url and pkg.model.icon_url.startswith("http"): + self._setup_file_downloader(max_workers=1, max_downloads=1) + self.file_downloader.get(pkg.model.icon_url, pkg.table_index) self._update_row(pkg, screen_width, change_update_col=change_update_col) @@ -219,14 +218,15 @@ class PackagesTable(QTableWidget): i18n=self.i18n).ask(): self.window.install(pkgv) - def _load_icon_and_cache(self, http_response: QNetworkReply): - icon_url = http_response.request().url().toString() + def _update_pkg_icon(self, url_: str, content: Optional[bytes], table_idx: int): + if not content: + return content - icon_data = self.icon_cache.get(icon_url) + icon_data = self.icon_cache.get(url_) icon_was_cached = True if not icon_data: - icon_bytes = http_response.readAll() + icon_bytes = content if not icon_bytes: return @@ -238,16 +238,16 @@ class PackagesTable(QTableWidget): if not pixmap.isNull(): icon = QIcon(pixmap) icon_data = {'icon': icon, 'bytes': icon_bytes} - self.icon_cache.add(icon_url, icon_data) + self.icon_cache.add(url_, icon_data) if icon_data: - for idx, app in enumerate(self.window.pkgs): - if app.model.icon_url == icon_url: - self._update_icon(self.cellWidget(idx, 0), icon_data['icon']) + for pkg in self.window.pkgs: + if pkg.table_index == table_idx: + self._update_icon(self.cellWidget(table_idx, 0), icon_data['icon']) - if app.model.supports_disk_cache() and app.model.get_disk_icon_path() and icon_data['bytes']: - if not icon_was_cached or not os.path.exists(app.model.get_disk_icon_path()): - self.window.manager.cache_to_disk(pkg=app.model, icon_bytes=icon_data['bytes'], + if pkg.model.supports_disk_cache() and pkg.model.get_disk_icon_path() and icon_data['bytes']: + if not icon_was_cached or not os.path.exists(pkg.model.get_disk_icon_path()): + self.window.manager.cache_to_disk(pkg=pkg.model, icon_bytes=icon_data['bytes'], only_icon=True) def update_packages(self, pkgs: List[PackageView], update_check_enabled: bool = True): @@ -259,13 +259,18 @@ class PackagesTable(QTableWidget): self.setColumnCount(self.COL_NUMBER if update_check_enabled else self.COL_NUMBER - 1) self.setRowCount(len(pkgs)) + file_downloader_defined = False + for idx, pkg in enumerate(pkgs): pkg.table_index = idx - if self.download_icons and pkg.model.status == PackageStatus.READY and pkg.model.icon_url: - icon_request = QNetworkRequest(QUrl(pkg.model.icon_url)) - icon_request.setAttribute(QNetworkRequest.FollowRedirectsAttribute, True) - self.network_man.get(icon_request) + if self.download_icons and pkg.model.status == PackageStatus.READY and pkg.model.icon_url \ + and pkg.model.icon_url.startswith("http"): + if not file_downloader_defined: + self._setup_file_downloader() + file_downloader_defined = True + + self.file_downloader.get(pkg.model.icon_url, idx) self._update_row(pkg, screen_width, update_check_enabled) @@ -593,3 +598,14 @@ class PackagesTable(QTableWidget): def get_width(self): return reduce(operator.add, [self.columnWidth(i) for i in range(self.columnCount())]) + + def _setup_file_downloader(self, max_workers: int = 50, max_downloads: int = -1) -> None: + self.file_downloader = URLFileDownloader(parent=self, + max_workers=max_workers, + max_downloads=max_downloads) + self.file_downloader.signal_downloaded.connect(self._update_pkg_icon) + self.file_downloader.start() + + def stop_file_downloader(self) -> None: + if self.file_downloader: + self.file_downloader.stop() diff --git a/bauh/view/qt/thread.py b/bauh/view/qt/thread.py index 6f38ecae..aae26de5 100644 --- a/bauh/view/qt/thread.py +++ b/bauh/view/qt/thread.py @@ -1,10 +1,13 @@ import os import re +import sys import time import traceback +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta from io import StringIO from pathlib import Path +from queue import Queue from typing import List, Type, Set, Tuple, Optional import requests @@ -1100,3 +1103,52 @@ class StartAsyncAction(QThread): self.msleep(self.delay) self.signal_start.emit() + + +class URLFileDownloader(QThread): + + signal_downloaded = pyqtSignal(str, bytes, object) + + def __init__(self, max_workers: int = 50, request_timeout: int = 30, inactivity_timeout: int = 5, + max_downloads: int = -1, parent: Optional[QWidget] = None): + super().__init__(parent) + self._queue = Queue() + self._max_workers = max_workers + self._request_timeout = request_timeout + self._inactivity_timeout = inactivity_timeout + self._max_downloads = max_downloads + self._stop = False + + def _get(self, url_: str, id_: Optional[object]): + try: + res = requests.get(url=url_, timeout=self._request_timeout) + content = res.content if res.status_code == 200 else None + self.signal_downloaded.emit(url_, content, id_) + except Exception as e: + sys.stderr.write(f"[ERROR] could not download file from '{url_}': " + f"{e.__class__.__name__}({str(e.args)})\n") + + def run(self) -> None: + download_count = 0 + with ThreadPoolExecutor(max_workers=self._max_workers) as executor: + while not self._stop and (self._max_downloads <= 0 or download_count < self._max_downloads): + try: + url_, id_ = self._queue.get(timeout=self._inactivity_timeout) + executor.submit(self._get, url_, id_) + download_count += 1 + except Exception: + self._stop = True + + executor.shutdown(wait=False, cancel_futures=True) + + def stop(self) -> None: + self._stop = True + + def get(self, url: str, id_: Optional[object]): + final_url = url.strip() if url else None + + if final_url: + if final_url.startswith("http"): + self._queue.put((final_url, id_)) + else: + self.signal_downloaded.emit(final_url, None, id_) diff --git a/bauh/view/qt/window.py b/bauh/view/qt/window.py index 7a061f9c..2a941e20 100755 --- a/bauh/view/qt/window.py +++ b/bauh/view/qt/window.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import List, Type, Set, Tuple, Optional from PyQt5.QtCore import QEvent, Qt, pyqtSignal, QRect -from PyQt5.QtGui import QIcon, QWindowStateChangeEvent, QCursor, QMoveEvent +from PyQt5.QtGui import QIcon, QWindowStateChangeEvent, QCursor from PyQt5.QtWidgets import QWidget, QVBoxLayout, QCheckBox, QHeaderView, QToolBar, \ QLabel, QPlainTextEdit, QProgressBar, QPushButton, QComboBox, QApplication, QListView, QSizePolicy, \ QMenu, QHBoxLayout @@ -598,6 +598,7 @@ class ManageWindow(QWidget): def _begin_loading_installed(self): if self.installed_loaded: + self.table_apps.stop_file_downloader() self.search_bar.clear() self.input_name.set_text('') self._begin_action(self.i18n['manage_window.status.installed']) @@ -725,6 +726,7 @@ class ManageWindow(QWidget): self.textarea_details.hide() def begin_refresh_packages(self, pkg_types: Optional[Set[Type[SoftwarePackage]]] = None): + self.table_apps.stop_file_downloader() self.search_bar.clear() self._begin_action(self.i18n['manage_window.status.refreshing']) @@ -761,6 +763,7 @@ class ManageWindow(QWidget): self._finish_refresh_packages({'installed': None, 'types': None}, as_installed=False) def begin_load_suggestions(self, filter_installed: bool): + self.table_apps.stop_file_downloader() self.search_bar.clear() self._begin_action(self.i18n['manage_window.status.suggestions']) self._handle_console_option(False) @@ -1265,8 +1268,9 @@ class ManageWindow(QWidget): if not proceed: return - self._begin_action(action_label='{} {}'.format(self.i18n['manage_window.status.downgrading'], pkg.model.name), - action_id=ACTION_DOWNGRADE) + self.table_apps.stop_file_downloader() + label = f"{self.i18n['manage_window.status.downgrading']} {pkg.model.name}" + self._begin_action(action_label=label, action_id=ACTION_DOWNGRADE) self.comp_manager.set_components_visible(False) self._handle_console_option(True) @@ -1360,16 +1364,15 @@ class ManageWindow(QWidget): dialog_history = HistoryDialog(res['history'], self.icon_cache, self.i18n) dialog_history.exec_() - def _begin_search(self, word, action_id: int = None): - self.filter_updates = False - self.filter_installed = False - self._begin_action('{} {}'.format(self.i18n['manage_window.status.searching'], word if word else ''), action_id=action_id) - def search(self): word = self.search_bar.text().strip() if word: + self.table_apps.stop_file_downloader() self._handle_console(False) - self._begin_search(word, action_id=ACTION_SEARCH) + self.filter_updates = False + self.filter_installed = False + label = f"{self.i18n['manage_window.status.searching']} {word if word else ''}" + self._begin_action(action_label=label, action_id=ACTION_SEARCH) self.comp_manager.set_components_visible(False) self.thread_search.word = word self.thread_search.start() @@ -1545,6 +1548,9 @@ class ManageWindow(QWidget): else: action_label += f' {pkg.model.name}' + if action.refresh: + self.table_apps.stop_file_downloader() + self._begin_action(action_label=action_label, action_id=ACTION_CUSTOM_ACTION) self.comp_manager.set_components_visible(False) self._handle_console_option(True)