From 656661b7b83b8b884d0e60fd5e4a0cca56d0a19e Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Tue, 28 Nov 2023 17:07:19 -0300 Subject: [PATCH] [view.qt] enhancement: faster packages filtering --- CHANGELOG.md | 1 + bauh/view/core/controller.py | 3 - bauh/view/qt/commons.py | 83 ++++++++++++++-------- bauh/view/qt/thread.py | 44 ++++++------ bauh/view/qt/view_index.py | 130 +++++++++++++++++++++++++++++++++++ bauh/view/qt/window.py | 58 ++++++++++------ 6 files changed, 245 insertions(+), 74 deletions(-) create mode 100644 bauh/view/qt/view_index.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 565900d4..436f77f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - UI - the "Skip" button on the initialization panel is now enabled after 10 seconds [#310](https://github.com/vinifmor/bauh/issues/310) - faster package icons download + - faster packages filtering (`type`, `category`, `name`, etc... up to **95% less time**) ### Fixes - AppImage diff --git a/bauh/view/core/controller.py b/bauh/view/core/controller.py index 8c9c9396..db902c02 100755 --- a/bauh/view/core/controller.py +++ b/bauh/view/core/controller.py @@ -181,9 +181,6 @@ class GenericSoftwareManager(SoftwareManager, SettingsController): if disk_loader: disk_loader.stop_working() disk_loader.join() - - # res.installed = self._sort(res.installed, norm_word) - # res.new = self._sort(res.new, norm_word) else: raise NoInternetException() diff --git a/bauh/view/qt/commons.py b/bauh/view/qt/commons.py index e5384a13..1cd470f8 100644 --- a/bauh/view/qt/commons.py +++ b/bauh/view/qt/commons.py @@ -1,10 +1,30 @@ -from typing import Iterable, List +from typing import List, Dict, Any, NamedTuple, Optional, Union, Collection, Iterable from bauh.api.abstract.model import SoftwarePackage from bauh.view.qt.view_model import PackageView -def new_pkgs_info() -> dict: +class PackageFilters(NamedTuple): + """ + It represents the manage window selected filters + """ + + display_limit: int + category: str + name: Optional[str] + only_apps: bool + only_installed: bool + only_updates: bool + search: Optional[str] # initial search term + type: str + + @property + def anything(self) -> bool: + return not self.only_installed and not self.only_updates and not self.only_apps \ + and not self.name and self.type == "any" and self.category == "any" + + +def new_pkgs_info() -> Dict[str, Any]: return {'apps_count': 0, # number of application packages 'napps_count': 0, # number of not application packages (libraries, runtimes or something else) 'available_types': {}, # available package types in 'new_pkgs' @@ -18,7 +38,7 @@ def new_pkgs_info() -> dict: 'pkgs': []} # total packages -def update_info(pkgv: PackageView, pkgs_info: dict): +def update_info(pkgv: PackageView, pkgs_info: Dict[str, Any]): pkgs_info['available_types'][pkgv.model.get_type()] = {'icon': pkgv.model.get_type_icon_path(), 'label': pkgv.get_type_label()} if pkgv.model.is_application(): @@ -49,8 +69,8 @@ def update_info(pkgv: PackageView, pkgs_info: dict): pkgs_info['not_installed'] += 1 -def apply_filters(pkg: PackageView, filters: dict, info: dict, limit: bool = True): - if not limit or not filters['display_limit'] or len(info['pkgs_displayed']) < filters['display_limit']: +def apply_filters(pkg: PackageView, filters: PackageFilters, info: dict, limit: bool = True): + if not limit or not filters.display_limit or len(info['pkgs_displayed']) < filters.display_limit: if not is_package_hidden(pkg, filters): info['pkgs_displayed'].append(pkg) @@ -65,41 +85,48 @@ def sum_updates_displayed(info: dict) -> int: return updates -def is_package_hidden(pkg: PackageView, filters: dict) -> bool: - hidden = filters['only_installed'] and not pkg.model.installed +def is_package_hidden(pkg: PackageView, filters: PackageFilters) -> bool: + hidden = filters.only_installed and not pkg.model.installed - if not hidden and filters['only_apps']: + if not hidden and filters.only_apps: hidden = pkg.model.installed and not pkg.model.is_application() - if not hidden and filters['updates']: + if not hidden and filters.only_updates: hidden = not pkg.model.update or pkg.model.is_update_ignored() - if not hidden and filters['type'] is not None and filters['type'] != 'any': - hidden = pkg.model.get_type() != filters['type'] + if not hidden and filters.type is not None and filters.type != "any": + hidden = pkg.model.get_type() != filters.type - if not hidden and filters['category'] is not None and filters['category'] != 'any': - hidden = not pkg.model.categories or not [c for c in pkg.model.categories if c.lower() == filters['category']] + if not hidden and filters.category is not None and filters.category != "any": + hidden = not pkg.model.categories or not next((c for c in pkg.model.categories if c.lower() == filters.category), None) - if not hidden and filters['name']: - hidden = not filters['name'] in pkg.model.name.lower() + if not hidden and filters.name: + hidden = not filters.name.startswith(pkg.model.name.lower()) return hidden -def sort_packages(pkgs: Iterable[SoftwarePackage], word: str, limit: int = 0) -> List[SoftwarePackage]: +def _by_name(pkg: Union[SoftwarePackage, PackageView]): + return pkg.name.lower() + + +def sort_packages(pkgs: Iterable[Union[SoftwarePackage, PackageView]], word: Optional[str], + limit: int = 0) -> List[SoftwarePackage]: exact, starts_with, contains, others = [], [], [], [] - for p in pkgs: - lower_name = p.name.lower() - - if word == lower_name: - exact.append(p) - elif lower_name.startswith(word): - starts_with.append(p) - elif word in lower_name: - contains.append(p) - else: - others.append(p) + if not word: + others.extend(pkgs) + else: + for p in pkgs: + lower_name = p.name.lower() + if word == lower_name: + exact.append(p) + elif lower_name.startswith(word): + starts_with.append(p) + elif word in lower_name: + contains.append(p) + else: + others.append(p) res = [] for app_list in (exact, starts_with, contains, others): @@ -110,7 +137,7 @@ def sort_packages(pkgs: Iterable[SoftwarePackage], word: str, limit: int = 0) -> break to_add = app_list[0:last] - to_add.sort(key=lambda a: a.name.lower()) + to_add.sort(key=_by_name) res.extend(to_add) return res diff --git a/bauh/view/qt/thread.py b/bauh/view/qt/thread.py index f43f3f54..63a538a1 100644 --- a/bauh/view/qt/thread.py +++ b/bauh/view/qt/thread.py @@ -32,8 +32,9 @@ from bauh.commons.view_utils import get_human_size_str from bauh.view.core import timeshift from bauh.view.core.config import CoreConfigManager, BACKUP_REMOVE_METHODS, BACKUP_DEFAULT_REMOVE_METHOD from bauh.view.qt import commons -from bauh.view.qt.commons import sort_packages +from bauh.view.qt.commons import sort_packages, PackageFilters from bauh.view.qt.qt_utils import get_current_screen_geometry +from bauh.view.qt.view_index import query_packages from bauh.view.qt.view_model import PackageView, PackageViewStatus from bauh.view.util.translation import I18n @@ -960,41 +961,42 @@ class LaunchPackage(AsyncAction): class ApplyFilters(AsyncAction): - signal_table = pyqtSignal(object) + signal_table = pyqtSignal(list) - def __init__(self, i18n: I18n, logger: Logger, filters: dict = None, pkgs: List[PackageView] = None): + def __init__(self, i18n: I18n, logger: Logger, filters: Optional[PackageFilters] = None, + pkgs: Optional[List[PackageView]] = None, index: Optional[dict] = None): super(ApplyFilters, self).__init__(i18n=i18n) self.logger = logger - self.pkgs = pkgs + self.index = index self.filters = filters + self.pkgs = pkgs self.wait_table_update = False def stop_waiting(self): self.wait_table_update = False def run(self): - if self.pkgs: - ti = time.time() - pkgs_info = commons.new_pkgs_info() + if self.index and self.filters and self.pkgs: + if self.filters.anything: + # it means no filter should be applied, and when can rely on the firstly displayed packages + sorted_pkgs = self.pkgs - name_filtering = bool(self.filters['name']) - - for pkgv in self.pkgs: - commons.update_info(pkgv, pkgs_info) - commons.apply_filters(pkgv, self.filters, pkgs_info, limit=not name_filtering) - - if name_filtering and pkgs_info['pkgs_displayed']: - pkgs_info['pkgs_displayed'] = sort_packages(word=self.filters['name'], - pkgs=pkgs_info['pkgs_displayed'], - limit=self.filters['display_limit']) + if self.filters.display_limit > 0: + sorted_pkgs = sorted_pkgs[0:self.filters.display_limit] + + else: + ti = time.time() + sort_term = self.filters.name or self.filters.search # improves displayed matches when no name typed + matched_pkgs = query_packages(index=self.index, filters=self.filters) + sorted_pkgs = commons.sort_packages(pkgs=matched_pkgs, word=sort_term) + tf = time.time() + self.logger.info(f"Took {tf - ti:.9f} seconds to filter and sort packages") self.wait_table_update = True - self.signal_table.emit(pkgs_info) - tf = time.time() - self.logger.info(f"Took {tf - ti:.4f} seconds to filter packages") + self.signal_table.emit(sorted_pkgs) while self.wait_table_update: - super(ApplyFilters, self).msleep(5) + super(ApplyFilters, self).msleep(1) self.notify_finished() diff --git a/bauh/view/qt/view_index.py b/bauh/view/qt/view_index.py new file mode 100644 index 00000000..d2a89595 --- /dev/null +++ b/bauh/view/qt/view_index.py @@ -0,0 +1,130 @@ +from collections import defaultdict +from typing import Dict, List, Generator, Tuple, Optional, Union + +from bauh.view.qt.commons import PackageFilters +from bauh.view.qt.view_model import PackageView + + +def new_character_idx() -> Dict[str, List[PackageView]]: + return defaultdict(list) + + +def new_category_idx() -> Dict[str, Dict[str, List[PackageView]]]: + return defaultdict(new_character_idx) + + +def new_type_index() -> Dict[str, Dict[str, Dict[str, List[PackageView]]]]: + return defaultdict(new_category_idx) + + +def new_update_index() -> Dict[int, Dict[str, Dict[str, Dict[str, List[PackageView]]]]]: + return defaultdict(new_type_index) + + +def new_app_index() -> Dict[int, Dict[int, Dict[str, Dict[str, Dict[str, List[PackageView]]]]]]: + return defaultdict(new_update_index) + + +def new_package_index() -> Dict[int, Dict[int, Dict[int, Dict[str, Dict[str, Dict[str, List[PackageView]]]]]]]: + return defaultdict(new_app_index) + + +def add_to_index(pkgv: PackageView, index: dict) -> None: + # root keys: (1) installed | (0) not installed + root_idx = index[1 if pkgv.model.installed else 0] + + # app keys: (1) app | (0) not app + app_lvl = root_idx[1 if pkgv.model.is_application() else 0] + + # update keys: (1) update | (0) no update + if pkgv.model.installed and pkgv.model.update and not pkgv.model.is_update_ignored(): + update_lvl = app_lvl[1] + else: + update_lvl = app_lvl[0] + + norm_name = pkgv.name.strip().lower() + starts_with_chars = tuple(norm_name[0:i] for i in range(1, len(norm_name) + 1)) + + for cat in ("any", *(pkgv.model.categories if pkgv.model.categories else tuple())): + category = cat.lower().strip() + + # any type > specific category > any character (None) + update_lvl["any"][category][None].append(pkgv) + + # any type > specific category > characters (start + for chars in starts_with_chars: + update_lvl["any"][category][chars].append(pkgv) + + type_lvl = update_lvl[pkgv.model.get_type()] + + # specific type > specific category > any character (None) + type_lvl[category][None].append(pkgv) + + # specific type > any category > first character + for chars in starts_with_chars: + type_lvl[category][chars].append(pkgv) + + +def generate_queries(filters: PackageFilters) -> Generator[Tuple[Optional[Union[int, str]]], None, None]: + chars_query = None + + if filters.name: + chars_query = filters.name.lower() + + installed_queries = (1,) if filters.only_installed else (1, 0) + apps_queries = (1,) if filters.only_apps else (1, 0) + updates_queries = (1,) if filters.only_updates else (1, 0) + + for installed in installed_queries: + for app in apps_queries: + for update in updates_queries: + yield installed, app, update, filters.type, filters.category, chars_query + + +def query_packages(index: dict, filters: PackageFilters) -> Generator[PackageView, None, None]: + yield_count = 0 + yield_limit = filters.display_limit if filters.display_limit and filters.display_limit > 0 else -1 + + queries = tuple(generate_queries(filters)) + + yielded_pkgs = defaultdict(set) + + for query in queries: + packages = index[query[0]][query[1]][query[2]][query[3]][query[4]][query[5]] + + for pkgv in packages: + yield pkgv + yield_count += 1 + yielded_pkgs[pkgv.model.get_type()].add(pkgv.model.id) + + # checking if the package display limit has been reached + if 0 < yield_limit <= yield_count: + break + + # if there is a limit and the number of yielded packages is not reached, performs also a "contains" query + # checking if the queries target "any character" (none), if so, there is no need to perform the "contains" query + any_char_query = next((True for q in queries if q[-1] is None), False) + + if not any_char_query and 0 < yield_limit > yield_count: + for query in queries: + # checking if the package display limit has been reached + if 0 < yield_limit <= yield_count: + break + + packages = index[query[0]][query[1]][query[2]][query[3]][query[4]][None] + + for pkgv in packages: + # checking if the package has already been yielded + yield_type_idx = yielded_pkgs.get(pkgv.model.get_type()) + if yield_type_idx and pkgv.model.id in yield_type_idx: + continue + + # checking if the package name contains the chars query + if query[5] in pkgv.model.name: + yield pkgv + yield_count += 1 + yielded_pkgs[pkgv.model.get_type()].add(pkgv.model.id) + + # checking if the package display limit has been reached + if 0 < yield_limit <= yield_count: + break diff --git a/bauh/view/qt/window.py b/bauh/view/qt/window.py index 71f9cd4b..40d1e848 100755 --- a/bauh/view/qt/window.py +++ b/bauh/view/qt/window.py @@ -3,7 +3,7 @@ import operator import os.path import time from pathlib import Path -from typing import List, Type, Set, Tuple, Optional +from typing import List, Type, Set, Tuple, Optional, Dict, Any from PyQt5.QtCore import QEvent, Qt, pyqtSignal, QRect from PyQt5.QtGui import QIcon, QWindowStateChangeEvent, QCursor @@ -27,7 +27,7 @@ from bauh.view.core.tray_client import notify_tray from bauh.view.qt import dialog, commons, qt_utils from bauh.view.qt.about import AboutDialog from bauh.view.qt.apps_table import PackagesTable, UpgradeToggleButton -from bauh.view.qt.commons import sum_updates_displayed +from bauh.view.qt.commons import sum_updates_displayed, PackageFilters from bauh.view.qt.components import new_spacer, IconButton, QtComponentsManager, to_widget, QSearchBar, \ QCustomMenuAction, QCustomToolbar from bauh.view.qt.dialog import ConfirmationDialog @@ -43,6 +43,7 @@ from bauh.view.qt.thread import UpgradeSelected, RefreshApps, UninstallPackage, AsyncAction, LaunchPackage, ApplyFilters, CustomSoftwareAction, ShowScreenshots, CustomAction, \ NotifyInstalledLoaded, \ IgnorePackageUpdates, SaveTheme, StartAsyncAction +from bauh.view.qt.view_index import add_to_index, new_package_index from bauh.view.qt.view_model import PackageView, PackageViewStatus from bauh.view.util import util, resource from bauh.view.util.translation import I18n @@ -110,6 +111,7 @@ class ManageWindow(QWidget): self.pkgs = [] # packages current loaded in the table self.pkgs_available = [] # all packages loaded in memory self.pkgs_installed = [] # cached installed packages + self.pkg_idx: Optional[Dict[str, Any]] = None # all packages available indexed by the available filters self.display_limit = config['ui']['table']['max_displayed'] self.icon_cache = icon_cache self.config = config @@ -455,6 +457,7 @@ class ManageWindow(QWidget): self.thread_load_installed.signal_loaded.connect(self._finish_loading_installed) self._register_groups() self._screen_geometry: Optional[QRect] = None + self.searched_term: Optional[str] = None # last searched term def _register_groups(self): common_filters = (CHECK_APPS, CHECK_UPDATES, COMBO_CATEGORIES, COMBO_TYPES, INP_NAME) @@ -503,8 +506,9 @@ class ManageWindow(QWidget): self.comp_manager.disable_visible_from_groups(GROUP_UPPER_BAR, GROUP_LOWER_BTS) self.comp_manager.set_component_read_only(INP_NAME, True) - self.thread_apply_filters.filters = self._gen_filters() self.thread_apply_filters.pkgs = self.pkgs_available + self.thread_apply_filters.index = self.pkg_idx + self.thread_apply_filters.filters = self._gen_filters() self.thread_apply_filters.start() self.setFocus(Qt.NoFocusReason) @@ -518,8 +522,13 @@ class ManageWindow(QWidget): self.signal_stop_notifying.emit() self.thread_notify_pkgs_ready.wait(1000) - def _update_table_and_upgrades(self, pkgs_info: dict): - self._update_table(pkgs_info=pkgs_info, signal=True) + def _update_table_and_upgrades(self, packages_displayed: List[PackageView]): + # generating a mocked info to keep compatibility with '_update_table' inputs. + # 'not_installed' is only used for a quick check and does not require an exact number (only 0 or 1) + info = {"pkgs_displayed": packages_displayed, + "not_installed": 1 if len(self.pkgs_installed) != len(self.pkgs_available) else 0} + + self._update_table(pkgs_info=info, signal=True) if self.pkgs: self._update_state_when_pkgs_ready() @@ -887,9 +896,9 @@ class ManageWindow(QWidget): self._resize(accept_lower_width=len(self.pkgs) > 0) def _update_table(self, pkgs_info: dict, signal: bool = False): - self.pkgs = pkgs_info['pkgs_displayed'] + self.pkgs = pkgs_info["pkgs_displayed"] - if pkgs_info['not_installed'] == 0: + if pkgs_info["not_installed"] == 0: update_check = sum_updates_displayed(pkgs_info) > 0 else: update_check = False @@ -903,9 +912,9 @@ class ManageWindow(QWidget): self._resize(accept_lower_width=len(self.pkgs) > 0) if len(self.pkgs) == 0 and len(self.pkgs_available) == 0: - self.label_displayed.setText('') + self.label_displayed.setText("") else: - self.label_displayed.setText('{} / {}'.format(len(self.pkgs), len(self.pkgs_available))) + self.label_displayed.setText(f"{len(self.pkgs)} / {len(self.pkgs_available)}") else: self.label_displayed.hide() @@ -955,20 +964,20 @@ class ManageWindow(QWidget): setattr(self, attr, checked) checkbox.blockSignals(False) - def _gen_filters(self, ignore_updates: bool = False) -> dict: - return { - 'only_apps': False if self.search_performed else self.filter_only_apps, - 'type': self.type_filter, - 'category': self.category_filter, - 'updates': False if ignore_updates else self.filter_updates, - 'name': self.input_name.text().lower() if self.input_name.text() else None, - 'display_limit': None if self.filter_updates else self.display_limit, - 'only_installed': self.filter_installed - } + def _gen_filters(self, ignore_updates: bool = False) -> PackageFilters: + return PackageFilters(category=self.category_filter, + display_limit=0 if self.filter_updates else self.display_limit, + name=self.input_name.text().strip(), + only_apps=False if self.search_performed else self.filter_only_apps, + only_updates=False if ignore_updates else self.filter_updates, + only_installed=self.filter_installed, + search=self.searched_term, + type=self.type_filter) def update_pkgs(self, new_pkgs: Optional[List[SoftwarePackage]], as_installed: bool, types: Optional[Set[type]] = None, ignore_updates: bool = False, keep_filters: bool = False) -> bool: self.input_name.set_text('') pkgs_info = commons.new_pkgs_info() + pkg_idx = new_package_index() filters = self._gen_filters(ignore_updates=ignore_updates) if new_pkgs is not None: @@ -979,19 +988,22 @@ class ManageWindow(QWidget): self.pkgs_installed = [] for pkg in new_pkgs: - app_model = PackageView(model=pkg, i18n=self.i18n) - commons.update_info(app_model, pkgs_info) - commons.apply_filters(app_model, filters, pkgs_info) + pkgv = PackageView(model=pkg, i18n=self.i18n) + commons.update_info(pkgv, pkgs_info) + add_to_index(pkgv, pkg_idx) + commons.apply_filters(pkgv, filters, pkgs_info) if old_installed and types: for pkgv in old_installed: if pkgv.model.__class__ not in types: commons.update_info(pkgv, pkgs_info) + add_to_index(pkgv, pkg_idx) commons.apply_filters(pkgv, filters, pkgs_info) else: # use installed for pkgv in self.pkgs_installed: commons.update_info(pkgv, pkgs_info) + add_to_index(pkgv, pkg_idx) commons.apply_filters(pkgv, filters, pkgs_info) if pkgs_info['apps_count'] == 0 and not self.suggestions_requested: @@ -1019,6 +1031,7 @@ class ManageWindow(QWidget): self.change_update_state(pkgs_info=pkgs_info, trigger_filters=False, keep_selected=keep_filters and bool(pkgs_info['pkgs_displayed'])) self.pkgs_available = pkgs_info['pkgs'] + self.pkg_idx = pkg_idx if as_installed: self.pkgs_installed = pkgs_info['pkgs'] @@ -1374,6 +1387,7 @@ class ManageWindow(QWidget): label = f"{self.i18n['manage_window.status.searching']} {word if word else ''}" self._begin_action(action_label=label, action_id=ACTION_SEARCH) self.comp_manager.set_components_visible(False) + self.searched_term = word self.thread_search.word = word self.thread_search.start()