[view.qt] enhancement: faster packages filtering

CHANGELOG.md

@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - UI
   - the "Skip" button on the initialization panel is now enabled after 10 seconds [#310](https://github.com/vinifmor/bauh/issues/310)
   - faster package icons download
+  - faster packages filtering (`type`, `category`, `name`, etc... up to **95% less time**)
 
 ### Fixes
 - AppImage

bauh/view/core/controller.py

@@ -181,9 +181,6 @@ class GenericSoftwareManager(SoftwareManager, SettingsController):
                 if disk_loader:
                     disk_loader.stop_working()
                     disk_loader.join()
-
-            # res.installed = self._sort(res.installed, norm_word)
-            # res.new = self._sort(res.new, norm_word)
         else:
             raise NoInternetException()
 

bauh/view/qt/commons.py

@@ -1,10 +1,30 @@
-from typing import Iterable, List
+from typing import List, Dict, Any, NamedTuple, Optional, Union, Collection, Iterable
 
 from bauh.api.abstract.model import SoftwarePackage
 from bauh.view.qt.view_model import PackageView
 
 
-def new_pkgs_info() -> dict:
+class PackageFilters(NamedTuple):
+    """
+    It represents the manage window selected filters
+    """
+
+    display_limit: int
+    category: str
+    name: Optional[str]
+    only_apps: bool
+    only_installed: bool
+    only_updates: bool
+    search: Optional[str]  # initial search term
+    type: str
+
+    @property
+    def anything(self) -> bool:
+        return not self.only_installed and not self.only_updates and not self.only_apps \
+            and not self.name and self.type == "any" and self.category == "any"
+
+
+def new_pkgs_info() -> Dict[str, Any]:
     return {'apps_count': 0,  # number of application packages
             'napps_count': 0,  # number of not application packages (libraries, runtimes or something else)
             'available_types': {},  # available package types in 'new_pkgs'
@@ -18,7 +38,7 @@ def new_pkgs_info() -> dict:
             'pkgs': []}  # total packages
 
 
-def update_info(pkgv: PackageView, pkgs_info: dict):
+def update_info(pkgv: PackageView, pkgs_info: Dict[str, Any]):
     pkgs_info['available_types'][pkgv.model.get_type()] = {'icon': pkgv.model.get_type_icon_path(), 'label': pkgv.get_type_label()}
 
     if pkgv.model.is_application():
@@ -49,8 +69,8 @@ def update_info(pkgv: PackageView, pkgs_info: dict):
         pkgs_info['not_installed'] += 1
 
 
-def apply_filters(pkg: PackageView, filters: dict, info: dict, limit: bool = True):
+def apply_filters(pkg: PackageView, filters: PackageFilters, info: dict, limit: bool = True):
-    if not limit or not filters['display_limit'] or len(info['pkgs_displayed']) < filters['display_limit']:
+    if not limit or not filters.display_limit or len(info['pkgs_displayed']) < filters.display_limit:
         if not is_package_hidden(pkg, filters):
             info['pkgs_displayed'].append(pkg)
 
@@ -65,41 +85,48 @@ def sum_updates_displayed(info: dict) -> int:
     return updates
 
 
-def is_package_hidden(pkg: PackageView, filters: dict) -> bool:
+def is_package_hidden(pkg: PackageView, filters: PackageFilters) -> bool:
-    hidden = filters['only_installed'] and not pkg.model.installed
+    hidden = filters.only_installed and not pkg.model.installed
 
-    if not hidden and filters['only_apps']:
+    if not hidden and filters.only_apps:
         hidden = pkg.model.installed and not pkg.model.is_application()
 
-    if not hidden and filters['updates']:
+    if not hidden and filters.only_updates:
         hidden = not pkg.model.update or pkg.model.is_update_ignored()
 
-    if not hidden and filters['type'] is not None and filters['type'] != 'any':
+    if not hidden and filters.type is not None and filters.type != "any":
-        hidden = pkg.model.get_type() != filters['type']
+        hidden = pkg.model.get_type() != filters.type
 
-    if not hidden and filters['category'] is not None and filters['category'] != 'any':
+    if not hidden and filters.category is not None and filters.category != "any":
-        hidden = not pkg.model.categories or not [c for c in pkg.model.categories if c.lower() == filters['category']]
+        hidden = not pkg.model.categories or not next((c for c in pkg.model.categories if c.lower() == filters.category), None)
 
-    if not hidden and filters['name']:
+    if not hidden and filters.name:
-        hidden = not filters['name'] in pkg.model.name.lower()
+        hidden = not filters.name.startswith(pkg.model.name.lower())
 
     return hidden
 
 
-def sort_packages(pkgs: Iterable[SoftwarePackage], word: str, limit: int = 0) -> List[SoftwarePackage]:
+def _by_name(pkg: Union[SoftwarePackage, PackageView]):
+    return pkg.name.lower()
+
+
+def sort_packages(pkgs: Iterable[Union[SoftwarePackage, PackageView]], word: Optional[str],
+                  limit: int = 0) -> List[SoftwarePackage]:
     exact, starts_with, contains, others = [], [], [], []
 
-    for p in pkgs:
+    if not word:
-        lower_name = p.name.lower()
+        others.extend(pkgs)
-
+    else:
-        if word == lower_name:
+        for p in pkgs:
-            exact.append(p)
+            lower_name = p.name.lower()
-        elif lower_name.startswith(word):
+            if word == lower_name:
-            starts_with.append(p)
+                exact.append(p)
-        elif word in lower_name:
+            elif lower_name.startswith(word):
-            contains.append(p)
+                starts_with.append(p)
-        else:
+            elif word in lower_name:
-            others.append(p)
+                contains.append(p)
+            else:
+                others.append(p)
 
     res = []
     for app_list in (exact, starts_with, contains, others):
@@ -110,7 +137,7 @@ def sort_packages(pkgs: Iterable[SoftwarePackage], word: str, limit: int = 0) ->
                 break
 
             to_add = app_list[0:last]
-            to_add.sort(key=lambda a: a.name.lower())
+            to_add.sort(key=_by_name)
             res.extend(to_add)
 
     return res

bauh/view/qt/thread.py

@@ -32,8 +32,9 @@ from bauh.commons.view_utils import get_human_size_str
 from bauh.view.core import timeshift
 from bauh.view.core.config import CoreConfigManager, BACKUP_REMOVE_METHODS, BACKUP_DEFAULT_REMOVE_METHOD
 from bauh.view.qt import commons
-from bauh.view.qt.commons import sort_packages
+from bauh.view.qt.commons import sort_packages, PackageFilters
 from bauh.view.qt.qt_utils import get_current_screen_geometry
+from bauh.view.qt.view_index import query_packages
 from bauh.view.qt.view_model import PackageView, PackageViewStatus
 from bauh.view.util.translation import I18n
 
@@ -960,41 +961,42 @@ class LaunchPackage(AsyncAction):
 
 class ApplyFilters(AsyncAction):
 
-    signal_table = pyqtSignal(object)
+    signal_table = pyqtSignal(list)
 
-    def __init__(self, i18n: I18n, logger: Logger, filters: dict = None, pkgs: List[PackageView] = None):
+    def __init__(self, i18n: I18n, logger: Logger, filters: Optional[PackageFilters] = None,
+                 pkgs: Optional[List[PackageView]] = None, index: Optional[dict] = None):
         super(ApplyFilters, self).__init__(i18n=i18n)
         self.logger = logger
-        self.pkgs = pkgs
+        self.index = index
         self.filters = filters
+        self.pkgs = pkgs
         self.wait_table_update = False
 
     def stop_waiting(self):
         self.wait_table_update = False
 
     def run(self):
-        if self.pkgs:
+        if self.index and self.filters and self.pkgs:
-            ti = time.time()
+            if self.filters.anything:
-            pkgs_info = commons.new_pkgs_info()
+                # it means no filter should be applied, and when can rely on the firstly displayed packages
+                sorted_pkgs = self.pkgs
 
-            name_filtering = bool(self.filters['name'])
+                if self.filters.display_limit > 0:
+                    sorted_pkgs = sorted_pkgs[0:self.filters.display_limit]
                 
-            for pkgv in self.pkgs:
+            else:
-                commons.update_info(pkgv, pkgs_info)
+                ti = time.time()
-                commons.apply_filters(pkgv, self.filters, pkgs_info, limit=not name_filtering)
+                sort_term = self.filters.name or self.filters.search  # improves displayed matches when no name typed
-
+                matched_pkgs = query_packages(index=self.index, filters=self.filters)
-            if name_filtering and pkgs_info['pkgs_displayed']:
+                sorted_pkgs = commons.sort_packages(pkgs=matched_pkgs, word=sort_term)
-                pkgs_info['pkgs_displayed'] = sort_packages(word=self.filters['name'],
+                tf = time.time()
-                                                            pkgs=pkgs_info['pkgs_displayed'],
+                self.logger.info(f"Took {tf - ti:.9f} seconds to filter and sort packages")
-                                                            limit=self.filters['display_limit'])
 
             self.wait_table_update = True
-            self.signal_table.emit(pkgs_info)
+            self.signal_table.emit(sorted_pkgs)
-            tf = time.time()
-            self.logger.info(f"Took {tf - ti:.4f} seconds to filter packages")
 
             while self.wait_table_update:
-                super(ApplyFilters, self).msleep(5)
+                super(ApplyFilters, self).msleep(1)
 
         self.notify_finished()
 

bauh/view/qt/view_index.py

@@ -0,0 +1,130 @@
+from collections import defaultdict
+from typing import Dict, List, Generator, Tuple, Optional, Union
+
+from bauh.view.qt.commons import PackageFilters
+from bauh.view.qt.view_model import PackageView
+
+
+def new_character_idx() -> Dict[str, List[PackageView]]:
+    return defaultdict(list)
+
+
+def new_category_idx() -> Dict[str, Dict[str, List[PackageView]]]:
+    return defaultdict(new_character_idx)
+
+
+def new_type_index() -> Dict[str, Dict[str, Dict[str, List[PackageView]]]]:
+    return defaultdict(new_category_idx)
+
+
+def new_update_index() -> Dict[int, Dict[str, Dict[str, Dict[str, List[PackageView]]]]]:
+    return defaultdict(new_type_index)
+
+
+def new_app_index() -> Dict[int, Dict[int, Dict[str, Dict[str, Dict[str, List[PackageView]]]]]]:
+    return defaultdict(new_update_index)
+
+
+def new_package_index() -> Dict[int, Dict[int, Dict[int, Dict[str, Dict[str, Dict[str, List[PackageView]]]]]]]:
+    return defaultdict(new_app_index)
+
+
+def add_to_index(pkgv: PackageView, index: dict) -> None:
+    # root keys: (1) installed | (0) not installed
+    root_idx = index[1 if pkgv.model.installed else 0]
+
+    # app keys: (1) app | (0) not app
+    app_lvl = root_idx[1 if pkgv.model.is_application() else 0]
+
+    # update keys: (1) update | (0) no update
+    if pkgv.model.installed and pkgv.model.update and not pkgv.model.is_update_ignored():
+        update_lvl = app_lvl[1]
+    else:
+        update_lvl = app_lvl[0]
+
+    norm_name = pkgv.name.strip().lower()
+    starts_with_chars = tuple(norm_name[0:i] for i in range(1, len(norm_name) + 1))
+
+    for cat in ("any", *(pkgv.model.categories if pkgv.model.categories else tuple())):
+        category = cat.lower().strip()
+
+        # any type > specific category > any character (None)
+        update_lvl["any"][category][None].append(pkgv)
+
+        # any type > specific category > characters (start
+        for chars in starts_with_chars:
+            update_lvl["any"][category][chars].append(pkgv)
+
+        type_lvl = update_lvl[pkgv.model.get_type()]
+
+        # specific type > specific category > any character (None)
+        type_lvl[category][None].append(pkgv)
+
+        # specific type > any category > first character
+        for chars in starts_with_chars:
+            type_lvl[category][chars].append(pkgv)
+
+
+def generate_queries(filters: PackageFilters) -> Generator[Tuple[Optional[Union[int, str]]], None, None]:
+    chars_query = None
+
+    if filters.name:
+        chars_query = filters.name.lower()
+
+    installed_queries = (1,) if filters.only_installed else (1, 0)
+    apps_queries = (1,) if filters.only_apps else (1, 0)
+    updates_queries = (1,) if filters.only_updates else (1, 0)
+
+    for installed in installed_queries:
+        for app in apps_queries:
+            for update in updates_queries:
+                yield installed, app, update, filters.type, filters.category, chars_query
+
+
+def query_packages(index: dict, filters: PackageFilters) -> Generator[PackageView, None, None]:
+    yield_count = 0
+    yield_limit = filters.display_limit if filters.display_limit and filters.display_limit > 0 else -1
+
+    queries = tuple(generate_queries(filters))
+
+    yielded_pkgs = defaultdict(set)
+
+    for query in queries:
+        packages = index[query[0]][query[1]][query[2]][query[3]][query[4]][query[5]]
+
+        for pkgv in packages:
+            yield pkgv
+            yield_count += 1
+            yielded_pkgs[pkgv.model.get_type()].add(pkgv.model.id)
+
+            # checking if the package display limit has been reached
+            if 0 < yield_limit <= yield_count:
+                break
+
+    # if there is a limit and the number of yielded packages is not reached, performs also a "contains" query
+    # checking if the queries target "any character" (none), if so, there is no need to perform the "contains" query
+    any_char_query = next((True for q in queries if q[-1] is None), False)
+
+    if not any_char_query and 0 < yield_limit > yield_count:
+        for query in queries:
+            # checking if the package display limit has been reached
+            if 0 < yield_limit <= yield_count:
+                break
+
+            packages = index[query[0]][query[1]][query[2]][query[3]][query[4]][None]
+
+            for pkgv in packages:
+                # checking if the package has already been yielded
+                yield_type_idx = yielded_pkgs.get(pkgv.model.get_type())
+                if yield_type_idx and pkgv.model.id in yield_type_idx:
+                    continue
+
+                # checking if the package name contains the chars query
+                if query[5] in pkgv.model.name:
+                    yield pkgv
+                    yield_count += 1
+                    yielded_pkgs[pkgv.model.get_type()].add(pkgv.model.id)
+
+                # checking if the package display limit has been reached
+                if 0 < yield_limit <= yield_count:
+                    break

bauh/view/qt/window.py

@@ -3,7 +3,7 @@ import operator
 import os.path
 import time
 from pathlib import Path
-from typing import List, Type, Set, Tuple, Optional
+from typing import List, Type, Set, Tuple, Optional, Dict, Any
 
 from PyQt5.QtCore import QEvent, Qt, pyqtSignal, QRect
 from PyQt5.QtGui import QIcon, QWindowStateChangeEvent, QCursor
@@ -27,7 +27,7 @@ from bauh.view.core.tray_client import notify_tray
 from bauh.view.qt import dialog, commons, qt_utils
 from bauh.view.qt.about import AboutDialog
 from bauh.view.qt.apps_table import PackagesTable, UpgradeToggleButton
-from bauh.view.qt.commons import sum_updates_displayed
+from bauh.view.qt.commons import sum_updates_displayed, PackageFilters
 from bauh.view.qt.components import new_spacer, IconButton, QtComponentsManager, to_widget, QSearchBar, \
     QCustomMenuAction, QCustomToolbar
 from bauh.view.qt.dialog import ConfirmationDialog
@@ -43,6 +43,7 @@ from bauh.view.qt.thread import UpgradeSelected, RefreshApps, UninstallPackage,
     AsyncAction, LaunchPackage, ApplyFilters, CustomSoftwareAction, ShowScreenshots, CustomAction, \
     NotifyInstalledLoaded, \
     IgnorePackageUpdates, SaveTheme, StartAsyncAction
+from bauh.view.qt.view_index import add_to_index, new_package_index
 from bauh.view.qt.view_model import PackageView, PackageViewStatus
 from bauh.view.util import util, resource
 from bauh.view.util.translation import I18n
@@ -110,6 +111,7 @@ class ManageWindow(QWidget):
         self.pkgs = []  # packages current loaded in the table
         self.pkgs_available = []  # all packages loaded in memory
         self.pkgs_installed = []  # cached installed packages
+        self.pkg_idx: Optional[Dict[str, Any]] = None  # all packages available indexed by the available filters
         self.display_limit = config['ui']['table']['max_displayed']
         self.icon_cache = icon_cache
         self.config = config
@@ -455,6 +457,7 @@ class ManageWindow(QWidget):
         self.thread_load_installed.signal_loaded.connect(self._finish_loading_installed)
         self._register_groups()
         self._screen_geometry: Optional[QRect] = None
+        self.searched_term: Optional[str] = None  # last searched term
 
     def _register_groups(self):
         common_filters = (CHECK_APPS, CHECK_UPDATES, COMBO_CATEGORIES, COMBO_TYPES, INP_NAME)
@@ -503,8 +506,9 @@ class ManageWindow(QWidget):
         self.comp_manager.disable_visible_from_groups(GROUP_UPPER_BAR, GROUP_LOWER_BTS)
         self.comp_manager.set_component_read_only(INP_NAME, True)
 
-        self.thread_apply_filters.filters = self._gen_filters()
         self.thread_apply_filters.pkgs = self.pkgs_available
+        self.thread_apply_filters.index = self.pkg_idx
+        self.thread_apply_filters.filters = self._gen_filters()
         self.thread_apply_filters.start()
         self.setFocus(Qt.NoFocusReason)
 
@@ -518,8 +522,13 @@ class ManageWindow(QWidget):
             self.signal_stop_notifying.emit()
             self.thread_notify_pkgs_ready.wait(1000)
 
-    def _update_table_and_upgrades(self, pkgs_info: dict):
+    def _update_table_and_upgrades(self, packages_displayed: List[PackageView]):
-        self._update_table(pkgs_info=pkgs_info, signal=True)
+        # generating a mocked info to keep compatibility with '_update_table' inputs.
+        # 'not_installed' is only used for a quick check and does not require an exact number (only 0 or 1)
+        info = {"pkgs_displayed": packages_displayed,
+                "not_installed": 1 if len(self.pkgs_installed) != len(self.pkgs_available) else 0}
+
+        self._update_table(pkgs_info=info, signal=True)
 
         if self.pkgs:
             self._update_state_when_pkgs_ready()
@@ -887,9 +896,9 @@ class ManageWindow(QWidget):
             self._resize(accept_lower_width=len(self.pkgs) > 0)
 
     def _update_table(self, pkgs_info: dict, signal: bool = False):
-        self.pkgs = pkgs_info['pkgs_displayed']
+        self.pkgs = pkgs_info["pkgs_displayed"]
 
-        if pkgs_info['not_installed'] == 0:
+        if pkgs_info["not_installed"] == 0:
             update_check = sum_updates_displayed(pkgs_info) > 0
         else:
             update_check = False
@@ -903,9 +912,9 @@ class ManageWindow(QWidget):
             self._resize(accept_lower_width=len(self.pkgs) > 0)
 
             if len(self.pkgs) == 0 and len(self.pkgs_available) == 0:
-                self.label_displayed.setText('')
+                self.label_displayed.setText("")
             else:
-                self.label_displayed.setText('{} / {}'.format(len(self.pkgs), len(self.pkgs_available)))
+                self.label_displayed.setText(f"{len(self.pkgs)} / {len(self.pkgs_available)}")
         else:
             self.label_displayed.hide()
 
@@ -955,20 +964,20 @@ class ManageWindow(QWidget):
             setattr(self, attr, checked)
             checkbox.blockSignals(False)
 
-    def _gen_filters(self, ignore_updates: bool = False) -> dict:
+    def _gen_filters(self, ignore_updates: bool = False) -> PackageFilters:
-        return {
+        return PackageFilters(category=self.category_filter,
-            'only_apps': False if self.search_performed else self.filter_only_apps,
+                              display_limit=0 if self.filter_updates else self.display_limit,
-            'type': self.type_filter,
+                              name=self.input_name.text().strip(),
-            'category': self.category_filter,
+                              only_apps=False if self.search_performed else self.filter_only_apps,
-            'updates': False if ignore_updates else self.filter_updates,
+                              only_updates=False if ignore_updates else self.filter_updates,
-            'name': self.input_name.text().lower() if self.input_name.text() else None,
+                              only_installed=self.filter_installed,
-            'display_limit': None if self.filter_updates else self.display_limit,
+                              search=self.searched_term,
-            'only_installed': self.filter_installed
+                              type=self.type_filter)
-        }
 
     def update_pkgs(self, new_pkgs: Optional[List[SoftwarePackage]], as_installed: bool, types: Optional[Set[type]] = None, ignore_updates: bool = False, keep_filters: bool = False) -> bool:
         self.input_name.set_text('')
         pkgs_info = commons.new_pkgs_info()
+        pkg_idx = new_package_index()
         filters = self._gen_filters(ignore_updates=ignore_updates)
 
         if new_pkgs is not None:
@@ -979,19 +988,22 @@ class ManageWindow(QWidget):
                 self.pkgs_installed = []
 
             for pkg in new_pkgs:
-                app_model = PackageView(model=pkg, i18n=self.i18n)
+                pkgv = PackageView(model=pkg, i18n=self.i18n)
-                commons.update_info(app_model, pkgs_info)
+                commons.update_info(pkgv, pkgs_info)
-                commons.apply_filters(app_model, filters, pkgs_info)
+                add_to_index(pkgv, pkg_idx)
+                commons.apply_filters(pkgv, filters, pkgs_info)
 
             if old_installed and types:
                 for pkgv in old_installed:
                     if pkgv.model.__class__ not in types:
                         commons.update_info(pkgv, pkgs_info)
+                        add_to_index(pkgv, pkg_idx)
                         commons.apply_filters(pkgv, filters, pkgs_info)
 
         else:  # use installed
             for pkgv in self.pkgs_installed:
                 commons.update_info(pkgv, pkgs_info)
+                add_to_index(pkgv, pkg_idx)
                 commons.apply_filters(pkgv, filters, pkgs_info)
 
         if pkgs_info['apps_count'] == 0 and not self.suggestions_requested:
@@ -1019,6 +1031,7 @@ class ManageWindow(QWidget):
         self.change_update_state(pkgs_info=pkgs_info, trigger_filters=False, keep_selected=keep_filters and bool(pkgs_info['pkgs_displayed']))
 
         self.pkgs_available = pkgs_info['pkgs']
+        self.pkg_idx = pkg_idx
 
         if as_installed:
             self.pkgs_installed = pkgs_info['pkgs']
@@ -1374,6 +1387,7 @@ class ManageWindow(QWidget):
             label = f"{self.i18n['manage_window.status.searching']} {word if word else ''}"
             self._begin_action(action_label=label, action_id=ACTION_SEARCH)
             self.comp_manager.set_components_visible(False)
+            self.searched_term = word
             self.thread_search.word = word
             self.thread_search.start()