[flatpak] fix: not displaying new required runtimes as updates

CHANGELOG.md

@@ -18,7 +18,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
     <p align="center">
         <img src="https://raw.githubusercontent.com/vinifmor/bauh-files/master/pictures/releases/0.10.2/arch_aur_tabs.png">
     </p>  
-
+  
+- Flatpak
+  - faster updates reading (threaded)
   
 - Debian
   - install: installation/download sizes order (to follow the upgrade dialog order) 
@@ -47,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Flatpak
   - some updates not being displayed when there are new required runtimes for installed Flatpaks
+  - not displaying new required runtimes as updates (requires Flatpak >= 1.12)
 
 - UI
   - not displaying the right unit symbol for byte based sizes (kB, MB, TB, ...) [#250](https://github.com/vinifmor/bauh/issues/250)

bauh/commons/system.py

@@ -26,7 +26,9 @@ USE_GLOBAL_INTERPRETER = bool(os.getenv('VIRTUAL_ENV'))
 RE_SUDO_OUTPUT = re.compile(r'[sudo]\s*[\w\s]+:\s*')
 
 
-def gen_env(global_interpreter: bool, lang: Optional[str] = DEFAULT_LANG, extra_paths: Optional[Set[str]] = None) -> dict:
+def gen_env(global_interpreter: bool = USE_GLOBAL_INTERPRETER, lang: Optional[str] = DEFAULT_LANG,
+            extra_paths: Optional[Set[str]] = None) -> dict:
+
     custom_env = dict(os.environ)
 
     if lang is not None:

bauh/gems/flatpak/controller.py

@@ -5,7 +5,7 @@ from datetime import datetime
 from math import floor
 from pathlib import Path
 from threading import Thread
-from typing import List, Set, Type, Tuple, Optional, Generator
+from typing import List, Set, Type, Tuple, Optional, Generator, Dict
 
 from packaging.version import Version
 
@@ -23,7 +23,7 @@ from bauh.commons.html import strip_html, bold
 from bauh.commons.system import ProcessHandler
 from bauh.gems.flatpak import flatpak, SUGGESTIONS_FILE, CONFIG_FILE, UPDATES_IGNORED_FILE, FLATPAK_CONFIG_DIR, \
     EXPORTS_PATH, \
-    get_icon_path, VERSION_1_5, VERSION_1_2
+    get_icon_path, VERSION_1_5, VERSION_1_2, VERSION_1_12
 from bauh.gems.flatpak.config import FlatpakConfigManager
 from bauh.gems.flatpak.constants import FLATHUB_API_URL
 from bauh.gems.flatpak.model import FlatpakApplication
@@ -124,17 +124,38 @@ class FlatpakManager(SoftwareManager, SettingsController):
     def _add_updates(self, version: Version, output: list):
         output.append(flatpak.list_updates_as_str(version))
 
+    def _fill_required_runtimes(self, installation: str, output: List[Tuple[str, str]]):
+        runtimes = flatpak.list_required_runtime_updates(installation=installation)
+
+        if runtimes:
+            output.extend(runtimes)
+
+    def _fill_required_runtime_updates(self, output: Dict[str, List[Tuple[str, str]]]):
+        threads = []
+        for installation in ('system', 'user'):
+            runtimes = list()
+            output[installation] = runtimes
+            t = Thread(target=self._fill_required_runtimes, args=(installation, runtimes))
+            t.start()
+            threads.append(t)
+
+        for t in threads:
+            t.join()
+
     def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None,
                        internet_available: bool = None, wait_async_data: bool = False) -> SearchResult:
         version = flatpak.get_version()
 
-        updates = []
+        updates, required_runtimes = list(), dict()
 
+        thread_updates, thread_runtimes = None, None
         if internet_available:
             thread_updates = Thread(target=self._add_updates, args=(version, updates))
             thread_updates.start()
-        else:
+
-            thread_updates = None
+            if version >= VERSION_1_12:
+                thread_runtimes = Thread(target=self._fill_required_runtime_updates, args=(required_runtimes,))
+                thread_runtimes.start()
 
         installed = flatpak.list_installed(version)
 
@@ -192,6 +213,28 @@ class FlatpakManager(SoftwareManager, SettingsController):
                                 models[partial_update_id] = partial_model
                                 break
 
+        if thread_runtimes:
+            thread_runtimes.join()
+
+        if required_runtimes:
+            for installation in ('system', 'user'):
+                installation_runtimes = required_runtimes.get(installation)
+
+                if installation_runtimes:
+                    for ref, origin in installation_runtimes:
+                        ref_split = ref.split('/')
+                        models[f'{installation}.'] = FlatpakApplication(id=ref_split[1],
+                                                                        ref=ref,
+                                                                        origin=origin,
+                                                                        name=ref_split[1],
+                                                                        version=ref_split[-1],
+                                                                        latest_version=ref_split[-1],
+                                                                        runtime=True,
+                                                                        installation=installation,
+                                                                        installed=True,
+                                                                        update_component=True,
+                                                                        update=True)
+
         if models:
             ignored = self._read_ignored_updates()
 
@@ -307,7 +350,7 @@ class FlatpakManager(SoftwareManager, SettingsController):
                             'origin': app.origin,
                             'arch': app.arch,
                             'ref': app.ref,
-                            'type': self.i18n['unknown']}
+                            'type': 'runtime' if app.runtime else self.i18n['unknown']}
             else:
                 version = flatpak.get_version()
                 id_ = app.base_id if app.partial and version < VERSION_1_5 else app.id

bauh/gems/flatpak/flatpak.py

@@ -3,12 +3,14 @@ import re
 import subprocess
 import traceback
 from datetime import datetime
-from typing import List, Dict, Set, Iterable, Optional
+from threading import Thread
+from typing import List, Dict, Set, Iterable, Optional, Tuple
 
 from packaging.version import Version
 from packaging.version import parse as parse_version
 
 from bauh.api.exception import NoInternetException
+from bauh.commons import system
 from bauh.commons.system import new_subprocess, run_cmd, SimpleProcess, ProcessHandler, DEFAULT_LANG
 from bauh.commons.util import size_to_byte
 from bauh.gems.flatpak import EXPORTS_PATH, VERSION_1_3, VERSION_1_2, VERSION_1_5, VERSION_1_12
@@ -16,6 +18,7 @@ from bauh.gems.flatpak.constants import FLATHUB_URL
 
 RE_SEVERAL_SPACES = re.compile(r'\s+')
 RE_COMMIT = re.compile(r'(Latest commit|Commit)\s*:\s*(.+)')
+RE_REQUIRED_RUNTIME = re.compile(f'Required\s+runtime\s+.+\(([\w./]+)\)\s*.+\s+remote\s+([\w+./]+)')
 OPERATION_UPDATE_SYMBOLS = {'i', 'u'}
 
 
@@ -184,18 +187,46 @@ def uninstall(app_ref: str, installation: str, version: Version) -> SimpleProces
                          shell=True)
 
 
-def list_updates_as_str(version: Version) -> Dict[str, set]:
+def _new_updates() -> Dict[str, Set[str]]:
-    updates = read_updates(version, 'system')
+    return {'full': set(), 'partial': set()}
-    user_updates = read_updates(version, 'user')
-
-    for attr in ('full', 'partial'):
-        updates[attr].update(user_updates[attr])
-
-    return updates
 
 
-def read_updates(version: Version, installation: str) -> Dict[str, set]:
+def list_updates_as_str(version: Version) -> Dict[str, Set[str]]:
-    res = {'partial': set(), 'full': set()}
+    sys_updates, user_updates = _new_updates(), _new_updates()
+
+    threads = []
+    for type_, output in (('system', sys_updates), ('user', user_updates)):
+        fill = Thread(target=fill_updates, args=(version, type_, output))
+        fill.start()
+        threads.append(fill)
+
+    for t in threads:
+        t.join()
+
+    all_updates = _new_updates()
+
+    for updates in (sys_updates, user_updates):
+        if updates:
+            for key, val in updates.items():
+                if val:
+                    all_updates[key].update(val)
+
+    return all_updates
+
+
+def list_required_runtime_updates(installation: str) -> Optional[List[Tuple[str, str]]]:
+    """
+    Return a list of tuples composed by the reference and the origin.
+    e.g: ('runtime/org.gnome.Desktop/42/x86_64', 'flathub')
+    """
+    _, updates = system.execute(f'flatpak update --{installation}', shell=True,
+                                custom_env=system.gen_env())
+
+    if updates:
+        return RE_REQUIRED_RUNTIME.findall(updates)
+
+
+def fill_updates(version: Version, installation: str, res: Dict[str, Set[str]]):
     if version < VERSION_1_2:
         try:
             output = run_cmd(f'flatpak update --no-related --no-deps --{installation}', ignore_return_code=True)
@@ -235,8 +266,6 @@ def read_updates(version: Version, installation: str) -> Dict[str, set]:
         except:
             traceback.print_exc()
 
-    return res
-
 
 def downgrade(app_ref: str, commit: str, installation: str, root_password: Optional[str], version: Version) -> SimpleProcess:
     cmd = ('flatpak', 'update', '--no-related', '--no-deps', f'--commit={commit}', app_ref, '-y', f'--{installation}')

bauh/gems/flatpak/model.py

@@ -8,11 +8,13 @@ from bauh.view.util.translation import I18n
 
 class FlatpakApplication(SoftwarePackage):
 
-    def __init__(self, id: str = None, name: str = None, version: str = None, latest_version: str = None, description: str = None,
+    def __init__(self, id: str = None, name: str = None, version: str = None, latest_version: str = None,
-                 branch: str = None, arch: str = None, origin: str = None, runtime: bool = False, ref: str = None, commit: str = None,
+                 description: str = None, branch: str = None, arch: str = None, origin: str = None,
-                 installation: str = None, i18n: I18n = None, partial: bool = False, updates_ignored: bool = False):
+                 runtime: bool = False, ref: str = None, commit: str = None, installation: str = None,
-        super(FlatpakApplication, self).__init__(id=id, name=name, version=version,
+                 i18n: I18n = None, partial: bool = False, updates_ignored: bool = False,  installed: bool = False,
-                                                 latest_version=latest_version, description=description)
+                 update: bool = False, update_component: bool = False):
+        super(FlatpakApplication, self).__init__(id=id, name=name, version=version, latest_version=latest_version,
+                                                 description=description, installed=installed, update=update)
         self.ref = ref
         self.branch = branch
         self.arch = arch
@@ -25,7 +27,7 @@ class FlatpakApplication(SoftwarePackage):
         self.base_id = None
         self.base_ref = None
         self.updates_ignored = updates_ignored
-        self.update_component = False  # if it is a new app/runtime that has come as an update
+        self.update_component = update_component  # if it is a new app/runtime that has come as an update
 
         if runtime:
             self.categories = ['runtime']