improving installed Flatpaks reading time

CHANGELOG.md

@@ -6,9 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [0.6.0]
 ### Improvements:
-- Reading installed snaps now takes around 95% less time.
+- Reading installed Snaps now takes around 95% less time.
+- Reading installed Flatpaks now takes around 45% less time.
 
 ### Fixes:
+- cached Flatpak app current version
 
 ## [0.5.1] - 2019-08-12
 ### Improvements:

bauh/gems/flatpak/controller.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from threading import Thread
 from typing import List, Set, Type
 
 from bauh.api.abstract.controller import SearchResult, SoftwareManager, ApplicationContext
@@ -8,7 +9,6 @@ from bauh.api.abstract.model import PackageHistory, PackageUpdate, SoftwarePacka
 from bauh.api.abstract.view import MessageType
 from bauh.commons.html import strip_html
 from bauh.commons.system import SystemProcess, ProcessHandler
-
 from bauh.gems.flatpak import flatpak, suggestions
 from bauh.gems.flatpak.model import FlatpakApplication
 from bauh.gems.flatpak.worker import FlatpakAsyncDataLoader, FlatpakUpdateLoader
@@ -27,25 +27,14 @@ class FlatpakManager(SoftwareManager):
 
     def _map_to_model(self, app_json: dict, installed: bool, disk_loader: DiskCacheLoader) -> FlatpakApplication:
 
-        app = FlatpakApplication(arch=app_json.get('arch'),
+        app = FlatpakApplication(**app_json)
-                                 branch=app_json.get('branch'),
-                                 origin=app_json.get('origin'),
-                                 runtime=app_json.get('runtime'),
-                                 ref=app_json.get('ref'),
-                                 commit=app_json.get('commit'),
-                                 id=app_json.get('id'),
-                                 name=app_json.get('name'),
-                                 version=app_json.get('version'),
-                                 latest_version=app_json.get('latest_version'))
-
         app.installed = installed
-
         api_data = self.api_cache.get(app_json['id'])
 
         expired_data = api_data and api_data.get('expires_at') and api_data['expires_at'] <= datetime.utcnow()
 
         if not api_data or expired_data:
-            if not app_json['runtime']:
+            if not app.runtime:
                 if disk_loader:
                     disk_loader.fill(app)  # preloading cached disk data
                 FlatpakAsyncDataLoader(app=app, api_cache=self.api_cache, manager=self, context=self.context).start()
@@ -79,22 +68,32 @@ class FlatpakManager(SoftwareManager):
         res.total = len(res.installed) + len(res.new)
         return res
 
+    def _add_updates(self, version: str, output: list):
+        output.append(flatpak.list_updates_as_str(version))
+
     def read_installed(self, disk_loader: DiskCacheLoader, limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None) -> SearchResult:
-        installed = flatpak.list_installed()
+        version = flatpak.get_version()
+
+        updates = []
+        thread_updates = Thread(target=self._add_updates, args=(version, updates))
+        thread_updates.start()
+
+        installed = flatpak.list_installed(version)
         models = []
 
         if installed:
-
+            thread_updates.join()
-            available_updates = flatpak.list_updates_as_str()
 
             for app_json in installed:
                 model = self._map_to_model(app_json=app_json, installed=True, disk_loader=disk_loader)
-                model.update = app_json['id'] in available_updates
+                model.update = app_json['ref'] in updates[0]
                 models.append(model)
 
         return SearchResult(models, None, len(models))
 
     def downgrade(self, pkg: FlatpakApplication, root_password: str, watcher: ProcessWatcher) -> bool:
+        pkg.commit = flatpak.get_commit(pkg.id, pkg.branch)
+
         watcher.change_progress(10)
         watcher.change_substatus(self.i18n['flatpak.downgrade.commits'])
         commits = flatpak.get_app_commits(pkg.ref, pkg.origin)
@@ -127,10 +126,11 @@ class FlatpakManager(SoftwareManager):
         app_info = flatpak.get_app_info_fields(app.id, app.branch)
         app_info['name'] = app.name
         app_info['type'] = 'runtime' if app.runtime else 'app'
-        app_info['description'] = strip_html(app.description)
+        app_info['description'] = strip_html(app.description) if app.description else ''
         return app_info
 
     def get_history(self, pkg: FlatpakApplication) -> PackageHistory:
+        pkg.commit = flatpak.get_commit(pkg.id, pkg.branch)
         commits = flatpak.get_app_commits_data(pkg.ref, pkg.origin)
         status_idx = 0
 
@@ -154,32 +154,25 @@ class FlatpakManager(SoftwareManager):
         pass
 
     def list_updates(self) -> List[PackageUpdate]:
+        to_update = [app for app in self.read_installed(None).installed if app.update]
         updates = []
-        installed = flatpak.list_installed(extra_fields=False)
 
-        if installed:
+        if to_update:
-            available_updates = flatpak.list_updates_as_str()
-
-            if available_updates:
-                loaders = None
-
-                for app_json in installed:
-                    if app_json['id'] in available_updates:
-                        loader = FlatpakUpdateLoader(app=app_json, http_client=self.context.http_client)
-                        loader.start()
-
-                        if loaders is None:
                 loaders = []
 
+                for app in to_update:
+                    if app.is_application():
+                        loader = FlatpakUpdateLoader(app=app, http_client=self.context.http_client)
+                        loader.start()
                         loaders.append(loader)
 
-                if loaders:
                 for loader in loaders:
                     loader.join()
-                        app = loader.app
+
-                        updates.append(PackageUpdate(pkg_id='{}:{}'.format(app['id'], app['branch']),
+                for app in to_update:
+                    updates.append(PackageUpdate(pkg_id='{}:{}'.format(app.id, app.branch),
                                                  pkg_type='flatpak',
-                                                     version=app.get('version')))
+                                                 version=app.version))
 
         return updates
 

bauh/gems/flatpak/flatpak.py

@@ -1,5 +1,6 @@
 import re
 import subprocess
+from io import StringIO
 from typing import List
 
 from bauh.api.exception import NoInternetException
@@ -8,44 +9,6 @@ from bauh.commons.system import new_subprocess, run_cmd, new_root_subprocess
 BASE_CMD = 'flatpak'
 
 
-def app_str_to_json(line: str, version: str, extra_fields: bool = True) -> dict:
-
-    app_array = line.split('\t')
-
-    if version >= '1.3.0':
-        app = {'name': app_array[0],
-               'id': app_array[1],
-               'version': app_array[2],
-               'branch': app_array[3]}
-
-    elif '1.0' <= version < '1.1':
-
-        ref_data = app_array[0].split('/')
-
-        app = {'id': ref_data[0],
-               'arch': ref_data[1],
-               'name': ref_data[0].split('.')[-1],
-               'ref': app_array[0],
-               'options': app_array[1],
-               'branch': ref_data[2],
-               'version': None}
-    elif '1.2' <= version < '1.3':
-        app = {'id': app_array[1],
-               'name': app_array[1].strip().split('.')[-1],
-               'version': app_array[2],
-               'branch': app_array[3],
-               'arch': app_array[4],
-               'origin': app_array[5]}
-    else:
-        raise Exception('Unsupported version')
-
-    if extra_fields:
-        extra_fields = get_app_info_fields(app['id'], app['branch'], ['origin', 'arch', 'ref', 'commit'], check_runtime=True)
-        app.update(extra_fields)
-
-    return app
-
-
 def get_app_info_fields(app_id: str, branch: str, fields: List[str] = [], check_runtime: bool = False):
     info = re.findall(r'\w+:\s.+', get_app_info(app_id, branch))
     data = {}
@@ -85,15 +48,64 @@ def get_app_info(app_id: str, branch: str):
     return run_cmd('{} info {} {}'.format(BASE_CMD, app_id, branch))
 
 
-def list_installed(extra_fields: bool = True) -> List[dict]:
+def get_commit(app_id: str, branch: str) -> str:
-    apps_str = run_cmd('{} list'.format(BASE_CMD))
+    info = new_subprocess([BASE_CMD, 'info', app_id, branch])
 
-    if apps_str:
+    for o in new_subprocess(['grep', 'Commit:', '--color=never'], stdin=info.stdout).stdout:
-        version = get_version()
+        if o:
-        app_lines = apps_str.split('\n')
+            return o.decode().split(':')[1].strip()
-        return [app_str_to_json(line, version, extra_fields=extra_fields) for line in app_lines if line]
 
-    return []
+
+def list_installed(version: str) -> List[dict]:
+
+    apps = []
+
+    if version < '1.2':
+        app_list = new_subprocess([BASE_CMD, 'list', '-d'])
+
+        for o in app_list.stdout:
+            if o:
+                data = o.decode().strip().split('\t')
+                ref_split = data[0].split('/')
+                runtime = 'runtime' in data[5]
+
+                apps.append({
+                    'id': ref_split[0],
+                    'name': ref_split[0].split('.')[-1].capitalize(),
+                    'ref': data[0],
+                    'arch': ref_split[1],
+                    'branch': ref_split[2],
+                    'description': None,
+                    'origin': data[1],
+                    'runtime': runtime,
+                    'version': ref_split[2] if runtime else None
+                })
+
+    else:
+        app_list = new_subprocess([BASE_CMD, 'list', '--columns=application,name,ref,arch,branch,description,origin,options,version'])
+
+        for o in app_list.stdout:
+            if o:
+                data = o.decode().strip().split('\t')
+                runtime = 'runtime' in data[7]
+
+                if len(data) > 8 and data[8]:
+                    version = data[8]
+                elif runtime:
+                    version = data[4]
+                else:
+                    version = None
+
+                apps.append({'id': data[0],
+                             'name': data[1],
+                             'ref': data[2],
+                             'arch': data[3],
+                             'branch': data[4],
+                             'description': data[5],
+                             'origin': data[6],
+                             'runtime': runtime,
+                             'version': version})
+    return apps
 
 
 def update(app_ref: str):
@@ -114,8 +126,19 @@ def uninstall(app_ref: str):
     return new_subprocess([BASE_CMD, 'uninstall', app_ref, '-y'])
 
 
-def list_updates_as_str():
+def list_updates_as_str(version: str):
+    if version < '1.2':
         return run_cmd('{} update'.format(BASE_CMD), ignore_return_code=True)
+    else:
+        updates = new_subprocess([BASE_CMD, 'update']).stdout
+
+        out = StringIO()
+        for o in new_subprocess(['grep', '-E', r'[0-9]+.\s+(\w+|\.)+\s+\w+\s+(\w|\.)+', '-o', '--color=never'], stdin=updates).stdout:
+            if o:
+                out.write('/'.join(o.decode().strip().split('\t')[2:]) + '\n')
+
+        out.seek(0)
+        return out.read()
 
 
 def downgrade(app_ref: str, commit: str, root_password: str) -> subprocess.Popen:

bauh/gems/flatpak/model.py

@@ -7,7 +7,7 @@ from bauh.gems.flatpak import ROOT_DIR
 class FlatpakApplication(SoftwarePackage):
 
     def __init__(self, id: str = None, name: str = None, version: str = None, latest_version: str = None, description: str = None,
-                 branch: str = None, arch: str = None, origin: str = None, runtime: bool = False, ref: str = str, commit: str = str):
+                 branch: str = None, arch: str = None, origin: str = None, runtime: bool = False, ref: str = None, commit: str = None):
         super(FlatpakApplication, self).__init__(id=id, name=name, version=version,
                                                  latest_version=latest_version, description=description)
         self.ref = ref

bauh/gems/flatpak/worker.py

@@ -43,7 +43,7 @@ class FlatpakAsyncDataLoader(Thread):
                     self.app.icon_url = data.get('iconMobileUrl', None)
                     self.app.latest_version = data.get('currentReleaseVersion', self.app.version)
 
-                    if not self.app.version and self.app.latest_version:
+                    if self.app.latest_version and (not self.app.version or not self.app.update):
                         self.app.version = self.app.latest_version
 
                     if not self.app.installed and self.app.latest_version:
@@ -73,24 +73,17 @@ class FlatpakAsyncDataLoader(Thread):
 
 class FlatpakUpdateLoader(Thread):
 
-    def __init__(self, app: dict, http_client: HttpClient):
+    def __init__(self, app: FlatpakApplication, http_client: HttpClient):
         super(FlatpakUpdateLoader, self).__init__(daemon=True)
         self.app = app
         self.http_client = http_client
 
     def run(self):
-
-        if self.app.get('ref') is None:
-            self.app.update(flatpak.get_app_info_fields(self.app['id'], self.app['branch'], fields=['ref'], check_runtime=True))
-        else:
-            self.app['runtime'] = self.app['ref'].startswith('runtime/')
-
-        if not self.app['runtime']:
-
         try:
-                data = self.http_client.get_json('{}/apps/{}'.format(FLATHUB_API_URL, self.app['id']))
+            data = self.http_client.get_json('{}/apps/{}'.format(FLATHUB_API_URL, self.app.id))
 
             if data and data.get('currentReleaseVersion'):
-                    self.app['version'] = data['currentReleaseVersion']
+                self.app.version = data['currentReleaseVersion']
+                self.app.latest_version = self.app.version
         except:
             traceback.print_exc()

bauh/resources/locale/en

@@ -53,7 +53,7 @@ notification.update_selected.failed=Update failed
 notification.install.failed=installation failed
 notification.uninstall.failed=uninstallation failed
 notification.downgrade.failed=Failed to downgrade
-about.info.desc=Application management graphical interface for Linux
+about.info.desc=Graphical interface for managing your Linux applications
 about.info.link=More information at
 about.info.license=Free license
 about.info.rate=If this tool is useful for you, give it a star on Github to keep it up

bauh/view/qt/apps_table.py

@@ -295,7 +295,7 @@ class AppsTable(QTableWidget):
             label_version.setStyleSheet("color: #20A435; font-weight: bold")
             tooltip = self.i18n['version.installed_outdated']
 
-        if pkg.model.installed and pkg.model.version and pkg.model.latest_version and pkg.model.version < pkg.model.latest_version:
+        if pkg.model.installed and pkg.model.update and pkg.model.version and pkg.model.latest_version and pkg.model.version < pkg.model.latest_version:
             tooltip = '{}. {}: {}'.format(tooltip, self.i18n['version.latest'], pkg.model.latest_version)
             label_version.setText(label_version.text() + '  >  {}'.format(pkg.model.latest_version))
 

bauh/view/qt/thread.py

@@ -85,7 +85,7 @@ class UpdateSelectedApps(AsyncAction):
         if self.apps_to_update:
             updated, updated_types = 0, set()
             for app in self.apps_to_update:
-                self.change_status('{} {}...'.format(self.locale_keys['manage_window.status.upgrading'], app.model.name))
+                self.change_status('{} {} {}...'.format(self.locale_keys['manage_window.status.upgrading'], app.model.name, app.model.version))
                 success = bool(self.manager.update(app.model, self.root_password, self))
 
                 if not success:
@@ -405,7 +405,7 @@ class RunApp(AsyncAction):
 
         if self.app:
             try:
-                time.sleep(0.5)
+                time.sleep(0.25)
                 subprocess.Popen(self.app.model.get_command().split(' '))
                 self.notify_finished(True)
             except:

bauh/view/qt/window.py

@@ -231,6 +231,7 @@ class ManageWindow(QWidget):
         self.toolbar_bottom.addWidget(new_spacer())
 
         self.progress_bar = QProgressBar()
+        self.progress_bar.setMaximumHeight(7)
         self.progress_bar.setTextVisible(False)
         self.ref_progress_bar = self.toolbar_bottom.addWidget(self.progress_bar)
 
@@ -263,7 +264,7 @@ class ManageWindow(QWidget):
         self.combo_styles.show_panel_after_restart = bool(tray_icon)
 
     def _update_process_progress(self, val: int):
-        self.progress_bar.setTextVisible(True)
+        # self.progress_bar.setTextVisible(True)
         self.thread_animate_progress.set_progress(val)
 
     def apply_filters_async(self):