From 634f3dc9327c20f63274d872c6165b52bfdaa3c7 Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Fri, 21 Aug 2020 15:12:05 -0300 Subject: [PATCH] [arch] improvment -> initializing task 'Organizing data from installed packages' is taking about 80% less time --- CHANGELOG.md | 1 + bauh/gems/arch/controller.py | 6 +- bauh/gems/arch/disk.py | 249 +++++++++++------------------ bauh/gems/arch/pacman.py | 44 ++--- bauh/gems/arch/resources/locale/ca | 9 +- bauh/gems/arch/resources/locale/de | 9 +- bauh/gems/arch/resources/locale/en | 9 +- bauh/gems/arch/resources/locale/es | 9 +- bauh/gems/arch/resources/locale/it | 9 +- bauh/gems/arch/resources/locale/pt | 9 +- bauh/gems/arch/resources/locale/ru | 9 +- bauh/gems/arch/resources/locale/tr | 9 +- bauh/gems/arch/worker.py | 39 ++--- 13 files changed, 171 insertions(+), 240 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ccdbfff..94ad2806 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - default search path set to '~/Downloads' - trying to auto-fill the 'Name' and 'Version' fields - Arch + - initializing task "Organizing data from installed packages" is taking about 80% less time (now is called "Indexing packages data") - upgrade summary: displaying the reason a given package must be installed

diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index c7c0282f..1317a49c 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -886,7 +886,7 @@ class ArchManager(SoftwareManager): maintainer=repo, categories=self.categories.get(name)) - disk.save_several(pkg_map, overwrite=True, maintainer=None) + disk.write_several(pkgs=pkg_map, overwrite=True, maintainer=None) return True elif 'conflicting files' in upgrade_output: files = self._map_conflicting_file(upgrade_output) @@ -1532,7 +1532,7 @@ class ArchManager(SoftwareManager): if installed: pkg_map = {d[0]: ArchPackage(name=d[0], repository=d[1], maintainer=d[1], categories=self.categories.get(d[0])) for d in repo_deps} - disk.save_several(pkg_map, overwrite=True, maintainer=None) + disk.write_several(pkg_map, overwrite=True, maintainer=None) progress += len(repo_deps) * progress_increment self._update_progress(context, progress) else: @@ -2078,7 +2078,7 @@ class ArchManager(SoftwareManager): maintainer=dep[1] if dep[1] != 'aur' else (aur_data[dep[0]].get('Maintainer') if aur_data else None), categories=self.categories.get(context.name)) - disk.save_several(pkgs=cache_map, maintainer=None, overwrite=True) + disk.write_several(pkgs=cache_map, maintainer=None, overwrite=True) context.watcher.change_substatus('') self._update_progress(context, 100) diff --git a/bauh/gems/arch/disk.py b/bauh/gems/arch/disk.py index f0b1e9e3..657dda8f 100644 --- a/bauh/gems/arch/disk.py +++ b/bauh/gems/arch/disk.py @@ -2,184 +2,127 @@ import json import os import re from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Tuple, Optional, Callable from bauh.gems.arch import pacman from bauh.gems.arch.model import ArchPackage -RE_DESKTOP_ENTRY = re.compile(r'(Exec|Icon|NoDisplay)\s*=\s*(.+)') +RE_DESKTOP_ENTRY = re.compile(r'[\n^](Exec|Icon|NoDisplay)\s*=\s*(.+)') RE_CLEAN_NAME = re.compile(r'[+*?%]') -def write(pkg: ArchPackage): - data = pkg.get_data_to_cache() - - Path(pkg.get_disk_cache_path()).mkdir(parents=True, exist_ok=True) - - with open(pkg.get_disk_data_path(), 'w+') as f: - f.write(json.dumps(data)) - - -def fill_icon_path(pkg: ArchPackage, icon_paths: List[str], only_exact_match: bool): - clean_name = RE_CLEAN_NAME.sub('', pkg.name) - ends_with = re.compile(r'.+/{}\.(png|svg|xpm)$'.format(pkg.icon_path if pkg.icon_path else clean_name), re.IGNORECASE) - - for path in icon_paths: - if ends_with.match(path): - pkg.icon_path = path - return - - if not only_exact_match: - pkg_icons_path = pacman.list_icon_paths({pkg.name}) - - if pkg_icons_path: - pkg.set_icon(pkg_icons_path) - - -def set_icon_path(pkg: ArchPackage, icon_name: str = None): - installed_icons = pacman.list_icon_paths({pkg.name}) - - if installed_icons: - exact_match = re.compile(r'.+/{}\..+$'.format(icon_name.split('.')[0] if icon_name else pkg.name)) - for icon_path in installed_icons: - if exact_match.match(icon_path): - pkg.icon_path = icon_path - break - - -def save_several(pkgs: Dict[str, ArchPackage], overwrite: bool = True, maintainer: str = None, when_prepared=None, after_written=None) -> int: +def write_several(pkgs: Dict[str, ArchPackage], overwrite: bool = True, maintainer: str = None, after_desktop_files: Optional[Callable] = None, after_written: Optional[Callable[[str], None]] = None) -> int: if overwrite: to_cache = {p.name for p in pkgs.values()} else: to_cache = {p.name for p in pkgs.values() if not os.path.exists(p.get_disk_cache_path())} - desktop_files = pacman.list_desktop_entries(to_cache) + desktop_files = pacman.map_desktop_files(*to_cache) - no_desktop_files = set() + if after_desktop_files: + after_desktop_files() - to_write = [] + if not desktop_files: + for pkgname in to_cache: + write(pkg=pkgs[pkgname], maintainer=maintainer, after_written=after_written) - if desktop_files: - desktop_matches, no_exact_match = {}, set() - for pkg in to_cache: # first try to find exact matches - try: - clean_name = RE_CLEAN_NAME.sub('', pkg) - ends_with = re.compile(r'/usr/share/applications/{}.desktop$'.format(clean_name), re.IGNORECASE) - except: - raise - - for f in desktop_files: - if ends_with.match(f) and os.path.isfile(f): - desktop_matches[pkg] = f - break - - if pkg not in desktop_matches: - no_exact_match.add(pkg) - if no_exact_match: # check every not matched app individually - for pkgname in no_exact_match: - entries = pacman.list_desktop_entries({pkgname}) - - if entries: - if len(entries) > 1: - for e in entries: - if e.startswith('/usr/share/applications') and os.path.isfile(e): - desktop_matches[pkgname] = e - break - else: - if os.path.isfile(entries[0]): - desktop_matches[pkgname] = entries[0] - - if not desktop_matches: - no_desktop_files = to_cache - else: - if len(desktop_matches) != len(to_cache): - no_desktop_files = {p for p in to_cache if p not in desktop_matches} - - instances, apps_icons_noabspath = [], [] - - for pkgname, file in desktop_matches.items(): - p = pkgs[pkgname] - - with open(file) as f: - try: - desktop_entry = f.read() - p.desktop_entry = file - - for field in RE_DESKTOP_ENTRY.findall(desktop_entry): - if field[0] == 'Exec': - p.command = field[1].strip().replace('"', '') - elif field[0] == 'Icon': - p.icon_path = field[1].strip() - - if p.icon_path and '/' not in p.icon_path: # if the icon full path is not defined - apps_icons_noabspath.append(p) - elif field[0] == 'NoDisplay' and field[1].strip().lower() == 'true': - p.command = None - - if p.icon_path: - apps_icons_noabspath.remove(p.icon_path) - p.icon_path = None - except: - continue - - instances.append(p) - - if when_prepared: - when_prepared(p.name) - - if apps_icons_noabspath: - icon_paths = pacman.list_icon_paths({app.name for app in apps_icons_noabspath}) - if icon_paths: - for p in apps_icons_noabspath: - fill_icon_path(p, icon_paths, False) - - for p in instances: - to_write.append(p) + return len(to_cache) else: - no_desktop_files = {n for n in to_cache} + for pkgname in to_cache: + pkgfiles = desktop_files.get(pkgname) - if no_desktop_files: - bin_paths = pacman.list_bin_paths(no_desktop_files) - icon_paths = pacman.list_icon_paths(no_desktop_files) + if not pkgfiles: + write(pkg=pkgs[pkgname], maintainer=maintainer, after_written=after_written) + else: + desktop_entry = find_best_desktop_entry(pkgname, pkgfiles) - for n in no_desktop_files: - p = pkgs[n] + if desktop_entry: + write(pkg=pkgs[pkgname], maintainer=maintainer, after_written=after_written, + desktop_file=desktop_entry[0], command=desktop_entry[1], icon=desktop_entry[2]) + else: + write(pkg=pkgs[pkgname], maintainer=maintainer, after_written=after_written) - if bin_paths: - clean_name = RE_CLEAN_NAME.sub('', p.name) - ends_with = re.compile(r'.+/{}$'.format(clean_name), re.IGNORECASE) + return len(to_cache) - for path in bin_paths: - if ends_with.match(path): - p.command = path - break - if icon_paths: - fill_icon_path(p, icon_paths, only_exact_match=True) - to_write.append(p) +def find_best_desktop_entry(pkgname: str, desktop_files: List[str]) -> Optional[Tuple[str, str, str]]: + if len(desktop_files) == 1: + exec_icon = read_desktop_exec_and_icon(pkgname, desktop_files[0]) - if when_prepared: - when_prepared(p.name) + if exec_icon: + return desktop_files[0], exec_icon[0], exec_icon[1] + else: + # trying to find the exact name match: + for dfile in desktop_files: + if dfile.endswith('{}.desktop'.format(pkgname)): + exec_icon = read_desktop_exec_and_icon(pkgname, dfile) - if to_write: - written = set() - for p in to_write: - if maintainer and not p.maintainer: - p.maintainer = maintainer + if exec_icon: + return dfile, exec_icon[0], exec_icon[1] - write(p) + # trying to find a close name match: + clean_name = RE_CLEAN_NAME.sub('', pkgname) + for dfile in desktop_files: + if dfile.endswith('{}.desktop'.format(clean_name)): + exec_icon = read_desktop_exec_and_icon(clean_name, dfile) - if after_written: - after_written(p.name) + if exec_icon: + return dfile, exec_icon[0], exec_icon[1] - written.add(p.name) + # finding any match: + for dfile in desktop_files: + exec_icon = read_desktop_exec_and_icon(pkgname, dfile) - if len(to_write) != len(to_cache): - for pkgname in to_cache: - if pkgname not in written: - Path(ArchPackage.disk_cache_path(pkgname)).mkdir(parents=True, exist_ok=True) - if after_written: - after_written(pkgname) + if exec_icon: + return dfile, exec_icon[0], exec_icon[1] - return len(to_write) - return 0 + +def read_desktop_exec_and_icon(pkgname: str, desktop_file: str) -> Optional[Tuple[str, str]]: + if os.path.isfile(desktop_file): + with open(desktop_file) as f: + possibilities = set() + + content = f.read() + cmd, icon = None, None + for field in RE_DESKTOP_ENTRY.findall(content): + if field[0] == 'Exec': + cmd = field[1].strip().replace('"', '') + elif field[0] == 'Icon': + icon = field[1].strip() + elif field[0] == 'NoDisplay' and field[1].strip().lower() == 'true': + cmd, icon = None, None + + if cmd and icon: + possibilities.add((cmd, icon)) + cmd, icon = None, None + + if possibilities: + if len(possibilities) == 1: + return [*possibilities][0] + else: + # trying to find the exact name x command match + for p in possibilities: + if p[0].startswith('{} '.format(pkgname)): + return p + + return sorted(possibilities)[0] # returning any possibility + + +def write(pkg: ArchPackage, desktop_file: Optional[str] = None, command: Optional[str] = None, + icon: Optional[str] = None, maintainer: Optional[str] = None, after_written: Optional[callable] = None): + pkg.desktop_entry = desktop_file + pkg.command = command + pkg.icon_path = icon + + if maintainer and not pkg.maintainer: + pkg.maintainer = maintainer + + Path(pkg.get_disk_cache_path()).mkdir(parents=True, exist_ok=True) + + data = pkg.get_data_to_cache() + + with open(pkg.get_disk_data_path(), 'w+') as f: + f.write(json.dumps(data)) + + if after_written: + after_written(pkg.name) diff --git a/bauh/gems/arch/pacman.py b/bauh/gems/arch/pacman.py index f0cfd0ef..323722eb 100644 --- a/bauh/gems/arch/pacman.py +++ b/bauh/gems/arch/pacman.py @@ -19,6 +19,7 @@ RE_UPDATE_REQUIRED_FIELDS = re.compile(r'(\bProvides\b|\bInstalled Size\b|\bConf RE_REMOVE_TRANSITIVE_DEPS = re.compile(r'removing\s([\w\-_]+)\s.+required\sby\s([\w\-_]+)\n?') RE_AVAILABLE_MIRRORS = re.compile(r'.+\s+OK\s+.+\s+(\d+:\d+)\s+.+(http.+)') RE_PACMAN_SYNC_FIRST = re.compile(r'SyncFirst\s*=\s*(.+)') +RE_DESKTOP_FILES = re.compile(r'\n?([\w\-_]+)\s+(/usr/share/.+\.desktop)') def is_available() -> bool: @@ -153,42 +154,19 @@ def install_as_process(pkgpaths: Iterable[str], root_password: str, file: bool, shell=True) -def list_desktop_entries(pkgnames: Set[str]) -> List[str]: +def map_desktop_files(*pkgnames) -> Dict[str, List[str]]: + res = {} + if pkgnames: - installed_files = new_subprocess(['pacman', '-Qlq', *pkgnames]) + output = run_cmd('pacman -Ql {}'.format(' '.join(pkgnames)), print_error=False) - desktop_files = [] - for out in new_subprocess(['grep', '-E', ".desktop$"], stdin=installed_files.stdout).stdout: - if out: - desktop_files.append(out.decode().strip()) + if output: + for match in RE_DESKTOP_FILES.findall(output): + pkgfiles = res.get(match[0], []) + res[match[0]] = pkgfiles + pkgfiles.append(match[1]) - return desktop_files - - -def list_icon_paths(pkgnames: Set[str]) -> List[str]: - installed_files = new_subprocess(['pacman', '-Qlq', *pkgnames]) - - icon_files = [] - for out in new_subprocess(['grep', '-E', '.(png|svg|xpm)$'], stdin=installed_files.stdout).stdout: - if out: - line = out.decode().strip() - if line: - icon_files.append(line) - - return icon_files - - -def list_bin_paths(pkgnames: Set[str]) -> List[str]: - installed_files = new_subprocess(['pacman', '-Qlq', *pkgnames]) - - bin_paths = [] - for out in new_subprocess(['grep', '-E', '^/usr/bin/.+'], stdin=installed_files.stdout).stdout: - if out: - line = out.decode().strip() - if line: - bin_paths.append(line) - - return bin_paths + return res def list_installed_files(pkgname: str) -> List[str]: diff --git a/bauh/gems/arch/resources/locale/ca b/bauh/gems/arch/resources/locale/ca index a65f0405..c799c0fe 100644 --- a/bauh/gems/arch/resources/locale/ca +++ b/bauh/gems/arch/resources/locale/ca @@ -181,10 +181,11 @@ arch.substatus.loading_files=Loading package files arch.substatus.pre_hooks=Running pre-transaction hooks arch.sync_databases.substatus=Synchronizing package databases arch.sync_databases.substatus.error=It was not possible to synchronize the package database -arch.task.disk_cache=Organitzen dades dels paquets instal·lats -arch.task.disk_cache.indexed=Preparats -arch.task.disk_cache.prepared=Llegits -arch.task.disk_cache.reading=Determinant els paquets instal·lats +arch.task.disk_cache=Indexing packages data +arch.task.disk_cache.checking=Checking index +arch.task.disk_cache.indexed=Indexed +arch.task.disk_cache.indexing=Indexing +arch.task.disk_cache.reading_files=Reading files arch.task.mirrors=Refreshing mirrors arch.task.optimizing=Optimizing {} arch.task.sync_databases.waiting=Waiting for {} diff --git a/bauh/gems/arch/resources/locale/de b/bauh/gems/arch/resources/locale/de index ecfd6a5c..6fdada78 100644 --- a/bauh/gems/arch/resources/locale/de +++ b/bauh/gems/arch/resources/locale/de @@ -181,10 +181,11 @@ arch.substatus.loading_files=Loading package files arch.substatus.pre_hooks=Running pre-transaction hooks arch.sync_databases.substatus=Synchronizing package databases arch.sync_databases.substatus.error=It was not possible to synchronize the package database -arch.task.disk_cache=Organizing data from installed packages -arch.task.disk_cache.indexed=Ready -arch.task.disk_cache.prepared=Read -arch.task.disk_cache.reading=Determining installed packages +arch.task.disk_cache=Indexing packages data +arch.task.disk_cache.checking=Checking index +arch.task.disk_cache.indexed=Indexed +arch.task.disk_cache.indexing=Indexing +arch.task.disk_cache.reading_files=Reading files arch.task.mirrors=Refreshing mirrors arch.task.optimizing=Optimizing {} arch.task.sync_databases.waiting=Waiting for {} diff --git a/bauh/gems/arch/resources/locale/en b/bauh/gems/arch/resources/locale/en index 0151fcf9..e96cc144 100644 --- a/bauh/gems/arch/resources/locale/en +++ b/bauh/gems/arch/resources/locale/en @@ -182,10 +182,11 @@ arch.substatus.loading_files=Loading package files arch.substatus.pre_hooks=Running pre-transaction hooks arch.sync_databases.substatus=Synchronizing package databases arch.sync_databases.substatus.error=It was not possible to synchronize the package database -arch.task.disk_cache=Organizing data from installed packages -arch.task.disk_cache.indexed=Ready -arch.task.disk_cache.prepared=Read -arch.task.disk_cache.reading=Determining installed packages +arch.task.disk_cache=Indexing packages data +arch.task.disk_cache.checking=Checking index +arch.task.disk_cache.indexed=Indexed +arch.task.disk_cache.indexing=Indexing +arch.task.disk_cache.reading_files=Reading files arch.task.mirrors=Refreshing mirrors arch.task.optimizing=Optimizing {} arch.task.sync_databases.waiting=Waiting for {} diff --git a/bauh/gems/arch/resources/locale/es b/bauh/gems/arch/resources/locale/es index e255ef77..a3a0ff49 100644 --- a/bauh/gems/arch/resources/locale/es +++ b/bauh/gems/arch/resources/locale/es @@ -181,10 +181,11 @@ arch.substatus.loading_files=Cargando archivos de los paquetes arch.substatus.pre_hooks=Ejecutando ganchos pre-transacción arch.sync_databases.substatus=Sincronizando bases de paquetes arch.sync_databases.substatus.error=No fue posible sincronizar la base de paquetes -arch.task.disk_cache=Organizando datos de paquetes instalados -arch.task.disk_cache.indexed=Listos -arch.task.disk_cache.prepared=Lidos -arch.task.disk_cache.reading=Determinando los paquetes instalados +arch.task.disk_cache=Indexando datos de paquetes +arch.task.disk_cache.checking=Comprobando índice +arch.task.disk_cache.indexed=Indexados +arch.task.disk_cache.indexing=Indexando +arch.task.disk_cache.reading_files=Lendo archivos arch.task.mirrors=Actualizando espejos arch.task.optimizing=Optimizando {} arch.task.sync_databases.waiting=Esperando por {} diff --git a/bauh/gems/arch/resources/locale/it b/bauh/gems/arch/resources/locale/it index f3bf0c22..84dbe716 100644 --- a/bauh/gems/arch/resources/locale/it +++ b/bauh/gems/arch/resources/locale/it @@ -181,10 +181,11 @@ arch.substatus.loading_files=Loading package files arch.substatus.pre_hooks=Running pre-transaction hooks arch.sync_databases.substatus=Synchronizing package databases arch.sync_databases.substatus.error=It was not possible to synchronize the package database -arch.task.disk_cache=Organizzando i dati dai pacchetti installati -arch.task.disk_cache.indexed=Pronti -arch.task.disk_cache.prepared=Letti -arch.task.disk_cache.reading=Determinando i pacchetti installati +arch.task.disk_cache=Indexing packages data +arch.task.disk_cache.checking=Checking index +arch.task.disk_cache.indexed=Indexed +arch.task.disk_cache.indexing=Indexing +arch.task.disk_cache.reading_files=Reading files arch.task.mirrors=Aggiornando i mirror arch.task.optimizing=Ottimizzando {} arch.task.sync_databases.waiting=Waiting for {} diff --git a/bauh/gems/arch/resources/locale/pt b/bauh/gems/arch/resources/locale/pt index 5ee94705..3ef9422f 100644 --- a/bauh/gems/arch/resources/locale/pt +++ b/bauh/gems/arch/resources/locale/pt @@ -181,10 +181,11 @@ arch.substatus.loading_files=Carregando os arquivos dos pacotes arch.substatus.pre_hooks=Executando ganchos pré-transação arch.sync_databases.substatus=Sincronizando bases de pacotes arch.sync_databases.substatus.error=Não foi possível sincronizar as bases de pacotes -arch.task.disk_cache=Organizando dados dos pacotes instalados -arch.task.disk_cache.indexed=Prontos -arch.task.disk_cache.prepared=Lidos -arch.task.disk_cache.reading=Determinando pacotes instalados +arch.task.disk_cache=Indexando dados de pacotes +arch.task.disk_cache.checking=Verificando índice +arch.task.disk_cache.indexed=Indexados +arch.task.disk_cache.indexing=Indexando +arch.task.disk_cache.reading_files=Lendo arquivos arch.task.mirrors=Atualizando espelhos arch.task.optimizing=Otimizando {} arch.task.sync_databases.waiting=Aguardando por {} diff --git a/bauh/gems/arch/resources/locale/ru b/bauh/gems/arch/resources/locale/ru index 62700a0e..f353aa61 100644 --- a/bauh/gems/arch/resources/locale/ru +++ b/bauh/gems/arch/resources/locale/ru @@ -181,10 +181,11 @@ arch.substatus.loading_files=Loading package files arch.substatus.pre_hooks=Running pre-transaction hooks arch.sync_databases.substatus=Синхронизация баз данных пакетов arch.sync_databases.substatus.error=Синхронизировать базу данных пакета не удалось -arch.task.disk_cache=Organizing data from installed packages -arch.task.disk_cache.indexed=Ready -arch.task.disk_cache.prepared=Read -arch.task.disk_cache.reading=Determining installed packages +arch.task.disk_cache=Indexing packages data +arch.task.disk_cache.checking=Checking index +arch.task.disk_cache.indexed=Indexed +arch.task.disk_cache.indexing=Indexing +arch.task.disk_cache.reading_files=Reading files arch.task.mirrors=Обновление зеркал arch.task.optimizing=Optimizing {} arch.task.sync_databases.waiting=Waiting for {} diff --git a/bauh/gems/arch/resources/locale/tr b/bauh/gems/arch/resources/locale/tr index 8105a722..5b702b3c 100644 --- a/bauh/gems/arch/resources/locale/tr +++ b/bauh/gems/arch/resources/locale/tr @@ -181,10 +181,11 @@ arch.substatus.loading_files=Loading package files arch.substatus.pre_hooks=Running pre-transaction hooks arch.sync_databases.substatus=Paket veritabanı eşitleniyor arch.sync_databases.substatus.error=Paket veritabanı eşitlenemedi -arch.task.disk_cache=Kurulu paket verileri düzenleniyor -arch.task.disk_cache.indexed=Hazır -arch.task.disk_cache.prepared=okunan -arch.task.disk_cache.reading=Kurulu paketler belirleniyor +arch.task.disk_cache=Indexing packages data +arch.task.disk_cache.checking=Checking index +arch.task.disk_cache.indexed=Indexed +arch.task.disk_cache.indexing=Indexing +arch.task.disk_cache.reading_files=Reading files arch.task.mirrors=Yansılar yenileniyor arch.task.optimizing=Uygun hale getiriliyor {} arch.task.sync_databases.waiting=Waiting for {} diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index 29261571..2a505101 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -66,8 +66,6 @@ class ArchDiskCacheUpdater(Thread): self.task_man = task_man self.task_id = 'arch_cache_up' self.i18n = i18n - self.prepared = 0 - self.prepared_template = self.i18n['arch.task.disk_cache.prepared'] + ': {}/ {}' self.indexed = 0 self.indexed_template = self.i18n['arch.task.disk_cache.indexed'] + ': {}/ {}' self.to_index = 0 @@ -79,19 +77,18 @@ class ArchDiskCacheUpdater(Thread): self.installed_hash_path = '{}/installed.sha1'.format(ARCH_CACHE_PATH) self.installed_cache_dir = '{}/installed'.format(ARCH_CACHE_PATH) - def update_prepared(self, pkgname: str, add: bool = True): - if add: - self.prepared += 1 - - sub = self.prepared_template.format(self.prepared, self.to_index) - progress = ((self.prepared + self.indexed) / self.progress) * 100 if self.progress > 0 else 0 - self.task_man.update_progress(self.task_id, progress, sub) - def update_indexed(self, pkgname: str): self.indexed += 1 sub = self.indexed_template.format(self.indexed, self.to_index) - progress = ((self.prepared + self.indexed) / self.progress) * 100 if self.progress > 0 else 0 - self.task_man.update_progress(self.task_id, progress, sub) + self.progress = self.progress + (self.indexed / self.to_index) * 50 + self.task_man.update_progress(self.task_id, self.progress, sub) + + def _update_progress(self, progress: float, msg: str): + self.progress = progress + self.task_man.update_progress(self.task_id, self.progress, msg) + + def _notify_reading_files(self): + self._update_progress(50, self.i18n['arch.task.disk_cache.indexing']) def run(self): if not any([self.aur, self.repositories]): @@ -100,19 +97,21 @@ class ArchDiskCacheUpdater(Thread): ti = time.time() self.task_man.register_task(self.task_id, self.i18n['arch.task.disk_cache'], get_icon_path()) - self.task_man.update_progress(self.task_id, 1, '') - self.logger.info("Checking already cached package data") + self._update_progress(1, self.i18n['arch.task.disk_cache.checking']) cache_dirs = [fpath for fpath in glob.glob('{}/*'.format(self.installed_cache_dir)) if os.path.isdir(fpath)] not_cached_names = None + self._update_progress(15, self.i18n['arch.task.disk_cache.checking']) if cache_dirs: # if there are cache data installed_names = pacman.list_installed_names() cached_pkgs = {cache_dir.split('/')[-1] for cache_dir in cache_dirs} not_cached_names = installed_names.difference(cached_pkgs) + self._update_progress(20, self.i18n['arch.task.disk_cache.checking']) + if not not_cached_names: self.task_man.update_progress(self.task_id, 100, '') self.task_man.finish_task(self.task_id) @@ -123,21 +122,23 @@ class ArchDiskCacheUpdater(Thread): self.logger.info('Pre-caching installed Arch packages data to disk') + self._update_progress(20, self.i18n['arch.task.disk_cache.checking']) installed = self.controller.read_installed(disk_loader=None, internet_available=self.internet_available, only_apps=False, pkg_types=None, limit=-1, names=not_cached_names, wait_disk_cache=False).installed - self.task_man.update_progress(self.task_id, 0, self.i18n['arch.task.disk_cache.reading']) + self._update_progress(35, self.i18n['arch.task.disk_cache.checking']) saved = 0 - pkgs = {p.name: p for p in installed if ((self.aur and p.repository == 'aur') or (self.repositories and p.repository != 'aur')) and not os.path.exists(p.get_disk_cache_path())} + pkgs = {p.name: p for p in installed if ((self.aur and p.repository == 'aur') or (self.repositories and p.repository != 'aur')) and not os.path.exists(p.get_disk_cache_path())} self.to_index = len(pkgs) - self.progress = self.to_index * 2 - self.update_prepared(None, add=False) # overwrite == True because the verification already happened - saved += disk.save_several(pkgs, when_prepared=self.update_prepared, after_written=self.update_indexed, overwrite=True) + self._update_progress(40, self.i18n['arch.task.disk_cache.reading_files']) + saved += disk.write_several(pkgs=pkgs, + after_desktop_files=self._notify_reading_files, + after_written=self.update_indexed, overwrite=True) self.task_man.update_progress(self.task_id, 100, None) self.task_man.finish_task(self.task_id)