spalencsar/bearhub
1
0
Fork 0
mirror of https://github.com/spalencsar/bearhub.git synced 2026-07-07 02:24:16 +02:00
Code Issues Packages Projects Releases Wiki Activity

[arch] improvement -> AUR: the index is not always being updated during the initialization process

Browse Source
This commit is contained in:
Vinicius Moreira
2020-12-21 16:08:45 -03:00
parent 7057f683cd
commit a9b196522d
18 changed files with 113 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bauh/gems/arch/__init__.py
View File

@@ -12,7 +12,8 @@ URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/mas
URL_GPG_SERVERS = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/arch/gpgservers.txt'
CONFIG_DIR = '{}/.config/bauh/arch'.format(str(Path.home()))
CUSTOM_MAKEPKG_FILE = '{}/makepkg.conf'.format(CONFIG_DIR)
AUR_INDEX_FILE = '{}/arch.txt'.format(BUILD_DIR)
AUR_INDEX_FILE = '{}/aur/index.txt'.format(ARCH_CACHE_PATH)
AUR_INDEX_TS_FILE = '{}/aur/index.ts'.format(ARCH_CACHE_PATH)
CONFIG_FILE = '{}/arch.yml'.format(CONFIG_PATH)
SUGGESTIONS_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/arch/aur_suggestions.txt'
UPDATES_IGNORED_FILE = '{}/updates_ignored.txt'.format(CONFIG_DIR)

3
bauh/gems/arch/config.py
View File

@@ -21,7 +21,8 @@ def read_config(update_file: bool = False) -> dict:
'aur_build_only_chosen': True,
'check_dependency_breakage': True,
'suggest_unneeded_uninstall': False,
'suggest_optdep_uninstall': False}
'suggest_optdep_uninstall': False,
'aur_idx_exp': 720}
return read(CONFIG_FILE, template, update_file=update_file)

38
bauh/gems/arch/controller.py
View File

@@ -75,7 +75,7 @@ class TransactionContext:
disk_loader: DiskCacheLoader = None, disk_cache_updater: Thread = None,
new_pkg: bool = False, custom_pkgbuild_path: str = None,
pkgs_to_build: Set[str] = None, last_modified: Optional[int] = None,
commit: Optional[str] = None):
commit: Optional[str] = None, update_aur_index: bool = False):
self.aur_supported = aur_supported
self.name = name
self.base = base
@@ -108,6 +108,7 @@ class TransactionContext:
self.previous_change_progress = change_progress
self.last_modified = last_modified
self.commit = commit
self.update_aur_index = update_aur_index
@classmethod
def gen_context_from(cls, pkg: ArchPackage, arch_config: dict, root_password: str, handler: ProcessHandler, aur_supported: Optional[bool] = None) -> "TransactionContext":
@@ -1132,6 +1133,7 @@ class ArchManager(SoftwareManager):
if not pkgs_api_data:
self.logger.warning("Could not retrieve the 'last_modified' fields from the AUR API during the upgrade process")
any_upgraded = False
for pkg in aur_pkgs:
watcher.change_substatus("{} {} ({})...".format(self.i18n['manage_window.status.upgrading'], pkg.name, pkg.version))
@@ -1149,19 +1151,29 @@ class ArchManager(SoftwareManager):
try:
if not self.install(pkg=pkg, root_password=root_password, watcher=watcher, disk_loader=None, context=context).success:
if any_upgraded:
self._update_aur_index(watcher)
watcher.print(self.i18n['arch.upgrade.fail'].format('"{}"'.format(pkg.name)))
self.logger.error("Could not upgrade AUR package '{}'".format(pkg.name))
watcher.change_substatus('')
return False
else:
any_upgraded = True
watcher.print(self.i18n['arch.upgrade.success'].format('"{}"'.format(pkg.name)))
except:
if any_upgraded:
self._update_aur_index(watcher)
watcher.print(self.i18n['arch.upgrade.fail'].format('"{}"'.format(pkg.name)))
watcher.change_substatus('')
self.logger.error("An error occurred when upgrading AUR package '{}'".format(pkg.name))
traceback.print_exc()
return False
if any_upgraded:
self._update_aur_index(watcher)
watcher.change_substatus('')
return True
@@ -1905,6 +1917,9 @@ class ArchManager(SoftwareManager):
if self._install(context=context):
self._save_pkgbuild(context)
if context.update_aur_index:
self._update_aur_index(context.watcher)
if context.dependency or context.skip_opt_deps:
return True
@@ -1917,6 +1932,16 @@ class ArchManager(SoftwareManager):
return False
def _update_aur_index(self, watcher: ProcessWatcher):
if self.context.internet_checker.is_available():
if watcher:
watcher.change_substatus(self.i18n['arch.task.aur.index.status'])
idx_updater = AURIndexUpdater(context=self.context, taskman=TaskManager()) # null task manager
idx_updater.run()
else:
self.logger.warning("Could not update the AUR index: no internet connection detected")
def __fill_aur_output_files(self, context: TransactionContext):
self.logger.info("Determining output files of '{}'".format(context.name))
context.watcher.change_substatus(self.i18n['arch.aur.build.list_output'])
@@ -2483,6 +2508,7 @@ class ArchManager(SoftwareManager):
root_password=root_password)
install_context.skip_opt_deps = False
install_context.disk_loader = disk_loader
install_context.update_aur_index = pkg.repository == 'aur'
self._sync_databases(arch_config=install_context.config, aur_supported=install_context.aur_supported,
root_password=root_password, handler=handler)
@@ -2607,7 +2633,7 @@ class ArchManager(SoftwareManager):
def prepare(self, task_manager: TaskManager, root_password: str, internet_available: bool):
arch_config = read_config(update_file=True)
if internet_available:
if internet_available and AURIndexUpdater.should_update(arch_config):
self.index_aur = AURIndexUpdater(context=self.context, taskman=task_manager) # must all execute to properly determine the installed packages (even that AUR is disabled)
self.index_aur.start()
@@ -2811,6 +2837,13 @@ class ArchManager(SoftwareManager):
only_int=True,
max_width=max_width,
value=local_config['mirrors_sort_limit'] if isinstance(local_config['mirrors_sort_limit'], int) else ''),
TextInputComponent(id_='aur_idx_exp',
label=self.i18n['arch.config.aur_idx_exp'] + ' (AUR)',
tooltip=self.i18n['arch.config.aur_idx_exp.tip'],
max_width=max_width,
only_int=True,
capitalize_label=False,
value=local_config['aur_idx_exp'] if isinstance(local_config['aur_idx_exp'], int) else ''),
new_select(id_='aur_build_only_chosen',
label=self.i18n['arch.config.aur_build_only_chosen'],
tip=self.i18n['arch.config.aur_build_only_chosen.tip'],
@@ -2868,6 +2901,7 @@ class ArchManager(SoftwareManager):
config['aur_remove_build_dir'] = form_install.get_component('aur_remove_build_dir').get_selected()
config['aur_build_dir'] = form_install.get_component('aur_build_dir').file_path
config['aur_build_only_chosen'] = form_install.get_component('aur_build_only_chosen').get_selected()
config['aur_idx_exp'] = form_install.get_component('aur_idx_exp').get_int_value()
config['check_dependency_breakage'] = form_install.get_component('check_dependency_breakage').get_selected()
config['suggest_optdep_uninstall'] = form_install.get_component('suggest_optdep_uninstall').get_selected()
config['suggest_unneeded_uninstall'] = form_install.get_component('suggest_unneeded_uninstall').get_selected()

2
bauh/gems/arch/resources/locale/ca
View File

@@ -47,6 +47,8 @@ arch.config.check_dependency_breakage=Check dependency version breakage
arch.config.check_dependency_breakage.tip=If, during the verification of upgrade requirements, specific versions of dependencies must also be checked. Example: package A depends on version 1.0 of B.
arch.config.edit_aur_pkgbuild=Edit PKGBUILD (AUR)
arch.config.edit_aur_pkgbuild.tip=If the PKGBUILD file of an AUR package should be displayed for editing before its installation/upgrade/downgrade
arch.config.aur_idx_exp=Index expiration
arch.config.aur_idx_exp.tip=It defines the period (in minutes) for the AUR index stored in disc to be considered valid during the initialization process. Use 0 so that it is always updated.
arch.config.mirrors_sort_limit=Mirrors sort limit
arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting.
arch.config.optimize=optimize

2
bauh/gems/arch/resources/locale/de
View File

@@ -47,6 +47,8 @@ arch.config.check_dependency_breakage=Check dependency version breakage
arch.config.check_dependency_breakage.tip=If, during the verification of upgrade requirements, specific versions of dependencies must also be checked. Example: package A depends on version 1.0 of B.
arch.config.edit_aur_pkgbuild=Edit PKGBUILD (AUR)
arch.config.edit_aur_pkgbuild.tip=If the PKGBUILD file of an AUR package should be displayed for editing before its installation/upgrade/downgrade
arch.config.aur_idx_exp=Index expiration
arch.config.aur_idx_exp.tip=It defines the period (in minutes) for the AUR index stored in disc to be considered valid during the initialization process. Use 0 so that it is always updated.
arch.config.mirrors_sort_limit=Mirrors sort limit
arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting.
arch.config.optimize=optimize

2
bauh/gems/arch/resources/locale/en
View File

@@ -47,6 +47,8 @@ arch.config.check_dependency_breakage=Check dependency version breakage
arch.config.check_dependency_breakage.tip=If, during the verification of upgrade requirements, specific versions of dependencies must also be checked. Example: package A depends on version 1.0 of B.
arch.config.edit_aur_pkgbuild=Edit PKGBUILD (AUR)
arch.config.edit_aur_pkgbuild.tip=If the PKGBUILD file of an AUR package should be displayed for editing before its installation/upgrade/downgrade
arch.config.aur_idx_exp=Index expiration
arch.config.aur_idx_exp.tip=It defines the period (in minutes) for the AUR index stored in disc to be considered valid during the initialization process. Use 0 so that it is always updated.
arch.config.mirrors_sort_limit=Mirrors sort limit
arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting.
arch.config.optimize=optimize

2
bauh/gems/arch/resources/locale/es
View File

@@ -47,6 +47,8 @@ arch.config.check_dependency_breakage=Verificar rotura de versión de dependenci
arch.config.check_dependency_breakage.tip=Si, durante la verificación de los requisitos de actualización, también se deben verificar versiones específicas de las dependencias. Ejemplo: el paquete A depende de la versión 1.0 de B.
arch.config.edit_aur_pkgbuild=Editar PKGBUILD (AUR)
arch.config.edit_aur_pkgbuild.tip=Si el archivo PKGBUILD de un paquete AUR debe ser exhibido para edición antes de su instalación/actualización/degradación
arch.config.aur_idx_exp=Expiración del índice
arch.config.aur_idx_exp.tip=Define el período (en minutos) para que el índice AUR almacenado en el disco se considere válido durante el proceso de inicialización. Utilice 0 para que esté siempre actualizado.
arch.config.mirrors_sort_limit=Límite de ordenación de espejos
arch.config.mirrors_sort_limit.tip=Define el número máximo de espejos que se utilizarán para la ordenación por velocidad. Use 0 para no limitar o déjelo en blanco para deshabilitar la clasificación.
arch.config.optimize=optimizar

2
bauh/gems/arch/resources/locale/fr
View File

@@ -47,6 +47,8 @@ arch.config.check_dependency_breakage=Vérification pour éviter qu'une version
arch.config.check_dependency_breakage.tip=Si, durant la vérification des prérequis de mise à jour, des versions spécifiques de dépendances doivent aussi être vérifiées. Exemple: paquet A dépend de B version 1.0.
arch.config.edit_aur_pkgbuild=Éditer PKGBUILD (AUR)
arch.config.edit_aur_pkgbuild.tip=Si le fichier PKGBUILD d'un paquet AUR devrait être affiché pour édition avant installation/mise à jour/downgrade
arch.config.aur_idx_exp=Index expiration
arch.config.aur_idx_exp.tip=It defines the period (in minutes) for the AUR index stored in disc to be considered valid during the initialization process. Use 0 so that it is always updated.
arch.config.mirrors_sort_limit=Limite de tri des miroirs
arch.config.mirrors_sort_limit.tip=Définit le nombre maximal de miroirs utilisés pour trier vite. 0 pour aucune limite ou vide pour désactiver le tri.
arch.config.optimize=optimizer

2
bauh/gems/arch/resources/locale/it
View File

@@ -47,6 +47,8 @@ arch.config.check_dependency_breakage=Check dependency version breakage
arch.config.check_dependency_breakage.tip=If, during the verification of upgrade requirements, specific versions of dependencies must also be checked. Example: package A depends on version 1.0 of B.
arch.config.edit_aur_pkgbuild=Edit PKGBUILD (AUR)
arch.config.edit_aur_pkgbuild.tip=If the PKGBUILD file of an AUR package should be displayed for editing before its installation/upgrade/downgrade
arch.config.aur_idx_exp=Index expiration
arch.config.aur_idx_exp.tip=It defines the period (in minutes) for the AUR index stored in disc to be considered valid during the initialization process. Use 0 so that it is always updated.
arch.config.mirrors_sort_limit=Mirrors sort limit
arch.config.mirrors_sort_limit.tip=Defines the maximum number of mirrors that will be used for speed sorting. Use 0 for no limit or leave it blank to disable sorting.
arch.config.optimize=optimize

4
bauh/gems/arch/resources/locale/pt
View File

@@ -47,8 +47,10 @@ arch.config.check_dependency_breakage=Verificar quebra de dependência de versã
arch.config.check_dependency_breakage.tip=Se durante a verificação dos requisitos de atualização deve-se verificar também versões específicas de dependências. Exemplo: pacote A depende da versão 1.0 de B.
arch.config.edit_aur_pkgbuild=Editar PKGBUILD (AUR)
arch.config.edit_aur_pkgbuild.tip=Se o arquivo PKGBUILD de um pacote do AUR deve ser exibido para edição antes da instalação/atualização/reversão
arch.config.aur_idx_exp=Expiração do índice
arch.config.aur_idx_exp.tip=Define o período (em minutos) em que o índice do AUR armazenado em disco é considerado válido durante a inicialização. Utilize 0 para que ele sempre seja atualizado.
arch.config.mirrors_sort_limit=Limite de ordenação de espelhos
arch.config.mirrors_sort_limit.tip=Define o número máximo de espelhos que serão utilizados para a ordenação por velocidade. Use 0 para não limitar ou deixe em branco para desabilitar a ordenação.
arch.config.mirrors_sort_limit.tip=Define o número máximo de espelhos que serão utilizados para a ordenação por velocidade. Utilize 0 para não limitar ou deixe em branco para desabilitar a ordenação.
arch.config.optimize=Otimizar
arch.config.pacman_mthread_download=Download segmentado (repositórios)
arch.config.pacman_mthread_download.tip=Se os pacotes dos repositórios devem baixados através de uma ferramenta que trabalha com segmentação/threads (pode ser mais rápido). pacman-mirrors precisa estar instalado.

2
bauh/gems/arch/resources/locale/ru
View File

@@ -47,6 +47,8 @@ arch.config.check_dependency_breakage=Check dependency version breakage
arch.config.check_dependency_breakage.tip=If, during the verification of upgrade requirements, specific versions of dependencies must also be checked. Example: package A depends on version 1.0 of B.
arch.config.edit_aur_pkgbuild=Edit PKGBUILD (AUR)
arch.config.edit_aur_pkgbuild.tip=If the PKGBUILD file of an AUR package should be displayed for editing before its installation/upgrade/downgrade
arch.config.aur_idx_exp=Index expiration
arch.config.aur_idx_exp.tip=It defines the period (in minutes) for the AUR index stored in disc to be considered valid during the initialization process. Use 0 so that it is always updated.
arch.config.mirrors_sort_limit=Ограничение сортировки зеркал
arch.config.mirrors_sort_limit.tip=Определяет максимальное количество зеркал, которые будут использоваться для сортировки по скорости. Используйте 0 для No limit или оставьте его пустым, чтобы отключить сортировку.
arch.config.optimize=Оптимизация

2
bauh/gems/arch/resources/locale/tr
View File

@@ -47,6 +47,8 @@ arch.config.check_dependency_breakage=Check dependency version breakage
arch.config.check_dependency_breakage.tip=If, during the verification of upgrade requirements, specific versions of dependencies must also be checked. Example: package A depends on version 1.0 of B.
arch.config.edit_aur_pkgbuild=Edit PKGBUILD (AUR)
arch.config.edit_aur_pkgbuild.tip=If the PKGBUILD file of an AUR package should be displayed for editing before its installation/upgrade/downgrade
arch.config.aur_idx_exp=Index expiration
arch.config.aur_idx_exp.tip=It defines the period (in minutes) for the AUR index stored in disc to be considered valid during the initialization process. Use 0 so that it is always updated.
arch.config.mirrors_sort_limit=Yansı sıralama sınırı
arch.config.mirrors_sort_limit.tip=Hız sıralama için kullanılacak maksimum yansı sayısını tanımlar. Sınırsız olması için 0 kullanın veya sıralamayı devre dışı bırakmak için boş bırakın.
arch.config.optimize=optimize

38
bauh/gems/arch/worker.py
View File

@@ -4,6 +4,7 @@ import os
import re
import time
import traceback
from datetime import datetime, timedelta
from pathlib import Path
from threading import Thread
from typing import Optional
@@ -14,8 +15,9 @@ from bauh.api.abstract.context import ApplicationContext
from bauh.api.abstract.handler import TaskManager
from bauh.commons.html import bold
from bauh.commons.system import run_cmd, new_root_subprocess, ProcessHandler
from bauh.commons.util import datetime_as_milis
from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_FILE, CONFIG_DIR, AUR_INDEX_FILE, get_icon_path, database, \
mirrors, ARCH_CACHE_PATH, BUILD_DIR
mirrors, ARCH_CACHE_PATH, BUILD_DIR, AUR_INDEX_TS_FILE
from bauh.gems.arch.aur import URL_INDEX
from bauh.view.util.translation import I18n
@@ -37,12 +39,40 @@ class AURIndexUpdater(Thread):
self.taskman = taskman
self.task_id = 'index_aur'
@staticmethod
def should_update(arch_config: dict) -> bool:
try:
exp_minutes = int(arch_config['aur_idx_exp'])
except:
traceback.print_exc()
return True
if exp_minutes <= 0:
return True
if not os.path.exists(AUR_INDEX_FILE):
return True
if not os.path.exists(AUR_INDEX_TS_FILE):
return True
with open(AUR_INDEX_TS_FILE) as f:
timestamp_str = f.read()
try:
index_timestamp = datetime.fromtimestamp(float(timestamp_str))
return (index_timestamp + timedelta(minutes=exp_minutes)) <= datetime.utcnow()
except:
traceback.print_exc()
return True
def run(self):
ti = time.time()
self.logger.info('Indexing AUR packages')
self.taskman.register_task(self.task_id, self.i18n['arch.task.aur.index.status'], get_icon_path())
self.taskman.update_progress(self.task_id, 1, self.i18n['arch.task.aur.index.substatus.download'])
try:
index_ts = datetime.utcnow().timestamp()
res = self.http_client.get(URL_INDEX)
if res and res.text:
@@ -50,7 +80,8 @@ class AURIndexUpdater(Thread):
self.taskman.update_progress(self.task_id, index_progress,
self.i18n['arch.task.aur.index.substatus.gen_index'])
indexed = 0
Path(BUILD_DIR).mkdir(parents=True, exist_ok=True)
Path(os.path.dirname(AUR_INDEX_FILE)).mkdir(parents=True, exist_ok=True)
with open(AUR_INDEX_FILE, 'w+') as f:
lines = res.text.split('\n')
@@ -70,6 +101,9 @@ class AURIndexUpdater(Thread):
perc_count += 1
with open(AUR_INDEX_TS_FILE, 'w+') as f:
f.write(str(index_ts))
self.logger.info('Pre-indexed {} AUR package names at {}'.format(indexed, AUR_INDEX_FILE))
self.taskman.update_progress(self.task_id, 100, None)