From ce1f30365df7f5e923f64efe0329689456b2f9a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Moreira?= Date: Wed, 18 Dec 2019 19:13:02 -0300 Subject: [PATCH] [aur] generating the optimized makepkg file if it is not found during an installation --- CHANGELOG.md | 2 +- bauh/gems/arch/__init__.py | 2 +- bauh/gems/arch/controller.py | 6 +- bauh/gems/arch/makepkg.py | 18 ++-- bauh/gems/arch/resources/locale/ca | 1 + bauh/gems/arch/resources/locale/de | 3 +- bauh/gems/arch/resources/locale/en | 3 +- bauh/gems/arch/resources/locale/es | 3 +- bauh/gems/arch/resources/locale/it | 1 + bauh/gems/arch/resources/locale/pt | 3 +- bauh/gems/arch/worker.py | 129 +++++++++++++++-------------- 11 files changed, 92 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7758bc3..0e1365de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixes - AUR: - an exception happens when retrieving matches from the cached AUR index - + - not using the optimized compilation settings if the custom makepkg file is not found during the installation process ## [0.7.4] 2019-12-09 ### Improvements diff --git a/bauh/gems/arch/__init__.py b/bauh/gems/arch/__init__.py index bb582a62..dd8e2e9a 100644 --- a/bauh/gems/arch/__init__.py +++ b/bauh/gems/arch/__init__.py @@ -9,6 +9,6 @@ CATEGORIES_CACHE_DIR = ARCH_CACHE_PATH + '/categories' CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt' URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt' CONFIG_DIR = '{}/.config/bauh/arch'.format(HOME_PATH) -CUSTOM_MAKEPKG_PATH = '{}/makepkg.conf'.format(CONFIG_DIR) +CUSTOM_MAKEPKG_FILE = '{}/makepkg.conf'.format(CONFIG_DIR) AUR_INDEX_FILE = '{}/aur.txt'.format(BUILD_DIR) CONFIG_FILE = '{}/arch.yml'.format(CONFIG_PATH) diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 5cbe866e..88108328 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -20,7 +20,7 @@ from bauh.commons.html import bold from bauh.commons.system import SystemProcess, ProcessHandler, new_subprocess, run_cmd, new_root_subprocess, \ SimpleProcess from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, confirmation, disk, git, suggestions, \ - gpg, URL_CATEGORIES_FILE, CATEGORIES_CACHE_DIR, CATEGORIES_FILE_PATH + gpg, URL_CATEGORIES_FILE, CATEGORIES_CACHE_DIR, CATEGORIES_FILE_PATH, CUSTOM_MAKEPKG_FILE from bauh.gems.arch.aur import AURClient from bauh.gems.arch.config import read_config from bauh.gems.arch.depedencies import DependenciesAnalyser @@ -759,6 +759,10 @@ class ArchManager(SoftwareManager): self.local_config = read_config() clean_config = True + if self.local_config['optimize'] and not os.path.exists(CUSTOM_MAKEPKG_FILE): + watcher.change_substatus(self.i18n['arch.makepkg.optimizing']) + ArchCompilationOptimizer(self.context.logger).optimize() + res = self._install_from_aur(pkg.name, pkg.maintainer, root_password, ProcessHandler(watcher), dependency=False, skip_optdeps=skip_optdeps) if res: diff --git a/bauh/gems/arch/makepkg.py b/bauh/gems/arch/makepkg.py index c2a2ccfc..85dc4178 100644 --- a/bauh/gems/arch/makepkg.py +++ b/bauh/gems/arch/makepkg.py @@ -3,7 +3,7 @@ import re from typing import Tuple from bauh.commons.system import SimpleProcess, ProcessHandler -from bauh.gems.arch import CUSTOM_MAKEPKG_PATH +from bauh.gems.arch import CUSTOM_MAKEPKG_FILE RE_DEPS_PATTERN = re.compile(r'\n?\s+->\s(.+)\n') RE_UNKNOWN_GPG_KEY = re.compile(r'\(unknown public key (\w+)\)') @@ -15,11 +15,11 @@ def check(pkgdir: str, optimize: bool, handler: ProcessHandler) -> dict: cmd = ['makepkg', '-ALcf', '--check', '--noarchive', '--nobuild', '--noprepare'] if optimize: - if os.path.exists(CUSTOM_MAKEPKG_PATH): - handler.watcher.print('Using custom makepkg.conf -> {}'.format(CUSTOM_MAKEPKG_PATH)) - cmd.append('--config={}'.format(CUSTOM_MAKEPKG_PATH)) + if os.path.exists(CUSTOM_MAKEPKG_FILE): + handler.watcher.print('Using custom makepkg.conf -> {}'.format(CUSTOM_MAKEPKG_FILE)) + cmd.append('--config={}'.format(CUSTOM_MAKEPKG_FILE)) else: - handler.watcher.print('Custom optimized makepkg.conf ( {} ) not found'.format(CUSTOM_MAKEPKG_PATH)) + handler.watcher.print('Custom optimized makepkg.conf ( {} ) not found'.format(CUSTOM_MAKEPKG_FILE)) success, output = handler.handle_simple(SimpleProcess(cmd, cwd=pkgdir)) @@ -41,10 +41,10 @@ def make(pkgdir: str, optimize: bool, handler: ProcessHandler) -> Tuple[bool, st cmd = ['makepkg', '-ALcsmf'] if optimize: - if os.path.exists(CUSTOM_MAKEPKG_PATH): - handler.watcher.print('Using custom makepkg.conf -> {}'.format(CUSTOM_MAKEPKG_PATH)) - cmd.append('--config={}'.format(CUSTOM_MAKEPKG_PATH)) + if os.path.exists(CUSTOM_MAKEPKG_FILE): + handler.watcher.print('Using custom makepkg.conf -> {}'.format(CUSTOM_MAKEPKG_FILE)) + cmd.append('--config={}'.format(CUSTOM_MAKEPKG_FILE)) else: - handler.watcher.print('Custom optimized makepkg.conf ( {} ) not found'.format(CUSTOM_MAKEPKG_PATH)) + handler.watcher.print('Custom optimized makepkg.conf ( {} ) not found'.format(CUSTOM_MAKEPKG_FILE)) return handler.handle_simple(SimpleProcess(cmd, cwd=pkgdir)) diff --git a/bauh/gems/arch/resources/locale/ca b/bauh/gems/arch/resources/locale/ca index f71d1b72..67d1f3de 100644 --- a/bauh/gems/arch/resources/locale/ca +++ b/bauh/gems/arch/resources/locale/ca @@ -101,3 +101,4 @@ arch.aur.install.unknown_key.receive_error=No s’ha pogut rebre la clau públic arch.install.aur.unknown_key.body=Per a continuar amb la instal·lació de {} cal confiar en la clau pública següent {} arch.aur.install.validity_check.title=Problemes d’integritat arch.aur.install.validity_check.body=Alguns dels fitxers font necessaris per a la instal·lació de {} són malmesos. La instal·lació es cancel·larà per a evitar danys al vostre sistema. +arch.makepkg.optimizing=Optimitzant la recopilació \ No newline at end of file diff --git a/bauh/gems/arch/resources/locale/de b/bauh/gems/arch/resources/locale/de index adf008ca..ac287a6c 100644 --- a/bauh/gems/arch/resources/locale/de +++ b/bauh/gems/arch/resources/locale/de @@ -100,4 +100,5 @@ aur.info.license=Lizenz aur.info.validpgpkeys=gültige PGP Schlüssel aur.info.options=Optionen aur.info.provides=stellt bereit -aur.info.conflicts with=Konflikt mit \ No newline at end of file +aur.info.conflicts with=Konflikt mit +arch.makepkg.optimizing=Optimiert die Zusammenstellung \ No newline at end of file diff --git a/bauh/gems/arch/resources/locale/en b/bauh/gems/arch/resources/locale/en index f2a295de..8bc34829 100644 --- a/bauh/gems/arch/resources/locale/en +++ b/bauh/gems/arch/resources/locale/en @@ -100,4 +100,5 @@ aur.info.license=license aur.info.validpgpkeys=valid PGP keys aur.info.options=options aur.info.provides=provides -aur.info.conflicts with=conflicts with \ No newline at end of file +aur.info.conflicts with=conflicts with +arch.makepkg.optimizing=Optimizing the compilation \ No newline at end of file diff --git a/bauh/gems/arch/resources/locale/es b/bauh/gems/arch/resources/locale/es index 711e5c48..49d81205 100644 --- a/bauh/gems/arch/resources/locale/es +++ b/bauh/gems/arch/resources/locale/es @@ -100,4 +100,5 @@ arch.aur.install.unknown_key.status=Recibiendo la clave pública {} arch.aur.install.unknown_key.receive_error=No fue posible recibir la clave pública {} arch.install.aur.unknown_key.body=Para continuar la instalación de {} es necesario confiar en la siguiente clave pública {} arch.aur.install.validity_check.title=Problemas de integridad -arch.aur.install.validity_check.body=Algunos de los archivos fuente necesarios para la instalación de {} no están en buen estado. La instalación se cancelará para evitar daños a su sistema. \ No newline at end of file +arch.aur.install.validity_check.body=Algunos de los archivos fuente necesarios para la instalación de {} no están en buen estado. La instalación se cancelará para evitar daños a su sistema. +arch.makepkg.optimizing=Optimizando la compilación \ No newline at end of file diff --git a/bauh/gems/arch/resources/locale/it b/bauh/gems/arch/resources/locale/it index 23fd2b3e..302b8bf0 100644 --- a/bauh/gems/arch/resources/locale/it +++ b/bauh/gems/arch/resources/locale/it @@ -69,3 +69,4 @@ arch.aur.install.unknown_key.status=Ricezione della chiave pubblica {} arch.aur.install.unknown_key.receive_error=Impossibile ricevere la chiave pubblica {} arch.aur.install.validity_check.title=Problemi di integrità arch.aur.install.validity_check.body=Alcuni dei file di origine necessari per l'installazione di {} non sono integri. L'installazione verrà annullata per evitare danni al sistema. +arch.makepkg.optimizing=Ottimizzando la compilazione \ No newline at end of file diff --git a/bauh/gems/arch/resources/locale/pt b/bauh/gems/arch/resources/locale/pt index b38003fa..471ce188 100644 --- a/bauh/gems/arch/resources/locale/pt +++ b/bauh/gems/arch/resources/locale/pt @@ -100,4 +100,5 @@ arch.install.aur.unknown_key.body=Para continuar a instalação de {} é ncessá arch.aur.install.unknown_key.status=Recebendo a chave pública {} arch.aur.install.unknown_key.receive_error=Não fui possível receber a chave pública {} arch.aur.install.validity_check.title=Problemas de integridade -arch.aur.install.validity_check.body=Alguns dos arquivos-fonte necessários para instalação de {} não estão íntegros. A instalação será cancelada para evitar danos no seu sistema. \ No newline at end of file +arch.aur.install.validity_check.body=Alguns dos arquivos-fonte necessários para instalação de {} não estão íntegros. A instalação será cancelada para evitar danos no seu sistema. +arch.makepkg.optimizing=Otimizando a compilação \ No newline at end of file diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index 0494a298..3ca41c95 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -8,7 +8,7 @@ from threading import Thread import requests from bauh.api.abstract.context import ApplicationContext -from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_PATH, CONFIG_DIR, BUILD_DIR, \ +from bauh.gems.arch import pacman, disk, CUSTOM_MAKEPKG_FILE, CONFIG_DIR, BUILD_DIR, \ AUR_INDEX_FILE, config URL_INDEX = 'https://aur.archlinux.org/packages.gz' @@ -75,74 +75,77 @@ class ArchCompilationOptimizer(Thread): super(ArchCompilationOptimizer, self).__init__(daemon=True) self.logger = logger + def optimize(self): + try: + ncpus = os.cpu_count() + except: + self.logger.error('Could not determine the number of processors. Aborting...') + ncpus = None + + if os.path.exists(GLOBAL_MAKEPKG): + self.logger.info("Verifying if it is possible to optimize Arch packages compilation") + + with open(GLOBAL_MAKEPKG) as f: + global_makepkg = f.read() + + Path(CONFIG_DIR).mkdir(parents=True, exist_ok=True) + + custom_makepkg, optimizations = None, [] + + if ncpus: + makeflags = RE_MAKE_FLAGS.findall(global_makepkg) + + if makeflags: + not_commented = [f for f in makeflags if not f.startswith('#')] + + if not not_commented: + custom_makepkg = RE_MAKE_FLAGS.sub('', global_makepkg) + optimizations.append('MAKEFLAGS="-j$(nproc)"') + else: + self.logger.warning("It seems '{}' compilation flags are already customized".format(GLOBAL_MAKEPKG)) + else: + optimizations.append('MAKEFLAGS="-j$(nproc)"') + + compress_xz = RE_COMPRESS_XZ.findall(custom_makepkg if custom_makepkg else global_makepkg) + + if compress_xz: + not_eligible = [f for f in compress_xz if not f.startswith('#') and '--threads' in f] + + if not not_eligible: + custom_makepkg = RE_COMPRESS_XZ.sub('', global_makepkg) + optimizations.append('COMPRESSXZ=(xz -c -z - --threads=0)') + else: + self.logger.warning("It seems '{}' COMPRESSXZ is already customized".format(GLOBAL_MAKEPKG)) + else: + optimizations.append('COMPRESSXZ=(xz -c -z - --threads=0)') + + if optimizations: + generated_by = '# \n' + custom_makepkg = generated_by + custom_makepkg + '\n' + generated_by + '\n'.join(optimizations) + '\n' + + with open(CUSTOM_MAKEPKG_FILE, 'w+') as f: + f.write(custom_makepkg) + + self.logger.info("A custom optimized 'makepkg.conf' was generated at '{}'".format(CUSTOM_MAKEPKG_FILE)) + else: + self.logger.info("No optimizations are necessary") + + if os.path.exists(CUSTOM_MAKEPKG_FILE): + self.logger.info("Removing old optimized 'makepkg.conf' at '{}'".format(CUSTOM_MAKEPKG_FILE)) + os.remove(CUSTOM_MAKEPKG_FILE) + + self.logger.info('Finished') + def run(self): local_config = config.read_config(update_file=True) if not local_config['optimize']: self.logger.info("Arch packages compilation optimizations are disabled") - if os.path.exists(CUSTOM_MAKEPKG_PATH): - self.logger.info("Removing custom 'makepkg.conf' -> '{}'".format(CUSTOM_MAKEPKG_PATH)) - os.remove(CUSTOM_MAKEPKG_PATH) + if os.path.exists(CUSTOM_MAKEPKG_FILE): + self.logger.info("Removing custom 'makepkg.conf' -> '{}'".format(CUSTOM_MAKEPKG_FILE)) + os.remove(CUSTOM_MAKEPKG_FILE) self.logger.info('Finished') else: - try: - ncpus = os.cpu_count() - except: - self.logger.error('Could not determine the number of processors. Aborting...') - ncpus = None - - if os.path.exists(GLOBAL_MAKEPKG): - self.logger.info("Verifying if it is possible to optimize Arch packages compilation") - - with open(GLOBAL_MAKEPKG) as f: - global_makepkg = f.read() - - Path(CONFIG_DIR).mkdir(parents=True, exist_ok=True) - - custom_makepkg, optimizations = None, [] - - if ncpus: - makeflags = RE_MAKE_FLAGS.findall(global_makepkg) - - if makeflags: - not_commented = [f for f in makeflags if not f.startswith('#')] - - if not not_commented: - custom_makepkg = RE_MAKE_FLAGS.sub('', global_makepkg) - optimizations.append('MAKEFLAGS="-j$(nproc)"') - else: - self.logger.warning("It seems '{}' compilation flags are already customized".format(GLOBAL_MAKEPKG)) - else: - optimizations.append('MAKEFLAGS="-j$(nproc)"') - - compress_xz = RE_COMPRESS_XZ.findall(custom_makepkg if custom_makepkg else global_makepkg) - - if compress_xz: - not_eligible = [f for f in compress_xz if not f.startswith('#') and '--threads' in f] - - if not not_eligible: - custom_makepkg = RE_COMPRESS_XZ.sub('', global_makepkg) - optimizations.append('COMPRESSXZ=(xz -c -z - --threads=0)') - else: - self.logger.warning("It seems '{}' COMPRESSXZ is already customized".format(GLOBAL_MAKEPKG)) - else: - optimizations.append('COMPRESSXZ=(xz -c -z - --threads=0)') - - if optimizations: - generated_by = '# \n' - custom_makepkg = generated_by + custom_makepkg + '\n' + generated_by + '\n'.join(optimizations) + '\n' - - with open(CUSTOM_MAKEPKG_PATH, 'w+') as f: - f.write(custom_makepkg) - - self.logger.info("A custom optimized 'makepkg.conf' was generated at '{}'".format(CUSTOM_MAKEPKG_PATH)) - else: - self.logger.info("No optimizations are necessary") - - if os.path.exists(CUSTOM_MAKEPKG_PATH): - self.logger.info("Removing old optimized 'makepkg.conf' at '{}'".format(CUSTOM_MAKEPKG_PATH)) - os.remove(CUSTOM_MAKEPKG_PATH) - - self.logger.info('Finished') + self.optimize()