From cbc9080cb8514c422af4389462fbfeade9c485bc Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Thu, 10 Oct 2019 19:01:27 -0300 Subject: [PATCH] [aur][feature] reading package categories --- bauh/api/http.py | 13 +++++++++---- bauh/gems/arch/controller.py | 8 +++++++- bauh/gems/arch/worker.py | 30 ++++++++++++++++++++++++++++++ bauh/view/resources/locale/es | 6 +++++- bauh/view/resources/locale/pt | 8 +++++++- 5 files changed, 58 insertions(+), 7 deletions(-) diff --git a/bauh/api/http.py b/bauh/api/http.py index d5b94a37..f67fe866 100644 --- a/bauh/api/http.py +++ b/bauh/api/http.py @@ -16,14 +16,19 @@ class HttpClient: self.sleep = sleep self.logger = logger - def get(self, url: str): + def get(self, url: str, headers: dict = None): cur_attempts = 1 while cur_attempts <= self.max_attempts: cur_attempts += 1 try: - res = self.session.get(url, timeout=self.timeout) + args = {'timeout': self.timeout} + + if headers: + args['headers'] = headers + + res = self.session.get(url, **args) if res.status_code == 200: return res @@ -41,8 +46,8 @@ class HttpClient: self.logger.warning("Could not retrieve data from '{}'".format(url)) - def get_json(self, url: str): - res = self.get(url) + def get_json(self, url: str, headers: dict = None): + res = self.get(url, headers=headers) return res.json() if res else None def get_content_length(self, url: str) -> str: diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 0ffb069a..acb7909a 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -22,7 +22,7 @@ from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, c from bauh.gems.arch.aur import AURClient from bauh.gems.arch.mapper import ArchDataMapper from bauh.gems.arch.model import ArchPackage -from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer +from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, CategoriesDownloader URL_GIT = 'https://aur.archlinux.org/{}.git' URL_PKG_DOWNLOAD = 'https://aur.archlinux.org/cgit/aur.git/snapshot/{}.tar.gz' @@ -51,6 +51,8 @@ class ArchManager(SoftwareManager): self.logger = context.logger self.enabled = True self.arch_distro = context.distro == 'arch' + self.categories_downloader = CategoriesDownloader(context.http_client, context.logger) + self.categories_map = {} def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader): app = self.mapper.map_api_data(apidata, installed_pkgs['not_signed']) @@ -117,6 +119,7 @@ class ArchManager(SoftwareManager): for pkgdata in pkgsinfo: pkg = self.mapper.map_api_data(pkgdata, not_signed) pkg.downgrade_enabled = downgrade_enabled + pkg.categories = self.categories_map.get(pkg.name) if disk_loader: disk_loader.fill(pkg) @@ -133,6 +136,8 @@ class ArchManager(SoftwareManager): pkg = ArchPackage(name=name, version=data.get('version'), latest_version=data.get('version'), description=data.get('description'), installed=True, mirror='aur') + + pkg.categories = self.categories_map.get(pkg.name) pkg.downgrade_enabled = downgrade_enabled if disk_loader: @@ -685,6 +690,7 @@ class ArchManager(SoftwareManager): self.dcache_updater.start() self.comp_optimizer.start() self.aur_index_updater.start() + self.categories_map = self.categories_downloader.get_categories() def list_updates(self, internet_available: bool) -> List[PackageUpdate]: installed = self.read_installed(disk_loader=None, internet_available=internet_available).installed diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index c0a583a6..15ba604a 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -5,10 +5,14 @@ import time from math import ceil from multiprocessing import Process from threading import Thread +from typing import Dict, List + +import requests from bauh.api.abstract.context import ApplicationContext from bauh.api.abstract.controller import SoftwareManager from bauh.api.constants import HOME_PATH +from bauh.api.http import HttpClient from bauh.gems.arch import pacman, disk URL_INDEX = 'https://aur.archlinux.org/packages.gz' @@ -128,3 +132,29 @@ class ArchCompilationOptimizer(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else P self.logger.info("A custom optimized 'makepkg.conf' was generated at '{}'".format(HOME_PATH)) else: self.logger.warning("A custom 'makepkg.conf' is already defined at '{}'".format(HOME_PATH)) + + +class CategoriesDownloader: + + URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt' + + def __init__(self, http_client: HttpClient, logger: logging.Logger): + self.http_client = http_client + self.logger = logger + + def get_categories(self) -> Dict[str, List[str]]: + self.logger.info('Downloading AUR category definitions from {}'.format(self.URL_CATEGORIES_FILE)) + + res = self.http_client.get(self.URL_CATEGORIES_FILE, headers={'Authorization': 'token {}'.format(os.getenv('GITHUB_TOKEN'))}) + + if res: + categories_map = {} + for l in res.text.split('\n'): + if l: + data = l.split('=') + categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c] + + self.logger.info('Loaded categories for {} AUR packages'.format(len(categories_map))) + return categories_map + else: + self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE)) diff --git a/bauh/view/resources/locale/es b/bauh/view/resources/locale/es index 77eb5349..b22d813a 100644 --- a/bauh/view/resources/locale/es +++ b/bauh/view/resources/locale/es @@ -141,4 +141,8 @@ system=sistema utility=utilidad terminalemulator=terminal database=base de datos -databases=bases de datos \ No newline at end of file +databases=bases de datos +movies=películas +movie=película +library=biblioteca +books=libros \ No newline at end of file diff --git a/bauh/view/resources/locale/pt b/bauh/view/resources/locale/pt index c572d64f..01917f86 100644 --- a/bauh/view/resources/locale/pt +++ b/bauh/view/resources/locale/pt @@ -141,4 +141,10 @@ system=sistema utility=utilidade terminalemulator=terminal database=banco de dados -databases=bancos de dados \ No newline at end of file +databases=bancos de dados +text editor=editor de texto +movies=filmes +movie=filme +media=mídia +library=biblioteca +books=livros