From 64e4822b2d23fde6b3796e16bc1a487984f9c6f7 Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Tue, 22 Oct 2019 17:10:24 -0300 Subject: [PATCH] [refactoring] CategoriesDownloader as a common class --- bauh/commons/category.py | 99 ++++++++++++++++++++++++++++++++++++ bauh/gems/arch/__init__.py | 4 +- bauh/gems/arch/controller.py | 9 ++-- bauh/gems/arch/worker.py | 90 +------------------------------- bauh/gems/snap/__init__.py | 2 + bauh/gems/snap/controller.py | 8 +-- bauh/gems/snap/worker.py | 91 +-------------------------------- 7 files changed, 118 insertions(+), 185 deletions(-) create mode 100644 bauh/commons/category.py diff --git a/bauh/commons/category.py b/bauh/commons/category.py new file mode 100644 index 00000000..0f9e3831 --- /dev/null +++ b/bauh/commons/category.py @@ -0,0 +1,99 @@ +import logging +import os +import traceback +from pathlib import Path +from threading import Thread +from typing import Dict, List + +import requests + +from bauh.api.abstract.controller import SoftwareManager +from bauh.api.http import HttpClient + + +class CategoriesDownloader(Thread): + + def __init__(self, id_: str, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, + disk_cache: bool, url_categories_file: str, disk_cache_dir: str, categories_path: str): + super(CategoriesDownloader, self).__init__(daemon=True) + self.id_ = id_ + self.http_client = http_client + self.logger = logger + self.manager = manager + self.disk_cache = disk_cache + self.url_categories_file = url_categories_file + self.disk_cache_dir = disk_cache_dir + self.categories_path = categories_path + + def _msg(self, msg: str): + return '{}({}): {}'.format(self.__class__.__name__, self.id_, msg) + + def _read_categories_from_disk(self) -> Dict[str, List[str]]: + if self.disk_cache and os.path.exists(self.categories_path): + self.logger.info(self._msg("Reading cached categories from the disk")) + + with open(self.categories_path) as f: + categories = f.read() + + return self._map_categories(categories) + + return {} + + def _map_categories(self, categories: str) -> Dict[str, List[str]]: + categories_map = {} + for l in categories.split('\n'): + if l: + data = l.split('=') + categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c] + + return categories_map + + def _cache_categories_to_disk(self, categories: str): + self.logger.info(self._msg('Caching categories to the disk')) + + try: + Path(self.disk_cache_dir).mkdir(parents=True, exist_ok=True) + + with open(self.categories_path, 'w+') as f: + f.write(categories) + + self.logger.info(self._msg("Categories cached to the disk as '{}'".format(self.categories_path))) + except: + self.logger.error(self._msg("Could not cache categories to the disk as '{}'".format(self.categories_path))) + traceback.print_exc() + + def download_categories(self) -> Dict[str, List[str]]: + self.logger.info(self._msg('Downloading category definitions from {}'.format(self.url_categories_file))) + + try: + res = self.http_client.get(self.url_categories_file) + + if res: + try: + categories = self._map_categories(res.text) + self.logger.info(self._msg('Loaded categories for {} applications'.format(len(categories)))) + + if self.disk_cache and categories: + Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start() + + return categories + except: + self.logger.error(self._msg("Could not parse categories definitions")) + traceback.print_exc() + else: + self.logger.info(self._msg('Could not download {}'.format(self.url_categories_file))) + + except requests.exceptions.ConnectionError: + self.logger.warning(self._msg('The internet connection seems to be off.')) + + return {} + + def _set_categories(self, categories: dict): + if categories: + self.logger.info(self._msg("Settings categories to {}".format(self.manager.__class__.__name__))) + self.manager.categories = categories + + def run(self): + self._set_categories(self._read_categories_from_disk()) + self._set_categories(self.download_categories()) + self.logger.info(self._msg('Finished')) diff --git a/bauh/gems/arch/__init__.py b/bauh/gems/arch/__init__.py index 87c42aa7..5bd71924 100644 --- a/bauh/gems/arch/__init__.py +++ b/bauh/gems/arch/__init__.py @@ -5,4 +5,6 @@ from bauh.api.constants import CACHE_PATH ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) BUILD_DIR = '/tmp/bauh/aur' ARCH_CACHE_PATH = CACHE_PATH + '/arch' - +CATEGORIES_CACHE_DIR = ARCH_CACHE_PATH + '/categories' +CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt' +URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt' diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 37a645bd..2b7934a2 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -15,14 +15,16 @@ from bauh.api.abstract.disk import DiskCacheLoader from bauh.api.abstract.handler import ProcessWatcher from bauh.api.abstract.model import PackageUpdate, PackageHistory, SoftwarePackage, PackageSuggestion, PackageStatus from bauh.api.abstract.view import MessageType +from bauh.commons.category import CategoriesDownloader from bauh.commons.html import bold from bauh.commons.system import SystemProcess, ProcessHandler, new_subprocess, run_cmd, new_root_subprocess, \ SimpleProcess -from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, confirmation, disk, git, suggestions, gpg +from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, confirmation, disk, git, suggestions, \ + gpg, URL_CATEGORIES_FILE, CATEGORIES_CACHE_DIR, CATEGORIES_FILE_PATH from bauh.gems.arch.aur import AURClient from bauh.gems.arch.mapper import ArchDataMapper from bauh.gems.arch.model import ArchPackage -from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, AURCategoriesMapper +from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer URL_GIT = 'https://aur.archlinux.org/{}.git' URL_PKG_DOWNLOAD = 'https://aur.archlinux.org/cgit/aur.git/snapshot/{}.tar.gz' @@ -51,7 +53,8 @@ class ArchManager(SoftwareManager): self.logger = context.logger self.enabled = True self.arch_distro = context.distro == 'arch' - self.categories_mapper = AURCategoriesMapper(context.http_client, context.logger, self, self.context.disk_cache) + self.categories_mapper = CategoriesDownloader('AUR', context.http_client, context.logger, self, self.context.disk_cache, + URL_CATEGORIES_FILE, CATEGORIES_CACHE_DIR, CATEGORIES_FILE_PATH) self.categories = {} def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader): diff --git a/bauh/gems/arch/worker.py b/bauh/gems/arch/worker.py index 1a359d87..b7f9245f 100644 --- a/bauh/gems/arch/worker.py +++ b/bauh/gems/arch/worker.py @@ -2,20 +2,16 @@ import logging import os import re import time -import traceback from math import ceil from multiprocessing import Process -from pathlib import Path from threading import Thread -from typing import Dict, List import requests from bauh.api.abstract.context import ApplicationContext from bauh.api.abstract.controller import SoftwareManager -from bauh.api.constants import HOME_PATH, CACHE_PATH -from bauh.api.http import HttpClient -from bauh.gems.arch import pacman, disk, ARCH_CACHE_PATH +from bauh.api.constants import HOME_PATH +from bauh.gems.arch import pacman, disk URL_INDEX = 'https://aur.archlinux.org/packages.gz' URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg={}' @@ -138,85 +134,3 @@ class ArchCompilationOptimizer(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else P self.logger.info("A custom optimized 'makepkg.conf' was generated at '{}'".format(HOME_PATH)) else: self.logger.warning("A custom 'makepkg.conf' is already defined at '{}'".format(HOME_PATH)) - - -class AURCategoriesMapper(Thread): - - URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt' - CATEGORIES_CACHE_DIR = ARCH_CACHE_PATH + '/categories' - CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt' - - def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool): - super(AURCategoriesMapper, self).__init__(daemon=True) - self.http_client = http_client - self.logger = logger - self.manager = manager - self.disk_cache = disk_cache - - def _read_categories_from_disk(self) -> dict: - if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH): - self.logger.info("Reading cached categories from the disk") - - with open(self.CATEGORIES_FILE_PATH) as f: - categories = f.read() - - return self._map_categories(categories) - - return {} - - def _map_categories(self, categories: str) -> dict: - categories_map = {} - for l in categories.split('\n'): - if l: - data = l.split('=') - categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c] - return categories_map - - def _cache_categories_to_disk(self, categories: str): - self.logger.info('Caching AUR categories to the disk') - - try: - Path(self.CATEGORIES_CACHE_DIR).mkdir(parents=True, exist_ok=True) - - with open(self.CATEGORIES_FILE_PATH, 'w+') as f: - f.write(categories) - - self.logger.info("AUR categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH)) - except: - self.logger.error("Could not cache AUR categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH)) - traceback.print_exc() - - def get_categories(self) -> Dict[str, List[str]]: - self.logger.info('Downloading AUR category definitions from {}'.format(self.URL_CATEGORIES_FILE)) - - try: - res = self.http_client.get(self.URL_CATEGORIES_FILE) - - if res: - try: - categories_map = self._map_categories(res.text) - self.logger.info('Loaded categories for {} AUR packages'.format(len(categories_map))) - - if self.disk_cache and categories_map: - Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start() - - return categories_map - except: - self.logger.error("Could not parse AUR categories definitions") - traceback.print_exc() - else: - self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE)) - - except requests.exceptions.ConnectionError: - self.logger.warning('The internet connection seems to be off.') - - return self._read_categories_from_disk() - - def run(self): - categories = self.get_categories() - - if categories: - self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__)) - self.manager.categories = categories - - self.logger.info('Finished') diff --git a/bauh/gems/snap/__init__.py b/bauh/gems/snap/__init__.py index 6e7f3aaf..bd219398 100644 --- a/bauh/gems/snap/__init__.py +++ b/bauh/gems/snap/__init__.py @@ -4,3 +4,5 @@ from bauh.api.constants import CACHE_PATH ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) SNAP_CACHE_PATH = CACHE_PATH + '/snap' +CATEGORIES_FILE_PATH = SNAP_CACHE_PATH + '/categories.txt' +URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/categories.txt' diff --git a/bauh/gems/snap/controller.py b/bauh/gems/snap/controller.py index 006f3cdb..2ecdd7c1 100644 --- a/bauh/gems/snap/controller.py +++ b/bauh/gems/snap/controller.py @@ -10,12 +10,13 @@ from bauh.api.abstract.handler import ProcessWatcher from bauh.api.abstract.model import SoftwarePackage, PackageHistory, PackageUpdate, PackageSuggestion, \ SuggestionPriority from bauh.api.abstract.view import SingleSelectComponent, SelectViewType, InputOption +from bauh.commons.category import CategoriesDownloader from bauh.commons.html import bold from bauh.commons.system import SystemProcess, ProcessHandler, new_root_subprocess -from bauh.gems.snap import snap, suggestions +from bauh.gems.snap import snap, suggestions, URL_CATEGORIES_FILE, SNAP_CACHE_PATH, CATEGORIES_FILE_PATH from bauh.gems.snap.constants import SNAP_API_URL from bauh.gems.snap.model import SnapApplication -from bauh.gems.snap.worker import SnapAsyncDataLoader, CategoriesDownloader +from bauh.gems.snap.worker import SnapAsyncDataLoader RE_AVAILABLE_CHANNELS = re.compile(re.compile(r'(\w+)\s+(snap install.+)')) @@ -32,7 +33,8 @@ class SnapManager(SoftwareManager): self.logger = context.logger self.ubuntu_distro = context.distro == 'ubuntu' self.categories = {} - self.categories_downloader = CategoriesDownloader(self.http_client, self.logger, self, context.disk_cache) + self.categories_downloader = CategoriesDownloader('snap', self.http_client, self.logger, self, context.disk_cache, + URL_CATEGORIES_FILE, SNAP_CACHE_PATH, CATEGORIES_FILE_PATH) def map_json(self, app_json: dict, installed: bool, disk_loader: DiskCacheLoader, internet: bool = True) -> SnapApplication: app = SnapApplication(publisher=app_json.get('publisher'), diff --git a/bauh/gems/snap/worker.py b/bauh/gems/snap/worker.py index 17a609d0..72dd76c2 100644 --- a/bauh/gems/snap/worker.py +++ b/bauh/gems/snap/worker.py @@ -1,18 +1,11 @@ -import logging -import os import traceback -from pathlib import Path from threading import Thread -from typing import Dict, List - -import requests from bauh.api.abstract.cache import MemoryCache from bauh.api.abstract.context import ApplicationContext from bauh.api.abstract.controller import SoftwareManager from bauh.api.abstract.model import PackageStatus -from bauh.api.http import HttpClient -from bauh.gems.snap import snap, SNAP_CACHE_PATH +from bauh.gems.snap import snap from bauh.gems.snap.constants import SNAP_API_URL from bauh.gems.snap.model import SnapApplication @@ -75,85 +68,3 @@ class SnapAsyncDataLoader(Thread): if self.persist: self.manager.cache_to_disk(pkg=self.app, icon_bytes=None, only_icon=False) - - -class CategoriesDownloader(Thread): - - URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/categories.txt' - CATEGORIES_FILE_PATH = SNAP_CACHE_PATH + '/categories.txt' - - def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool): - super(CategoriesDownloader, self).__init__(daemon=True) - self.http_client = http_client - self.logger = logger - self.manager = manager - self.disk_cache = disk_cache - - def _read_categories_from_disk(self) -> dict: - if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH): - self.logger.info("Reading cached categories from the disk") - - with open(self.CATEGORIES_FILE_PATH) as f: - categories = f.read() - - return self._map_categories(categories) - - return {} - - def _map_categories(self, categories: str) -> dict: - categories_map = {} - for l in categories.split('\n'): - if l: - data = l.split('=') - categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c] - - return categories_map - - def _cache_categories_to_disk(self, categories: str): - self.logger.info('Caching Snap categories to the disk') - - try: - Path(SNAP_CACHE_PATH).mkdir(parents=True, exist_ok=True) - - with open(self.CATEGORIES_FILE_PATH, 'w+') as f: - f.write(categories) - - self.logger.info("Snap categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH)) - except: - self.logger.error("Could not cache Snap categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH)) - traceback.print_exc() - - def get_categories(self) -> Dict[str, List[str]]: - self.logger.info('Downloading Snap category definitions from {}'.format(self.URL_CATEGORIES_FILE)) - - try: - res = self.http_client.get(self.URL_CATEGORIES_FILE) - - if res: - try: - categories = self._map_categories(res.text) - self.logger.info('Loaded categories for {} Snap applications'.format(len(categories))) - - if self.disk_cache and categories: - Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start() - - return categories - except: - self.logger.error("Could not parse categories definitions") - traceback.print_exc() - else: - self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE)) - - except requests.exceptions.ConnectionError: - self.logger.warning('The internet connection seems to be off.') - - return self._read_categories_from_disk() - - def run(self): - categories = self.get_categories() - - if categories: - self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__)) - self.manager.categories = categories - - self.logger.info('Finished')