spalencsar/bearhub
1
0
Fork 0
mirror of https://github.com/spalencsar/bearhub.git synced 2026-07-07 01:14:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

[refactoring] CategoriesDownloader as a common class

Browse Source
This commit is contained in:
Vinicius Moreira
2019-10-22 17:10:24 -03:00
parent c358d077be
commit 64e4822b2d
7 changed files with 118 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
bauh/commons/category.py Normal file
View File

@@ -0,0 +1,99 @@
import logging
import os
import traceback
from pathlib import Path
from threading import Thread
from typing import Dict, List
import requests
from bauh.api.abstract.controller import SoftwareManager
from bauh.api.http import HttpClient
class CategoriesDownloader(Thread):
def __init__(self, id_: str, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager,
disk_cache: bool, url_categories_file: str, disk_cache_dir: str, categories_path: str):
super(CategoriesDownloader, self).__init__(daemon=True)
self.id_ = id_
self.http_client = http_client
self.logger = logger
self.manager = manager
self.disk_cache = disk_cache
self.url_categories_file = url_categories_file
self.disk_cache_dir = disk_cache_dir
self.categories_path = categories_path
def _msg(self, msg: str):
return '{}({}): {}'.format(self.__class__.__name__, self.id_, msg)
def _read_categories_from_disk(self) -> Dict[str, List[str]]:
if self.disk_cache and os.path.exists(self.categories_path):
self.logger.info(self._msg("Reading cached categories from the disk"))
with open(self.categories_path) as f:
categories = f.read()
return self._map_categories(categories)
return {}
def _map_categories(self, categories: str) -> Dict[str, List[str]]:
categories_map = {}
for l in categories.split('\n'):
if l:
data = l.split('=')
categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
return categories_map
def _cache_categories_to_disk(self, categories: str):
self.logger.info(self._msg('Caching categories to the disk'))
try:
Path(self.disk_cache_dir).mkdir(parents=True, exist_ok=True)
with open(self.categories_path, 'w+') as f:
f.write(categories)
self.logger.info(self._msg("Categories cached to the disk as '{}'".format(self.categories_path)))
except:
self.logger.error(self._msg("Could not cache categories to the disk as '{}'".format(self.categories_path)))
traceback.print_exc()
def download_categories(self) -> Dict[str, List[str]]:
self.logger.info(self._msg('Downloading category definitions from {}'.format(self.url_categories_file)))
try:
res = self.http_client.get(self.url_categories_file)
if res:
try:
categories = self._map_categories(res.text)
self.logger.info(self._msg('Loaded categories for {} applications'.format(len(categories))))
if self.disk_cache and categories:
Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start()
return categories
except:
self.logger.error(self._msg("Could not parse categories definitions"))
traceback.print_exc()
else:
self.logger.info(self._msg('Could not download {}'.format(self.url_categories_file)))
except requests.exceptions.ConnectionError:
self.logger.warning(self._msg('The internet connection seems to be off.'))
return {}
def _set_categories(self, categories: dict):
if categories:
self.logger.info(self._msg("Settings categories to {}".format(self.manager.__class__.__name__)))
self.manager.categories = categories
def run(self):
self._set_categories(self._read_categories_from_disk())
self._set_categories(self.download_categories())
self.logger.info(self._msg('Finished'))

4
bauh/gems/arch/__init__.py
View File

@@ -5,4 +5,6 @@ from bauh.api.constants import CACHE_PATH
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
BUILD_DIR = '/tmp/bauh/aur'
ARCH_CACHE_PATH = CACHE_PATH + '/arch'
CATEGORIES_CACHE_DIR = ARCH_CACHE_PATH + '/categories'
CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt'
URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt'

9
bauh/gems/arch/controller.py
View File

@@ -15,14 +15,16 @@ from bauh.api.abstract.disk import DiskCacheLoader
from bauh.api.abstract.handler import ProcessWatcher
from bauh.api.abstract.model import PackageUpdate, PackageHistory, SoftwarePackage, PackageSuggestion, PackageStatus
from bauh.api.abstract.view import MessageType
from bauh.commons.category import CategoriesDownloader
from bauh.commons.html import bold
from bauh.commons.system import SystemProcess, ProcessHandler, new_subprocess, run_cmd, new_root_subprocess, \
SimpleProcess
from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, confirmation, disk, git, suggestions, gpg
from bauh.gems.arch import BUILD_DIR, aur, pacman, makepkg, pkgbuild, message, confirmation, disk, git, suggestions, \
gpg, URL_CATEGORIES_FILE, CATEGORIES_CACHE_DIR, CATEGORIES_FILE_PATH
from bauh.gems.arch.aur import AURClient
from bauh.gems.arch.mapper import ArchDataMapper
from bauh.gems.arch.model import ArchPackage
from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer, AURCategoriesMapper
from bauh.gems.arch.worker import AURIndexUpdater, ArchDiskCacheUpdater, ArchCompilationOptimizer
URL_GIT = 'https://aur.archlinux.org/{}.git'
URL_PKG_DOWNLOAD = 'https://aur.archlinux.org/cgit/aur.git/snapshot/{}.tar.gz'
@@ -51,7 +53,8 @@ class ArchManager(SoftwareManager):
self.logger = context.logger
self.enabled = True
self.arch_distro = context.distro == 'arch'
self.categories_mapper = AURCategoriesMapper(context.http_client, context.logger, self, self.context.disk_cache)
self.categories_mapper = CategoriesDownloader('AUR', context.http_client, context.logger, self, self.context.disk_cache,
URL_CATEGORIES_FILE, CATEGORIES_CACHE_DIR, CATEGORIES_FILE_PATH)
self.categories = {}
def _upgrade_search_result(self, apidata: dict, installed_pkgs: dict, downgrade_enabled: bool, res: SearchResult, disk_loader: DiskCacheLoader):

90
bauh/gems/arch/worker.py
View File

@@ -2,20 +2,16 @@ import logging
import os
import re
import time
import traceback
from math import ceil
from multiprocessing import Process
from pathlib import Path
from threading import Thread
from typing import Dict, List
import requests
from bauh.api.abstract.context import ApplicationContext
from bauh.api.abstract.controller import SoftwareManager
from bauh.api.constants import HOME_PATH, CACHE_PATH
from bauh.api.http import HttpClient
from bauh.gems.arch import pacman, disk, ARCH_CACHE_PATH
from bauh.api.constants import HOME_PATH
from bauh.gems.arch import pacman, disk
URL_INDEX = 'https://aur.archlinux.org/packages.gz'
URL_INFO = 'https://aur.archlinux.org/rpc/?v=5&type=info&arg={}'
@@ -138,85 +134,3 @@ class ArchCompilationOptimizer(Thread if bool(os.getenv('BAUH_DEBUG', 0)) else P
self.logger.info("A custom optimized 'makepkg.conf' was generated at '{}'".format(HOME_PATH))
else:
self.logger.warning("A custom 'makepkg.conf' is already defined at '{}'".format(HOME_PATH))
class AURCategoriesMapper(Thread):
URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/aur/categories.txt'
CATEGORIES_CACHE_DIR = ARCH_CACHE_PATH + '/categories'
CATEGORIES_FILE_PATH = CATEGORIES_CACHE_DIR + '/aur.txt'
def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool):
super(AURCategoriesMapper, self).__init__(daemon=True)
self.http_client = http_client
self.logger = logger
self.manager = manager
self.disk_cache = disk_cache
def _read_categories_from_disk(self) -> dict:
if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH):
self.logger.info("Reading cached categories from the disk")
with open(self.CATEGORIES_FILE_PATH) as f:
categories = f.read()
return self._map_categories(categories)
return {}
def _map_categories(self, categories: str) -> dict:
categories_map = {}
for l in categories.split('\n'):
if l:
data = l.split('=')
categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
return categories_map
def _cache_categories_to_disk(self, categories: str):
self.logger.info('Caching AUR categories to the disk')
try:
Path(self.CATEGORIES_CACHE_DIR).mkdir(parents=True, exist_ok=True)
with open(self.CATEGORIES_FILE_PATH, 'w+') as f:
f.write(categories)
self.logger.info("AUR categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
except:
self.logger.error("Could not cache AUR categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
traceback.print_exc()
def get_categories(self) -> Dict[str, List[str]]:
self.logger.info('Downloading AUR category definitions from {}'.format(self.URL_CATEGORIES_FILE))
try:
res = self.http_client.get(self.URL_CATEGORIES_FILE)
if res:
try:
categories_map = self._map_categories(res.text)
self.logger.info('Loaded categories for {} AUR packages'.format(len(categories_map)))
if self.disk_cache and categories_map:
Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start()
return categories_map
except:
self.logger.error("Could not parse AUR categories definitions")
traceback.print_exc()
else:
self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE))
except requests.exceptions.ConnectionError:
self.logger.warning('The internet connection seems to be off.')
return self._read_categories_from_disk()
def run(self):
categories = self.get_categories()
if categories:
self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__))
self.manager.categories = categories
self.logger.info('Finished')

2
bauh/gems/snap/__init__.py
View File

@@ -4,3 +4,5 @@ from bauh.api.constants import CACHE_PATH
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
SNAP_CACHE_PATH = CACHE_PATH + '/snap'
CATEGORIES_FILE_PATH = SNAP_CACHE_PATH + '/categories.txt'
URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/categories.txt'

8
bauh/gems/snap/controller.py
View File

@@ -10,12 +10,13 @@ from bauh.api.abstract.handler import ProcessWatcher
from bauh.api.abstract.model import SoftwarePackage, PackageHistory, PackageUpdate, PackageSuggestion, \
SuggestionPriority
from bauh.api.abstract.view import SingleSelectComponent, SelectViewType, InputOption
from bauh.commons.category import CategoriesDownloader
from bauh.commons.html import bold
from bauh.commons.system import SystemProcess, ProcessHandler, new_root_subprocess
from bauh.gems.snap import snap, suggestions
from bauh.gems.snap import snap, suggestions, URL_CATEGORIES_FILE, SNAP_CACHE_PATH, CATEGORIES_FILE_PATH
from bauh.gems.snap.constants import SNAP_API_URL
from bauh.gems.snap.model import SnapApplication
from bauh.gems.snap.worker import SnapAsyncDataLoader, CategoriesDownloader
from bauh.gems.snap.worker import SnapAsyncDataLoader
RE_AVAILABLE_CHANNELS = re.compile(re.compile(r'(\w+)\s+(snap install.+)'))
@@ -32,7 +33,8 @@ class SnapManager(SoftwareManager):
self.logger = context.logger
self.ubuntu_distro = context.distro == 'ubuntu'
self.categories = {}
self.categories_downloader = CategoriesDownloader(self.http_client, self.logger, self, context.disk_cache)
self.categories_downloader = CategoriesDownloader('snap', self.http_client, self.logger, self, context.disk_cache,
URL_CATEGORIES_FILE, SNAP_CACHE_PATH, CATEGORIES_FILE_PATH)
def map_json(self, app_json: dict, installed: bool, disk_loader: DiskCacheLoader, internet: bool = True) -> SnapApplication:
app = SnapApplication(publisher=app_json.get('publisher'),

91
bauh/gems/snap/worker.py
View File

@@ -1,18 +1,11 @@
import logging
import os
import traceback
from pathlib import Path
from threading import Thread
from typing import Dict, List
import requests
from bauh.api.abstract.cache import MemoryCache
from bauh.api.abstract.context import ApplicationContext
from bauh.api.abstract.controller import SoftwareManager
from bauh.api.abstract.model import PackageStatus
from bauh.api.http import HttpClient
from bauh.gems.snap import snap, SNAP_CACHE_PATH
from bauh.gems.snap import snap
from bauh.gems.snap.constants import SNAP_API_URL
from bauh.gems.snap.model import SnapApplication
@@ -75,85 +68,3 @@ class SnapAsyncDataLoader(Thread):
if self.persist:
self.manager.cache_to_disk(pkg=self.app, icon_bytes=None, only_icon=False)
class CategoriesDownloader(Thread):
URL_CATEGORIES_FILE = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/snap/categories.txt'
CATEGORIES_FILE_PATH = SNAP_CACHE_PATH + '/categories.txt'
def __init__(self, http_client: HttpClient, logger: logging.Logger, manager: SoftwareManager, disk_cache: bool):
super(CategoriesDownloader, self).__init__(daemon=True)
self.http_client = http_client
self.logger = logger
self.manager = manager
self.disk_cache = disk_cache
def _read_categories_from_disk(self) -> dict:
if self.disk_cache and os.path.exists(self.CATEGORIES_FILE_PATH):
self.logger.info("Reading cached categories from the disk")
with open(self.CATEGORIES_FILE_PATH) as f:
categories = f.read()
return self._map_categories(categories)
return {}
def _map_categories(self, categories: str) -> dict:
categories_map = {}
for l in categories.split('\n'):
if l:
data = l.split('=')
categories_map[data[0]] = [c.strip() for c in data[1].split(',') if c]
return categories_map
def _cache_categories_to_disk(self, categories: str):
self.logger.info('Caching Snap categories to the disk')
try:
Path(SNAP_CACHE_PATH).mkdir(parents=True, exist_ok=True)
with open(self.CATEGORIES_FILE_PATH, 'w+') as f:
f.write(categories)
self.logger.info("Snap categories cached to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
except:
self.logger.error("Could not cache Snap categories to the disk as '{}'".format(self.CATEGORIES_FILE_PATH))
traceback.print_exc()
def get_categories(self) -> Dict[str, List[str]]:
self.logger.info('Downloading Snap category definitions from {}'.format(self.URL_CATEGORIES_FILE))
try:
res = self.http_client.get(self.URL_CATEGORIES_FILE)
if res:
try:
categories = self._map_categories(res.text)
self.logger.info('Loaded categories for {} Snap applications'.format(len(categories)))
if self.disk_cache and categories:
Thread(target=self._cache_categories_to_disk, args=(res.text,), daemon=True).start()
return categories
except:
self.logger.error("Could not parse categories definitions")
traceback.print_exc()
else:
self.logger.info('Could not download {}'.format(self.URL_CATEGORIES_FILE))
except requests.exceptions.ConnectionError:
self.logger.warning('The internet connection seems to be off.')
return self._read_categories_from_disk()
def run(self):
categories = self.get_categories()
if categories:
self.logger.info("Settings categories to {}".format(self.manager.__class__.__name__))
self.manager.categories = categories
self.logger.info('Finished')