From 22c002f435f078202cdf138b74bd95aef37758d2 Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Wed, 30 Dec 2020 15:51:11 -0300 Subject: [PATCH] [web] improvement -> code refactoring --- CHANGELOG.md | 1 + bauh/gems/web/controller.py | 16 ++++----- bauh/gems/web/search.py | 67 +++++++++++++++++++++++++++++++++++++ bauh/gems/web/worker.py | 61 +++++++-------------------------- 4 files changed, 87 insertions(+), 58 deletions(-) create mode 100644 bauh/gems/web/search.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d664393f..0be09666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

- displaying the "Indexing suggestions" task during the initialization process + - code refactoring - all types now display an initialization task "Checking configuration file" responsible to create/update the respective configuration file - minor translation improvements diff --git a/bauh/gems/web/controller.py b/bauh/gems/web/controller.py index d77a00a6..33ea6d6a 100644 --- a/bauh/gems/web/controller.py +++ b/bauh/gems/web/controller.py @@ -32,11 +32,12 @@ from bauh.commons.boot import CreateConfigFile from bauh.commons.html import bold from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str, SimpleProcess from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \ - SEARCH_INDEX_FILE, SUGGESTIONS_CACHE_FILE, ROOT_DIR, TEMP_PATH, FIXES_PATH, ELECTRON_PATH, \ + SUGGESTIONS_CACHE_FILE, ROOT_DIR, TEMP_PATH, FIXES_PATH, ELECTRON_PATH, \ get_icon_path from bauh.gems.web.config import WebConfigManager from bauh.gems.web.environment import EnvironmentUpdater, EnvironmentComponent from bauh.gems.web.model import WebApplication +from bauh.gems.web.search import SearchIndexManager from bauh.gems.web.worker import SuggestionsManager, UpdateEnvironmentSettings, \ SuggestionsLoader, SearchIndexGenerator @@ -72,6 +73,7 @@ class WebApplicationManager(SoftwareManager): self.suggestions_loader = suggestions_loader self.suggestions = {} self.configman = WebConfigManager() + self.idxman = SearchIndexManager(logger=context.logger) self.custom_actions = [CustomSoftwareAction(i18n_label_key='web.custom_action.clean_env', i18n_status_key='web.custom_action.clean_env.status', manager=self, @@ -255,11 +257,11 @@ class WebApplicationManager(SoftwareManager): lower_words = words.lower().strip() installed_matches = [app for app in installed if lower_words in app.name.lower()] - index = self._read_search_index() + index = self.idxman.read() if not index and self.suggestions_loader and self.suggestions_loader.is_alive(): self.suggestions_loader.join() - index = self._read_search_index() + index = self.idxman.read() if index: split_words = lower_words.split(' ') @@ -333,13 +335,6 @@ class WebApplicationManager(SoftwareManager): return res - def _read_search_index(self) -> dict: - if os.path.exists(SEARCH_INDEX_FILE): - with open(SEARCH_INDEX_FILE) as f: - return yaml.safe_load(f.read()) - else: - self.logger.warning("No search index found at {}".format(SEARCH_INDEX_FILE)) - def read_installed(self, disk_loader: Optional[DiskCacheLoader], limit: int = -1, only_apps: bool = False, pkg_types: Set[Type[SoftwarePackage]] = None, internet_available: bool = True) -> SearchResult: res = SearchResult([], [], 0) @@ -842,6 +837,7 @@ class WebApplicationManager(SoftwareManager): self.suggestions_loader.start() SearchIndexGenerator(taskman=task_manager, + idxman=self.idxman, suggestions_loader=self.suggestions_loader, i18n=self.i18n, logger=self.logger).start() diff --git a/bauh/gems/web/search.py b/bauh/gems/web/search.py new file mode 100644 index 00000000..f2ebeeb8 --- /dev/null +++ b/bauh/gems/web/search.py @@ -0,0 +1,67 @@ +import os +import time +import traceback +from logging import Logger +from typing import Optional + +import yaml + +from bauh.gems.web import SEARCH_INDEX_FILE + + +class SearchIndexManager: + + def __init__(self, logger: Logger): + self.logger = logger + + def generate(self, suggestions: dict) -> Optional[dict]: + if suggestions: + ti = time.time() + index = {} + + for key, sug in suggestions.items(): + name = sug.get('name') + + if name: + split_name = name.lower().strip().split(' ') + single_name = ''.join(split_name) + + for word in (*split_name, single_name): + mapped = index.get(word) + + if not mapped: + mapped = set() + index[word] = mapped + + mapped.add(key) + + tf = time.time() + self.logger.info("Took {0:.4f} seconds to generate the index".format(tf - ti)) + + return index + + def read(self) -> Optional[dict]: + if os.path.exists(SEARCH_INDEX_FILE): + with open(SEARCH_INDEX_FILE) as f: + return yaml.safe_load(f.read()) + else: + self.logger.warning("No search index found at {}".format(SEARCH_INDEX_FILE)) + + def write(self, index: dict) -> bool: + if index: + self.logger.info('Preparing search index for writing') # YAML does not work with 'sets' + + for key in index.keys(): + index[key] = list(index[key]) + + try: + self.logger.info('Writing {} indexed keys as {}'.format(len(index), SEARCH_INDEX_FILE)) + with open(SEARCH_INDEX_FILE, 'w+') as f: + f.write(yaml.safe_dump(index)) + self.logger.info("Search index successfully written at {}".format(SEARCH_INDEX_FILE)) + return True + except: + self.logger.error("Could not write the search index to {}".format(SEARCH_INDEX_FILE)) + traceback.print_exc() + + return False diff --git a/bauh/gems/web/worker.py b/bauh/gems/web/worker.py index 91b86f94..953dce59 100644 --- a/bauh/gems/web/worker.py +++ b/bauh/gems/web/worker.py @@ -5,13 +5,12 @@ from datetime import datetime from threading import Thread from typing import Optional -import yaml - from bauh.api.abstract.handler import TaskManager from bauh.commons.boot import CreateConfigFile from bauh.commons.html import bold -from bauh.gems.web import SEARCH_INDEX_FILE, get_icon_path +from bauh.gems.web import get_icon_path from bauh.gems.web.environment import EnvironmentUpdater +from bauh.gems.web.search import SearchIndexManager from bauh.gems.web.suggestions import SuggestionsManager from bauh.view.util.translation import I18n @@ -69,14 +68,15 @@ class SuggestionsLoader(Thread): self.taskman.update_progress(self.task_id, 100, None) self.taskman.finish_task(self.task_id) tf = time.time() - self.logger.info("Finished. Took {0:.2f} seconds".format(tf - ti)) + self.logger.info("Finished. Took {0:.4f} seconds".format(tf - ti)) class SearchIndexGenerator(Thread): - def __init__(self, taskman: TaskManager, suggestions_loader: SuggestionsLoader, i18n: I18n, logger: logging.Logger): + def __init__(self, taskman: TaskManager, idxman: SearchIndexManager, suggestions_loader: SuggestionsLoader, i18n: I18n, logger: logging.Logger): super(SearchIndexGenerator, self).__init__(daemon=True) self.taskman = taskman + self.idxman = idxman self.i18n = i18n self.logger = logger self.suggestions_loader = suggestions_loader @@ -89,53 +89,18 @@ class SearchIndexGenerator(Thread): self.suggestions_loader.join() if self.suggestions_loader.suggestions: - self.generate_index(self.suggestions_loader.suggestions) - else: - self.taskman.update_progress(self.task_id, 100, None) - self.taskman.finish_task(self.task_id) + self.taskman.update_progress(self.task_id, 1, None) + self.logger.info('Indexing suggestions') + index = self.idxman.generate(self.suggestions_loader.suggestions) - tf = time.time() - self.logger.info("Finished. Took {0:.2f} seconds".format(tf - ti)) - - def generate_index(self, suggestions: dict): - self.taskman.update_progress(self.task_id, 1, None) - self.logger.info('Indexing suggestions') - index = {} - - for key, sug in suggestions.items(): - name = sug.get('name') - - if name: - split_name = name.lower().strip().split(' ') - single_name = ''.join(split_name) - - for word in (*split_name, single_name): - mapped = index.get(word) - - if not mapped: - mapped = set() - index[word] = mapped - - mapped.add(key) - - if index: - self.taskman.update_progress(self.task_id, 50, self.i18n['web.task.suggestions.saving']) - self.logger.info('Preparing search index for writing') - - for key in index.keys(): - index[key] = list(index[key]) - - try: - self.logger.info('Writing {} indexed keys as {}'.format(len(index), SEARCH_INDEX_FILE)) - with open(SEARCH_INDEX_FILE, 'w+') as f: - f.write(yaml.safe_dump(index)) - self.logger.info("Search index successfully written at {}".format(SEARCH_INDEX_FILE)) - except: - self.logger.error("Could not write the seach index to {}".format(SEARCH_INDEX_FILE)) - traceback.print_exc() + if index: + self.taskman.update_progress(self.task_id, 50, self.i18n['web.task.suggestions.saving']) + self.idxman.write(index) self.taskman.update_progress(self.task_id, 100, None) self.taskman.finish_task(self.task_id) + tf = time.time() + self.logger.info("Finished. Took {0:.4f} seconds".format(tf - ti)) class UpdateEnvironmentSettings(Thread):