From ea14310ad71b85635ee9597c78e0ef7ae77046a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Moreira?= Date: Tue, 17 Dec 2019 12:33:47 -0300 Subject: [PATCH] [wgem] searching through the suggestions index | fix iso descriptions --- bauh/gems/web/__init__.py | 1 + bauh/gems/web/controller.py | 56 ++++++++++++++++++++++++++++++------ bauh/gems/web/worker.py | 30 +++++++++++++------ bauh/view/core/controller.py | 2 +- 4 files changed, 71 insertions(+), 18 deletions(-) diff --git a/bauh/gems/web/__init__.py b/bauh/gems/web/__init__.py index 14e9bd38..24bf0976 100644 --- a/bauh/gems/web/__init__.py +++ b/bauh/gems/web/__init__.py @@ -23,3 +23,4 @@ URL_SUGGESTIONS = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/ UA_CHROME = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36' TEMP_PATH = '/tmp/bauh/web' SEARCH_INDEX_FILE = '{}/index.yaml'.format(TEMP_PATH) +SUGGESTIONS_CACHE_FILE = '{}/suggestions.yaml'.format(TEMP_PATH) diff --git a/bauh/gems/web/controller.py b/bauh/gems/web/controller.py index 471a32ec..096d6aad 100644 --- a/bauh/gems/web/controller.py +++ b/bauh/gems/web/controller.py @@ -25,7 +25,7 @@ from bauh.api.constants import DESKTOP_ENTRIES_DIR from bauh.commons.html import bold from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \ - URL_SUGGESTIONS, SEARCH_INDEX_FILE + SEARCH_INDEX_FILE, SUGGESTIONS_CACHE_FILE from bauh.gems.web.environment import EnvironmentUpdater from bauh.gems.web.model import WebApplication from bauh.gems.web.worker import SuggestionsDownloader, SearchIndexGenerator @@ -122,6 +122,13 @@ class WebApplicationManager(SoftwareManager): desc_tag = soup.find('title') description = desc_tag.text if desc_tag else url + if description: + try: + utf8_desc = description.encode('iso-8859-1').decode('utf-8') + description = utf8_desc + except: + pass + return description def _get_fix_for(self, url_no_protocol: str) -> str: @@ -177,23 +184,56 @@ class WebApplicationManager(SoftwareManager): lower_words = words.lower().strip() installed_matches = [app for app in installed if lower_words in app.name.lower()] - if installed_matches: - res.installed.extend(installed_matches) - index = self._read_search_index() if index: split_words = lower_words.split(' ') singleword = ''.join(lower_words) - app_keys = set() + index_match_keys = set() for word in (*split_words, singleword): indexed = index.get(word) if indexed: - app_keys.update(indexed) + index_match_keys.update(indexed) - print(indexed) - # TODO stopped here -> write suggestions to disk and read them here + if index_match_keys: + if not os.path.exists(SUGGESTIONS_CACHE_FILE): + # if the suggestions cache was not found, it will not be possible to retrieve the matched apps + # so only the installed matches will be returned + self.logger.warning("Suggestion cached file {} was not found".format(SUGGESTIONS_CACHE_FILE)) + res.installed.extend(installed_matches) + else: + with open(SUGGESTIONS_CACHE_FILE) as f: + cached_suggestions = yaml.safe_load(f.read()) + + if not cached_suggestions: + # if no suggestion is found, it will not be possible to retrieve the matched apps + # so only the installed matches will be returned + self.logger.warning("No suggestion found in {}".format(SUGGESTIONS_CACHE_FILE)) + res.installed.extend(installed_matches) + else: + matched_suggestions = [cached_suggestions[key] for key in index_match_keys if cached_suggestions.get(key)] + + if not matched_suggestions: + self.logger.warning("No suggestion found for the search index keys: {}".format(index_match_keys)) + res.installed.extend(installed_matches) + else: + matched_suggestions.sort(key=lambda s: s.get('priority', 0), reverse=True) + + if installed_matches: + # checking if any of the installed matches is one of the matched suggestions + + for sug in matched_suggestions: + found = (installed for installed in installed_matches if i.url == sug.get('url')) + + if found: + res.installed.extend(found) + else: + res.new.append(self._map_suggestion(sug).package) + + else: + for sug in matched_suggestions: + res.new.append(self._map_suggestion(sug).package) res.total += len(res.installed) res.total += len(res.new) diff --git a/bauh/gems/web/worker.py b/bauh/gems/web/worker.py index 3a007aac..6dacbced 100644 --- a/bauh/gems/web/worker.py +++ b/bauh/gems/web/worker.py @@ -5,7 +5,7 @@ from pathlib import Path import yaml from bauh.api.http import HttpClient -from bauh.gems.web import URL_SUGGESTIONS, TEMP_PATH, SEARCH_INDEX_FILE +from bauh.gems.web import URL_SUGGESTIONS, TEMP_PATH, SEARCH_INDEX_FILE, SUGGESTIONS_CACHE_FILE class SuggestionsDownloader: @@ -33,7 +33,26 @@ class SearchIndexGenerator: self.logger = logger def generate_index(self, suggestions: dict): - self.logger.info('Indexing {} app suggestions'.format(len(suggestions))) + self.logger.info('Caching {} suggestions to the disk'.format(len(suggestions))) + + try: + Path(TEMP_PATH).mkdir(parents=True, exist_ok=True) + except: + self.logger.error("Could not generate the directory {}".format(TEMP_PATH)) + traceback.print_exc() + return + + try: + with open(SUGGESTIONS_CACHE_FILE, 'w+') as f: + f.write(yaml.safe_dump(suggestions)) + + self.logger.info('{} successfully cached to the disk as {}'.format(len(suggestions), SUGGESTIONS_CACHE_FILE)) + except: + self.logger.error("Could write to {}".format(SUGGESTIONS_CACHE_FILE)) + traceback.print_exc() + return + + self.logger.info('Indexing suggestions') index = {} for key, sug in suggestions.items(): @@ -58,13 +77,6 @@ class SearchIndexGenerator: for key in index.keys(): index[key] = list(index[key]) - try: - Path(TEMP_PATH).mkdir(parents=True, exist_ok=True) - except: - self.logger.error("Could not generate the directory {}".format(TEMP_PATH)) - traceback.print_exc() - return - try: self.logger.info('Writing {} indexed keys as {}'.format(len(index), SEARCH_INDEX_FILE)) with open(SEARCH_INDEX_FILE, 'w+') as f: diff --git a/bauh/view/core/controller.py b/bauh/view/core/controller.py index 39d1b6f1..65effc90 100755 --- a/bauh/view/core/controller.py +++ b/bauh/view/core/controller.py @@ -14,7 +14,7 @@ from bauh.commons import internet RE_IS_URL = re.compile(r'^https?://.+') -SUGGESTIONS_LIMIT = 5 +SUGGESTIONS_LIMIT = 10 class GenericSoftwareManager(SoftwareManager):