From e73e36b9c2947e1874fddf5758a119bb4a3d06b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Moreira?= Date: Mon, 16 Dec 2019 19:50:47 -0300 Subject: [PATCH] [wgem] downloading suggestions from the cloud and generating a search index --- bauh/api/http.py | 5 + bauh/app.py | 3 + bauh/gems/web/__init__.py | 3 + bauh/gems/web/controller.py | 48 +++++-- bauh/gems/web/resources/suggestions.yml | 177 ------------------------ bauh/gems/web/worker.py | 76 ++++++++++ 6 files changed, 126 insertions(+), 186 deletions(-) delete mode 100644 bauh/gems/web/resources/suggestions.yml create mode 100644 bauh/gems/web/worker.py diff --git a/bauh/api/http.py b/bauh/api/http.py index 07641a87..e025b7b9 100644 --- a/bauh/api/http.py +++ b/bauh/api/http.py @@ -3,6 +3,7 @@ import time import traceback import requests +import yaml from bauh.commons import system @@ -59,6 +60,10 @@ class HttpClient: res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects) return res.json() if res else None + def get_yaml(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True): + res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects) + return yaml.safe_load(res.text) if res else None + def get_content_length(self, url: str) -> str: """ :param url: diff --git a/bauh/app.py b/bauh/app.py index 488241e9..3ff4c75f 100755 --- a/bauh/app.py +++ b/bauh/app.py @@ -1,6 +1,7 @@ import os import sys +import urllib3 from PyQt5.QtGui import QIcon from PyQt5.QtWidgets import QApplication @@ -24,6 +25,8 @@ def main(): if not os.getenv('PYTHONUNBUFFERED'): os.environ['PYTHONUNBUFFERED'] = '1' + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + args = app_args.read() logger = logs.new_logger(__app_name__, bool(args.logs)) diff --git a/bauh/gems/web/__init__.py b/bauh/gems/web/__init__.py index 27eed23d..14e9bd38 100644 --- a/bauh/gems/web/__init__.py +++ b/bauh/gems/web/__init__.py @@ -19,4 +19,7 @@ ELECTRON_SHA256_URL = 'https://github.com/electron/electron/releases/download/v{ URL_ENVIRONMENT_SETTINGS = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/environment.yml' DESKTOP_ENTRY_PATH_PATTERN = DESKTOP_ENTRIES_DIR + '/bauh.web.{name}.desktop' URL_FIX_PATTERN = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/fix/{url}.js" +URL_SUGGESTIONS = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/suggestions.yaml" UA_CHROME = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36' +TEMP_PATH = '/tmp/bauh/web' +SEARCH_INDEX_FILE = '{}/index.yaml'.format(TEMP_PATH) diff --git a/bauh/gems/web/controller.py b/bauh/gems/web/controller.py index a2bb0c81..ce27994e 100644 --- a/bauh/gems/web/controller.py +++ b/bauh/gems/web/controller.py @@ -22,13 +22,13 @@ from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSugge from bauh.api.abstract.view import MessageType, MultipleSelectComponent, InputOption, SingleSelectComponent, \ SelectViewType, TextInputComponent, FormComponent, FileChooserComponent from bauh.api.constants import DESKTOP_ENTRIES_DIR -from bauh.commons import resource, internet from bauh.commons.html import bold from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \ - ROOT_DIR + URL_SUGGESTIONS from bauh.gems.web.environment import EnvironmentUpdater from bauh.gems.web.model import WebApplication +from bauh.gems.web.worker import SuggestionsDownloader, SearchIndexGenerator try: from bs4 import BeautifulSoup, SoupStrainer @@ -50,7 +50,8 @@ RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:.]') class WebApplicationManager(SoftwareManager): - def __init__(self, context: ApplicationContext, env_updater: Thread = None): + def __init__(self, context: ApplicationContext, env_updater: Thread = None, + suggestions_downloader: Thread = None, suggestions: dict = None): super(WebApplicationManager, self).__init__(context=context) self.http_client = context.http_client self.node_updater = EnvironmentUpdater(logger=context.logger, http_client=context.http_client, @@ -61,6 +62,8 @@ class WebApplicationManager(SoftwareManager): self.env_settings = {} self.logger = context.logger self.env_thread = None + self.suggestions_downloader = suggestions_downloader + self.suggestions = suggestions def _get_lang_header(self) -> str: try: @@ -177,6 +180,10 @@ class WebApplicationManager(SoftwareManager): if installed_matches: res.installed.extend(installed_matches) + if self.search_index: + # TODO + pass + res.total += len(res.installed) res.total += len(res.new) return res @@ -514,10 +521,21 @@ class WebApplicationManager(SoftwareManager): return False + def _download_suggestions(self): + downloader = SuggestionsDownloader(logger=self.logger, http_client=self.http_client) + self.suggestions = downloader.download() + + if self.suggestions: + index_gen = SearchIndexGenerator(logger=self.logger) + Thread(target=index_gen.generate_index, args=(self.suggestions,)).start() + def prepare(self): self.env_thread = Thread(target=self._update_environment) self.env_thread.start() + self.suggestions_downloader = Thread(target=self._download_suggestions) + self.suggestions_downloader.start() + def list_updates(self, internet_available: bool) -> List[PackageUpdate]: pass @@ -554,15 +572,27 @@ class WebApplicationManager(SoftwareManager): return PackageSuggestion(priority=SuggestionPriority(suggestion['priority']), package=app) def list_suggestions(self, limit: int) -> List[PackageSuggestion]: - with open(resource.get_path('suggestions.yml', ROOT_DIR), 'r') as f: - suggestions = yaml.safe_load(f.read()) + + if self.suggestions: + suggestions = self.suggestions + elif self.suggestions_downloader: + self.suggestions_downloader.join(5) + suggestions = self.suggestions + else: + suggestions = SuggestionsDownloader(logger=self.logger, http_client=self.http_client).download() + + # cleaning memory + self.suggestions_downloader = None + self.suggestions = None if suggestions: - suggestions = list(suggestions.values()) - suggestions.sort(key=lambda s: s.get('priority', 0), reverse=True) - to_map = suggestions if limit <= 0 else suggestions[0:limit] + suggestion_list = list(suggestions.values()) + suggestion_list.sort(key=lambda s: s.get('priority', 0), reverse=True) + to_map = suggestion_list if limit <= 0 else suggestion_list[0:limit] res = [self._map_suggestion(s) for s in to_map] - self.env_thread.join() + + if not self.env_settings: + self.env_thread.join() if self.env_settings: for s in res: diff --git a/bauh/gems/web/resources/suggestions.yml b/bauh/gems/web/resources/suggestions.yml deleted file mode 100644 index f895259d..00000000 --- a/bauh/gems/web/resources/suggestions.yml +++ /dev/null @@ -1,177 +0,0 @@ -youtube: - name: YouTube - priority: 1 - url: https://youtube.com - category: Network - options: [allow_urls] - -facebook: - name: Facebook - priority: 3 - url: https://facebook.com - category: Network - options: [allow_urls] - -twitch: - name: Twitch - priority: 2 - url: https://twitch.tv - category: Network - options: [allow_urls] - -whatsapp: - name: WhatsApp - priority: 3 - url: https://web.whatsapp.com - category: Network - options: [allow_urls,tray_min] - -telegram: - name: Telegram - priority: 3 - url: https://web.telegram.org - category: Network - options: [allow_urls,tray_min] - -slack: - name: Slack - priority: 3 - url: https://slack.com - category: Network - options: [allow_urls,tray_min] - - -office: - name: Microsoft Office - priority: 0 - url: https://office.com - category: office - options: [allow_urls] - -ms_excel: - name: Microsoft Excel - priority: 1 - url: https://www.office.com/launch/excel - icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-excel-svg-29b733e1d7.ico - category: office - options: [allow_urls] - -ms_word: - name: Microsoft Word - priority: 1 - url: https://www.office.com/launch/word - icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-word-svg-8eedb3d95e.ico - category: office - options: [allow_urls] - -ms_pp: - name: Microsoft Powerpoint - priority: 1 - url: https://www.office.com/launch/powerpoint - icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-powerpoint-svg-f44c23dac7.ico - category: office - options: [allow_urls] - -ms_outlook: - name: Microsoft Outlook - priority: 3 - url: https://outlook.live.com - category: Network - options: [allow_urls,tray_min] - -ms_drive: - name: Microsoft One Drive - priority: 0 - url: https://onedrive.live.com - category: Utility - options: [allow_urls] - -reddit: - name: Reddit - priority: 1 - url: https://reddit.com - category: network - options: [allow_urls] - -google-drive: - name: Google Drive - priority: 1 - url: https://drive.google.com - category: Utility - options: [allow_urls] - -google-keep: - name: Google Keep - priority: 2 - url: https://keep.google.com - icon_url: https://ssl.gstatic.com/keep/icon_128.png - category: Utility - options: [allow_urls,tray_min] - -google-cal: - name: Google Calendar - priority: 2 - url: https://calendar.google.com - icon_url: https://calendar.google.com/googlecalendar/images/favicon_v2014_16.ico - category: Utility - options: [allow_urls,tray_min] - -google-sheets: - name: Google Sheets - priority: 1 - url: https://docs.google.com/spreadsheets/ - icon_url: https://ssl.gstatic.com/docs/spreadsheets/favicon3.ico - category: Office - options: [allow_urls] - -google-docs: - name: Google Docs - priority: 1 - url: https://docs.google.com/document - icon_url: https://ssl.gstatic.com/docs/documents/images/kix-favicon7.ico - category: Office - options: [allow_urls] - -google-pres: - name: Google Presentation - priority: 1 - url: https://docs.google.com/presentation - icon_url: https://ssl.gstatic.com/docs/presentations/images/favicon5.ico - category: Office - options: [allow_urls] - -google-hangouts: - name: Google Hangouts - priority: 2 - url: https://hangouts.google.com - category: Network - options: [allow_urls,tray_min] - -gmail: - name: GMail - priority: 3 - url: https://gmail.com - icon_url: https://ssl.gstatic.com/ui/v1/icons/mail/images/favicon5.ico - category: Network - options: [allow_urls,tray_min] - -mega: - name: Mega - priority: 0 - url: https://mega.nz - category: Utility - options: [allow_urls] - -tweetdeck: - name: TweetDeck - priority: 3 - url: https://tweetdeck.twitter.com - category: Network - options: [allow_urls] - -twitter: - name: Twitter - url: https://twitter.com - priority: 3 - category: Network - options: [allow_urls] diff --git a/bauh/gems/web/worker.py b/bauh/gems/web/worker.py new file mode 100644 index 00000000..256c7d67 --- /dev/null +++ b/bauh/gems/web/worker.py @@ -0,0 +1,76 @@ +import logging +import traceback +from pathlib import Path + +import yaml + +from bauh.api.http import HttpClient +from bauh.gems.web import URL_SUGGESTIONS, TEMP_PATH, SEARCH_INDEX_FILE + + +class SuggestionsDownloader: + + def __init__(self, http_client: HttpClient, logger: logging.Logger): + super(SuggestionsDownloader, self).__init__() + self.http_client = http_client + self.logger = logger + + def download(self) -> dict: + self.logger.info("Reading suggestions from {}".format(URL_SUGGESTIONS)) + suggestions = self.http_client.get_yaml(URL_SUGGESTIONS) + + if suggestions: + self.logger.info("{} suggestions successfully read".format(len(suggestions))) + else: + self.logger.warning("Could not read suggestions from {}".format(URL_SUGGESTIONS)) + + return suggestions + + +class SearchIndexGenerator: + + def __init__(self, logger: logging.Logger): + self.logger = logger + + def generate_index(self, suggestions: dict): + self.logger.info('Indexing {} app suggestions'.format(len(suggestions))) + index = {} + + for key, sug in suggestions.items(): + name = sug.get('name') + + if name: + split_name = name.split(' ') + single_name = ''.join(split_name) + + for word in (*split_name, single_name): + word_key = word.lower().strip() + mapped = index.get(word_key) + + if not mapped: + mapped = set() + index[word_key] = mapped + + mapped.add(key) + + if index: + self.logger.info('Preparing search index for writing') + + for key in index.keys(): + index[key] = list(index[key]) + + try: + Path(TEMP_PATH).mkdir(parents=True, exist_ok=True) + except: + self.logger.error("Could not generate the directory {}".format(TEMP_PATH)) + traceback.print_exc() + return + + try: + self.logger.info('Writing {} indexed keys as {}'.format(len(index), SEARCH_INDEX_FILE)) + with open(SEARCH_INDEX_FILE, 'w+') as f: + f.write(yaml.safe_dump(index)) + self.logger.info("Search index successfully written at {}".format(SEARCH_INDEX_FILE)) + except: + self.logger.error("Could not write the seach index to {}".format(SEARCH_INDEX_FILE)) + traceback.print_exc()