[wgem] downloading suggestions from the cloud and generating a search index

bauh/api/http.py

@@ -3,6 +3,7 @@ import time
 import traceback
 
 import requests
+import yaml
 
 from bauh.commons import system
 
@@ -59,6 +60,10 @@ class HttpClient:
         res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects)
         return res.json() if res else None
 
+    def get_yaml(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True):
+        res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects)
+        return yaml.safe_load(res.text) if res else None
+
     def get_content_length(self, url: str) -> str:
         """
         :param url:

bauh/app.py

@@ -1,6 +1,7 @@
 import os
 import sys
 
+import urllib3
 from PyQt5.QtGui import QIcon
 from PyQt5.QtWidgets import QApplication
 
@@ -24,6 +25,8 @@ def main():
     if not os.getenv('PYTHONUNBUFFERED'):
         os.environ['PYTHONUNBUFFERED'] = '1'
 
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
     args = app_args.read()
 
     logger = logs.new_logger(__app_name__, bool(args.logs))

bauh/gems/web/__init__.py

@@ -19,4 +19,7 @@ ELECTRON_SHA256_URL = 'https://github.com/electron/electron/releases/download/v{
 URL_ENVIRONMENT_SETTINGS = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/environment.yml'
 DESKTOP_ENTRY_PATH_PATTERN = DESKTOP_ENTRIES_DIR + '/bauh.web.{name}.desktop'
 URL_FIX_PATTERN = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/fix/{url}.js"
+URL_SUGGESTIONS = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/suggestions.yaml"
 UA_CHROME = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
+TEMP_PATH = '/tmp/bauh/web'
+SEARCH_INDEX_FILE = '{}/index.yaml'.format(TEMP_PATH)

bauh/gems/web/controller.py

@@ -22,13 +22,13 @@ from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSugge
 from bauh.api.abstract.view import MessageType, MultipleSelectComponent, InputOption, SingleSelectComponent, \
     SelectViewType, TextInputComponent, FormComponent, FileChooserComponent
 from bauh.api.constants import DESKTOP_ENTRIES_DIR
-from bauh.commons import resource, internet
 from bauh.commons.html import bold
 from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str
 from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \
-    ROOT_DIR
+    URL_SUGGESTIONS
 from bauh.gems.web.environment import EnvironmentUpdater
 from bauh.gems.web.model import WebApplication
+from bauh.gems.web.worker import SuggestionsDownloader, SearchIndexGenerator
 
 try:
     from bs4 import BeautifulSoup, SoupStrainer
@@ -50,7 +50,8 @@ RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:.]')
 
 class WebApplicationManager(SoftwareManager):
 
-    def __init__(self, context: ApplicationContext, env_updater: Thread = None):
+    def __init__(self, context: ApplicationContext, env_updater: Thread = None,
+                 suggestions_downloader: Thread = None, suggestions: dict = None):
         super(WebApplicationManager, self).__init__(context=context)
         self.http_client = context.http_client
         self.node_updater = EnvironmentUpdater(logger=context.logger, http_client=context.http_client,
@@ -61,6 +62,8 @@ class WebApplicationManager(SoftwareManager):
         self.env_settings = {}
         self.logger = context.logger
         self.env_thread = None
+        self.suggestions_downloader = suggestions_downloader
+        self.suggestions = suggestions
 
     def _get_lang_header(self) -> str:
         try:
@@ -177,6 +180,10 @@ class WebApplicationManager(SoftwareManager):
             if installed_matches:
                 res.installed.extend(installed_matches)
 
+            if self.search_index:
+                # TODO
+                pass
+
         res.total += len(res.installed)
         res.total += len(res.new)
         return res
@@ -514,10 +521,21 @@ class WebApplicationManager(SoftwareManager):
 
         return False
 
+    def _download_suggestions(self):
+        downloader = SuggestionsDownloader(logger=self.logger, http_client=self.http_client)
+        self.suggestions = downloader.download()
+
+        if self.suggestions:
+            index_gen = SearchIndexGenerator(logger=self.logger)
+            Thread(target=index_gen.generate_index, args=(self.suggestions,)).start()
+
     def prepare(self):
         self.env_thread = Thread(target=self._update_environment)
         self.env_thread.start()
 
+        self.suggestions_downloader = Thread(target=self._download_suggestions)
+        self.suggestions_downloader.start()
+
     def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
         pass
 
@@ -554,15 +572,27 @@ class WebApplicationManager(SoftwareManager):
         return PackageSuggestion(priority=SuggestionPriority(suggestion['priority']), package=app)
 
     def list_suggestions(self, limit: int) -> List[PackageSuggestion]:
-        with open(resource.get_path('suggestions.yml', ROOT_DIR), 'r') as f:
+
-            suggestions = yaml.safe_load(f.read())
+        if self.suggestions:
+            suggestions = self.suggestions
+        elif self.suggestions_downloader:
+            self.suggestions_downloader.join(5)
+            suggestions = self.suggestions
+        else:
+            suggestions = SuggestionsDownloader(logger=self.logger, http_client=self.http_client).download()
+
+        # cleaning memory
+        self.suggestions_downloader = None
+        self.suggestions = None
 
         if suggestions:
-            suggestions = list(suggestions.values())
+            suggestion_list = list(suggestions.values())
-            suggestions.sort(key=lambda s: s.get('priority', 0), reverse=True)
+            suggestion_list.sort(key=lambda s: s.get('priority', 0), reverse=True)
-            to_map = suggestions if limit <= 0 else suggestions[0:limit]
+            to_map = suggestion_list if limit <= 0 else suggestion_list[0:limit]
             res = [self._map_suggestion(s) for s in to_map]
-            self.env_thread.join()
+
+            if not self.env_settings:
+                self.env_thread.join()
 
             if self.env_settings:
                 for s in res:

bauh/gems/web/resources/suggestions.yml

@@ -1,177 +0,0 @@
-youtube:
-  name: YouTube
-  priority: 1
-  url: https://youtube.com
-  category: Network
-  options: [allow_urls]
-
-facebook:
-  name: Facebook
-  priority: 3
-  url: https://facebook.com
-  category: Network
-  options: [allow_urls]
-
-twitch:
-  name: Twitch
-  priority: 2
-  url: https://twitch.tv
-  category: Network
-  options: [allow_urls]
-
-whatsapp:
-  name: WhatsApp
-  priority: 3
-  url: https://web.whatsapp.com
-  category: Network
-  options: [allow_urls,tray_min]
-
-telegram:
-  name: Telegram
-  priority: 3
-  url: https://web.telegram.org
-  category: Network
-  options: [allow_urls,tray_min]
-
-slack:
-  name: Slack
-  priority: 3
-  url: https://slack.com
-  category: Network
-  options: [allow_urls,tray_min]
-
-
-office:
-  name: Microsoft Office
-  priority: 0
-  url: https://office.com
-  category: office
-  options: [allow_urls]
-
-ms_excel:
-  name: Microsoft Excel
-  priority: 1
-  url: https://www.office.com/launch/excel
-  icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-excel-svg-29b733e1d7.ico
-  category: office
-  options: [allow_urls]
-
-ms_word:
-  name: Microsoft Word
-  priority: 1
-  url: https://www.office.com/launch/word
-  icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-word-svg-8eedb3d95e.ico
-  category: office
-  options: [allow_urls]
-
-ms_pp:
-  name: Microsoft Powerpoint
-  priority: 1
-  url: https://www.office.com/launch/powerpoint
-  icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-powerpoint-svg-f44c23dac7.ico
-  category: office
-  options: [allow_urls]
-
-ms_outlook:
-  name: Microsoft Outlook
-  priority: 3
-  url: https://outlook.live.com
-  category: Network
-  options: [allow_urls,tray_min]
-
-ms_drive:
-  name: Microsoft One Drive
-  priority: 0
-  url: https://onedrive.live.com
-  category: Utility
-  options: [allow_urls]
-
-reddit:
-  name: Reddit
-  priority: 1
-  url: https://reddit.com
-  category: network
-  options: [allow_urls]
-
-google-drive:
-  name: Google Drive
-  priority: 1
-  url: https://drive.google.com
-  category: Utility
-  options: [allow_urls]
-
-google-keep:
-  name: Google Keep
-  priority: 2
-  url: https://keep.google.com
-  icon_url: https://ssl.gstatic.com/keep/icon_128.png
-  category: Utility
-  options: [allow_urls,tray_min]
-
-google-cal:
-  name: Google Calendar
-  priority: 2
-  url: https://calendar.google.com
-  icon_url: https://calendar.google.com/googlecalendar/images/favicon_v2014_16.ico
-  category: Utility
-  options: [allow_urls,tray_min]
-
-google-sheets:
-  name: Google Sheets
-  priority: 1
-  url: https://docs.google.com/spreadsheets/
-  icon_url: https://ssl.gstatic.com/docs/spreadsheets/favicon3.ico
-  category: Office
-  options: [allow_urls]
-
-google-docs:
-  name: Google Docs
-  priority: 1
-  url: https://docs.google.com/document
-  icon_url: https://ssl.gstatic.com/docs/documents/images/kix-favicon7.ico
-  category: Office
-  options: [allow_urls]
-
-google-pres:
-  name: Google Presentation
-  priority: 1
-  url: https://docs.google.com/presentation
-  icon_url: https://ssl.gstatic.com/docs/presentations/images/favicon5.ico
-  category: Office
-  options: [allow_urls]
-
-google-hangouts:
-  name: Google Hangouts
-  priority: 2
-  url: https://hangouts.google.com
-  category: Network
-  options: [allow_urls,tray_min]
-
-gmail:
-  name: GMail
-  priority: 3
-  url: https://gmail.com
-  icon_url: https://ssl.gstatic.com/ui/v1/icons/mail/images/favicon5.ico
-  category: Network
-  options: [allow_urls,tray_min]
-
-mega:
-  name: Mega
-  priority: 0
-  url: https://mega.nz
-  category: Utility
-  options: [allow_urls]
-
-tweetdeck:
-  name: TweetDeck
-  priority: 3
-  url: https://tweetdeck.twitter.com
-  category: Network
-  options: [allow_urls]
-
-twitter:
-  name: Twitter
-  url: https://twitter.com
-  priority: 3
-  category: Network
-  options: [allow_urls]

bauh/gems/web/worker.py

@@ -0,0 +1,76 @@
+import logging
+import traceback
+from pathlib import Path
+
+import yaml
+
+from bauh.api.http import HttpClient
+from bauh.gems.web import URL_SUGGESTIONS, TEMP_PATH, SEARCH_INDEX_FILE
+
+
+class SuggestionsDownloader:
+
+    def __init__(self, http_client: HttpClient, logger: logging.Logger):
+        super(SuggestionsDownloader, self).__init__()
+        self.http_client = http_client
+        self.logger = logger
+
+    def download(self) -> dict:
+        self.logger.info("Reading suggestions from {}".format(URL_SUGGESTIONS))
+        suggestions = self.http_client.get_yaml(URL_SUGGESTIONS)
+
+        if suggestions:
+            self.logger.info("{} suggestions successfully read".format(len(suggestions)))
+        else:
+            self.logger.warning("Could not read suggestions from {}".format(URL_SUGGESTIONS))
+
+        return suggestions
+
+
+class SearchIndexGenerator:
+
+    def __init__(self, logger: logging.Logger):
+        self.logger = logger
+
+    def generate_index(self, suggestions: dict):
+        self.logger.info('Indexing {} app suggestions'.format(len(suggestions)))
+        index = {}
+
+        for key, sug in suggestions.items():
+            name = sug.get('name')
+
+            if name:
+                split_name = name.split(' ')
+                single_name = ''.join(split_name)
+
+                for word in (*split_name, single_name):
+                    word_key = word.lower().strip()
+                    mapped = index.get(word_key)
+
+                    if not mapped:
+                        mapped = set()
+                        index[word_key] = mapped
+
+                    mapped.add(key)
+
+        if index:
+            self.logger.info('Preparing search index for writing')
+
+            for key in index.keys():
+                index[key] = list(index[key])
+
+            try:
+                Path(TEMP_PATH).mkdir(parents=True, exist_ok=True)
+            except:
+                self.logger.error("Could not generate the directory {}".format(TEMP_PATH))
+                traceback.print_exc()
+                return
+
+            try:
+                self.logger.info('Writing {} indexed keys as {}'.format(len(index), SEARCH_INDEX_FILE))
+                with open(SEARCH_INDEX_FILE, 'w+') as f:
+                    f.write(yaml.safe_dump(index))
+                self.logger.info("Search index successfully written at {}".format(SEARCH_INDEX_FILE))
+            except:
+                self.logger.error("Could not write the seach index to {}".format(SEARCH_INDEX_FILE))
+                traceback.print_exc()