[wgem] reading local suggestions

bauh/api/http.py

@@ -71,3 +71,8 @@ class HttpClient:
 
             if size is not None:
                 return system.get_human_size_str(size)
+
+    def exists(self, url: str) -> bool:
+        res = self.session.get(url=url, allow_redirects=True, ignore_ssl=True)
+        return res.status_code == 200
+

bauh/gems/web/controller.py

@@ -17,13 +17,16 @@ from bauh.api.abstract.context import ApplicationContext
 from bauh.api.abstract.controller import SoftwareManager, SearchResult
 from bauh.api.abstract.disk import DiskCacheLoader
 from bauh.api.abstract.handler import ProcessWatcher
-from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSuggestion, PackageUpdate, PackageHistory
+from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSuggestion, PackageUpdate, PackageHistory, \
+    SuggestionPriority, PackageStatus
 from bauh.api.abstract.view import MessageType, MultipleSelectComponent, InputOption, SingleSelectComponent, \
     SelectViewType, TextInputComponent, FormComponent, FileChooserComponent
 from bauh.api.constants import DESKTOP_ENTRIES_DIR
+from bauh.commons import resource
 from bauh.commons.html import bold
 from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str
-from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME
+from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \
+    ROOT_DIR
 from bauh.gems.web.environment import EnvironmentUpdater
 from bauh.gems.web.model import WebApplication
 
@@ -42,7 +45,7 @@ except:
 
 RE_PROTOCOL_STRIP = re.compile(r'[a-zA-Z]+://')
 RE_SEVERAL_SPACES = re.compile(r'\s+')
-RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:]')
+RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:.]')
 
 
 class WebApplicationManager(SoftwareManager):
@@ -57,6 +60,7 @@ class WebApplicationManager(SoftwareManager):
         self.env_updater = env_updater
         self.env_settings = {}
         self.logger = context.logger
+        self.env_thread = None
 
     def _get_lang_header(self) -> str:
         try:
@@ -87,14 +91,22 @@ class WebApplicationManager(SoftwareManager):
             icon_url = icon_tag.get('href') if icon_tag else None
 
             if icon_url and not icon_url.startswith('http'):
-                icon_url = url + (icon_url if icon_url.startswith('/') else '/{}'.format(icon_url))
+                if icon_url.startswith('//'):
+                    icon_url = 'https:{}'.format(icon_url)
+                elif icon_url.startswith('/'):
+                    icon_url = url + icon_url
+                else:
+                    icon_url = url + '/{}'.format(icon_url)
+
+            if icon_url:
+                return icon_url
 
         if not icon_url:
             icon_tag = soup.head.find('meta', attrs={"property": 'og:image'})
             icon_url = icon_tag.get('content') if icon_tag else None
 
-        if icon_url:
+            if icon_url:
-            return icon_url
+                return icon_url
 
     def _get_fix_for(self, url_no_protocol: str) -> str:
         res = self.http_client.get(URL_FIX_PATTERN.format(url=url_no_protocol))
@@ -108,6 +120,14 @@ class WebApplicationManager(SoftwareManager):
     def serialize_to_disk(self, pkg: SoftwarePackage, icon_bytes: bytes, only_icon: bool):
         super(WebApplicationManager, self).serialize_to_disk(pkg=pkg, icon_bytes=None, only_icon=False)
 
+    def _map_url(self, url: str) -> BeautifulSoup:
+        url_res = self.http_client.get(url,
+                                       headers={'Accept-language': self._get_lang_header(), 'User-Agent': UA_CHROME},
+                                       ignore_ssl=True, single_call=True)
+
+        if url_res:
+            return BeautifulSoup(url_res.text, 'lxml', parse_only=SoupStrainer('head'))
+
     def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1, is_url: bool = False) -> SearchResult:
         res = SearchResult([], [], 0)
 
@@ -123,11 +143,9 @@ class WebApplicationManager(SoftwareManager):
             if installed_matches:
                 res.installed.extend(installed_matches)
             else:
-                url_res = self.http_client.get(url, headers={'Accept-language': self._get_lang_header(), 'User-Agent': UA_CHROME}, ignore_ssl=True, single_call=True)
+                soup = self._map_url(url)
-
-                if url_res:
-                    soup = BeautifulSoup(url_res.text, 'lxml', parse_only=SoupStrainer('head'))
 
+                if soup:
                     name = self._get_app_name(url_no_protocol, soup)
 
                     desc_tag = soup.head.find('meta', attrs={'name': 'description'})
@@ -461,7 +479,8 @@ class WebApplicationManager(SoftwareManager):
         return False
 
     def prepare(self):
-        Thread(target=self._update_environment).start()
+        self.env_thread = Thread(target=self._update_environment)
+        self.env_thread.start()
 
     def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
         pass
@@ -469,8 +488,59 @@ class WebApplicationManager(SoftwareManager):
     def list_warnings(self, internet_available: bool) -> List[str]:
         pass
 
+    def _fill_suggestion(self, app: WebApplication):
+        soup = self._map_url(app.url)
+
+        if soup:
+            if not app.name:
+                app.name = self._get_app_name(app.url, soup)
+
+            if not app.description:
+                desc_tag = soup.head.find('meta', attrs={'name': 'description'})
+
+                if desc_tag:
+                    app.description = desc_tag.get('content')
+
+                if not app.description:
+                    desc_tag = soup.find('title')
+                    app.description = desc_tag.text if desc_tag else app.url
+
+            find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url))
+
+            if find_url:
+                app.icon_url = self._get_app_icon_url(app.url, soup)
+
+        app.status = PackageStatus.READY
+
+    def _map_suggestion(self, suggestion: dict) -> PackageSuggestion:
+        app = WebApplication(name=suggestion.get('name'),
+                             url=suggestion.get('url'),
+                             icon_url=suggestion.get('icon_url'),
+                             categories=[suggestion['category']] if suggestion.get('category') else None,
+                             preset_options=suggestion.get('options'))
+        app.status = PackageStatus.LOADING_DATA
+
+        Thread(target=self._fill_suggestion, args=(app,)).start()
+
+        return PackageSuggestion(priority=SuggestionPriority(suggestion['priority']), package=app)
+
     def list_suggestions(self, limit: int) -> List[PackageSuggestion]:
-        pass
+        with open(resource.get_path('suggestions.yml', ROOT_DIR), 'r') as f:
+            suggestions = yaml.safe_load(f.read())
+
+        if suggestions:
+            suggestions = list(suggestions.values())
+            suggestions.sort(key=lambda s: s.get('priority', 0), reverse=True)
+            to_map = suggestions if limit <= 0 else suggestions[0:limit]
+            res = [self._map_suggestion(s) for s in to_map]
+            self.env_thread.join()
+
+            if self.env_settings:
+                for s in res:
+                    s.package.version = self.env_settings['electron']['version']
+                    s.package.latest_version = s.package.version
+
+            return res
 
     def execute_custom_action(self, action: PackageAction, pkg: SoftwarePackage, root_password: str, watcher: ProcessWatcher) -> bool:
         pass

bauh/gems/web/model.py

@@ -10,7 +10,7 @@ class WebApplication(SoftwarePackage):
 
     def __init__(self, id: str = None, url: str = None, name: str = None, description: str = None, icon_url: str = None,
                  installation_dir: str = None, desktop_entry: str = None, installed: bool = False, version: str = None,
-                 categories: List[str] = None, custom_icon: str = None):
+                 categories: List[str] = None, custom_icon: str = None, preset_options: List[str] = None):
         super(WebApplication, self).__init__(id=id if id else url, name=name, description=description,
                                              icon_url=icon_url, installed=installed, version=version,
                                              categories=categories)
@@ -18,6 +18,7 @@ class WebApplication(SoftwarePackage):
         self.installation_dir = installation_dir
         self.desktop_entry = desktop_entry
         self.set_custom_icon(custom_icon)
+        self.preset_categories = preset_options
 
     def has_history(self):
         return False
@@ -92,7 +93,7 @@ class WebApplication(SoftwarePackage):
         return False
 
     def get_publisher(self) -> str:
-        pass
+        return 'bauh'
 
     def has_screenshots(self) -> bool:
         return False

bauh/gems/web/resources/suggestions.yml

@@ -1,36 +1,30 @@
 youtube:
   name: YouTube
+  priority: 1
   url: https://youtube.com
   category: Network
 
 facebook:
   name: Facebook
+  priority: 3
   url: https://facebook.com
   category: Network
-  options:
-    - tray_min
-
-outlook:
-  name: Outlook
-  url: https://outlook.live.com
-  category: Network
-  options:
-    - allow_urls
 
 twitch:
   name: Twitch
+  priority: 2
   url: https://twitch.tv
   category: Network
 
 whatsapp:
   name: WhatsApp
+  priority: 3
   url: https://web.whatsapp.com
   category: Network
-  options:
-    - tray_min
 
 telegram:
   name: Telegram
+  priority: 3
   url: https://web.telegram.org
   category: Network
   options:
@@ -38,35 +32,158 @@ telegram:
 
 slack:
   name: Slack
+  priority: 3
   url: https://slack.com
   category: Network
-  options:
-    - tray_min
 
-hangouts:
-  name: Hangouts
-  url: https://hangouts.google.com
-  category: Network
-  options:
-    - tray_min
 
 office:
   name: Microsoft Office
+  priority: 0
   url: https://office.com
   category: office
   options:
     - allow_urls
 
+ms_excel:
+  name: Microsoft Excel
+  priority: 1
+  url: https://www.office.com/launch/excel
+  icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-excel-svg-29b733e1d7.ico
+  category: office
+  options:
+    - allow_urls
+
+ms_word:
+  name: Microsoft Word
+  priority: 1
+  url: https://www.office.com/launch/word
+  icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-word-svg-8eedb3d95e.ico
+  category: office
+  options:
+    - allow_urls
+
+ms_pp:
+  name: Microsoft Powerpoint
+  priority: 1
+  url: https://www.office.com/launch/powerpoint
+  icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-powerpoint-svg-f44c23dac7.ico
+  category: office
+  options:
+    - allow_urls
+
+ms_outlook:
+  name: Microsoft Outlook
+  priority: 3
+  url: https://outlook.live.com
+  category: Network
+  options:
+    - allow_urls
+
+ms_drive:
+  name: Microsoft One Drive
+  priority: 0
+  url: https://onedrive.live.com
+  category: Utility
+  options:
+    - allow_urls
+
 reddit:
   name: Reddit
+  priority: 1
   url: https://reddit.com
   category: network
-  options:
-    - tray_min
 
 google-drive:
-  name: Drive
+  name: Google Drive
+  priority: 1
   url: https://drive.google.com
   category: Utility
   options:
-    - tray_min
+    - allow_urls
+
+google-keep:
+  name: Google Keep
+  priority: 2
+  url: https://keep.google.com
+  icon_url: https://ssl.gstatic.com/keep/icon_128.png
+  category: Utility
+  options:
+    - allow_urls
+
+google-cal:
+  name: Google Calendar
+  priority: 2
+  url: https://calendar.google.com
+  icon_url: https://calendar.google.com/googlecalendar/images/favicon_v2014_16.ico
+  category: Utility
+  options:
+    - allow_urls
+
+google-sheets:
+  name: Google Sheets
+  priority: 1
+  url: https://docs.google.com/spreadsheets/
+  icon_url: https://ssl.gstatic.com/docs/spreadsheets/favicon3.ico
+  category: Office
+  options:
+    - allow_urls
+
+google-docs:
+  name: Google Docs
+  priority: 1
+  url: https://docs.google.com/document
+  icon_url: https://ssl.gstatic.com/docs/documents/images/kix-favicon7.ico
+  category: Office
+  options:
+    - allow_urls
+
+google-pres:
+  name: Google Presentation
+  priority: 1
+  url: https://docs.google.com/presentation
+  icon_url: https://ssl.gstatic.com/docs/presentations/images/favicon5.ico
+  category: Office
+  options:
+    - allow_urls
+
+google-hangouts:
+  name: Google Hangouts
+  priority: 2
+  url: https://hangouts.google.com
+  category: Network
+  options:
+    - allow_urls
+
+gmail:
+  name: GMail
+  priority: 3
+  url: https://gmail.com
+  icon_url: https://ssl.gstatic.com/ui/v1/icons/mail/images/favicon5.ico
+  category: Network
+  options:
+    - allow_urls
+
+mega:
+  name: Mega
+  priority: 0
+  url: https://mega.nz
+  category: Utility
+  options:
+    - allow_urls
+
+tweetdeck:
+  name: TweetDeck
+  priority: 3
+  url: https://tweetdeck.twitter.com
+  category: Network
+  options:
+    - allow_urls
+
+twitter:
+  name: Twitter
+  url: https://twitter.com
+  priority: 3
+  category: Network
+  options:
+    - allow_urls