[wgem] reading local suggestions

2026-07-07 01:14:15 +02:00 · 2019-12-16 16:55:48 -03:00
parent f7f1f32eb3
commit 2cdd5b59f5
4 changed files with 230 additions and 37 deletions
--- a/bauh/api/http.py
+++ b/bauh/api/http.py
@@ -71,3 +71,8 @@ class HttpClient:

            if size is not None:
                return system.get_human_size_str(size)
+
+    def exists(self, url: str) -> bool:
+        res = self.session.get(url=url, allow_redirects=True, ignore_ssl=True)
+        return res.status_code == 200
+
--- a/bauh/gems/web/controller.py
+++ b/bauh/gems/web/controller.py
@@ -17,13 +17,16 @@ from bauh.api.abstract.context import ApplicationContext
 from bauh.api.abstract.controller import SoftwareManager, SearchResult
 from bauh.api.abstract.disk import DiskCacheLoader
 from bauh.api.abstract.handler import ProcessWatcher
-from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSuggestion, PackageUpdate, PackageHistory
+from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSuggestion, PackageUpdate, PackageHistory, \
+    SuggestionPriority, PackageStatus
 from bauh.api.abstract.view import MessageType, MultipleSelectComponent, InputOption, SingleSelectComponent, \
    SelectViewType, TextInputComponent, FormComponent, FileChooserComponent
 from bauh.api.constants import DESKTOP_ENTRIES_DIR
+from bauh.commons import resource
 from bauh.commons.html import bold
 from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str
-from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME
+from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \
+    ROOT_DIR
 from bauh.gems.web.environment import EnvironmentUpdater
 from bauh.gems.web.model import WebApplication

@@ -42,7 +45,7 @@ except:

 RE_PROTOCOL_STRIP = re.compile(r'[a-zA-Z]+://')
 RE_SEVERAL_SPACES = re.compile(r'\s+')
-RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:]')
+RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:.]')


 class WebApplicationManager(SoftwareManager):
@@ -57,6 +60,7 @@ class WebApplicationManager(SoftwareManager):
        self.env_updater = env_updater
        self.env_settings = {}
        self.logger = context.logger
+        self.env_thread = None

    def _get_lang_header(self) -> str:
        try:
@@ -87,14 +91,22 @@ class WebApplicationManager(SoftwareManager):
            icon_url = icon_tag.get('href') if icon_tag else None

            if icon_url and not icon_url.startswith('http'):
-                icon_url = url + (icon_url if icon_url.startswith('/') else '/{}'.format(icon_url))
+                if icon_url.startswith('//'):
+                    icon_url = 'https:{}'.format(icon_url)
+                elif icon_url.startswith('/'):
+                    icon_url = url + icon_url
+                else:
+                    icon_url = url + '/{}'.format(icon_url)
+
+            if icon_url:
+                return icon_url

        if not icon_url:
            icon_tag = soup.head.find('meta', attrs={"property": 'og:image'})
            icon_url = icon_tag.get('content') if icon_tag else None

-        if icon_url:
-            return icon_url
+            if icon_url:
+                return icon_url

    def _get_fix_for(self, url_no_protocol: str) -> str:
        res = self.http_client.get(URL_FIX_PATTERN.format(url=url_no_protocol))
@@ -108,6 +120,14 @@ class WebApplicationManager(SoftwareManager):
    def serialize_to_disk(self, pkg: SoftwarePackage, icon_bytes: bytes, only_icon: bool):
        super(WebApplicationManager, self).serialize_to_disk(pkg=pkg, icon_bytes=None, only_icon=False)

+    def _map_url(self, url: str) -> BeautifulSoup:
+        url_res = self.http_client.get(url,
+                                       headers={'Accept-language': self._get_lang_header(), 'User-Agent': UA_CHROME},
+                                       ignore_ssl=True, single_call=True)
+
+        if url_res:
+            return BeautifulSoup(url_res.text, 'lxml', parse_only=SoupStrainer('head'))
+
    def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1, is_url: bool = False) -> SearchResult:
        res = SearchResult([], [], 0)

@@ -123,11 +143,9 @@ class WebApplicationManager(SoftwareManager):
            if installed_matches:
                res.installed.extend(installed_matches)
            else:
-                url_res = self.http_client.get(url, headers={'Accept-language': self._get_lang_header(), 'User-Agent': UA_CHROME}, ignore_ssl=True, single_call=True)
-
-                if url_res:
-                    soup = BeautifulSoup(url_res.text, 'lxml', parse_only=SoupStrainer('head'))
+                soup = self._map_url(url)

+                if soup:
                    name = self._get_app_name(url_no_protocol, soup)

                    desc_tag = soup.head.find('meta', attrs={'name': 'description'})
@@ -461,7 +479,8 @@ class WebApplicationManager(SoftwareManager):
        return False

    def prepare(self):
-        Thread(target=self._update_environment).start()
+        self.env_thread = Thread(target=self._update_environment)
+        self.env_thread.start()

    def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
        pass
@@ -469,8 +488,59 @@ class WebApplicationManager(SoftwareManager):
    def list_warnings(self, internet_available: bool) -> List[str]:
        pass

+    def _fill_suggestion(self, app: WebApplication):
+        soup = self._map_url(app.url)
+
+        if soup:
+            if not app.name:
+                app.name = self._get_app_name(app.url, soup)
+
+            if not app.description:
+                desc_tag = soup.head.find('meta', attrs={'name': 'description'})
+
+                if desc_tag:
+                    app.description = desc_tag.get('content')
+
+                if not app.description:
+                    desc_tag = soup.find('title')
+                    app.description = desc_tag.text if desc_tag else app.url
+
+            find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url))
+
+            if find_url:
+                app.icon_url = self._get_app_icon_url(app.url, soup)
+
+        app.status = PackageStatus.READY
+
+    def _map_suggestion(self, suggestion: dict) -> PackageSuggestion:
+        app = WebApplication(name=suggestion.get('name'),
+                             url=suggestion.get('url'),
+                             icon_url=suggestion.get('icon_url'),
+                             categories=[suggestion['category']] if suggestion.get('category') else None,
+                             preset_options=suggestion.get('options'))
+        app.status = PackageStatus.LOADING_DATA
+
+        Thread(target=self._fill_suggestion, args=(app,)).start()
+
+        return PackageSuggestion(priority=SuggestionPriority(suggestion['priority']), package=app)
+
    def list_suggestions(self, limit: int) -> List[PackageSuggestion]:
-        pass
+        with open(resource.get_path('suggestions.yml', ROOT_DIR), 'r') as f:
+            suggestions = yaml.safe_load(f.read())
+
+        if suggestions:
+            suggestions = list(suggestions.values())
+            suggestions.sort(key=lambda s: s.get('priority', 0), reverse=True)
+            to_map = suggestions if limit <= 0 else suggestions[0:limit]
+            res = [self._map_suggestion(s) for s in to_map]
+            self.env_thread.join()
+
+            if self.env_settings:
+                for s in res:
+                    s.package.version = self.env_settings['electron']['version']
+                    s.package.latest_version = s.package.version
+
+            return res

    def execute_custom_action(self, action: PackageAction, pkg: SoftwarePackage, root_password: str, watcher: ProcessWatcher) -> bool:
        pass
--- a/bauh/gems/web/model.py
+++ b/bauh/gems/web/model.py
@@ -10,7 +10,7 @@ class WebApplication(SoftwarePackage):

    def __init__(self, id: str = None, url: str = None, name: str = None, description: str = None, icon_url: str = None,
                 installation_dir: str = None, desktop_entry: str = None, installed: bool = False, version: str = None,
-                 categories: List[str] = None, custom_icon: str = None):
+                 categories: List[str] = None, custom_icon: str = None, preset_options: List[str] = None):
        super(WebApplication, self).__init__(id=id if id else url, name=name, description=description,
                                             icon_url=icon_url, installed=installed, version=version,
                                             categories=categories)
@@ -18,6 +18,7 @@ class WebApplication(SoftwarePackage):
        self.installation_dir = installation_dir
        self.desktop_entry = desktop_entry
        self.set_custom_icon(custom_icon)
+        self.preset_categories = preset_options

    def has_history(self):
        return False
@@ -92,7 +93,7 @@ class WebApplication(SoftwarePackage):
        return False

    def get_publisher(self) -> str:
-        pass
+        return 'bauh'

    def has_screenshots(self) -> bool:
        return False
--- a/bauh/gems/web/resources/suggestions.yml
+++ b/bauh/gems/web/resources/suggestions.yml
@@ -1,36 +1,30 @@
 youtube:
  name: YouTube
+  priority: 1
  url: https://youtube.com
  category: Network

 facebook:
  name: Facebook
+  priority: 3
  url: https://facebook.com
  category: Network
-  options:
-    - tray_min
-
-outlook:
-  name: Outlook
-  url: https://outlook.live.com
-  category: Network
-  options:
-    - allow_urls

 twitch:
  name: Twitch
+  priority: 2
  url: https://twitch.tv
  category: Network

 whatsapp:
  name: WhatsApp
+  priority: 3
  url: https://web.whatsapp.com
  category: Network
-  options:
-    - tray_min

 telegram:
  name: Telegram
+  priority: 3
  url: https://web.telegram.org
  category: Network
  options:
@@ -38,35 +32,158 @@ telegram:

 slack:
  name: Slack
+  priority: 3
  url: https://slack.com
  category: Network
-  options:
-    - tray_min

-hangouts:
-  name: Hangouts
-  url: https://hangouts.google.com
-  category: Network
-  options:
-    - tray_min

 office:
  name: Microsoft Office
+  priority: 0
  url: https://office.com
  category: office
  options:
    - allow_urls

+ms_excel:
+  name: Microsoft Excel
+  priority: 1
+  url: https://www.office.com/launch/excel
+  icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-excel-svg-29b733e1d7.ico
+  category: office
+  options:
+    - allow_urls
+
+ms_word:
+  name: Microsoft Word
+  priority: 1
+  url: https://www.office.com/launch/word
+  icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-word-svg-8eedb3d95e.ico
+  category: office
+  options:
+    - allow_urls
+
+ms_pp:
+  name: Microsoft Powerpoint
+  priority: 1
+  url: https://www.office.com/launch/powerpoint
+  icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-powerpoint-svg-f44c23dac7.ico
+  category: office
+  options:
+    - allow_urls
+
+ms_outlook:
+  name: Microsoft Outlook
+  priority: 3
+  url: https://outlook.live.com
+  category: Network
+  options:
+    - allow_urls
+
+ms_drive:
+  name: Microsoft One Drive
+  priority: 0
+  url: https://onedrive.live.com
+  category: Utility
+  options:
+    - allow_urls
+
 reddit:
  name: Reddit
+  priority: 1
  url: https://reddit.com
  category: network
-  options:
-    - tray_min

 google-drive:
-  name: Drive
+  name: Google Drive
+  priority: 1
  url: https://drive.google.com
  category: Utility
  options:
-    - tray_min
+    - allow_urls
+
+google-keep:
+  name: Google Keep
+  priority: 2
+  url: https://keep.google.com
+  icon_url: https://ssl.gstatic.com/keep/icon_128.png
+  category: Utility
+  options:
+    - allow_urls
+
+google-cal:
+  name: Google Calendar
+  priority: 2
+  url: https://calendar.google.com
+  icon_url: https://calendar.google.com/googlecalendar/images/favicon_v2014_16.ico
+  category: Utility
+  options:
+    - allow_urls
+
+google-sheets:
+  name: Google Sheets
+  priority: 1
+  url: https://docs.google.com/spreadsheets/
+  icon_url: https://ssl.gstatic.com/docs/spreadsheets/favicon3.ico
+  category: Office
+  options:
+    - allow_urls
+
+google-docs:
+  name: Google Docs
+  priority: 1
+  url: https://docs.google.com/document
+  icon_url: https://ssl.gstatic.com/docs/documents/images/kix-favicon7.ico
+  category: Office
+  options:
+    - allow_urls
+
+google-pres:
+  name: Google Presentation
+  priority: 1
+  url: https://docs.google.com/presentation
+  icon_url: https://ssl.gstatic.com/docs/presentations/images/favicon5.ico
+  category: Office
+  options:
+    - allow_urls
+
+google-hangouts:
+  name: Google Hangouts
+  priority: 2
+  url: https://hangouts.google.com
+  category: Network
+  options:
+    - allow_urls
+
+gmail:
+  name: GMail
+  priority: 3
+  url: https://gmail.com
+  icon_url: https://ssl.gstatic.com/ui/v1/icons/mail/images/favicon5.ico
+  category: Network
+  options:
+    - allow_urls
+
+mega:
+  name: Mega
+  priority: 0
+  url: https://mega.nz
+  category: Utility
+  options:
+    - allow_urls
+
+tweetdeck:
+  name: TweetDeck
+  priority: 3
+  url: https://tweetdeck.twitter.com
+  category: Network
+  options:
+    - allow_urls
+
+twitter:
+  name: Twitter
+  url: https://twitter.com
+  priority: 3
+  category: Network
+  options:
+    - allow_urls