From 2cdd5b59f5a7dc40021b4846d496c5c326911790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Moreira?= Date: Mon, 16 Dec 2019 16:55:48 -0300 Subject: [PATCH] [wgem] reading local suggestions --- bauh/api/http.py | 5 + bauh/gems/web/controller.py | 94 ++++++++++++-- bauh/gems/web/model.py | 5 +- bauh/gems/web/resources/suggestions.yml | 163 ++++++++++++++++++++---- 4 files changed, 230 insertions(+), 37 deletions(-) diff --git a/bauh/api/http.py b/bauh/api/http.py index 4ad6cada..69631677 100644 --- a/bauh/api/http.py +++ b/bauh/api/http.py @@ -71,3 +71,8 @@ class HttpClient: if size is not None: return system.get_human_size_str(size) + + def exists(self, url: str) -> bool: + res = self.session.get(url=url, allow_redirects=True, ignore_ssl=True) + return res.status_code == 200 + diff --git a/bauh/gems/web/controller.py b/bauh/gems/web/controller.py index 90396c52..93714f48 100644 --- a/bauh/gems/web/controller.py +++ b/bauh/gems/web/controller.py @@ -17,13 +17,16 @@ from bauh.api.abstract.context import ApplicationContext from bauh.api.abstract.controller import SoftwareManager, SearchResult from bauh.api.abstract.disk import DiskCacheLoader from bauh.api.abstract.handler import ProcessWatcher -from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSuggestion, PackageUpdate, PackageHistory +from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSuggestion, PackageUpdate, PackageHistory, \ + SuggestionPriority, PackageStatus from bauh.api.abstract.view import MessageType, MultipleSelectComponent, InputOption, SingleSelectComponent, \ SelectViewType, TextInputComponent, FormComponent, FileChooserComponent from bauh.api.constants import DESKTOP_ENTRIES_DIR +from bauh.commons import resource from bauh.commons.html import bold from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str -from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME +from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \ + ROOT_DIR from bauh.gems.web.environment import EnvironmentUpdater from bauh.gems.web.model import WebApplication @@ -42,7 +45,7 @@ except: RE_PROTOCOL_STRIP = re.compile(r'[a-zA-Z]+://') RE_SEVERAL_SPACES = re.compile(r'\s+') -RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:]') +RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:.]') class WebApplicationManager(SoftwareManager): @@ -57,6 +60,7 @@ class WebApplicationManager(SoftwareManager): self.env_updater = env_updater self.env_settings = {} self.logger = context.logger + self.env_thread = None def _get_lang_header(self) -> str: try: @@ -87,14 +91,22 @@ class WebApplicationManager(SoftwareManager): icon_url = icon_tag.get('href') if icon_tag else None if icon_url and not icon_url.startswith('http'): - icon_url = url + (icon_url if icon_url.startswith('/') else '/{}'.format(icon_url)) + if icon_url.startswith('//'): + icon_url = 'https:{}'.format(icon_url) + elif icon_url.startswith('/'): + icon_url = url + icon_url + else: + icon_url = url + '/{}'.format(icon_url) + + if icon_url: + return icon_url if not icon_url: icon_tag = soup.head.find('meta', attrs={"property": 'og:image'}) icon_url = icon_tag.get('content') if icon_tag else None - if icon_url: - return icon_url + if icon_url: + return icon_url def _get_fix_for(self, url_no_protocol: str) -> str: res = self.http_client.get(URL_FIX_PATTERN.format(url=url_no_protocol)) @@ -108,6 +120,14 @@ class WebApplicationManager(SoftwareManager): def serialize_to_disk(self, pkg: SoftwarePackage, icon_bytes: bytes, only_icon: bool): super(WebApplicationManager, self).serialize_to_disk(pkg=pkg, icon_bytes=None, only_icon=False) + def _map_url(self, url: str) -> BeautifulSoup: + url_res = self.http_client.get(url, + headers={'Accept-language': self._get_lang_header(), 'User-Agent': UA_CHROME}, + ignore_ssl=True, single_call=True) + + if url_res: + return BeautifulSoup(url_res.text, 'lxml', parse_only=SoupStrainer('head')) + def search(self, words: str, disk_loader: DiskCacheLoader, limit: int = -1, is_url: bool = False) -> SearchResult: res = SearchResult([], [], 0) @@ -123,11 +143,9 @@ class WebApplicationManager(SoftwareManager): if installed_matches: res.installed.extend(installed_matches) else: - url_res = self.http_client.get(url, headers={'Accept-language': self._get_lang_header(), 'User-Agent': UA_CHROME}, ignore_ssl=True, single_call=True) - - if url_res: - soup = BeautifulSoup(url_res.text, 'lxml', parse_only=SoupStrainer('head')) + soup = self._map_url(url) + if soup: name = self._get_app_name(url_no_protocol, soup) desc_tag = soup.head.find('meta', attrs={'name': 'description'}) @@ -461,7 +479,8 @@ class WebApplicationManager(SoftwareManager): return False def prepare(self): - Thread(target=self._update_environment).start() + self.env_thread = Thread(target=self._update_environment) + self.env_thread.start() def list_updates(self, internet_available: bool) -> List[PackageUpdate]: pass @@ -469,8 +488,59 @@ class WebApplicationManager(SoftwareManager): def list_warnings(self, internet_available: bool) -> List[str]: pass + def _fill_suggestion(self, app: WebApplication): + soup = self._map_url(app.url) + + if soup: + if not app.name: + app.name = self._get_app_name(app.url, soup) + + if not app.description: + desc_tag = soup.head.find('meta', attrs={'name': 'description'}) + + if desc_tag: + app.description = desc_tag.get('content') + + if not app.description: + desc_tag = soup.find('title') + app.description = desc_tag.text if desc_tag else app.url + + find_url = not app.icon_url or (app.icon_url and not self.http_client.exists(app.icon_url)) + + if find_url: + app.icon_url = self._get_app_icon_url(app.url, soup) + + app.status = PackageStatus.READY + + def _map_suggestion(self, suggestion: dict) -> PackageSuggestion: + app = WebApplication(name=suggestion.get('name'), + url=suggestion.get('url'), + icon_url=suggestion.get('icon_url'), + categories=[suggestion['category']] if suggestion.get('category') else None, + preset_options=suggestion.get('options')) + app.status = PackageStatus.LOADING_DATA + + Thread(target=self._fill_suggestion, args=(app,)).start() + + return PackageSuggestion(priority=SuggestionPriority(suggestion['priority']), package=app) + def list_suggestions(self, limit: int) -> List[PackageSuggestion]: - pass + with open(resource.get_path('suggestions.yml', ROOT_DIR), 'r') as f: + suggestions = yaml.safe_load(f.read()) + + if suggestions: + suggestions = list(suggestions.values()) + suggestions.sort(key=lambda s: s.get('priority', 0), reverse=True) + to_map = suggestions if limit <= 0 else suggestions[0:limit] + res = [self._map_suggestion(s) for s in to_map] + self.env_thread.join() + + if self.env_settings: + for s in res: + s.package.version = self.env_settings['electron']['version'] + s.package.latest_version = s.package.version + + return res def execute_custom_action(self, action: PackageAction, pkg: SoftwarePackage, root_password: str, watcher: ProcessWatcher) -> bool: pass diff --git a/bauh/gems/web/model.py b/bauh/gems/web/model.py index 019a678a..9cc932c9 100644 --- a/bauh/gems/web/model.py +++ b/bauh/gems/web/model.py @@ -10,7 +10,7 @@ class WebApplication(SoftwarePackage): def __init__(self, id: str = None, url: str = None, name: str = None, description: str = None, icon_url: str = None, installation_dir: str = None, desktop_entry: str = None, installed: bool = False, version: str = None, - categories: List[str] = None, custom_icon: str = None): + categories: List[str] = None, custom_icon: str = None, preset_options: List[str] = None): super(WebApplication, self).__init__(id=id if id else url, name=name, description=description, icon_url=icon_url, installed=installed, version=version, categories=categories) @@ -18,6 +18,7 @@ class WebApplication(SoftwarePackage): self.installation_dir = installation_dir self.desktop_entry = desktop_entry self.set_custom_icon(custom_icon) + self.preset_categories = preset_options def has_history(self): return False @@ -92,7 +93,7 @@ class WebApplication(SoftwarePackage): return False def get_publisher(self) -> str: - pass + return 'bauh' def has_screenshots(self) -> bool: return False diff --git a/bauh/gems/web/resources/suggestions.yml b/bauh/gems/web/resources/suggestions.yml index e21d3822..c4c71eaa 100644 --- a/bauh/gems/web/resources/suggestions.yml +++ b/bauh/gems/web/resources/suggestions.yml @@ -1,36 +1,30 @@ youtube: name: YouTube + priority: 1 url: https://youtube.com category: Network facebook: name: Facebook + priority: 3 url: https://facebook.com category: Network - options: - - tray_min - -outlook: - name: Outlook - url: https://outlook.live.com - category: Network - options: - - allow_urls twitch: name: Twitch + priority: 2 url: https://twitch.tv category: Network whatsapp: name: WhatsApp + priority: 3 url: https://web.whatsapp.com category: Network - options: - - tray_min telegram: name: Telegram + priority: 3 url: https://web.telegram.org category: Network options: @@ -38,35 +32,158 @@ telegram: slack: name: Slack + priority: 3 url: https://slack.com category: Network - options: - - tray_min -hangouts: - name: Hangouts - url: https://hangouts.google.com - category: Network - options: - - tray_min office: name: Microsoft Office + priority: 0 url: https://office.com category: office options: - allow_urls +ms_excel: + name: Microsoft Excel + priority: 1 + url: https://www.office.com/launch/excel + icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-excel-svg-29b733e1d7.ico + category: office + options: + - allow_urls + +ms_word: + name: Microsoft Word + priority: 1 + url: https://www.office.com/launch/word + icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-word-svg-8eedb3d95e.ico + category: office + options: + - allow_urls + +ms_pp: + name: Microsoft Powerpoint + priority: 1 + url: https://www.office.com/launch/powerpoint + icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-powerpoint-svg-f44c23dac7.ico + category: office + options: + - allow_urls + +ms_outlook: + name: Microsoft Outlook + priority: 3 + url: https://outlook.live.com + category: Network + options: + - allow_urls + +ms_drive: + name: Microsoft One Drive + priority: 0 + url: https://onedrive.live.com + category: Utility + options: + - allow_urls + reddit: name: Reddit + priority: 1 url: https://reddit.com category: network - options: - - tray_min google-drive: - name: Drive + name: Google Drive + priority: 1 url: https://drive.google.com category: Utility options: - - tray_min + - allow_urls + +google-keep: + name: Google Keep + priority: 2 + url: https://keep.google.com + icon_url: https://ssl.gstatic.com/keep/icon_128.png + category: Utility + options: + - allow_urls + +google-cal: + name: Google Calendar + priority: 2 + url: https://calendar.google.com + icon_url: https://calendar.google.com/googlecalendar/images/favicon_v2014_16.ico + category: Utility + options: + - allow_urls + +google-sheets: + name: Google Sheets + priority: 1 + url: https://docs.google.com/spreadsheets/ + icon_url: https://ssl.gstatic.com/docs/spreadsheets/favicon3.ico + category: Office + options: + - allow_urls + +google-docs: + name: Google Docs + priority: 1 + url: https://docs.google.com/document + icon_url: https://ssl.gstatic.com/docs/documents/images/kix-favicon7.ico + category: Office + options: + - allow_urls + +google-pres: + name: Google Presentation + priority: 1 + url: https://docs.google.com/presentation + icon_url: https://ssl.gstatic.com/docs/presentations/images/favicon5.ico + category: Office + options: + - allow_urls + +google-hangouts: + name: Google Hangouts + priority: 2 + url: https://hangouts.google.com + category: Network + options: + - allow_urls + +gmail: + name: GMail + priority: 3 + url: https://gmail.com + icon_url: https://ssl.gstatic.com/ui/v1/icons/mail/images/favicon5.ico + category: Network + options: + - allow_urls + +mega: + name: Mega + priority: 0 + url: https://mega.nz + category: Utility + options: + - allow_urls + +tweetdeck: + name: TweetDeck + priority: 3 + url: https://tweetdeck.twitter.com + category: Network + options: + - allow_urls + +twitter: + name: Twitter + url: https://twitter.com + priority: 3 + category: Network + options: + - allow_urls \ No newline at end of file