spalencsar/bearhub
1
0
Fork 0
mirror of https://github.com/spalencsar/bearhub.git synced 2026-07-07 01:14:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

[wgem] downloading suggestions from the cloud and generating a search index

Browse Source
This commit is contained in:
Vinícius Moreira
2019-12-16 19:50:47 -03:00
parent 813a6aa0af
commit e73e36b9c2
6 changed files with 126 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
bauh/api/http.py
View File

@@ -3,6 +3,7 @@ import time
import traceback
import requests
import yaml
from bauh.commons import system
@@ -59,6 +60,10 @@ class HttpClient:
res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects)
return res.json() if res else None
def get_yaml(self, url: str, params: dict = None, headers: dict = None, allow_redirects: bool = True):
res = self.get(url, params=params, headers=headers, allow_redirects=allow_redirects)
return yaml.safe_load(res.text) if res else None
def get_content_length(self, url: str) -> str:
"""
:param url:

3
bauh/app.py
View File

@@ -1,6 +1,7 @@
import os
import sys
import urllib3
from PyQt5.QtGui import QIcon
from PyQt5.QtWidgets import QApplication
@@ -24,6 +25,8 @@ def main():
if not os.getenv('PYTHONUNBUFFERED'):
os.environ['PYTHONUNBUFFERED'] = '1'
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
args = app_args.read()
logger = logs.new_logger(__app_name__, bool(args.logs))

3
bauh/gems/web/__init__.py
View File

@@ -19,4 +19,7 @@ ELECTRON_SHA256_URL = 'https://github.com/electron/electron/releases/download/v{
URL_ENVIRONMENT_SETTINGS = 'https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/environment.yml'
DESKTOP_ENTRY_PATH_PATTERN = DESKTOP_ENTRIES_DIR + '/bauh.web.{name}.desktop'
URL_FIX_PATTERN = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/fix/{url}.js"
URL_SUGGESTIONS = "https://raw.githubusercontent.com/vinifmor/bauh-files/master/web/suggestions.yaml"
UA_CHROME = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
TEMP_PATH = '/tmp/bauh/web'
SEARCH_INDEX_FILE = '{}/index.yaml'.format(TEMP_PATH)

48
bauh/gems/web/controller.py
View File

@@ -22,13 +22,13 @@ from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSugge
from bauh.api.abstract.view import MessageType, MultipleSelectComponent, InputOption, SingleSelectComponent, \
SelectViewType, TextInputComponent, FormComponent, FileChooserComponent
from bauh.api.constants import DESKTOP_ENTRIES_DIR
from bauh.commons import resource, internet
from bauh.commons.html import bold
from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str
from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \
ROOT_DIR
URL_SUGGESTIONS
from bauh.gems.web.environment import EnvironmentUpdater
from bauh.gems.web.model import WebApplication
from bauh.gems.web.worker import SuggestionsDownloader, SearchIndexGenerator
try:
from bs4 import BeautifulSoup, SoupStrainer
@@ -50,7 +50,8 @@ RE_SYMBOLS_SPLIT = re.compile(r'[\-|_\s:.]')
class WebApplicationManager(SoftwareManager):
def __init__(self, context: ApplicationContext, env_updater: Thread = None):
def __init__(self, context: ApplicationContext, env_updater: Thread = None,
suggestions_downloader: Thread = None, suggestions: dict = None):
super(WebApplicationManager, self).__init__(context=context)
self.http_client = context.http_client
self.node_updater = EnvironmentUpdater(logger=context.logger, http_client=context.http_client,
@@ -61,6 +62,8 @@ class WebApplicationManager(SoftwareManager):
self.env_settings = {}
self.logger = context.logger
self.env_thread = None
self.suggestions_downloader = suggestions_downloader
self.suggestions = suggestions
def _get_lang_header(self) -> str:
try:
@@ -177,6 +180,10 @@ class WebApplicationManager(SoftwareManager):
if installed_matches:
res.installed.extend(installed_matches)
if self.search_index:
# TODO
pass
res.total += len(res.installed)
res.total += len(res.new)
return res
@@ -514,10 +521,21 @@ class WebApplicationManager(SoftwareManager):
return False
def _download_suggestions(self):
downloader = SuggestionsDownloader(logger=self.logger, http_client=self.http_client)
self.suggestions = downloader.download()
if self.suggestions:
index_gen = SearchIndexGenerator(logger=self.logger)
Thread(target=index_gen.generate_index, args=(self.suggestions,)).start()
def prepare(self):
self.env_thread = Thread(target=self._update_environment)
self.env_thread.start()
self.suggestions_downloader = Thread(target=self._download_suggestions)
self.suggestions_downloader.start()
def list_updates(self, internet_available: bool) -> List[PackageUpdate]:
pass
@@ -554,15 +572,27 @@ class WebApplicationManager(SoftwareManager):
return PackageSuggestion(priority=SuggestionPriority(suggestion['priority']), package=app)
def list_suggestions(self, limit: int) -> List[PackageSuggestion]:
with open(resource.get_path('suggestions.yml', ROOT_DIR), 'r') as f:
suggestions = yaml.safe_load(f.read())
if self.suggestions:
suggestions = self.suggestions
elif self.suggestions_downloader:
self.suggestions_downloader.join(5)
suggestions = self.suggestions
else:
suggestions = SuggestionsDownloader(logger=self.logger, http_client=self.http_client).download()
# cleaning memory
self.suggestions_downloader = None
self.suggestions = None
if suggestions:
suggestions = list(suggestions.values())
suggestions.sort(key=lambda s: s.get('priority', 0), reverse=True)
to_map = suggestions if limit <= 0 else suggestions[0:limit]
suggestion_list = list(suggestions.values())
suggestion_list.sort(key=lambda s: s.get('priority', 0), reverse=True)
to_map = suggestion_list if limit <= 0 else suggestion_list[0:limit]
res = [self._map_suggestion(s) for s in to_map]
self.env_thread.join()
if not self.env_settings:
self.env_thread.join()
if self.env_settings:
for s in res:

177
bauh/gems/web/resources/suggestions.yml
View File

@@ -1,177 +0,0 @@
youtube:
name: YouTube
priority: 1
url: https://youtube.com
category: Network
options: [allow_urls]
facebook:
name: Facebook
priority: 3
url: https://facebook.com
category: Network
options: [allow_urls]
twitch:
name: Twitch
priority: 2
url: https://twitch.tv
category: Network
options: [allow_urls]
whatsapp:
name: WhatsApp
priority: 3
url: https://web.whatsapp.com
category: Network
options: [allow_urls,tray_min]
telegram:
name: Telegram
priority: 3
url: https://web.telegram.org
category: Network
options: [allow_urls,tray_min]
slack:
name: Slack
priority: 3
url: https://slack.com
category: Network
options: [allow_urls,tray_min]
office:
name: Microsoft Office
priority: 0
url: https://office.com
category: office
options: [allow_urls]
ms_excel:
name: Microsoft Excel
priority: 1
url: https://www.office.com/launch/excel
icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-excel-svg-29b733e1d7.ico
category: office
options: [allow_urls]
ms_word:
name: Microsoft Word
priority: 1
url: https://www.office.com/launch/word
icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-word-svg-8eedb3d95e.ico
category: office
options: [allow_urls]
ms_pp:
name: Microsoft Powerpoint
priority: 1
url: https://www.office.com/launch/powerpoint
icon_url: https://blobs.officehome.msocdn.com/images/content/images/favicons/favicon-powerpoint-svg-f44c23dac7.ico
category: office
options: [allow_urls]
ms_outlook:
name: Microsoft Outlook
priority: 3
url: https://outlook.live.com
category: Network
options: [allow_urls,tray_min]
ms_drive:
name: Microsoft One Drive
priority: 0
url: https://onedrive.live.com
category: Utility
options: [allow_urls]
reddit:
name: Reddit
priority: 1
url: https://reddit.com
category: network
options: [allow_urls]
google-drive:
name: Google Drive
priority: 1
url: https://drive.google.com
category: Utility
options: [allow_urls]
google-keep:
name: Google Keep
priority: 2
url: https://keep.google.com
icon_url: https://ssl.gstatic.com/keep/icon_128.png
category: Utility
options: [allow_urls,tray_min]
google-cal:
name: Google Calendar
priority: 2
url: https://calendar.google.com
icon_url: https://calendar.google.com/googlecalendar/images/favicon_v2014_16.ico
category: Utility
options: [allow_urls,tray_min]
google-sheets:
name: Google Sheets
priority: 1
url: https://docs.google.com/spreadsheets/
icon_url: https://ssl.gstatic.com/docs/spreadsheets/favicon3.ico
category: Office
options: [allow_urls]
google-docs:
name: Google Docs
priority: 1
url: https://docs.google.com/document
icon_url: https://ssl.gstatic.com/docs/documents/images/kix-favicon7.ico
category: Office
options: [allow_urls]
google-pres:
name: Google Presentation
priority: 1
url: https://docs.google.com/presentation
icon_url: https://ssl.gstatic.com/docs/presentations/images/favicon5.ico
category: Office
options: [allow_urls]
google-hangouts:
name: Google Hangouts
priority: 2
url: https://hangouts.google.com
category: Network
options: [allow_urls,tray_min]
gmail:
name: GMail
priority: 3
url: https://gmail.com
icon_url: https://ssl.gstatic.com/ui/v1/icons/mail/images/favicon5.ico
category: Network
options: [allow_urls,tray_min]
mega:
name: Mega
priority: 0
url: https://mega.nz
category: Utility
options: [allow_urls]
tweetdeck:
name: TweetDeck
priority: 3
url: https://tweetdeck.twitter.com
category: Network
options: [allow_urls]
twitter:
name: Twitter
url: https://twitter.com
priority: 3
category: Network
options: [allow_urls]

76
bauh/gems/web/worker.py Normal file
View File

@@ -0,0 +1,76 @@
import logging
import traceback
from pathlib import Path
import yaml
from bauh.api.http import HttpClient
from bauh.gems.web import URL_SUGGESTIONS, TEMP_PATH, SEARCH_INDEX_FILE
class SuggestionsDownloader:
def __init__(self, http_client: HttpClient, logger: logging.Logger):
super(SuggestionsDownloader, self).__init__()
self.http_client = http_client
self.logger = logger
def download(self) -> dict:
self.logger.info("Reading suggestions from {}".format(URL_SUGGESTIONS))
suggestions = self.http_client.get_yaml(URL_SUGGESTIONS)
if suggestions:
self.logger.info("{} suggestions successfully read".format(len(suggestions)))
else:
self.logger.warning("Could not read suggestions from {}".format(URL_SUGGESTIONS))
return suggestions
class SearchIndexGenerator:
def __init__(self, logger: logging.Logger):
self.logger = logger
def generate_index(self, suggestions: dict):
self.logger.info('Indexing {} app suggestions'.format(len(suggestions)))
index = {}
for key, sug in suggestions.items():
name = sug.get('name')
if name:
split_name = name.split(' ')
single_name = ''.join(split_name)
for word in (*split_name, single_name):
word_key = word.lower().strip()
mapped = index.get(word_key)
if not mapped:
mapped = set()
index[word_key] = mapped
mapped.add(key)
if index:
self.logger.info('Preparing search index for writing')
for key in index.keys():
index[key] = list(index[key])
try:
Path(TEMP_PATH).mkdir(parents=True, exist_ok=True)
except:
self.logger.error("Could not generate the directory {}".format(TEMP_PATH))
traceback.print_exc()
return
try:
self.logger.info('Writing {} indexed keys as {}'.format(len(index), SEARCH_INDEX_FILE))
with open(SEARCH_INDEX_FILE, 'w+') as f:
f.write(yaml.safe_dump(index))
self.logger.info("Search index successfully written at {}".format(SEARCH_INDEX_FILE))
except:
self.logger.error("Could not write the seach index to {}".format(SEARCH_INDEX_FILE))
traceback.print_exc()