mirror of
https://github.com/spalencsar/bearhub.git
synced 2026-07-07 04:44:15 +02:00
improving initialization time and reducing async overload
This commit is contained in:
Vinícius Moreira
GitHub
2
fpakman/core/constants.py
Normal file
2
fpakman/core/constants.py
Normal file
@@ -0,0 +1,2 @@
|
||||
FLATHUB_URL = 'https://flathub.org'
|
||||
FLATHUB_API_URL = FLATHUB_URL + '/api/v1'
|
||||
@@ -1,80 +1,53 @@
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime
|
||||
from threading import Lock
|
||||
from typing import List
|
||||
|
||||
import requests
|
||||
|
||||
from fpakman.core import flatpak
|
||||
|
||||
__FLATHUB_URL__ = 'https://flathub.org'
|
||||
__FLATHUB_API_URL__ = __FLATHUB_URL__ + '/api/v1'
|
||||
from fpakman.core.model import FlatpakApplication, ApplicationData, ApplicationStatus
|
||||
from fpakman.core.worker import FlatpakAsyncDataLoaderManager
|
||||
from fpakman.util.cache import Cache
|
||||
|
||||
|
||||
class FlatpakManager:
|
||||
|
||||
def __init__(self, cache_expire: int = 60 * 60):
|
||||
self.cache_apps = {}
|
||||
self.cache_expire = cache_expire
|
||||
def __init__(self, api_cache: Cache):
|
||||
self.api_cache = api_cache
|
||||
self.http_session = requests.Session()
|
||||
self.lock_db_read = Lock()
|
||||
self.lock_read = Lock()
|
||||
self.async_data_loader = FlatpakAsyncDataLoaderManager(api_cache=self.api_cache)
|
||||
|
||||
def load_full_database(self):
|
||||
def _map_to_model(self, app: dict) -> FlatpakApplication:
|
||||
|
||||
self.lock_db_read.acquire()
|
||||
model = FlatpakApplication(arch=app.get('arch'),
|
||||
branch=app.get('branch'),
|
||||
origin=app.get('origin'),
|
||||
runtime=app.get('runtime'),
|
||||
ref=app.get('ref'),
|
||||
commit=app.get('commit'),
|
||||
base_data=ApplicationData(id=app.get('id'),
|
||||
name=app.get('name'),
|
||||
version=app.get('version'),
|
||||
latest_version=app.get('latest_version')))
|
||||
|
||||
try:
|
||||
res = self.http_session.get(__FLATHUB_API_URL__ + '/apps', timeout=30)
|
||||
api_data = self.api_cache.get(app['id'])
|
||||
|
||||
if res.status_code == 200:
|
||||
for app in res.json():
|
||||
self.cache_apps[app['flatpakAppId']] = app
|
||||
finally:
|
||||
self.lock_db_read.release()
|
||||
expired_data = api_data and api_data.get('expires_at') and api_data['expires_at'] <= datetime.utcnow()
|
||||
|
||||
def _request_app_data(self, app_id: str):
|
||||
if not api_data or expired_data:
|
||||
if not app['runtime']:
|
||||
model.status = ApplicationStatus.LOADING_DATA
|
||||
self.async_data_loader.load(model)
|
||||
|
||||
try:
|
||||
res = self.http_session.get('{}/apps/{}'.format(__FLATHUB_API_URL__, app_id), timeout=30)
|
||||
else: # filling cached data
|
||||
for attr, val in api_data.items():
|
||||
if attr != 'expires_at' and val:
|
||||
setattr(model.base_data, attr, val)
|
||||
|
||||
if res.status_code == 200:
|
||||
return res.json()
|
||||
else:
|
||||
print("Could not retrieve app data for id '{}'. Server response: {}".format(app_id, res.status_code))
|
||||
except:
|
||||
print("Could not retrieve app data for id '{}'. Timeout".format(app_id))
|
||||
return None
|
||||
return model
|
||||
|
||||
def _fill_api_data(self, app: dict):
|
||||
|
||||
api_data = self.cache_apps.get(app['id'])
|
||||
|
||||
if (not app['runtime'] and not api_data) or (api_data and api_data.get('expires_at') and api_data['expires_at'] <= datetime.utcnow()): # if api data is not cached or expired, tries to retrieve it
|
||||
api_data = self._request_app_data(app['id'])
|
||||
|
||||
if api_data:
|
||||
|
||||
if self.cache_expire > 0:
|
||||
api_data['expires_at'] = datetime.utcnow() + timedelta(seconds=self.cache_expire)
|
||||
|
||||
self.cache_apps[app['id']] = api_data
|
||||
|
||||
if not api_data:
|
||||
for attr in ('latest_version', 'icon', 'description'):
|
||||
if attr not in app:
|
||||
app[attr] = None
|
||||
else:
|
||||
app['latest_version'] = api_data.get('currentReleaseVersion')
|
||||
app['icon'] = api_data.get('iconMobileUrl')
|
||||
|
||||
for attr in ('name', 'description'):
|
||||
if not app.get(attr):
|
||||
app[attr] = api_data.get(attr)
|
||||
|
||||
if app['icon'].startswith('/'):
|
||||
app['icon'] = __FLATHUB_URL__ + app['icon']
|
||||
|
||||
def search(self, word: str) -> List[dict]:
|
||||
def search(self, word: str) -> List[FlatpakApplication]:
|
||||
|
||||
res = []
|
||||
apps_found = flatpak.search(word)
|
||||
@@ -82,25 +55,24 @@ class FlatpakManager:
|
||||
if apps_found:
|
||||
|
||||
already_read = set()
|
||||
installed_apps = self.read_installed()
|
||||
installed_apps = self.read_installed(keep_workers=True)
|
||||
|
||||
if installed_apps:
|
||||
for app in apps_found:
|
||||
for app_found in apps_found:
|
||||
for installed_app in installed_apps:
|
||||
if app['id'] == installed_app['id']:
|
||||
if app_found['id'] == installed_app.base_data.id:
|
||||
res.append(installed_app)
|
||||
already_read.add(app['id'])
|
||||
already_read.add(app_found['id'])
|
||||
|
||||
for app in apps_found:
|
||||
if app['id'] not in already_read:
|
||||
app['update'] = False
|
||||
app['installed'] = False
|
||||
self._fill_api_data(app)
|
||||
res.append(app)
|
||||
for app_found in apps_found:
|
||||
if app_found['id'] not in already_read:
|
||||
res.append(self._map_to_model(app_found))
|
||||
|
||||
self.async_data_loader.stop_current_workers()
|
||||
|
||||
return res
|
||||
|
||||
def read_installed(self) -> List[dict]:
|
||||
def read_installed(self, keep_workers: bool = False) -> List[FlatpakApplication]:
|
||||
|
||||
self.lock_read.acquire()
|
||||
|
||||
@@ -112,23 +84,34 @@ class FlatpakManager:
|
||||
|
||||
available_updates = flatpak.list_updates_as_str()
|
||||
|
||||
for app in installed:
|
||||
self._fill_api_data(app)
|
||||
app['update'] = app['id'] in available_updates
|
||||
app['installed'] = True
|
||||
models = []
|
||||
|
||||
return installed
|
||||
for app in installed:
|
||||
model = self._map_to_model(app)
|
||||
model.installed = True
|
||||
model.update = app['id'] in available_updates
|
||||
models.append(model)
|
||||
|
||||
if not keep_workers:
|
||||
self.async_data_loader.stop_current_workers()
|
||||
|
||||
return models
|
||||
|
||||
return []
|
||||
|
||||
finally:
|
||||
self.lock_read.release()
|
||||
|
||||
def downgrade_app(self, app: dict, root_password: str):
|
||||
def downgrade_app(self, app: FlatpakApplication, root_password: str):
|
||||
|
||||
commits = flatpak.get_app_commits(app['ref'], app['origin'])
|
||||
commit_idx = commits.index(app['commit'])
|
||||
commits = flatpak.get_app_commits(app.ref, app.origin)
|
||||
commit_idx = commits.index(app.commit)
|
||||
|
||||
# downgrade is not possible if the app current commit in the first one:
|
||||
if commit_idx == len(commits) - 1:
|
||||
return None
|
||||
|
||||
return flatpak.downgrade_and_stream(app['ref'], commits[commit_idx + 1], root_password)
|
||||
return flatpak.downgrade_and_stream(app.ref, commits[commit_idx + 1], root_password)
|
||||
|
||||
def clean_cache_for(self, app_id: str):
|
||||
self.api_cache.delete(app_id)
|
||||
|
||||
@@ -3,6 +3,7 @@ import subprocess
|
||||
from typing import List
|
||||
|
||||
from fpakman.core import system
|
||||
from fpakman.core.model import Application
|
||||
|
||||
BASE_CMD = 'flatpak'
|
||||
|
||||
|
||||
42
fpakman/core/model.py
Normal file
42
fpakman/core/model.py
Normal file
@@ -0,0 +1,42 @@
|
||||
from abc import ABC
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class ApplicationStatus(Enum):
|
||||
READY = 1
|
||||
LOADING_DATA = 2
|
||||
|
||||
|
||||
class ApplicationData:
|
||||
|
||||
def __init__(self, id: str, version: str, name: str = None, description: str = None, latest_version: str = None, icon_url: str = None):
|
||||
self.id = id
|
||||
self.name = name
|
||||
self.version = version
|
||||
self.description = description
|
||||
self.latest_version = latest_version
|
||||
self.icon_url = icon_url
|
||||
|
||||
|
||||
class Application(ABC):
|
||||
|
||||
def __init__(self, base_data: ApplicationData, status: ApplicationStatus = ApplicationStatus.READY, installed: bool = False, update: bool = False):
|
||||
self.base_data = base_data
|
||||
self.status = status
|
||||
self.installed = installed
|
||||
self.update = update
|
||||
|
||||
|
||||
class FlatpakApplication(Application):
|
||||
|
||||
def __init__(self, base_data: ApplicationData, branch: str, arch: str, origin: str, runtime: bool, ref: str, commit: str):
|
||||
super(FlatpakApplication, self).__init__(base_data=base_data)
|
||||
self.ref = ref
|
||||
self.branch = branch
|
||||
self.arch = arch
|
||||
self.origin = origin
|
||||
self.runtime = runtime
|
||||
self.commit = commit
|
||||
|
||||
def is_incomplete(self):
|
||||
return self.base_data.description is None and self.base_data.icon_url
|
||||
@@ -1,47 +0,0 @@
|
||||
import locale
|
||||
|
||||
from fpakman.core import resource
|
||||
import glob
|
||||
import re
|
||||
|
||||
HTML_RE = re.compile(r'<[^>]+>')
|
||||
|
||||
|
||||
def get_locale_keys(key: str = None):
|
||||
|
||||
locale_path = None
|
||||
|
||||
if key is None:
|
||||
current_locale = locale.getdefaultlocale()
|
||||
else:
|
||||
current_locale = [key.strip().lower()]
|
||||
|
||||
if current_locale:
|
||||
current_locale = current_locale[0]
|
||||
|
||||
locale_dir = resource.get_path('locale')
|
||||
|
||||
for locale_file in glob.glob(locale_dir + '/*'):
|
||||
name = locale_file.split('/')[-1]
|
||||
|
||||
if current_locale == name or current_locale.startswith(name + '_'):
|
||||
locale_path = locale_file
|
||||
break
|
||||
|
||||
if not locale_path:
|
||||
locale_path = resource.get_path('locale/en')
|
||||
|
||||
with open(locale_path, 'r') as f:
|
||||
locale_keys = f.readlines()
|
||||
|
||||
locale_obj = {}
|
||||
for line in locale_keys:
|
||||
if line:
|
||||
keyval = line.strip().split('=')
|
||||
locale_obj[keyval[0].strip()] = keyval[1].strip()
|
||||
|
||||
return locale_obj
|
||||
|
||||
|
||||
def strip_html(string: str):
|
||||
return HTML_RE.sub('', string)
|
||||
119
fpakman/core/worker.py
Normal file
119
fpakman/core/worker.py
Normal file
@@ -0,0 +1,119 @@
|
||||
import traceback
|
||||
from io import StringIO
|
||||
from threading import Thread
|
||||
from typing import List
|
||||
|
||||
import requests
|
||||
from colorama import Fore
|
||||
|
||||
from fpakman.core.constants import FLATHUB_API_URL, FLATHUB_URL
|
||||
from fpakman.core.model import FlatpakApplication, ApplicationStatus
|
||||
from fpakman.util.cache import Cache
|
||||
|
||||
|
||||
class FlatpakAsyncDataLoader(Thread):
|
||||
|
||||
def __init__(self, http_session, api_cache: Cache, attempts: int = 3):
|
||||
super(FlatpakAsyncDataLoader, self).__init__(daemon=True)
|
||||
self.apps = []
|
||||
self.http_session = http_session
|
||||
self.attempts = attempts
|
||||
self.api_cache = api_cache
|
||||
self.id_ = '{}#{}'.format(self.__class__.__name__, id(self))
|
||||
self.stop = False
|
||||
|
||||
def log_msg(self, msg: str, color: int = None):
|
||||
final_msg = StringIO()
|
||||
|
||||
if color:
|
||||
final_msg.write(str(color))
|
||||
|
||||
final_msg.write('[{}] '.format(self.id_))
|
||||
|
||||
final_msg.write(msg)
|
||||
|
||||
if color:
|
||||
final_msg.write(Fore.RESET)
|
||||
|
||||
final_msg.seek(0)
|
||||
|
||||
print(final_msg.read())
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
if not self.apps and self.stop:
|
||||
break # stop working
|
||||
else:
|
||||
if self.apps:
|
||||
app = self.apps[0]
|
||||
app.status = ApplicationStatus.LOADING_DATA
|
||||
|
||||
for _ in range(0, self.attempts):
|
||||
try:
|
||||
res = self.http_session.get('{}/apps/{}'.format(FLATHUB_API_URL, app.base_data.id), timeout=30)
|
||||
|
||||
if res.status_code == 200 and res.text:
|
||||
data = res.json()
|
||||
|
||||
if not app.base_data.version:
|
||||
app.base_data.version = data.get('version')
|
||||
|
||||
app.base_data.description = data.get('description', data.get('summary', None))
|
||||
app.base_data.icon_url = data.get('iconMobileUrl', None)
|
||||
app.base_data.latest_version = data.get('currentReleaseVersion', app.base_data.version)
|
||||
|
||||
if app.base_data.icon_url and app.base_data.icon_url.startswith('/'):
|
||||
app.base_data.icon_url = FLATHUB_URL + app.base_data.icon_url
|
||||
|
||||
self.api_cache.add(app.base_data.id, {
|
||||
'description': app.base_data.description,
|
||||
'icon_url': app.base_data.icon_url,
|
||||
'latest_version': app.base_data.latest_version,
|
||||
'version': app.base_data.version
|
||||
})
|
||||
|
||||
app.status = ApplicationStatus.READY
|
||||
break
|
||||
else:
|
||||
self.log_msg("Could not retrieve app data for id '{}'. Server response: {}. Body: {}".format(app.base_data.id, res.status_code, res.content.decode()), Fore.RED)
|
||||
except:
|
||||
self.log_msg("Could not retrieve app data for id '{}'".format(app.base_data.id), Fore.YELLOW)
|
||||
traceback.print_exc()
|
||||
|
||||
del self.apps[0]
|
||||
|
||||
def add(self, app: FlatpakApplication):
|
||||
self.apps.append(app)
|
||||
|
||||
def current_load(self):
|
||||
return len(self.apps)
|
||||
|
||||
|
||||
class FlatpakAsyncDataLoaderManager:
|
||||
|
||||
def __init__(self, api_cache: Cache, worker_load: int = 3, workers: List[FlatpakAsyncDataLoader] = []):
|
||||
self.worker_load = worker_load
|
||||
self.current_workers = workers
|
||||
self.http_session = requests.Session()
|
||||
self.api_cache = api_cache
|
||||
|
||||
def load(self, app: FlatpakApplication):
|
||||
|
||||
available_workers = [w for w in self.current_workers if w.current_load() < self.worker_load]
|
||||
|
||||
if available_workers:
|
||||
worker = available_workers[0]
|
||||
else: # new worker
|
||||
worker = FlatpakAsyncDataLoader(http_session=self.http_session,
|
||||
api_cache=self.api_cache)
|
||||
worker.start()
|
||||
self.current_workers.append(worker)
|
||||
|
||||
worker.add(app)
|
||||
|
||||
def stop_current_workers(self):
|
||||
|
||||
for w in self.current_workers:
|
||||
w.stop = True
|
||||
|
||||
self.current_workers = []
|
||||
Reference in New Issue
Block a user