From be71434f2ecee983499e0c262736188d8e033539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Moreira?= Date: Tue, 17 Dec 2019 19:27:53 -0300 Subject: [PATCH] [wgem] display required environment files to download and install --- bauh/gems/web/__init__.py | 2 + bauh/gems/web/controller.py | 69 ++++---- bauh/gems/web/environment.py | 265 +++++++++++++++++++++++++----- bauh/gems/web/resources/locale/en | 2 + bauh/gems/web/resources/locale/pt | 2 + 5 files changed, 268 insertions(+), 72 deletions(-) diff --git a/bauh/gems/web/__init__.py b/bauh/gems/web/__init__.py index 26946647..436c063d 100644 --- a/bauh/gems/web/__init__.py +++ b/bauh/gems/web/__init__.py @@ -25,3 +25,5 @@ TEMP_PATH = '/tmp/bauh/web' SEARCH_INDEX_FILE = '{}/index.yml'.format(TEMP_PATH) SUGGESTIONS_CACHE_FILE = '{}/suggestions.yml'.format(TEMP_PATH) CONFIG_FILE = '{}/web.yml'.format(CONFIG_PATH) +NATIVEFIER_URL = 'https://github.com/jiahaog/nativefier/archive/v7.7.0.tar.gz' + diff --git a/bauh/gems/web/controller.py b/bauh/gems/web/controller.py index b2302f76..7c5883c1 100644 --- a/bauh/gems/web/controller.py +++ b/bauh/gems/web/controller.py @@ -22,12 +22,13 @@ from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSugge from bauh.api.abstract.view import MessageType, MultipleSelectComponent, InputOption, SingleSelectComponent, \ SelectViewType, TextInputComponent, FormComponent, FileChooserComponent from bauh.api.constants import DESKTOP_ENTRIES_DIR +from bauh.commons import resource from bauh.commons.html import bold from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \ - SEARCH_INDEX_FILE, SUGGESTIONS_CACHE_FILE + SEARCH_INDEX_FILE, SUGGESTIONS_CACHE_FILE, ROOT_DIR from bauh.gems.web.config import read_config -from bauh.gems.web.environment import EnvironmentUpdater +from bauh.gems.web.environment import EnvironmentUpdater, EnvironmentComponent from bauh.gems.web.model import WebApplication from bauh.gems.web.worker import SuggestionsDownloader, SearchIndexGenerator @@ -54,8 +55,8 @@ class WebApplicationManager(SoftwareManager): def __init__(self, context: ApplicationContext, suggestions_downloader: Thread = None): super(WebApplicationManager, self).__init__(context=context) self.http_client = context.http_client - self.node_updater = EnvironmentUpdater(logger=context.logger, http_client=context.http_client, - file_downloader=context.file_downloader, i18n=context.i18n) + self.env_updater = EnvironmentUpdater(logger=context.logger, http_client=context.http_client, + file_downloader=context.file_downloader, i18n=context.i18n) self.enabled = True self.i18n = context.i18n self.env_settings = {} @@ -450,6 +451,20 @@ class WebApplicationManager(SoftwareManager): base_id = '{}_{}'.format(app_id, counter) counter += 1 + def _ask_update_permission(self, to_update: List[EnvironmentComponent], watcher: ProcessWatcher) -> bool: + + icon = resource.get_path('img/web.png', ROOT_DIR) + opts = [InputOption(label='{} ( {} )'.format(f.name, f.size or '?'), + tooltip=f.url, icon_path=icon, read_only=True, value=f.name) for f in to_update] + + comps = MultipleSelectComponent(label=None, options=opts, default_options=set(opts)) + + return watcher.request_confirmation(title=self.i18n['web.install.env_update.title'], + body=self.i18n['web.install.env_update.body'], + components=[comps], + confirmation_label=self.i18n['continue'].capitalize(), + deny_label=self.i18n['cancel'].capitalize()) + def install(self, pkg: WebApplication, root_password: str, watcher: ProcessWatcher) -> bool: continue_install, install_options = self._ask_install_options(pkg, watcher) @@ -460,10 +475,19 @@ class WebApplicationManager(SoftwareManager): watcher.change_substatus(self.i18n['web.env.checking']) handler = ProcessHandler(watcher) - if not self._update_environment(handler=handler): + + env_settings, to_update = self.env_updater.check_environment(app=pkg, is_x86_x64_arch=self.context.is_system_x86_64()) + + if to_update: + if not self._ask_update_permission(to_update, watcher): + return False + + if not self.env_updater.update(components=to_update, handler=handler): watcher.show_message(title=self.i18n['error'], body=self.i18n['web.env.error'].format(bold(pkg.name)), type_=MessageType.ERROR) return False + # TODO stopped here + Path(INSTALLED_PATH).mkdir(parents=True, exist_ok=True) app_id, treated_name = self._gen_app_id(pkg.name) @@ -480,25 +504,8 @@ class WebApplicationManager(SoftwareManager): watcher.print('Fix found for {}'.format(pkg.url)) install_options.append('--inject={}'.format(fix_path)) - config = read_config() - watcher.change_substatus(self.i18n['web.install.substatus.call_nativefier'].format(bold('nativefier'))) - system_env = config['environment'].get('system', False) - - if system_env: - self.logger.warning("Using system's nativefier to install {}".format(pkg.url)) - - electron_version = self.env_settings['electron']['version'] - - if pkg.version and pkg.version != electron_version: - self.logger.info('A preset Electron version is defined for {}: {}'.format(pkg.url, pkg.version)) - electron_version = pkg.version - - if config['environment']['electron']['version']: - self.logger.warning("A custom Electron version will be used {} to install {}".format(electron_version, pkg.url)) - electron_version = config['environment']['electron']['version'] - installed = handler.handle_simple(nativefier.install(url=pkg.url, name=app_id, output_dir=app_dir, electron_version=electron_version, system=system_env, @@ -597,17 +604,17 @@ class WebApplicationManager(SoftwareManager): def requires_root(self, action: str, pkg: SoftwarePackage): return False - def _update_environment(self, handler: ProcessHandler = None) -> bool: - updated_settings = self.node_updater.update_environment(self.context.is_system_x86_64(), handler=handler) - - if updated_settings is not None: - self.env_settings = updated_settings - return True - - return False + # def _update_environment(self, handler: ProcessHandler = None) -> bool: + # updated_settings = self.env_updater.update_environment(self.context.is_system_x86_64(), handler=handler) + # + # if updated_settings is not None: + # self.env_settings = updated_settings + # return True + # + # return False def _update_env_settings(self): - self.env_settings = self.node_updater.read_settings() + self.env_settings = self.env_updater.read_settings() def _download_suggestions(self): downloader = SuggestionsDownloader(logger=self.logger, http_client=self.http_client) diff --git a/bauh/gems/web/environment.py b/bauh/gems/web/environment.py index e041aa03..d6ef6612 100644 --- a/bauh/gems/web/environment.py +++ b/bauh/gems/web/environment.py @@ -3,6 +3,8 @@ import os import shutil import tarfile from pathlib import Path +from threading import Thread +from typing import Dict, List, Tuple import requests import yaml @@ -15,11 +17,22 @@ from bauh.commons.html import bold from bauh.commons.system import SimpleProcess, ProcessHandler from bauh.gems.web import ENV_PATH, NODE_DIR_PATH, NODE_BIN_PATH, NODE_MODULES_PATH, NATIVEFIER_BIN_PATH, \ ELECTRON_PATH, ELECTRON_DOWNLOAD_URL, ELECTRON_SHA256_URL, URL_ENVIRONMENT_SETTINGS, NPM_BIN_PATH, NODE_PATHS, \ - nativefier + nativefier, NATIVEFIER_URL from bauh.gems.web.config import read_config +from bauh.gems.web.model import WebApplication from bauh.view.util.translation import I18n +class EnvironmentComponent: + + def __init__(self, id: str, name: str, size: str, version: str, url: str): + self.id = id + self.name = name + self.size = size + self.version = version + self.url = url + + class EnvironmentUpdater: def __init__(self, logger: logging.Logger, http_client: HttpClient, file_downloader: FileDownloader, i18n: I18n): @@ -45,6 +58,15 @@ class EnvironmentUpdater: extracted_file = '{}/{}'.format(ENV_PATH, tf.getnames()[0]) os.rename(extracted_file, NODE_DIR_PATH) + + if os.path.exists(NODE_MODULES_PATH): + self.logger.info('Deleting {}'.format(NODE_MODULES_PATH)) + try: + shutil.rmtree(NODE_MODULES_PATH) + except: + self.logger.error("Could not delete the directory {}".format(NODE_MODULES_PATH)) + return False + return True except: self.logger.error('Could not extract {}'.format(tarf_path)) @@ -56,6 +78,31 @@ class EnvironmentUpdater: except: self.logger.error('Could not delete file {}'.format(tarf_path)) + + def check_node_installed(self, version: str) -> bool: + if not os.path.exists(NODE_DIR_PATH): + return False + else: + installed_version = system.run_cmd('{} --version'.format(NODE_BIN_PATH), print_error=False) + + if installed_version: + installed_version = installed_version.strip() + + if installed_version.startswith('v'): + installed_version = installed_version[1:] + + self.logger.info('Node versions: installed ({}), cloud ({})'.format(installed_version, version)) + + if version != installed_version: + self.logger.info("The NodeJs installed version is different from the Cloud.") + return False + else: + self.logger.info("Node is already up to date") + return True + else: + self.logger.warning("Could not determine the current NodeJS installed version") + return False + def update_node(self, version: str, version_url: str, watcher: ProcessWatcher = None) -> bool: Path(ENV_PATH).mkdir(parents=True, exist_ok=True) @@ -149,57 +196,42 @@ class EnvironmentUpdater: self.logger.info("Nativefier is already installed and up to date") return True - def _download_electron(self, version: str, is_x86_x64_arch: bool, watcher: ProcessWatcher) -> bool: + def download_electron(self, version: str, url: str, watcher: ProcessWatcher) -> bool: self.logger.info("Downloading Electron {}".format(version)) - arch = 'ia32' if not is_x86_x64_arch else 'x64' + electron_path = '{}/{}'.format(ELECTRON_PATH, url.split('/')[-1]) + return self.file_downloader.download(file_url=url, watcher=watcher, output_path=electron_path, cwd=ELECTRON_PATH) - electron_url = ELECTRON_DOWNLOAD_URL.format(arch=arch, version=version) - electron_path = '{}/{}'.format(ELECTRON_PATH, electron_url.split('/')[-1]) - - return self.file_downloader.download(file_url=electron_url, watcher=watcher, output_path=electron_path, cwd=ELECTRON_PATH) - - def _download_electron_sha256(self, version: str, watcher: ProcessWatcher) -> bool: + def download_electron_sha256(self, version: str, url: str, watcher: ProcessWatcher) -> bool: self.logger.info("Downloading Electron {} sha526".format(version)) + sha256_path = '{}/{}'.format(ELECTRON_PATH, url.split('/')[-1]) + return self.file_downloader.download(file_url=url, watcher=watcher, output_path=sha256_path, cwd=ELECTRON_PATH) - sha256_url = ELECTRON_SHA256_URL.format(version=version) - sha256_path = '{}/{}'.format(ELECTRON_PATH, sha256_url.split('/')[-1]) - return self.file_downloader.download(file_url=sha256_url, watcher=watcher, output_path=sha256_path, cwd=ELECTRON_PATH) + def _get_electron_url(self, version: str, is_x86_x64_arch: bool) -> str: + return ELECTRON_DOWNLOAD_URL.format(version=version, arch='x64' if is_x86_x64_arch else 'ia32') - def install_electron(self, version: str, is_x86_x64_arch: bool, watcher: ProcessWatcher) -> bool: - self.logger.info("Checking installed Electron") + def check_installed_electron(self, version: str, is_x86_x64_arch: bool) -> Dict[str, bool]: + self.logger.info("Checking if Electron {} is installed".format(version)) + res = {'electron': False, 'sha256': False} if not os.path.exists(ELECTRON_PATH): - self.logger.info("Electron is not installed") - Path(ELECTRON_PATH).mkdir(parents=True, exist_ok=True) - if self._download_electron(version=version, is_x86_x64_arch=is_x86_x64_arch, watcher=watcher): - return self._download_electron_sha256(version=version, watcher=watcher) + self.logger.info("The Electron folder {} was not found".format(ELECTRON_PATH)) else: files = os.listdir(ELECTRON_PATH) if files: - file_name = ELECTRON_DOWNLOAD_URL.format(version=version, arch='x64' if is_x86_x64_arch else 'ia32').split('/')[-1] - electron_file = [f for f in files if f == file_name] - - if electron_file: - self.logger.info("Electron {} already downloaded".format(version)) - else: - if not self._download_electron(version=version, is_x86_x64_arch=is_x86_x64_arch, watcher=watcher): - return False + file_name = self._get_electron_url(version, is_x86_x64_arch).split('/')[-1] + res['electron'] = bool([f for f in files if f == file_name]) file_name = ELECTRON_SHA256_URL.split('/')[-1] - sha256_file = [f for f in files if f == file_name] + res['sha256'] = bool([f for f in files if f == file_name]) + else: + self.logger.info('No Electron file found in {}'.format(ELECTRON_PATH)) - if sha256_file: - self.logger.info("Electron {} sha256 already downloaded".format(version)) - return True - else: - return self._download_electron_sha256(version=version, watcher=watcher) + for att in ('electron', 'sha256'): + if res[att]: + self.logger.info('{} ({}) already downloaded'.format(att, version)) - self.logger.info('No Electron file found') - if self._download_electron(version=version, is_x86_x64_arch=is_x86_x64_arch, watcher=watcher): - return self._download_electron_sha256(version=version, watcher=watcher) - - return False + return res def read_settings(self) -> dict: try: @@ -217,8 +249,116 @@ class EnvironmentUpdater: except requests.exceptions.ConnectionError: return - def update_environment(self, is_x86_x64_arch: bool, config: dict = None, handler: ProcessHandler = None) -> dict: + def _fill_electron_to_install(self, electron_version: str, x86_x64: bool, output: List[EnvironmentComponent]): + electron_url = self._get_electron_url(version=electron_version, is_x86_x64_arch=x86_x64) + output.append(EnvironmentComponent(name=electron_url.split('/')[-1], + version=electron_version, + url=electron_url, + size=self.http_client.get_content_length(electron_url), + id='electron')) + def _fill_electron_sha256_to_install(self, electron_version: str, output: List[EnvironmentComponent]): + sha_url = ELECTRON_SHA256_URL.format(version=electron_version) + output.append(EnvironmentComponent(name=sha_url.split('/')[-1], + version=electron_version, + url=sha_url, + size=self.http_client.get_content_length(sha_url), + id='electron_sha256')) + + def _check_electron_async(self, pkg: WebApplication, env: dict, local_config: dict, x86_x64: bool, output: List[EnvironmentComponent]): + electron_version = env['electron']['version'] + + if pkg.version and pkg.version != electron_version: + self.logger.info('A preset Electron version is defined for {}: {}'.format(pkg.url, pkg.version)) + electron_version = pkg.version + + if local_config['environment']['electron']['version']: + self.logger.warning("A custom Electron version will be used {} to install {}".format(electron_version, pkg.url)) + electron_version = local_config['environment']['electron']['version'] + + electron_status = self.check_installed_electron(version=electron_version, is_x86_x64_arch=x86_x64) + threads = [] + + if not electron_status['electron']: + t = Thread(target=self._fill_electron_to_install, args=(electron_version, x86_x64, output)) + t.start() + threads.append(t) + + if not electron_status['sha256']: + t = Thread(target=self._fill_electron_sha256_to_install, args=(electron_version, output)) + t.start() + threads.append(t) + + for t in threads: + t.join() + + def _check_node_async(self, env: dict, output: List[EnvironmentComponent]): + if not self.check_node_installed(env['nodejs']['version']): + output.append(EnvironmentComponent(name=env['nodejs']['url'].split('/')[-1], + url=env['nodejs']['url'], + size=self.http_client.get_content_length(env['nodejs']['url']), + version=env['nodejs']['version'], + id='nodejs')) + output.append(self._map_nativefier_file(env['nativefier'])) + else: + if not self._check_nativefier_installed(env['nativefier']): + output.append(self._map_nativefier_file(env['nativefier'])) + + def _check_nativefier_installed(self, nativefier_settings: dict) -> bool: + if not os.path.exists(NODE_MODULES_PATH): + self.logger.info('Node modules path {} not found'.format(NODE_MODULES_PATH)) + return False + else: + if not self._is_nativefier_installed(): + return False + + installed_version = nativefier.get_version() + + if installed_version: + installed_version = installed_version.strip() + + self.logger.info("Nativefier versions: installed ({}), cloud ({})".format(installed_version, nativefier_settings['version'])) + + if nativefier_settings['version'] != installed_version: + self.logger.info("Installed nativefier version is different from cloud's. Changing version.") + return False + + self.logger.info("Nativefier is already installed and up to date") + return True + + def _map_nativefier_file(self, nativefier_settings: dict) -> EnvironmentComponent: + url = NATIVEFIER_URL.format(version=nativefier_settings['version']) + return EnvironmentComponent(name='nativefier@{}'.format(nativefier_settings['version']), + url=url, + size=self.http_client.get_content_length(url), + version=nativefier_settings['version'], + id='nativefier') + + def check_environment(self, app: WebApplication, is_x86_x64_arch: bool) -> Tuple[dict, List[EnvironmentComponent]]: + to_update, check_threads = [], [] + + env = self.read_settings() + local_config = read_config() + + system_env = local_config['environment'].get('system', False) + + if system_env: + self.logger.warning("Using system's nativefier to install {}".format(app.url)) + else: + node_check = Thread(target=self._check_node_async, args=(env, to_update)) + node_check.start() + check_threads.append(node_check) + + elec_check = Thread(target=self._check_electron_async, args=(app, env, local_config, is_x86_x64_arch, to_update)) + elec_check.start() + check_threads.append(elec_check) + + for t in check_threads: + t.join() + + return env, to_update + + def update_environment(self, is_x86_x64_arch: bool, config: dict = None, handler: ProcessHandler = None) -> dict: settings = self.read_settings() if settings is None: @@ -226,9 +366,10 @@ class EnvironmentUpdater: current_config = config if config else read_config() system_env = current_config['environment']['system'] + watcher = handler.watcher if handler else None if not system_env and not self.update_node(version=settings['nodejs']['version'], version_url=settings['nodejs']['url'], - watcher=handler.watcher if handler else None): + watcher=watcher): self.logger.warning('Could not install / update NodeJS') return @@ -243,8 +384,17 @@ class EnvironmentUpdater: else: electron_version = settings['electron']['version'] - res = self.install_electron(version=electron_version, is_x86_x64_arch=is_x86_x64_arch, - watcher=handler.watcher if handler else None) + installed = self.check_installed_electron(version=electron_version, is_x86_x64_arch=is_x86_x64_arch) + + if not installed['electron']: + if self.download_electron(version=electron_version, is_x86_x64_arch=is_x86_x64_arch, watcher=watcher): + res = self.download_electron_sha256(version=electron_version, watcher=watcher) + else: + res = False + elif not installed['sha256']: + res = self.download_electron_sha256(version=electron_version, watcher=watcher) + else: + res = True if res: self.logger.info('Environment updated') @@ -252,3 +402,36 @@ class EnvironmentUpdater: self.logger.warning('Could not update the environment') return settings + + def update(self, components: List[EnvironmentComponent], handler: ProcessHandler) -> bool: + self.logger.info('Updating environment') + + comp_map = {c.id: c for c in components} + + node_data = comp_map.get('node') + nativefier_data = comp_map.get('nativefier') + + if node_data: + if not self._download_and_install(version=node_data.version, version_url=node_data.url, watcher=handler.watcher): + return False + + if not self._install_nativefier(version=nativefier_data.version, handler=handler): + return False + else: + if nativefier_data and not self._install_nativefier(version=nativefier_data.version, handler=handler): + return False + + electron_data = comp_map.get('electron') + + if electron_data: + if not self.download_electron(version=electron_data.version, url=electron_data.url, watcher=handler.watcher): + return False + + sha256_data = comp_map.get('electron_sha256') + + if sha256_data: + if not self.download_electron_sha256(version=sha256_data.version, url=sha256_data.url, watcher=handler.watcher): + return False + + self.logger.info('Environment successfully updated') + return True diff --git a/bauh/gems/web/resources/locale/en b/bauh/gems/web/resources/locale/en index 0fe8da0f..7d6a6e23 100644 --- a/bauh/gems/web/resources/locale/en +++ b/bauh/gems/web/resources/locale/en @@ -4,6 +4,8 @@ web.environment.install=Installing {} web.waiting.env_updater=Updating environment web.env.checking=Checking the Web installation environment web.env.error=It seems there are issues with the Web installation environment. It wil not be possible to install {}. +web.install.env_update.title=Environment update +web.install.env_update.title=The following files must be downloaded and installed web.install.options.basic=basic web.install.options.advanced=advanced web.install.options_dialog.title=Installation options diff --git a/bauh/gems/web/resources/locale/pt b/bauh/gems/web/resources/locale/pt index 06bac2f1..6c3ebdd7 100644 --- a/bauh/gems/web/resources/locale/pt +++ b/bauh/gems/web/resources/locale/pt @@ -4,6 +4,8 @@ web.environment.nativefier=Instalando {} web.waiting.env_updater=Atualizando ambiente web.env.checking=Verificando o ambiente de instalação Web web.env.error=Parce que existem problemas com o ambiente de instalação Web. Não será possível instalar {}. +web.install.env_update.title=Atualização de ambiente +web.install.env_update.body=Os seguintes arquivo precisam ser baixados e instalados web.install.options.basic=básicas web.install.options.advanced=avançadas web.install.options_dialog.title=Opções de instalação