[wgem] display required environment files to download and install

bauh/gems/web/__init__.py

@@ -25,3 +25,5 @@ TEMP_PATH = '/tmp/bauh/web'
 SEARCH_INDEX_FILE = '{}/index.yml'.format(TEMP_PATH)
 SUGGESTIONS_CACHE_FILE = '{}/suggestions.yml'.format(TEMP_PATH)
 CONFIG_FILE = '{}/web.yml'.format(CONFIG_PATH)
+NATIVEFIER_URL = 'https://github.com/jiahaog/nativefier/archive/v7.7.0.tar.gz'
+

bauh/gems/web/controller.py

@@ -22,12 +22,13 @@ from bauh.api.abstract.model import SoftwarePackage, PackageAction, PackageSugge
 from bauh.api.abstract.view import MessageType, MultipleSelectComponent, InputOption, SingleSelectComponent, \
     SelectViewType, TextInputComponent, FormComponent, FileChooserComponent
 from bauh.api.constants import DESKTOP_ENTRIES_DIR
+from bauh.commons import resource
 from bauh.commons.html import bold
 from bauh.commons.system import ProcessHandler, get_dir_size, get_human_size_str
 from bauh.gems.web import INSTALLED_PATH, nativefier, DESKTOP_ENTRY_PATH_PATTERN, URL_FIX_PATTERN, ENV_PATH, UA_CHROME, \
-    SEARCH_INDEX_FILE, SUGGESTIONS_CACHE_FILE
+    SEARCH_INDEX_FILE, SUGGESTIONS_CACHE_FILE, ROOT_DIR
 from bauh.gems.web.config import read_config
-from bauh.gems.web.environment import EnvironmentUpdater
+from bauh.gems.web.environment import EnvironmentUpdater, EnvironmentComponent
 from bauh.gems.web.model import WebApplication
 from bauh.gems.web.worker import SuggestionsDownloader, SearchIndexGenerator
 
@@ -54,8 +55,8 @@ class WebApplicationManager(SoftwareManager):
     def __init__(self, context: ApplicationContext, suggestions_downloader: Thread = None):
         super(WebApplicationManager, self).__init__(context=context)
         self.http_client = context.http_client
-        self.node_updater = EnvironmentUpdater(logger=context.logger, http_client=context.http_client,
+        self.env_updater = EnvironmentUpdater(logger=context.logger, http_client=context.http_client,
-                                               file_downloader=context.file_downloader, i18n=context.i18n)
+                                              file_downloader=context.file_downloader, i18n=context.i18n)
         self.enabled = True
         self.i18n = context.i18n
         self.env_settings = {}
@@ -450,6 +451,20 @@ class WebApplicationManager(SoftwareManager):
                 base_id = '{}_{}'.format(app_id, counter)
                 counter += 1
 
+    def _ask_update_permission(self, to_update: List[EnvironmentComponent], watcher: ProcessWatcher) -> bool:
+
+        icon = resource.get_path('img/web.png', ROOT_DIR)
+        opts = [InputOption(label='{} ( {} )'.format(f.name, f.size or '?'),
+                            tooltip=f.url, icon_path=icon, read_only=True, value=f.name) for f in to_update]
+
+        comps = MultipleSelectComponent(label=None, options=opts, default_options=set(opts))
+
+        return watcher.request_confirmation(title=self.i18n['web.install.env_update.title'],
+                                            body=self.i18n['web.install.env_update.body'],
+                                            components=[comps],
+                                            confirmation_label=self.i18n['continue'].capitalize(),
+                                            deny_label=self.i18n['cancel'].capitalize())
+
     def install(self, pkg: WebApplication, root_password: str, watcher: ProcessWatcher) -> bool:
 
         continue_install, install_options = self._ask_install_options(pkg, watcher)
@@ -460,10 +475,19 @@ class WebApplicationManager(SoftwareManager):
 
         watcher.change_substatus(self.i18n['web.env.checking'])
         handler = ProcessHandler(watcher)
-        if not self._update_environment(handler=handler):
+
+        env_settings, to_update = self.env_updater.check_environment(app=pkg, is_x86_x64_arch=self.context.is_system_x86_64())
+
+        if to_update:
+            if not self._ask_update_permission(to_update, watcher):
+                return False
+
+        if not self.env_updater.update(components=to_update,  handler=handler):
             watcher.show_message(title=self.i18n['error'], body=self.i18n['web.env.error'].format(bold(pkg.name)), type_=MessageType.ERROR)
             return False
 
+        # TODO stopped here
+
         Path(INSTALLED_PATH).mkdir(parents=True, exist_ok=True)
 
         app_id, treated_name = self._gen_app_id(pkg.name)
@@ -480,25 +504,8 @@ class WebApplicationManager(SoftwareManager):
             watcher.print('Fix found for {}'.format(pkg.url))
             install_options.append('--inject={}'.format(fix_path))
 
-        config = read_config()
-
         watcher.change_substatus(self.i18n['web.install.substatus.call_nativefier'].format(bold('nativefier')))
 
-        system_env = config['environment'].get('system', False)
-
-        if system_env:
-            self.logger.warning("Using system's nativefier to install {}".format(pkg.url))
-
-        electron_version = self.env_settings['electron']['version']
-
-        if pkg.version and pkg.version != electron_version:
-            self.logger.info('A preset Electron version is defined for {}: {}'.format(pkg.url, pkg.version))
-            electron_version = pkg.version
-
-        if config['environment']['electron']['version']:
-            self.logger.warning("A custom Electron version will be used {} to install {}".format(electron_version, pkg.url))
-            electron_version = config['environment']['electron']['version']
-
         installed = handler.handle_simple(nativefier.install(url=pkg.url, name=app_id, output_dir=app_dir,
                                                              electron_version=electron_version,
                                                              system=system_env,
@@ -597,17 +604,17 @@ class WebApplicationManager(SoftwareManager):
     def requires_root(self, action: str, pkg: SoftwarePackage):
         return False
 
-    def _update_environment(self, handler: ProcessHandler = None) -> bool:
+    # def _update_environment(self, handler: ProcessHandler = None) -> bool:
-        updated_settings = self.node_updater.update_environment(self.context.is_system_x86_64(), handler=handler)
+    #     updated_settings = self.env_updater.update_environment(self.context.is_system_x86_64(), handler=handler)
-
+    #
-        if updated_settings is not None:
+    #     if updated_settings is not None:
-            self.env_settings = updated_settings
+    #         self.env_settings = updated_settings
-            return True
+    #         return True
-
+    #
-        return False
+    #     return False
 
     def _update_env_settings(self):
-        self.env_settings = self.node_updater.read_settings()
+        self.env_settings = self.env_updater.read_settings()
 
     def _download_suggestions(self):
         downloader = SuggestionsDownloader(logger=self.logger, http_client=self.http_client)

bauh/gems/web/environment.py

@@ -3,6 +3,8 @@ import os
 import shutil
 import tarfile
 from pathlib import Path
+from threading import Thread
+from typing import Dict, List, Tuple
 
 import requests
 import yaml
@@ -15,11 +17,22 @@ from bauh.commons.html import bold
 from bauh.commons.system import SimpleProcess, ProcessHandler
 from bauh.gems.web import ENV_PATH, NODE_DIR_PATH, NODE_BIN_PATH, NODE_MODULES_PATH, NATIVEFIER_BIN_PATH, \
     ELECTRON_PATH, ELECTRON_DOWNLOAD_URL, ELECTRON_SHA256_URL, URL_ENVIRONMENT_SETTINGS, NPM_BIN_PATH, NODE_PATHS, \
-    nativefier
+    nativefier, NATIVEFIER_URL
 from bauh.gems.web.config import read_config
+from bauh.gems.web.model import WebApplication
 from bauh.view.util.translation import I18n
 
 
+class EnvironmentComponent:
+
+    def __init__(self, id: str, name: str, size: str, version: str, url: str):
+        self.id = id
+        self.name = name
+        self.size = size
+        self.version = version
+        self.url = url
+
+
 class EnvironmentUpdater:
 
     def __init__(self, logger: logging.Logger, http_client: HttpClient, file_downloader: FileDownloader, i18n: I18n):
@@ -45,6 +58,15 @@ class EnvironmentUpdater:
                 extracted_file = '{}/{}'.format(ENV_PATH, tf.getnames()[0])
 
                 os.rename(extracted_file, NODE_DIR_PATH)
+
+                if os.path.exists(NODE_MODULES_PATH):
+                    self.logger.info('Deleting {}'.format(NODE_MODULES_PATH))
+                    try:
+                        shutil.rmtree(NODE_MODULES_PATH)
+                    except:
+                        self.logger.error("Could not delete the directory {}".format(NODE_MODULES_PATH))
+                        return False
+
                 return True
             except:
                 self.logger.error('Could not extract {}'.format(tarf_path))
@@ -56,6 +78,31 @@ class EnvironmentUpdater:
                     except:
                         self.logger.error('Could not delete file {}'.format(tarf_path))
 
+
+    def check_node_installed(self, version: str) -> bool:
+        if not os.path.exists(NODE_DIR_PATH):
+            return False
+        else:
+            installed_version = system.run_cmd('{} --version'.format(NODE_BIN_PATH), print_error=False)
+
+            if installed_version:
+                installed_version = installed_version.strip()
+
+                if installed_version.startswith('v'):
+                    installed_version = installed_version[1:]
+
+                self.logger.info('Node versions: installed ({}), cloud ({})'.format(installed_version, version))
+
+                if version != installed_version:
+                    self.logger.info("The NodeJs installed version is different from the Cloud.")
+                    return False
+                else:
+                    self.logger.info("Node is already up to date")
+                    return True
+            else:
+                self.logger.warning("Could not determine the current NodeJS installed version")
+                return False
+
     def update_node(self, version: str, version_url: str, watcher: ProcessWatcher = None) -> bool:
         Path(ENV_PATH).mkdir(parents=True, exist_ok=True)
 
@@ -149,57 +196,42 @@ class EnvironmentUpdater:
             self.logger.info("Nativefier is already installed and up to date")
             return True
 
-    def _download_electron(self, version: str, is_x86_x64_arch: bool, watcher: ProcessWatcher) -> bool:
+    def download_electron(self, version: str, url: str, watcher: ProcessWatcher) -> bool:
         self.logger.info("Downloading Electron {}".format(version))
-        arch = 'ia32' if not is_x86_x64_arch else 'x64'
+        electron_path = '{}/{}'.format(ELECTRON_PATH, url.split('/')[-1])
+        return self.file_downloader.download(file_url=url, watcher=watcher, output_path=electron_path, cwd=ELECTRON_PATH)
 
-        electron_url = ELECTRON_DOWNLOAD_URL.format(arch=arch, version=version)
+    def download_electron_sha256(self, version: str, url: str, watcher: ProcessWatcher) -> bool:
-        electron_path = '{}/{}'.format(ELECTRON_PATH, electron_url.split('/')[-1])
-
-        return self.file_downloader.download(file_url=electron_url, watcher=watcher, output_path=electron_path, cwd=ELECTRON_PATH)
-
-    def _download_electron_sha256(self, version: str, watcher: ProcessWatcher) -> bool:
         self.logger.info("Downloading Electron {} sha526".format(version))
+        sha256_path = '{}/{}'.format(ELECTRON_PATH, url.split('/')[-1])
+        return self.file_downloader.download(file_url=url, watcher=watcher, output_path=sha256_path, cwd=ELECTRON_PATH)
 
-        sha256_url = ELECTRON_SHA256_URL.format(version=version)
+    def _get_electron_url(self, version: str, is_x86_x64_arch: bool) -> str:
-        sha256_path = '{}/{}'.format(ELECTRON_PATH, sha256_url.split('/')[-1])
+        return ELECTRON_DOWNLOAD_URL.format(version=version, arch='x64' if is_x86_x64_arch else 'ia32')
-        return self.file_downloader.download(file_url=sha256_url, watcher=watcher, output_path=sha256_path, cwd=ELECTRON_PATH)
 
-    def install_electron(self, version: str, is_x86_x64_arch: bool, watcher: ProcessWatcher) -> bool:
+    def check_installed_electron(self, version: str, is_x86_x64_arch: bool) -> Dict[str, bool]:
-        self.logger.info("Checking installed Electron")
+        self.logger.info("Checking if Electron {} is installed".format(version))
+        res = {'electron': False, 'sha256': False}
 
         if not os.path.exists(ELECTRON_PATH):
-            self.logger.info("Electron is not installed")
+            self.logger.info("The Electron folder {} was not found".format(ELECTRON_PATH))
-            Path(ELECTRON_PATH).mkdir(parents=True, exist_ok=True)
-            if self._download_electron(version=version, is_x86_x64_arch=is_x86_x64_arch, watcher=watcher):
-                return self._download_electron_sha256(version=version, watcher=watcher)
         else:
             files = os.listdir(ELECTRON_PATH)
 
             if files:
-                file_name = ELECTRON_DOWNLOAD_URL.format(version=version, arch='x64' if is_x86_x64_arch else 'ia32').split('/')[-1]
+                file_name = self._get_electron_url(version, is_x86_x64_arch).split('/')[-1]
-                electron_file = [f for f in files if f == file_name]
+                res['electron'] = bool([f for f in files if f == file_name])
-
-                if electron_file:
-                    self.logger.info("Electron {} already downloaded".format(version))
-                else:
-                    if not self._download_electron(version=version, is_x86_x64_arch=is_x86_x64_arch, watcher=watcher):
-                        return False
 
                 file_name = ELECTRON_SHA256_URL.split('/')[-1]
-                sha256_file = [f for f in files if f == file_name]
+                res['sha256'] = bool([f for f in files if f == file_name])
+            else:
+                self.logger.info('No Electron file found in {}'.format(ELECTRON_PATH))
 
-                if sha256_file:
+            for att in ('electron', 'sha256'):
-                    self.logger.info("Electron {} sha256 already downloaded".format(version))
+                if res[att]:
-                    return True
+                    self.logger.info('{} ({}) already downloaded'.format(att, version))
-                else:
-                    return self._download_electron_sha256(version=version, watcher=watcher)
 
-            self.logger.info('No Electron file found')
+        return res
-            if self._download_electron(version=version, is_x86_x64_arch=is_x86_x64_arch, watcher=watcher):
-                return self._download_electron_sha256(version=version, watcher=watcher)
-
-            return False
 
     def read_settings(self) -> dict:
         try:
@@ -217,8 +249,116 @@ class EnvironmentUpdater:
         except requests.exceptions.ConnectionError:
             return
 
-    def update_environment(self, is_x86_x64_arch: bool, config: dict = None, handler: ProcessHandler = None) -> dict:
+    def _fill_electron_to_install(self, electron_version: str, x86_x64: bool, output: List[EnvironmentComponent]):
+        electron_url = self._get_electron_url(version=electron_version, is_x86_x64_arch=x86_x64)
+        output.append(EnvironmentComponent(name=electron_url.split('/')[-1],
+                                           version=electron_version,
+                                           url=electron_url,
+                                           size=self.http_client.get_content_length(electron_url),
+                                           id='electron'))
 
+    def _fill_electron_sha256_to_install(self, electron_version: str, output: List[EnvironmentComponent]):
+        sha_url = ELECTRON_SHA256_URL.format(version=electron_version)
+        output.append(EnvironmentComponent(name=sha_url.split('/')[-1],
+                                           version=electron_version,
+                                           url=sha_url,
+                                           size=self.http_client.get_content_length(sha_url),
+                                           id='electron_sha256'))
+
+    def _check_electron_async(self, pkg: WebApplication, env: dict, local_config: dict, x86_x64: bool, output: List[EnvironmentComponent]):
+        electron_version = env['electron']['version']
+
+        if pkg.version and pkg.version != electron_version:
+            self.logger.info('A preset Electron version is defined for {}: {}'.format(pkg.url, pkg.version))
+            electron_version = pkg.version
+
+        if local_config['environment']['electron']['version']:
+            self.logger.warning("A custom Electron version will be used {} to install {}".format(electron_version, pkg.url))
+            electron_version = local_config['environment']['electron']['version']
+
+        electron_status = self.check_installed_electron(version=electron_version, is_x86_x64_arch=x86_x64)
+        threads = []
+
+        if not electron_status['electron']:
+            t = Thread(target=self._fill_electron_to_install, args=(electron_version, x86_x64, output))
+            t.start()
+            threads.append(t)
+
+        if not electron_status['sha256']:
+            t = Thread(target=self._fill_electron_sha256_to_install, args=(electron_version, output))
+            t.start()
+            threads.append(t)
+
+        for t in threads:
+            t.join()
+
+    def _check_node_async(self, env: dict, output: List[EnvironmentComponent]):
+        if not self.check_node_installed(env['nodejs']['version']):
+            output.append(EnvironmentComponent(name=env['nodejs']['url'].split('/')[-1],
+                                               url=env['nodejs']['url'],
+                                               size=self.http_client.get_content_length(env['nodejs']['url']),
+                                               version=env['nodejs']['version'],
+                                               id='nodejs'))
+            output.append(self._map_nativefier_file(env['nativefier']))
+        else:
+            if not self._check_nativefier_installed(env['nativefier']):
+                output.append(self._map_nativefier_file(env['nativefier']))
+
+    def _check_nativefier_installed(self, nativefier_settings: dict) -> bool:
+        if not os.path.exists(NODE_MODULES_PATH):
+            self.logger.info('Node modules path {} not found'.format(NODE_MODULES_PATH))
+            return False
+        else:
+            if not self._is_nativefier_installed():
+                return False
+
+            installed_version = nativefier.get_version()
+
+            if installed_version:
+                installed_version = installed_version.strip()
+
+            self.logger.info("Nativefier versions: installed ({}), cloud ({})".format(installed_version, nativefier_settings['version']))
+
+            if nativefier_settings['version'] != installed_version:
+                self.logger.info("Installed nativefier version is different from cloud's. Changing version.")
+                return False
+
+            self.logger.info("Nativefier is already installed and up to date")
+            return True
+
+    def _map_nativefier_file(self, nativefier_settings: dict) -> EnvironmentComponent:
+        url = NATIVEFIER_URL.format(version=nativefier_settings['version'])
+        return EnvironmentComponent(name='nativefier@{}'.format(nativefier_settings['version']),
+                                    url=url,
+                                    size=self.http_client.get_content_length(url),
+                                    version=nativefier_settings['version'],
+                                    id='nativefier')
+
+    def check_environment(self, app: WebApplication, is_x86_x64_arch: bool) -> Tuple[dict, List[EnvironmentComponent]]:
+        to_update, check_threads = [], []
+
+        env = self.read_settings()
+        local_config = read_config()
+
+        system_env = local_config['environment'].get('system', False)
+
+        if system_env:
+            self.logger.warning("Using system's nativefier to install {}".format(app.url))
+        else:
+            node_check = Thread(target=self._check_node_async, args=(env, to_update))
+            node_check.start()
+            check_threads.append(node_check)
+
+        elec_check = Thread(target=self._check_electron_async, args=(app, env, local_config, is_x86_x64_arch, to_update))
+        elec_check.start()
+        check_threads.append(elec_check)
+
+        for t in check_threads:
+            t.join()
+
+        return env, to_update
+
+    def update_environment(self, is_x86_x64_arch: bool, config: dict = None, handler: ProcessHandler = None) -> dict:
         settings = self.read_settings()
 
         if settings is None:
@@ -226,9 +366,10 @@ class EnvironmentUpdater:
 
         current_config = config if config else read_config()
         system_env = current_config['environment']['system']
+        watcher = handler.watcher if handler else None
 
         if not system_env and not self.update_node(version=settings['nodejs']['version'], version_url=settings['nodejs']['url'],
-                                                   watcher=handler.watcher if handler else None):
+                                                   watcher=watcher):
             self.logger.warning('Could not install / update NodeJS')
             return
 
@@ -243,8 +384,17 @@ class EnvironmentUpdater:
         else:
             electron_version = settings['electron']['version']
 
-        res = self.install_electron(version=electron_version, is_x86_x64_arch=is_x86_x64_arch,
+        installed = self.check_installed_electron(version=electron_version, is_x86_x64_arch=is_x86_x64_arch)
-                                    watcher=handler.watcher if handler else None)
+
+        if not installed['electron']:
+            if self.download_electron(version=electron_version, is_x86_x64_arch=is_x86_x64_arch, watcher=watcher):
+                res = self.download_electron_sha256(version=electron_version, watcher=watcher)
+            else:
+                res = False
+        elif not installed['sha256']:
+            res = self.download_electron_sha256(version=electron_version, watcher=watcher)
+        else:
+            res = True
 
         if res:
             self.logger.info('Environment updated')
@@ -252,3 +402,36 @@ class EnvironmentUpdater:
             self.logger.warning('Could not update the environment')
 
         return settings
+
+    def update(self, components: List[EnvironmentComponent], handler: ProcessHandler) -> bool:
+        self.logger.info('Updating  environment')
+
+        comp_map = {c.id: c for c in components}
+
+        node_data = comp_map.get('node')
+        nativefier_data = comp_map.get('nativefier')
+
+        if node_data:
+            if not self._download_and_install(version=node_data.version, version_url=node_data.url, watcher=handler.watcher):
+                return False
+
+            if not self._install_nativefier(version=nativefier_data.version, handler=handler):
+                return False
+        else:
+            if nativefier_data and not self._install_nativefier(version=nativefier_data.version, handler=handler):
+                return False
+
+        electron_data = comp_map.get('electron')
+
+        if electron_data:
+            if not self.download_electron(version=electron_data.version, url=electron_data.url, watcher=handler.watcher):
+                return False
+
+        sha256_data = comp_map.get('electron_sha256')
+
+        if sha256_data:
+            if not self.download_electron_sha256(version=sha256_data.version, url=sha256_data.url, watcher=handler.watcher):
+                return False
+
+        self.logger.info('Environment successfully updated')
+        return True

bauh/gems/web/resources/locale/en

@@ -4,6 +4,8 @@ web.environment.install=Installing {}
 web.waiting.env_updater=Updating environment
 web.env.checking=Checking the Web installation environment
 web.env.error=It seems there are issues with the Web installation environment. It wil not be possible to install {}.
+web.install.env_update.title=Environment update
+web.install.env_update.title=The following files must be downloaded and installed
 web.install.options.basic=basic
 web.install.options.advanced=advanced
 web.install.options_dialog.title=Installation options

bauh/gems/web/resources/locale/pt

@@ -4,6 +4,8 @@ web.environment.nativefier=Instalando {}
 web.waiting.env_updater=Atualizando ambiente
 web.env.checking=Verificando o ambiente de instalação Web
 web.env.error=Parce que existem problemas com o ambiente de instalação Web. Não será possível instalar {}.
+web.install.env_update.title=Atualização de ambiente
+web.install.env_update.body=Os seguintes arquivo precisam ser baixados e instalados
 web.install.options.basic=básicas
 web.install.options.advanced=avançadas
 web.install.options_dialog.title=Opções de instalação