bearhub/bauh/gems/web/environment.py

import glob
import logging
import os
import shutil
import tarfile
import traceback
from datetime import datetime, timedelta
from pathlib import Path
from threading import Thread
from typing import Dict, List, Optional

import requests
import yaml

from bauh.api.abstract.download import FileDownloader
from bauh.api.abstract.handler import ProcessWatcher, TaskManager
from bauh.api.abstract.view import MessageType
from bauh.api.http import HttpClient
from bauh.commons import system
from bauh.commons.html import bold
from bauh.commons.system import SimpleProcess, ProcessHandler
from bauh.gems.web import ENV_PATH, NODE_DIR_PATH, NODE_BIN_PATH, NODE_MODULES_PATH, NATIVEFIER_BIN_PATH, \
    ELECTRON_CACHE_DIR, URL_ENVIRONMENT_SETTINGS, NPM_BIN_PATH, NODE_PATHS, \
    nativefier, ENVIRONMENT_SETTINGS_CACHED_FILE, ENVIRONMENT_SETTINGS_TS_FILE, get_icon_path
from bauh.gems.web.model import WebApplication
from bauh.view.util.translation import I18n


class EnvironmentComponent:

    def __init__(self, id: str, name: str, size: str, version: str, url: str, update: bool = False, properties: Optional[dict] = None):
        self.id = id
        self.name = name
        self.size = size
        self.version = version
        self.url = url
        self.update = update
        self.properties = properties


class EnvironmentUpdater:

    def __init__(self, logger: logging.Logger, http_client: HttpClient, file_downloader: FileDownloader, i18n: I18n, taskman: Optional[TaskManager] = None):
        self.logger = logger
        self.file_downloader = file_downloader
        self.i18n = i18n
        self.http_client = http_client
        self.task_read_settings_id = 'web_read_settings'
        self.taskman = taskman

    def _install_nodejs(self, version: str, version_url: str, watcher: ProcessWatcher) -> bool:
        self.logger.info(f"Downloading NodeJS {version}: {version_url}")

        tarf_path = f"{ENV_PATH}/{version_url.split('/')[-1]}"
        downloaded = self.file_downloader.download(version_url, watcher=watcher, output_path=tarf_path, cwd=ENV_PATH)

        if not downloaded:
            self.logger.error(f"Could not download '{version_url}'. Aborting...")
            return False
        else:
            try:
                tf = tarfile.open(tarf_path)
                tf.extractall(path=ENV_PATH)

                extracted_file = f'{ENV_PATH}/{tf.getnames()[0]}'

                if os.path.exists(NODE_DIR_PATH):
                    self.logger.info(f"Removing old NodeJS version installation dir -> {NODE_DIR_PATH}")
                    try:
                        shutil.rmtree(NODE_DIR_PATH)
                    except Exception:
                        self.logger.error(f"Could not delete old NodeJS version dir -> {NODE_DIR_PATH}")
                        traceback.print_exc()
                        return False

                try:
                    os.rename(extracted_file, NODE_DIR_PATH)
                except Exception:
                    self.logger.error(f"Could not rename the NodeJS version file {extracted_file} as {NODE_DIR_PATH}")
                    traceback.print_exc()
                    return False

                if os.path.exists(NODE_MODULES_PATH):
                    self.logger.info(f'Deleting {NODE_MODULES_PATH}')
                    try:
                        shutil.rmtree(NODE_MODULES_PATH)
                    except Exception:
                        self.logger.error(f"Could not delete the directory {NODE_MODULES_PATH}")
                        return False

                return True
            except Exception:
                self.logger.error(f'Could not extract {tarf_path}')
                traceback.print_exc()
                return False
            finally:
                if os.path.exists(tarf_path):
                    try:
                        os.remove(tarf_path)
                    except Exception:
                        self.logger.error(f'Could not delete file {tarf_path}')

    def check_node_installed(self, version: str) -> bool:
        if not os.path.exists(NODE_DIR_PATH):
            return False
        else:
            installed_version = system.run_cmd(f'{NODE_BIN_PATH} --version', print_error=False)

            if installed_version:
                installed_version = installed_version.strip()

                if installed_version.startswith('v'):
                    installed_version = installed_version[1:]

                self.logger.info(f'Node versions: installed ({installed_version}), cloud ({version})')

                if version != installed_version:
                    self.logger.info("The NodeJs installed version is different from the Cloud.")
                    return False
                else:
                    self.logger.info("Node is already up to date")
                    return True
            else:
                self.logger.warning("Could not determine the current NodeJS installed version")
                return False

    def update_node(self, version: str, version_url: str, watcher: ProcessWatcher = None) -> bool:
        Path(ENV_PATH).mkdir(parents=True, exist_ok=True)

        if not os.path.exists(NODE_DIR_PATH):
            return self._install_nodejs(version=version, version_url=version_url, watcher=watcher)
        else:
            installed_version = system.run_cmd('{} --version'.format(NODE_BIN_PATH), print_error=False)

            if installed_version:
                installed_version = installed_version.strip()

                if installed_version.startswith('v'):
                    installed_version = installed_version[1:]

                self.logger.info(f'Node versions: installed ({installed_version}), cloud ({version})')

                if version != installed_version:
                    self.logger.info("The NodeJs installed version is different from the Cloud.")
                    return self._install_nodejs(version=version, version_url=version_url, watcher=watcher)
                else:
                    self.logger.info("Node is already up to date")
                    return True
            else:
                self.logger.warning("Could not determine the current NodeJS installed version")
                self.logger.info(f"Removing {NODE_DIR_PATH}")
                try:
                    shutil.rmtree(NODE_DIR_PATH)
                    return self._install_nodejs(version=version, version_url=version_url, watcher=watcher)
                except Exception:
                    self.logger.error(f'Could not delete the dir {NODE_DIR_PATH}')
                    return False

    def _install_node_lib(self, name: str, version: str, handler: ProcessHandler):
        lib_repr = f"{name}{'@{}'.format(version) if version else ''}"
        self.logger.info(f"Installing {lib_repr}")

        if handler and handler.watcher:
            handler.watcher.change_substatus(self.i18n['web.environment.install'].format(bold(lib_repr)))

        proc = SimpleProcess([NPM_BIN_PATH, 'install', lib_repr], cwd=ENV_PATH, extra_paths=NODE_PATHS)

        installed = handler.handle_simple(proc)[0]

        if installed:
            self.logger.info(f"{lib_repr} successfully installed")

        return installed

    def _install_nativefier(self, version: str, url: str, handler: ProcessHandler) -> bool:
        self.logger.info(f"Checking if nativefier@{version} exists")

        if not url or not self.http_client.exists(url):
            self.logger.warning(f"The file {url} seems not to exist")
            handler.watcher.show_message(title=self.i18n['message.file.not_exist'],
                                         body=self.i18n['message.file.not_exist.body'].format(bold(url)),
                                         type_=MessageType.ERROR)
            return False

        success = self._install_node_lib('nativefier', version, handler)

        if success:
            return self._is_nativefier_installed()

    def _is_nativefier_installed(self) -> bool:
        return os.path.exists(NATIVEFIER_BIN_PATH)

    def _get_electron_url(self, version: str, base_url: str, is_x86_x64_arch: bool) -> str:
        return base_url.format(version=version, arch='x64' if is_x86_x64_arch else 'ia32')

    def check_electron_installed(self, version: str, base_url: str, is_x86_x64_arch: bool, widevine: bool) -> Dict[str, bool]:
        self.logger.info(f"Checking if Electron {version} (widevine={widevine}) is installed")
        res = {'electron': False, 'sha256': False}

        if not os.path.exists(ELECTRON_CACHE_DIR):
            self.logger.info(f"Electron cache directory {ELECTRON_CACHE_DIR} not found")
        else:
            files = {os.path.basename(f) for f in glob.glob(f'{ELECTRON_CACHE_DIR}/**', recursive=True) if os.path.isfile(f)}

            if files:
                electron_url = self._get_electron_url(version=version, base_url=base_url, is_x86_x64_arch=is_x86_x64_arch)
                res['electron'] = os.path.basename(electron_url) in files
                res['sha256'] = res['electron']
            else:
                self.logger.info(f"No Electron file found in '{ELECTRON_CACHE_DIR}'")

            for att in ('electron', 'sha256'):
                if res[att]:
                    self.logger.info(f'{att} ({version}) already downloaded')

        return res

    def _finish_task_download_settings(self):
        if self.taskman:
            self.taskman.update_progress(self.task_read_settings_id, 100, None)
            self.taskman.finish_task(self.task_read_settings_id)

    def should_download_settings(self, web_config: dict) -> bool:
        try:
            settings_exp = int(web_config['environment']['cache_exp'])
        except ValueError:
            self.logger.error(f"Could not parse settings property 'environment.cache_exp': {web_config['environment']['cache_exp']}")
            return True

        if settings_exp <= 0:
            self.logger.info("No expiration time configured for the environment settings cache file.")
            return True

        self.logger.info("Checking cached environment settings file")

        if not os.path.exists(ENVIRONMENT_SETTINGS_CACHED_FILE):
            self.logger.warning("Environment settings file not cached.")
            return True

        if not os.path.exists(ENVIRONMENT_SETTINGS_TS_FILE):
            self.logger.warning("Environment settings file has no timestamp associated with it.")
            return True

        with open(ENVIRONMENT_SETTINGS_TS_FILE) as f:
            env_ts_str = f.read()

        try:
            env_timestamp = datetime.fromtimestamp(float(env_ts_str))
        except Exception:
            self.logger.error(f"Could not parse environment settings file timestamp: {env_ts_str}")
            return True

        expired = env_timestamp + timedelta(hours=settings_exp) <= datetime.utcnow()

        if expired:
            self.logger.info("Environment settings file has expired. It should be re-downloaded")
            return True
        else:
            self.logger.info("Cached environment settings file is up to date")
            return False

    def read_cached_settings(self, web_config: dict) -> Optional[dict]:
        if not self.should_download_settings(web_config):
            with open(ENVIRONMENT_SETTINGS_CACHED_FILE) as f:
                cached_settings_str = f.read()

            try:
                return yaml.safe_load(cached_settings_str)
            except yaml.YAMLError:
                self.logger.error(f'Could not parse the cache environment settings file: {cached_settings_str}')

    def read_settings(self, web_config: dict, cache: bool = True) -> Optional[dict]:
        if self.taskman:
            self.taskman.register_task(self.task_read_settings_id, self.i18n['web.task.download_settings'], get_icon_path())
            self.taskman.update_progress(self.task_read_settings_id, 1, None)

        cached_settings = self.read_cached_settings(web_config) if cache else None

        if cached_settings:
            return cached_settings

        try:
            if self.taskman:
                self.taskman.update_progress(self.task_read_settings_id, 10, None)

            self.logger.info("Downloading environment settings")
            res = self.http_client.get(URL_ENVIRONMENT_SETTINGS)

            if not res:
                self.logger.warning('Could not retrieve the environments settings from the cloud')
                self._finish_task_download_settings()
                return

            try:
                settings = yaml.safe_load(res.content)
                nodejs_settings = settings.get('nodejs')

                if nodejs_settings:
                    nodejs_settings['url'] = nodejs_settings['url'].format(version=nodejs_settings['version'])

            except yaml.YAMLError:
                self.logger.error(f'Could not parse environment settings: {res.text}')
                self._finish_task_download_settings()
                return

            self.logger.info("Caching environment settings to disk")
            cache_dir = os.path.dirname(ENVIRONMENT_SETTINGS_CACHED_FILE)

            try:
                Path(cache_dir).mkdir(parents=True, exist_ok=True)
            except OSError:
                self.logger.error(f"Could not create Web cache directory: {cache_dir}")
                self.logger.info('Finished')
                self._finish_task_download_settings()
                return

            cache_timestamp = datetime.utcnow().timestamp()
            with open(ENVIRONMENT_SETTINGS_CACHED_FILE, 'w+') as f:
                f.write(yaml.safe_dump(settings))

            with open(ENVIRONMENT_SETTINGS_TS_FILE, 'w+') as f:
                f.write(str(cache_timestamp))

            self._finish_task_download_settings()
            self.logger.info("Finished")
            return settings

        except requests.exceptions.ConnectionError:
            self._finish_task_download_settings()
            return

    def _check_and_fill_electron(self, pkg: WebApplication, env: dict, local_config: dict, x86_x64: bool, widevine: bool, output: List[EnvironmentComponent]):
        electron_settings = env['electron-wvvmp' if widevine else 'electron']
        electron_version = electron_settings['version']

        if not widevine and pkg.version and pkg.version != electron_version:  # this feature does not support custom widevine electron at the moment
            self.logger.info(f'A preset Electron version is defined for {pkg.url}: {pkg.version}')
            electron_version = pkg.version

        if not widevine and local_config['environment']['electron']['version']:
            self.logger.warning(f"A custom Electron version will be used {electron_version} to install {pkg.url}")
            electron_version = local_config['environment']['electron']['version']

        electron_status = self.check_electron_installed(version=electron_version, base_url=electron_settings['url'],
                                                        is_x86_x64_arch=x86_x64, widevine=widevine)

        electron_url = self._get_electron_url(version=electron_version, base_url=electron_settings['url'], is_x86_x64_arch=x86_x64)

        output.append(EnvironmentComponent(name=electron_url.split('/')[-1],
                                           version=electron_version,
                                           url=electron_url,
                                           size=self.http_client.get_content_length(electron_url),
                                           id='electron',
                                           update=not electron_status['electron'],
                                           properties={'widevine': widevine}))

        sha_url = electron_settings['sha_url'].format(version=electron_version)

        output.append(EnvironmentComponent(name=sha_url.split('/')[-1],
                                           version=electron_version,
                                           url=sha_url,
                                           size=self.http_client.get_content_length(sha_url),
                                           id='electron_sha256',
                                           update=not electron_status['electron'] or not electron_status['sha256'],
                                           properties={'widevine': widevine}))

    def _check_and_fill_node(self, env: dict, output: List[EnvironmentComponent]):
        node = EnvironmentComponent(name=env['nodejs']['url'].split('/')[-1],
                                    url=env['nodejs']['url'],
                                    size=self.http_client.get_content_length(env['nodejs']['url']),
                                    version=env['nodejs']['version'],
                                    id='nodejs')
        output.append(node)

        native = self._map_nativefier_file(env['nativefier'])
        output.append(native)

        if not self.check_node_installed(env['nodejs']['version']):
            node.update, native.update = True, True
        else:
            if not self._check_nativefier_installed(env['nativefier']):
                native.update = True

    def _check_nativefier_installed(self, nativefier_settings: dict) -> bool:
        if not os.path.exists(NODE_MODULES_PATH):
            self.logger.info(f'Node modules path {NODE_MODULES_PATH} not found')
            return False
        else:
            if not self._is_nativefier_installed():
                return False

            installed_version = nativefier.get_version()

            if installed_version:
                installed_version = installed_version.strip()

            self.logger.info(f"Nativefier versions: installed ({installed_version}), cloud ({nativefier_settings['version']})")

            if nativefier_settings['version'] != installed_version:
                self.logger.info("Installed nativefier version is different from cloud's. Changing version.")
                return False

            self.logger.info("Nativefier is already installed and up to date")
            return True

    def _map_nativefier_file(self, nativefier_settings: dict) -> EnvironmentComponent:
        url = nativefier_settings['url'].format(version=nativefier_settings['version'])
        return EnvironmentComponent(name=f"nativefier@{nativefier_settings['version']}",
                                    url=url,
                                    size=self.http_client.get_content_length(url),
                                    version=nativefier_settings['version'],
                                    id='nativefier')

    def check_environment(self, env: dict, local_config: dict, app: WebApplication,
                          is_x86_x64_arch: bool, widevine: bool) -> List[EnvironmentComponent]:

        components, check_threads = [], []

        system_env = local_config['environment'].get('system', False)

        if system_env:
            self.logger.warning(f"Using system's nativefier to install {app.url}")
        else:
            node_check = Thread(target=self._check_and_fill_node, args=(env, components))
            node_check.start()
            check_threads.append(node_check)

        elec_check = Thread(target=self._check_and_fill_electron, args=(app, env, local_config, is_x86_x64_arch, widevine, components))
        elec_check.start()
        check_threads.append(elec_check)

        for t in check_threads:
            t.join()

        return components

    def update(self, components: List[EnvironmentComponent], handler: ProcessHandler) -> bool:
        self.logger.info('Updating  environment')
        Path(ENV_PATH).mkdir(parents=True, exist_ok=True)

        comp_map = {c.id: c for c in components}

        node_data = comp_map.get('nodejs')
        nativefier_data = comp_map.get('nativefier')

        if node_data:
            if not self._install_nodejs(version=node_data.version, version_url=node_data.url, watcher=handler.watcher):
                return False

            if not self._install_nativefier(version=nativefier_data.version, url=nativefier_data.url, handler=handler):
                return False
        else:
            if nativefier_data and not self._install_nativefier(version=nativefier_data.version, url=nativefier_data.url, handler=handler):
                return False

        self.logger.info('Environment successfully updated')
        return True