diff --git a/bauh/app.py b/bauh/app.py index a8bce3c3..e947cafb 100755 --- a/bauh/app.py +++ b/bauh/app.py @@ -1,3 +1,4 @@ +import os import sys from PyQt5.QtGui import QIcon @@ -17,6 +18,9 @@ from bauh.view.util.disk import DefaultDiskCacheLoaderFactory def main(): + if not os.getenv('PYTHONUNBUFFERED'): + os.environ['PYTHONUNBUFFERED'] = '1' + args = app_args.read() logger = logs.new_logger(__app_name__, bool(args.logs)) app_args.validate(args, logger) diff --git a/bauh/commons/system.py b/bauh/commons/system.py index ab199d5c..70ec1171 100644 --- a/bauh/commons/system.py +++ b/bauh/commons/system.py @@ -11,6 +11,7 @@ PY_VERSION = "{}.{}".format(sys.version_info.major, sys.version_info.minor) GLOBAL_PY_LIBS = '/usr/lib/python{}'.format(PY_VERSION) PATH = os.getenv('PATH') +DEFAULT_LANG = 'en' GLOBAL_INTERPRETER_PATH = ':'.join(PATH.split(':')[1:]) @@ -20,8 +21,11 @@ if GLOBAL_PY_LIBS not in PATH: USE_GLOBAL_INTERPRETER = bool(os.getenv('VIRTUAL_ENV')) -def gen_env(global_interpreter: bool) -> dict: - res = {'LANG': 'en'} +def gen_env(global_interpreter: bool, lang: str = DEFAULT_LANG) -> dict: + res = {} + + if lang: + res['LANG'] = lang if global_interpreter: # to avoid subprocess calls to the virtualenv python interpreter instead of the global one. res['PATH'] = GLOBAL_INTERPRETER_PATH @@ -37,11 +41,12 @@ class SystemProcess: Represents a system process being executed. """ - def __init__(self, subproc: subprocess.Popen, success_phrase: str = None, wrong_error_phrase: str = '[sudo] password for', check_error_output: bool = True): + def __init__(self, subproc: subprocess.Popen, success_phrase: str = None, wrong_error_phrase: str = '[sudo] password for', check_error_output: bool = True, skip_stdout: bool = False): self.subproc = subproc self.success_phrase = success_phrase self.wrong_error_phrase = wrong_error_phrase self.check_error_output = check_error_output + self.skip_stdout = skip_stdout def wait(self): self.subproc.wait() @@ -65,28 +70,35 @@ class ProcessHandler: already_succeeded = False - for output in process.subproc.stdout: - line = output.decode().strip() - if line: - self._notify_watcher(line) + if not process.skip_stdout: + for output in process.subproc.stdout: + line = output.decode().strip() + if line: + self._notify_watcher(line) - if process.success_phrase and process.success_phrase in line: - already_succeeded = True + if process.success_phrase and process.success_phrase in line: + already_succeeded = True + + if already_succeeded: + return True + + for output in process.subproc.stderr: + if output: + line = output.decode().strip() + if line: + self._notify_watcher(line) + + if process.check_error_output: + if process.wrong_error_phrase and process.wrong_error_phrase in line: + continue + else: + return False + elif process.skip_stdout and process.success_phrase and process.success_phrase in line: + already_succeeded = True if already_succeeded: return True - for output in process.subproc.stderr: - line = output.decode().strip() - if line: - self._notify_watcher(line) - - if process.check_error_output: - if process.wrong_error_phrase and process.wrong_error_phrase in line: - continue - else: - return False - return process.subproc.returncode is None or process.subproc.returncode == 0 @@ -116,13 +128,13 @@ def run_cmd(cmd: str, expected_code: int = 0, ignore_return_code: bool = False, def new_subprocess(cmd: List[str], cwd: str = '.', shell: bool = False, stdin = None, - global_interpreter: bool = USE_GLOBAL_INTERPRETER) -> subprocess.Popen: + global_interpreter: bool = USE_GLOBAL_INTERPRETER, lang: str = DEFAULT_LANG) -> subprocess.Popen: args = { "stdout": PIPE, "stderr": PIPE, "cwd": cwd, "shell": shell, - "env": gen_env(global_interpreter) + "env": gen_env(global_interpreter, lang) } if input: @@ -132,12 +144,12 @@ def new_subprocess(cmd: List[str], cwd: str = '.', shell: bool = False, stdin = def new_root_subprocess(cmd: List[str], root_password: str, cwd: str = '.', - global_interpreter: bool = USE_GLOBAL_INTERPRETER) -> subprocess.Popen: + global_interpreter: bool = USE_GLOBAL_INTERPRETER, lang: str = DEFAULT_LANG) -> subprocess.Popen: pwdin, final_cmd = None, [] if root_password is not None: final_cmd.extend(['sudo', '-S']) - pwdin = new_subprocess(['echo', root_password], global_interpreter=global_interpreter).stdout + pwdin = new_subprocess(['echo', root_password], global_interpreter=global_interpreter, lang=lang).stdout final_cmd.extend(cmd) return subprocess.Popen(final_cmd, stdin=pwdin, stdout=PIPE, stderr=PIPE, cwd=cwd) diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 34d57655..10efea31 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -391,7 +391,7 @@ class ArchManager(SoftwareManager): # building main package handler.watcher.change_substatus(self.i18n['arch.building.package'].format(bold(pkgname))) - pkgbuilt = handler.handle(SystemProcess(new_subprocess(['makepkg', '-ALcsmf'], cwd=project_dir), check_error_output=False)) + pkgbuilt = handler.handle(SystemProcess(new_subprocess(['makepkg', '-ALcsmf'], cwd=project_dir, lang=None), check_error_output=False)) self._update_progress(handler.watcher, 65, change_progress) if pkgbuilt: diff --git a/bauh/view/core/downloader.py b/bauh/view/core/downloader.py index fa33c8a5..7888d2e5 100644 --- a/bauh/view/core/downloader.py +++ b/bauh/view/core/downloader.py @@ -1,7 +1,8 @@ import logging +import os import time +import traceback from multiprocessing import cpu_count -from typing import List from bauh.api.abstract.download import FileDownloader from bauh.api.abstract.handler import ProcessWatcher @@ -11,43 +12,100 @@ from bauh.commons.system import run_cmd, new_subprocess, ProcessHandler, SystemP class AdaptableFileDownloader(FileDownloader): def __init__(self, logger: logging.Logger, multithread_enabled: bool): - self.download_threads = cpu_count() * 2 self.logger = logger self.multithread_enabled = multithread_enabled + if multithread_enabled: + try: + self.download_threads = int(cpu_count() * 2) + except: + self.download_threads = 4 + + self.download_threads = 16 if self.download_threads > 16 else self.download_threads + else: + self.download_threads = 1 + def is_axel_available(self) -> bool: return bool(run_cmd('which axel')) - def _get_download_command(self, url: str, output_path: str) -> List: - if self.is_multithreaded(): - cmd = ['axel', '-k', '-n', str(self.download_threads), url] + def is_aria2c_available(self) -> bool: + return bool(run_cmd('which aria2c')) - if output_path: - cmd.append('-o') - cmd.append(output_path) + def _get_aria2c_process(self, url: str, output_path: str, cwd: str) -> SystemProcess: + cmd = ['aria2c', '-x{}'.format(self.download_threads), url, + '--enable-color=false', + '--stderr=true', + '--summary-interval=0', + '--disable-ipv6', + '--min-split-size=1M'] - return cmd + if output_path: + output_split = output_path.split('/') + cmd.append('-d=' + '/'.join(output_split[:-1])) + cmd.append('-o=' + output_split[-1]) + return SystemProcess(new_subprocess(cmd=cmd, cwd=cwd), + skip_stdout=True, + check_error_output=False, + success_phrase='download completed') + + def _get_axel_cmd(self, url: str, output_path: str, cwd: str) -> SystemProcess: + cmd = ['axel', '-k', '-n', str(self.download_threads), url] + + if output_path: + cmd.append('-o') + cmd.append(output_path) + + return SystemProcess(new_subprocess(cmd, cwd=cwd)) + + def _get_wget_process(self, url: str, output_path: str, cwd: str) -> SystemProcess: cmd = ['wget', url] if output_path: cmd.append('-O') cmd.append(output_path) - return cmd + return SystemProcess(new_subprocess(cmd, cwd=cwd)) - def download(self, file_url: str, watcher: ProcessWatcher, output_path: str, cwd: str): - handler = ProcessHandler(watcher) - cmd = self._get_download_command(file_url, output_path) + def _rm_bad_file(self, file_name: str, output_path: str, cwd): + to_delete = output_path if output_path else '{}/{}'.format(cwd, file_name) + + if to_delete and os.path.exists(to_delete): + self.logger.info('Removing downloaded file {}'.format(to_delete)) + os.remove(to_delete) + + def download(self, file_url: str, watcher: ProcessWatcher, output_path: str, cwd: str) -> bool: self.logger.info('Downloading {}'.format(file_url)) - watcher.print("[{}] downloading {}{}".format(cmd[0], file_url, " as {}".format(output_path) if output_path else '')) + handler = ProcessHandler(watcher) + file_name = file_url.split('/')[-1] + final_cwd = cwd if cwd else '.' + + success = False ti = time.time() - res = handler.handle(SystemProcess(new_subprocess(cmd=cmd, cwd=cwd if cwd else '.'))) + try: + if self.is_multithreaded(): + ti = time.time() + process = self._get_aria2c_process(file_url, output_path, final_cwd) + # process = self._get_axel_cmd(file_url, output_path, final_cwd) + else: + ti = time.time() + process = self._get_wget_process(file_url, output_path, final_cwd) + + success = handler.handle(process) + except: + traceback.print_exc() + self._rm_bad_file(file_name, output_path, final_cwd) + tf = time.time() - self.logger.info(file_url.split('/')[-1] + ' download took {0:.2f} minutes'.format((tf - ti) / 60)) - return res + self.logger.info(file_name + ' download took {0:.2f} minutes'.format((tf - ti) / 60)) + + if not success: + self.logger.error("Could not download '{}'".format(file_name)) + self._rm_bad_file(file_name, output_path, final_cwd) + + return success def is_multithreaded(self) -> bool: - return self.multithread_enabled and self.is_axel_available() + return self.multithread_enabled and self.is_aria2c_available() diff --git a/bauh/view/qt/window.py b/bauh/view/qt/window.py index fb269157..542f5d87 100755 --- a/bauh/view/qt/window.py +++ b/bauh/view/qt/window.py @@ -869,18 +869,6 @@ class ManageWindow(QWidget): self.thread_install.start() def _finish_install(self, pkgv: PackageView): - console_output = self.textarea_output.toPlainText() - - if console_output: - log_path = '/tmp/bauh/logs/install/{}/{}'.format(pkgv.model.get_type(), pkgv.model.name) - try: - Path(log_path).mkdir(parents=True, exist_ok=True) - - with open(log_path + '/{}.log'.format(int(time.time())), 'w+') as f: - f.write(console_output) - except: - self.textarea_output.appendPlainText("[warning] Could not write install log file to '{}'".format(log_path)) - self.input_search.setText('') self.finish_action() @@ -888,6 +876,19 @@ class ManageWindow(QWidget): if self._can_notify_user(): util.notify_user(msg='{} ({}) {}'.format(pkgv.model.name, pkgv.model.get_type(), self.i18n['installed'])) + console_output = self.textarea_output.toPlainText() + + if console_output: + log_path = '/tmp/bauh/logs/install/{}/{}'.format(pkgv.model.get_type(), pkgv.model.name) + try: + Path(log_path).mkdir(parents=True, exist_ok=True) + + with open(log_path + '/{}.log'.format(int(time.time())), 'w+') as f: + f.write(console_output) + except: + self.textarea_output.appendPlainText( + "[warning] Could not write install log file to '{}'".format(log_path)) + self.refresh_apps(top_app=pkgv, pkg_types={pkgv.model.__class__}) else: if self._can_notify_user():