spalencsar/bearhub
1
0
Fork 0
mirror of https://github.com/spalencsar/bearhub.git synced 2026-07-07 00:04:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

axel and aria2c for faster AUR downloads

Browse Source
This commit is contained in:
Vinicius Moreira
2019-09-22 23:12:34 -03:00
parent 917c6ec5d7
commit c5215c8236
5 changed files with 130 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bauh/app.py
View File

@@ -1,3 +1,4 @@
import os
import sys
from PyQt5.QtGui import QIcon
@@ -17,6 +18,9 @@ from bauh.view.util.disk import DefaultDiskCacheLoaderFactory
def main():
if not os.getenv('PYTHONUNBUFFERED'):
os.environ['PYTHONUNBUFFERED'] = '1'
args = app_args.read()
logger = logs.new_logger(__app_name__, bool(args.logs))
app_args.validate(args, logger)

60
bauh/commons/system.py
View File

@@ -11,6 +11,7 @@ PY_VERSION = "{}.{}".format(sys.version_info.major, sys.version_info.minor)
GLOBAL_PY_LIBS = '/usr/lib/python{}'.format(PY_VERSION)
PATH = os.getenv('PATH')
DEFAULT_LANG = 'en'
GLOBAL_INTERPRETER_PATH = ':'.join(PATH.split(':')[1:])
@@ -20,8 +21,11 @@ if GLOBAL_PY_LIBS not in PATH:
USE_GLOBAL_INTERPRETER = bool(os.getenv('VIRTUAL_ENV'))
def gen_env(global_interpreter: bool) -> dict:
res = {'LANG': 'en'}
def gen_env(global_interpreter: bool, lang: str = DEFAULT_LANG) -> dict:
res = {}
if lang:
res['LANG'] = lang
if global_interpreter: # to avoid subprocess calls to the virtualenv python interpreter instead of the global one.
res['PATH'] = GLOBAL_INTERPRETER_PATH
@@ -37,11 +41,12 @@ class SystemProcess:
Represents a system process being executed.
"""
def __init__(self, subproc: subprocess.Popen, success_phrase: str = None, wrong_error_phrase: str = '[sudo] password for', check_error_output: bool = True):
def __init__(self, subproc: subprocess.Popen, success_phrase: str = None, wrong_error_phrase: str = '[sudo] password for', check_error_output: bool = True, skip_stdout: bool = False):
self.subproc = subproc
self.success_phrase = success_phrase
self.wrong_error_phrase = wrong_error_phrase
self.check_error_output = check_error_output
self.skip_stdout = skip_stdout
def wait(self):
self.subproc.wait()
@@ -65,28 +70,35 @@ class ProcessHandler:
already_succeeded = False
for output in process.subproc.stdout:
line = output.decode().strip()
if line:
self._notify_watcher(line)
if not process.skip_stdout:
for output in process.subproc.stdout:
line = output.decode().strip()
if line:
self._notify_watcher(line)
if process.success_phrase and process.success_phrase in line:
already_succeeded = True
if process.success_phrase and process.success_phrase in line:
already_succeeded = True
if already_succeeded:
return True
for output in process.subproc.stderr:
if output:
line = output.decode().strip()
if line:
self._notify_watcher(line)
if process.check_error_output:
if process.wrong_error_phrase and process.wrong_error_phrase in line:
continue
else:
return False
elif process.skip_stdout and process.success_phrase and process.success_phrase in line:
already_succeeded = True
if already_succeeded:
return True
for output in process.subproc.stderr:
line = output.decode().strip()
if line:
self._notify_watcher(line)
if process.check_error_output:
if process.wrong_error_phrase and process.wrong_error_phrase in line:
continue
else:
return False
return process.subproc.returncode is None or process.subproc.returncode == 0
@@ -116,13 +128,13 @@ def run_cmd(cmd: str, expected_code: int = 0, ignore_return_code: bool = False,
def new_subprocess(cmd: List[str], cwd: str = '.', shell: bool = False, stdin = None,
global_interpreter: bool = USE_GLOBAL_INTERPRETER) -> subprocess.Popen:
global_interpreter: bool = USE_GLOBAL_INTERPRETER, lang: str = DEFAULT_LANG) -> subprocess.Popen:
args = {
"stdout": PIPE,
"stderr": PIPE,
"cwd": cwd,
"shell": shell,
"env": gen_env(global_interpreter)
"env": gen_env(global_interpreter, lang)
}
if input:
@@ -132,12 +144,12 @@ def new_subprocess(cmd: List[str], cwd: str = '.', shell: bool = False, stdin =
def new_root_subprocess(cmd: List[str], root_password: str, cwd: str = '.',
global_interpreter: bool = USE_GLOBAL_INTERPRETER) -> subprocess.Popen:
global_interpreter: bool = USE_GLOBAL_INTERPRETER, lang: str = DEFAULT_LANG) -> subprocess.Popen:
pwdin, final_cmd = None, []
if root_password is not None:
final_cmd.extend(['sudo', '-S'])
pwdin = new_subprocess(['echo', root_password], global_interpreter=global_interpreter).stdout
pwdin = new_subprocess(['echo', root_password], global_interpreter=global_interpreter, lang=lang).stdout
final_cmd.extend(cmd)
return subprocess.Popen(final_cmd, stdin=pwdin, stdout=PIPE, stderr=PIPE, cwd=cwd)

2
bauh/gems/arch/controller.py
View File

@@ -391,7 +391,7 @@ class ArchManager(SoftwareManager):
# building main package
handler.watcher.change_substatus(self.i18n['arch.building.package'].format(bold(pkgname)))
pkgbuilt = handler.handle(SystemProcess(new_subprocess(['makepkg', '-ALcsmf'], cwd=project_dir), check_error_output=False))
pkgbuilt = handler.handle(SystemProcess(new_subprocess(['makepkg', '-ALcsmf'], cwd=project_dir, lang=None), check_error_output=False))
self._update_progress(handler.watcher, 65, change_progress)
if pkgbuilt:

94
bauh/view/core/downloader.py
View File

@@ -1,7 +1,8 @@
import logging
import os
import time
import traceback
from multiprocessing import cpu_count
from typing import List
from bauh.api.abstract.download import FileDownloader
from bauh.api.abstract.handler import ProcessWatcher
@@ -11,43 +12,100 @@ from bauh.commons.system import run_cmd, new_subprocess, ProcessHandler, SystemP
class AdaptableFileDownloader(FileDownloader):
def __init__(self, logger: logging.Logger, multithread_enabled: bool):
self.download_threads = cpu_count() * 2
self.logger = logger
self.multithread_enabled = multithread_enabled
if multithread_enabled:
try:
self.download_threads = int(cpu_count() * 2)
except:
self.download_threads = 4
self.download_threads = 16 if self.download_threads > 16 else self.download_threads
else:
self.download_threads = 1
def is_axel_available(self) -> bool:
return bool(run_cmd('which axel'))
def _get_download_command(self, url: str, output_path: str) -> List:
if self.is_multithreaded():
cmd = ['axel', '-k', '-n', str(self.download_threads), url]
def is_aria2c_available(self) -> bool:
return bool(run_cmd('which aria2c'))
if output_path:
cmd.append('-o')
cmd.append(output_path)
def _get_aria2c_process(self, url: str, output_path: str, cwd: str) -> SystemProcess:
cmd = ['aria2c', '-x{}'.format(self.download_threads), url,
'--enable-color=false',
'--stderr=true',
'--summary-interval=0',
'--disable-ipv6',
'--min-split-size=1M']
return cmd
if output_path:
output_split = output_path.split('/')
cmd.append('-d=' + '/'.join(output_split[:-1]))
cmd.append('-o=' + output_split[-1])
return SystemProcess(new_subprocess(cmd=cmd, cwd=cwd),
skip_stdout=True,
check_error_output=False,
success_phrase='download completed')
def _get_axel_cmd(self, url: str, output_path: str, cwd: str) -> SystemProcess:
cmd = ['axel', '-k', '-n', str(self.download_threads), url]
if output_path:
cmd.append('-o')
cmd.append(output_path)
return SystemProcess(new_subprocess(cmd, cwd=cwd))
def _get_wget_process(self, url: str, output_path: str, cwd: str) -> SystemProcess:
cmd = ['wget', url]
if output_path:
cmd.append('-O')
cmd.append(output_path)
return cmd
return SystemProcess(new_subprocess(cmd, cwd=cwd))
def download(self, file_url: str, watcher: ProcessWatcher, output_path: str, cwd: str):
handler = ProcessHandler(watcher)
cmd = self._get_download_command(file_url, output_path)
def _rm_bad_file(self, file_name: str, output_path: str, cwd):
to_delete = output_path if output_path else '{}/{}'.format(cwd, file_name)
if to_delete and os.path.exists(to_delete):
self.logger.info('Removing downloaded file {}'.format(to_delete))
os.remove(to_delete)
def download(self, file_url: str, watcher: ProcessWatcher, output_path: str, cwd: str) -> bool:
self.logger.info('Downloading {}'.format(file_url))
watcher.print("[{}] downloading {}{}".format(cmd[0], file_url, " as {}".format(output_path) if output_path else ''))
handler = ProcessHandler(watcher)
file_name = file_url.split('/')[-1]
final_cwd = cwd if cwd else '.'
success = False
ti = time.time()
res = handler.handle(SystemProcess(new_subprocess(cmd=cmd, cwd=cwd if cwd else '.')))
try:
if self.is_multithreaded():
ti = time.time()
process = self._get_aria2c_process(file_url, output_path, final_cwd)
# process = self._get_axel_cmd(file_url, output_path, final_cwd)
else:
ti = time.time()
process = self._get_wget_process(file_url, output_path, final_cwd)
success = handler.handle(process)
except:
traceback.print_exc()
self._rm_bad_file(file_name, output_path, final_cwd)
tf = time.time()
self.logger.info(file_url.split('/')[-1] + ' download took {0:.2f} minutes'.format((tf - ti) / 60))
return res
self.logger.info(file_name + ' download took {0:.2f} minutes'.format((tf - ti) / 60))
if not success:
self.logger.error("Could not download '{}'".format(file_name))
self._rm_bad_file(file_name, output_path, final_cwd)
return success
def is_multithreaded(self) -> bool:
return self.multithread_enabled and self.is_axel_available()
return self.multithread_enabled and self.is_aria2c_available()

25
bauh/view/qt/window.py
View File

@@ -869,18 +869,6 @@ class ManageWindow(QWidget):
self.thread_install.start()
def _finish_install(self, pkgv: PackageView):
console_output = self.textarea_output.toPlainText()
if console_output:
log_path = '/tmp/bauh/logs/install/{}/{}'.format(pkgv.model.get_type(), pkgv.model.name)
try:
Path(log_path).mkdir(parents=True, exist_ok=True)
with open(log_path + '/{}.log'.format(int(time.time())), 'w+') as f:
f.write(console_output)
except:
self.textarea_output.appendPlainText("[warning] Could not write install log file to '{}'".format(log_path))
self.input_search.setText('')
self.finish_action()
@@ -888,6 +876,19 @@ class ManageWindow(QWidget):
if self._can_notify_user():
util.notify_user(msg='{} ({}) {}'.format(pkgv.model.name, pkgv.model.get_type(), self.i18n['installed']))
console_output = self.textarea_output.toPlainText()
if console_output:
log_path = '/tmp/bauh/logs/install/{}/{}'.format(pkgv.model.get_type(), pkgv.model.name)
try:
Path(log_path).mkdir(parents=True, exist_ok=True)
with open(log_path + '/{}.log'.format(int(time.time())), 'w+') as f:
f.write(console_output)
except:
self.textarea_output.appendPlainText(
"[warning] Could not write install log file to '{}'".format(log_path))
self.refresh_apps(top_app=pkgv, pkg_types={pkgv.model.__class__})
else:
if self._can_notify_user():