From 16eb8445fbd711b3e9a7b7d01c58c77771193f14 Mon Sep 17 00:00:00 2001 From: Vinicius Moreira Date: Fri, 28 Aug 2020 13:53:59 -0300 Subject: [PATCH] [arch] fix -> AUR: not properly extracting srcinfo data when several pkgnames are declared --- CHANGELOG.md | 3 +- bauh/gems/arch/aur.py | 79 ++++-- bauh/gems/arch/controller.py | 16 +- tests/gems/arch/resources/bauh_srcinfo | 27 +++ tests/gems/arch/resources/mangohud_srcinfo | 36 +++ tests/gems/arch/test_aur.py | 266 +++++++++++++++++++++ 6 files changed, 395 insertions(+), 32 deletions(-) create mode 100644 tests/gems/arch/resources/bauh_srcinfo create mode 100644 tests/gems/arch/resources/mangohud_srcinfo create mode 100644 tests/gems/arch/test_aur.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ef7aad0..1bcfa425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,7 +77,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - multi-threaded download: not retrieving correctly some source files URLs (e.g: linux-xanmod-lts) - importing PGP keys (Generic error). Now the key server is specified: `gpg --keyserver SERVER --recv-key KEYID` (the server address is retrieved from [bauh-files](https://github.com/vinifmor/bauh-files/blob/master/arch/gpgservers.txt)) - not installing the correct package built when several are generated (e.g: linux-xanmod-lts) - - some packages dependencies cannot be downloaded due to the wrong download URL (missing the 'pkgbase' field to determine the proper url) + - some packages dependencies cannot be downloaded due to the wrong download URL (missing the 'pkgbase' field to determine the proper url) + - not properly extracting srcinfo data when several pkgnames are declared (leads to wrong dependencies requirements) - some environment variables are not available during the common operations (install, upgrade, downgrade, uninstall, makepkg, launch) - Flatpak - downgrading crashing with version 1.8.X diff --git a/bauh/gems/arch/aur.py b/bauh/gems/arch/aur.py index 7e4cd493..adfb54c6 100644 --- a/bauh/gems/arch/aur.py +++ b/bauh/gems/arch/aur.py @@ -2,7 +2,7 @@ import logging import os import re import urllib.parse -from typing import Set, List, Iterable, Dict +from typing import Set, List, Iterable, Dict, Optional import requests @@ -28,6 +28,8 @@ KNOWN_LIST_FIELDS = ('validpgpkeys', 'optdepends', 'optdepends_x86_64', 'optdepends_i686', + 'sha256sums', + 'sha256sums_x86_64', 'sha512sums', 'sha512sums_x86_64', 'source', @@ -44,30 +46,61 @@ def map_pkgbuild(pkgbuild: str) -> dict: return {attr: val.replace('"', '').replace("'", '').replace('(', '').replace(')', '') for attr, val in re.findall(r'\n(\w+)=(.+)', pkgbuild)} -def map_srcinfo(string: str, fields: Set[str] = None) -> dict: +def map_srcinfo(string: str, pkgname: str, fields: Set[str] = None) -> dict: + subinfos, subinfo = [], {} + + key_fields = {'pkgname', 'pkgbase'} + + for field in RE_SRCINFO_KEYS.findall(string): + key = field[0].strip() + val = field[1].strip() + + if subinfo and key in key_fields: + subinfos.append(subinfo) + subinfo = {key: val} + elif not fields or key in fields: + if key not in subinfo: + subinfo[key] = {val} if key in KNOWN_LIST_FIELDS else val + else: + if not isinstance(subinfo[key], set): + subinfo[key] = {subinfo[key]} + + subinfo[key].add(val) + + if subinfo: + subinfos.append(subinfo) + + pkgnames = {s['pkgname'] for s in subinfos if 'pkgname' in s} + return merge_subinfos(subinfos=subinfos, + pkgname=None if (len(pkgnames) == 1 or pkgname not in pkgnames) else pkgname, + fields=fields) + + +def merge_subinfos(subinfos: List[dict], pkgname: Optional[str] = None, fields: Optional[Set[str]] = None) -> dict: info = {} + for subinfo in subinfos: + if not pkgname or subinfo.get('pkgname') in {None, pkgname}: + for key, val in subinfo.items(): + if not fields or key in fields: + current_val = info.get(key) - if fields: - field_re = re.compile(r'({})\s+=\s+(.+)\n'.format('|'.join(fields))) - else: - field_re = RE_SRCINFO_KEYS + if current_val is None: + info[key] = val + else: + if not isinstance(current_val, set): + current_val = {current_val} + info[key] = current_val - for tupl in field_re.findall(string): - key = tupl[0].strip() - val = tupl[1].strip() + if isinstance(val, set): + current_val.update(val) + else: + current_val.add(val) - if key not in info: - info[key] = [val] if key in KNOWN_LIST_FIELDS else val - else: - if not isinstance(info[key], list): - info[key] = [info[key]] + for field in info.keys(): + val = info.get(field) - info[key].append(val) - - pkgname = info.get('pkgname') - - if isinstance(pkgname, list): - info['pkgname'] = pkgname[0] + if isinstance(val, set): + info[field] = [*val] return info @@ -90,7 +123,7 @@ class AURClient: except: return [] - def get_src_info(self, name: str) -> dict: + def get_src_info(self, name: str, real_name: Optional[str] = None) -> dict: srcinfo = self.srcinfo_cache.get(name) if srcinfo: @@ -99,7 +132,7 @@ class AURClient: res = self.http_client.get(URL_SRC_INFO + urllib.parse.quote(name)) if res and res.text: - srcinfo = map_srcinfo(res.text) + srcinfo = map_srcinfo(string=res.text, pkgname=real_name if real_name else name) if srcinfo: self.srcinfo_cache[name] = srcinfo @@ -118,7 +151,7 @@ class AURClient: info_base = info.get('PackageBase') if info_name and info_base and info_name != info_base: self.logger.info('{p} is based on {b}. Retrieving {b} .SRCINFO'.format(p=info_name, b=info_base)) - srcinfo = self.get_src_info(info_base) + srcinfo = self.get_src_info(name=info_base, real_name=info_name) if srcinfo: self.srcinfo_cache[name] = srcinfo diff --git a/bauh/gems/arch/controller.py b/bauh/gems/arch/controller.py index 167178dd..9339ebe3 100644 --- a/bauh/gems/arch/controller.py +++ b/bauh/gems/arch/controller.py @@ -611,7 +611,7 @@ class ArchManager(SoftwareManager): for idx in range(1, len(commit_list)): commit = commit_list[idx] with open(srcinfo_path) as f: - pkgsrc = aur.map_srcinfo(f.read(), srcfields) + pkgsrc = aur.map_srcinfo(string=f.read(), pkgname=context.name ,fields=srcfields) reset_proc = new_subprocess(['git', 'reset', '--hard', commit], cwd=clone_path) if not context.handler.handle(SystemProcess(reset_proc, check_error_output=False)): @@ -1350,7 +1350,7 @@ class ArchManager(SoftwareManager): for idx, commit in enumerate(commits): with open(srcinfo_path) as f: - pkgsrc = aur.map_srcinfo(f.read(), srcfields) + pkgsrc = aur.map_srcinfo(string=f.read(), pkgname=pkg.name, fields=srcfields) if status_idx < 0 and '{}-{}'.format(pkgsrc.get('pkgver'), pkgsrc.get('pkgrel')) == pkg.version: status_idx = idx @@ -1566,10 +1566,10 @@ class ArchManager(SoftwareManager): return pkg_repos - def _pre_download_source(self, project_dir: str, watcher: ProcessWatcher) -> bool: + def _pre_download_source(self, pkgname: str, project_dir: str, watcher: ProcessWatcher) -> bool: if self.context.file_downloader.is_multithreaded(): with open('{}/.SRCINFO'.format(project_dir)) as f: - srcinfo = aur.map_srcinfo(f.read()) + srcinfo = aur.map_srcinfo(string=f.read(), pkgname=pkgname) pre_download_files = [] @@ -1639,7 +1639,7 @@ class ArchManager(SoftwareManager): watcher=context.watcher, pkgbuild_path='{}/PKGBUILD'.format(context.project_dir)): context.pkgbuild_edited = True - srcinfo = aur.map_srcinfo(makepkg.gen_srcinfo(context.project_dir)) + srcinfo = aur.map_srcinfo(string=makepkg.gen_srcinfo(context.project_dir), pkgname=context.name) if srcinfo: context.name = srcinfo['pkgname'] @@ -1656,7 +1656,7 @@ class ArchManager(SoftwareManager): def _build(self, context: TransactionContext) -> bool: self._edit_pkgbuild_and_update_context(context) - self._pre_download_source(context.project_dir, context.watcher) + self._pre_download_source(context.name, context.project_dir, context.watcher) self._update_progress(context, 50) if not self._handle_aur_package_deps_and_keys(context): @@ -1691,7 +1691,7 @@ class ArchManager(SoftwareManager): file_to_install = gen_file[0] if len(gen_file) > 1: - srcinfo = aur.map_srcinfo(makepkg.gen_srcinfo(context.project_dir)) + srcinfo = aur.map_srcinfo(string=makepkg.gen_srcinfo(context.project_dir), pkgname=context.name) pkgver = '-{}'.format(srcinfo['pkgver']) if srcinfo.get('pkgver') else '' pkgrel = '-{}'.format(srcinfo['pkgrel']) if srcinfo.get('pkgrel') else '' arch = '-{}'.format(srcinfo['arch']) if srcinfo.get('arch') else '' @@ -1762,7 +1762,7 @@ class ArchManager(SoftwareManager): if context.repository == 'aur': with open('{}/.SRCINFO'.format(context.project_dir)) as f: - srcinfo = aur.map_srcinfo(f.read()) + srcinfo = aur.map_srcinfo(string=f.read(), pkgname=context.name) pkgs_data = {context.name: self.aur_client.map_update_data(context.name, context.get_version(), srcinfo)} else: diff --git a/tests/gems/arch/resources/bauh_srcinfo b/tests/gems/arch/resources/bauh_srcinfo new file mode 100644 index 00000000..ba7e53ae --- /dev/null +++ b/tests/gems/arch/resources/bauh_srcinfo @@ -0,0 +1,27 @@ +pkgbase = bauh + pkgdesc = Graphical interface for managing your applications ( AppImage, Flatpak, Snap, Arch/AUR, Web ) + pkgver = 0.9.6 + pkgrel = 2 + url = https://github.com/vinifmor/bauh + arch = any + license = zlib/libpng + makedepends = git + makedepends = python + makedepends = python-pip + makedepends = python-setuptools + depends = python + depends = python-pyqt5 + depends = python-pyqt5-sip + depends = python-requests + depends = python-colorama + depends = python-pyaml + depends = qt5-svg + optdepends = flatpak: required for Flatpak support + optdepends = snapd: required for Snap support + optdepends = python-beautifulsoup4: for Native Web applications support + optdepends = python-lxml: for Native Web applications support + source = https://github.com/vinifmor/bauh/archive/0.9.6.tar.gz + sha512sums = cb1820b8a41dccec746d91d71b7f524c2e3caf6b30b0cd9666598b8ad49302654d9ce9bd1a0a2a9612afebc27ef78a2a94ac10e4e6c183742effe4feeabaa7b2 + +pkgname = bauh + diff --git a/tests/gems/arch/resources/mangohud_srcinfo b/tests/gems/arch/resources/mangohud_srcinfo new file mode 100644 index 00000000..bec0adc2 --- /dev/null +++ b/tests/gems/arch/resources/mangohud_srcinfo @@ -0,0 +1,36 @@ +pkgbase = mangohud + pkgver = 0.5.1 + pkgrel = 3 + url = https://github.com/flightlessmango/MangoHud + arch = x86_64 + license = MIT + makedepends = meson + makedepends = python-mako + makedepends = glslang + makedepends = libglvnd + makedepends = lib32-libglvnd + makedepends = vulkan-headers + makedepends = vulkan-icd-loader + makedepends = lib32-vulkan-icd-loader + makedepends = libxnvctrl + source = mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz + sha256sums = 3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6 + +pkgname = mangohud + pkgdesc = A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more + depends = gcc-libs + depends = mangohud-common + optdepends = bash: mangohud helper script + optdepends = libxnvctrl: support for older NVIDIA GPUs + +pkgname = lib32-mangohud + pkgdesc = A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more (32-bit) + depends = lib32-gcc-libs + depends = mangohud + depends = mangohud-common + optdepends = lib32-libxnvctrl: support for older NVIDIA GPUs + +pkgname = mangohud-common + pkgdesc = Common files for mangohud and lib32-mangohud + optdepends = bash: mangohud helper script + diff --git a/tests/gems/arch/test_aur.py b/tests/gems/arch/test_aur.py new file mode 100644 index 00000000..6f34cebe --- /dev/null +++ b/tests/gems/arch/test_aur.py @@ -0,0 +1,266 @@ +import os +from unittest import TestCase + +from bauh.gems.arch import aur + +FILE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class AURModuleTest(TestCase): + + def test_map_srcinfo__only_one_pkgname(self): + expected_fields = { + 'pkgbase': 'bauh', + 'pkgname': 'bauh', + 'pkgver': '0.9.6', + 'pkgrel': '2', + 'url': 'https://github.com/vinifmor/bauh', + 'arch': 'any', + 'license': 'zlib/libpng', + 'makedepends': ['git', 'python', 'python-pip', 'python-setuptools'], + 'depends': [ + 'python', 'python-colorama', 'python-pyaml', 'python-pyqt5', 'python-pyqt5-sip', 'python-requests', 'qt5-svg' + ], + 'optdepends': [ + 'flatpak: required for Flatpak support', + 'python-beautifulsoup4: for Native Web applications support', + 'python-lxml: for Native Web applications support', + 'snapd: required for Snap support' + ], + 'source': ['https://github.com/vinifmor/bauh/archive/0.9.6.tar.gz'], + 'sha512sums': ['cb1820b8a41dccec746d91d71b7f524c2e3caf6b30b0cd9666598b8ad49302654d9ce9bd1a0a2a9612afebc27ef78a2a94ac10e4e6c183742effe4feeabaa7b2'] + } + + with open(FILE_DIR + '/resources/bauh_srcinfo') as f: + srcinfo = f.read() + + res = aur.map_srcinfo(srcinfo, 'bauh') + + for key, val in expected_fields.items(): + self.assertIn(key, res, "key '{}' not in res".format(key)) + + if isinstance(val, list): + val.sort() + res[key].sort() + + self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key])) + + def test_map_srcinfo__one_name__only_specific_fields(self): + expected_fields = { + 'pkgver': '0.9.6', + 'pkgrel': '2' + } + + with open(FILE_DIR + '/resources/bauh_srcinfo') as f: + srcinfo = f.read() + + res = aur.map_srcinfo(srcinfo, 'bauh', fields={*expected_fields.keys()}) + + self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res))) + + for key, val in expected_fields.items(): + self.assertIn(key, res, "key '{}' not in res".format(key)) + + if isinstance(val, list): + val.sort() + res[key].sort() + + self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key])) + + def test_map_srcinfo__several_pkgnames__pkgname_specified_case_1(self): + expected_fields = { + 'pkgbase': 'mangohud', + 'pkgname': 'mangohud', + 'pkgver': '0.5.1', + 'pkgrel': '3', + 'pkgdesc': 'A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more', + 'source': ['mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz'], + 'sha256sums': ['3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6'], + 'license': 'MIT', + 'arch': 'x86_64', + 'url': 'https://github.com/flightlessmango/MangoHud', + 'makedepends': [ + 'glslang', 'libglvnd', 'lib32-libglvnd', 'meson', 'python-mako', 'vulkan-headers', 'vulkan-icd-loader', + 'lib32-vulkan-icd-loader', 'libxnvctrl' + ], + 'depends': ['gcc-libs', 'mangohud-common'], + 'optdepends': ['bash: mangohud helper script', 'libxnvctrl: support for older NVIDIA GPUs'] + } + + with open(FILE_DIR + '/resources/mangohud_srcinfo') as f: + srcinfo = f.read() + + res = aur.map_srcinfo(srcinfo, 'mangohud') + + self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res))) + + for key, val in expected_fields.items(): + self.assertIn(key, res, "key '{}' not in res".format(key)) + + if isinstance(val, list): + val.sort() + + if isinstance(res[key], list): + res[key].sort() + + self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key])) + + def test_map_srcinfo__several_pkgnames__pkgname_specified_case_2(self): + expected_fields = { + 'pkgbase': 'mangohud', + 'pkgname': 'mangohud-common', + 'pkgver': '0.5.1', + 'pkgrel': '3', + 'pkgdesc': 'Common files for mangohud and lib32-mangohud', + 'source': ['mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz'], + 'sha256sums': ['3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6'], + 'license': 'MIT', + 'url': 'https://github.com/flightlessmango/MangoHud', + 'arch': 'x86_64', + 'makedepends': [ + 'glslang', 'libglvnd', 'lib32-libglvnd', 'meson', 'python-mako', 'vulkan-headers', 'vulkan-icd-loader', + 'lib32-vulkan-icd-loader', 'libxnvctrl' + ], + 'optdepends': ['bash: mangohud helper script'] + } + + with open(FILE_DIR + '/resources/mangohud_srcinfo') as f: + srcinfo = f.read() + + res = aur.map_srcinfo(srcinfo, 'mangohud-common') + self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res))) + + for key, val in expected_fields.items(): + self.assertIn(key, res, "key '{}' not in res".format(key)) + + if isinstance(val, list): + val.sort() + + if isinstance(res[key], list): + res[key].sort() + + self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key])) + + def test_map_srcinfo__several_pkgnames__pkgname_specified_case_3(self): + expected_fields = { + 'pkgbase': 'mangohud', + 'pkgname': 'lib32-mangohud', + 'pkgver': '0.5.1', + 'pkgrel': '3', + 'pkgdesc': 'A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more (32-bit)', + 'source': ['mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz'], + 'sha256sums': ['3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6'], + 'license': 'MIT', + 'url': 'https://github.com/flightlessmango/MangoHud', + 'arch': 'x86_64', + 'makedepends': [ + 'glslang', 'libglvnd', 'lib32-libglvnd', 'meson', 'python-mako', 'vulkan-headers', 'vulkan-icd-loader', + 'lib32-vulkan-icd-loader', 'libxnvctrl' + ], + 'depends': ['mangohud', 'mangohud-common', 'lib32-gcc-libs'], + 'optdepends': ['lib32-libxnvctrl: support for older NVIDIA GPUs'] + } + + with open(FILE_DIR + '/resources/mangohud_srcinfo') as f: + srcinfo = f.read() + + res = aur.map_srcinfo(srcinfo, 'lib32-mangohud') + self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res))) + + for key, val in expected_fields.items(): + self.assertIn(key, res, "key '{}' not in res".format(key)) + + if isinstance(val, list): + val.sort() + + if isinstance(res[key], list): + res[key].sort() + + self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key])) + + def test_map_srcinfo__several_pkgnames__different_pkgname(self): + expected_fields = { + 'pkgbase': 'mangohud', + 'pkgname': ['lib32-mangohud', 'mangohud', 'mangohud-common'], + 'pkgver': '0.5.1', + 'pkgrel': '3', + 'pkgdesc': [ + 'A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more (32-bit)', + 'Common files for mangohud and lib32-mangohud', + 'A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more', + ], + 'source': ['mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz'], + 'sha256sums': ['3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6'], + 'license': 'MIT', + 'url': 'https://github.com/flightlessmango/MangoHud', + 'arch': 'x86_64', + 'makedepends': [ + 'glslang', 'libglvnd', 'lib32-libglvnd', 'meson', 'python-mako', 'vulkan-headers', 'vulkan-icd-loader', + 'lib32-vulkan-icd-loader', 'libxnvctrl' + ], + 'depends': ['mangohud', 'mangohud-common', 'lib32-gcc-libs', 'gcc-libs'], + 'optdepends': ['lib32-libxnvctrl: support for older NVIDIA GPUs', + 'bash: mangohud helper script', + 'libxnvctrl: support for older NVIDIA GPUs'] + } + + with open(FILE_DIR + '/resources/mangohud_srcinfo') as f: + srcinfo = f.read() + + res = aur.map_srcinfo(srcinfo, 'xpto') + self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res))) + + for key, val in expected_fields.items(): + self.assertIn(key, res, "key '{}' not in res".format(key)) + + if isinstance(val, list): + val.sort() + + if isinstance(res[key], list): + res[key].sort() + + self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key])) + + def test_map_srcinfo__several_names__pkgname_present__only_specific_fields(self): + expected_fields = { + 'pkgver': '0.5.1', + 'pkgrel': '3' + } + + with open(FILE_DIR + '/resources/mangohud_srcinfo') as f: + srcinfo = f.read() + + res = aur.map_srcinfo(srcinfo, 'mangohud-commons', fields={*expected_fields.keys()}) + + self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res))) + + for key, val in expected_fields.items(): + self.assertIn(key, res, "key '{}' not in res".format(key)) + + if isinstance(val, list): + val.sort() + res[key].sort() + + self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key])) + + def test_map_srcinfo__several_names__pkgname_not_present__only_specific_fields(self): + expected_fields = { + 'pkgname': ['mangohud', 'lib32-mangohud', 'mangohud-common'], + 'pkgver': '0.5.1' + } + + with open(FILE_DIR + '/resources/mangohud_srcinfo') as f: + srcinfo = f.read() + + res = aur.map_srcinfo(srcinfo, 'xpto', fields={*expected_fields.keys()}) + + self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res))) + + for key, val in expected_fields.items(): + self.assertIn(key, res, "key '{}' not in res".format(key)) + + if isinstance(val, list): + val.sort() + res[key].sort() + + self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key]))