[arch] fix -> AUR: not properly extracting srcinfo data when several pkgnames are declared

This commit is contained in:
Vinicius Moreira
2020-08-28 13:53:59 -03:00
parent 4d526e5488
commit 16eb8445fb
6 changed files with 395 additions and 32 deletions

View File

@@ -77,7 +77,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- multi-threaded download: not retrieving correctly some source files URLs (e.g: linux-xanmod-lts)
- importing PGP keys (Generic error). Now the key server is specified: `gpg --keyserver SERVER --recv-key KEYID` (the server address is retrieved from [bauh-files](https://github.com/vinifmor/bauh-files/blob/master/arch/gpgservers.txt))
- not installing the correct package built when several are generated (e.g: linux-xanmod-lts)
- some packages dependencies cannot be downloaded due to the wrong download URL (missing the 'pkgbase' field to determine the proper url)
- some packages dependencies cannot be downloaded due to the wrong download URL (missing the 'pkgbase' field to determine the proper url)
- not properly extracting srcinfo data when several pkgnames are declared (leads to wrong dependencies requirements)
- some environment variables are not available during the common operations (install, upgrade, downgrade, uninstall, makepkg, launch)
- Flatpak
- downgrading crashing with version 1.8.X

View File

@@ -2,7 +2,7 @@ import logging
import os
import re
import urllib.parse
from typing import Set, List, Iterable, Dict
from typing import Set, List, Iterable, Dict, Optional
import requests
@@ -28,6 +28,8 @@ KNOWN_LIST_FIELDS = ('validpgpkeys',
'optdepends',
'optdepends_x86_64',
'optdepends_i686',
'sha256sums',
'sha256sums_x86_64',
'sha512sums',
'sha512sums_x86_64',
'source',
@@ -44,30 +46,61 @@ def map_pkgbuild(pkgbuild: str) -> dict:
return {attr: val.replace('"', '').replace("'", '').replace('(', '').replace(')', '') for attr, val in re.findall(r'\n(\w+)=(.+)', pkgbuild)}
def map_srcinfo(string: str, fields: Set[str] = None) -> dict:
def map_srcinfo(string: str, pkgname: str, fields: Set[str] = None) -> dict:
subinfos, subinfo = [], {}
key_fields = {'pkgname', 'pkgbase'}
for field in RE_SRCINFO_KEYS.findall(string):
key = field[0].strip()
val = field[1].strip()
if subinfo and key in key_fields:
subinfos.append(subinfo)
subinfo = {key: val}
elif not fields or key in fields:
if key not in subinfo:
subinfo[key] = {val} if key in KNOWN_LIST_FIELDS else val
else:
if not isinstance(subinfo[key], set):
subinfo[key] = {subinfo[key]}
subinfo[key].add(val)
if subinfo:
subinfos.append(subinfo)
pkgnames = {s['pkgname'] for s in subinfos if 'pkgname' in s}
return merge_subinfos(subinfos=subinfos,
pkgname=None if (len(pkgnames) == 1 or pkgname not in pkgnames) else pkgname,
fields=fields)
def merge_subinfos(subinfos: List[dict], pkgname: Optional[str] = None, fields: Optional[Set[str]] = None) -> dict:
info = {}
for subinfo in subinfos:
if not pkgname or subinfo.get('pkgname') in {None, pkgname}:
for key, val in subinfo.items():
if not fields or key in fields:
current_val = info.get(key)
if fields:
field_re = re.compile(r'({})\s+=\s+(.+)\n'.format('|'.join(fields)))
else:
field_re = RE_SRCINFO_KEYS
if current_val is None:
info[key] = val
else:
if not isinstance(current_val, set):
current_val = {current_val}
info[key] = current_val
for tupl in field_re.findall(string):
key = tupl[0].strip()
val = tupl[1].strip()
if isinstance(val, set):
current_val.update(val)
else:
current_val.add(val)
if key not in info:
info[key] = [val] if key in KNOWN_LIST_FIELDS else val
else:
if not isinstance(info[key], list):
info[key] = [info[key]]
for field in info.keys():
val = info.get(field)
info[key].append(val)
pkgname = info.get('pkgname')
if isinstance(pkgname, list):
info['pkgname'] = pkgname[0]
if isinstance(val, set):
info[field] = [*val]
return info
@@ -90,7 +123,7 @@ class AURClient:
except:
return []
def get_src_info(self, name: str) -> dict:
def get_src_info(self, name: str, real_name: Optional[str] = None) -> dict:
srcinfo = self.srcinfo_cache.get(name)
if srcinfo:
@@ -99,7 +132,7 @@ class AURClient:
res = self.http_client.get(URL_SRC_INFO + urllib.parse.quote(name))
if res and res.text:
srcinfo = map_srcinfo(res.text)
srcinfo = map_srcinfo(string=res.text, pkgname=real_name if real_name else name)
if srcinfo:
self.srcinfo_cache[name] = srcinfo
@@ -118,7 +151,7 @@ class AURClient:
info_base = info.get('PackageBase')
if info_name and info_base and info_name != info_base:
self.logger.info('{p} is based on {b}. Retrieving {b} .SRCINFO'.format(p=info_name, b=info_base))
srcinfo = self.get_src_info(info_base)
srcinfo = self.get_src_info(name=info_base, real_name=info_name)
if srcinfo:
self.srcinfo_cache[name] = srcinfo

View File

@@ -611,7 +611,7 @@ class ArchManager(SoftwareManager):
for idx in range(1, len(commit_list)):
commit = commit_list[idx]
with open(srcinfo_path) as f:
pkgsrc = aur.map_srcinfo(f.read(), srcfields)
pkgsrc = aur.map_srcinfo(string=f.read(), pkgname=context.name ,fields=srcfields)
reset_proc = new_subprocess(['git', 'reset', '--hard', commit], cwd=clone_path)
if not context.handler.handle(SystemProcess(reset_proc, check_error_output=False)):
@@ -1350,7 +1350,7 @@ class ArchManager(SoftwareManager):
for idx, commit in enumerate(commits):
with open(srcinfo_path) as f:
pkgsrc = aur.map_srcinfo(f.read(), srcfields)
pkgsrc = aur.map_srcinfo(string=f.read(), pkgname=pkg.name, fields=srcfields)
if status_idx < 0 and '{}-{}'.format(pkgsrc.get('pkgver'), pkgsrc.get('pkgrel')) == pkg.version:
status_idx = idx
@@ -1566,10 +1566,10 @@ class ArchManager(SoftwareManager):
return pkg_repos
def _pre_download_source(self, project_dir: str, watcher: ProcessWatcher) -> bool:
def _pre_download_source(self, pkgname: str, project_dir: str, watcher: ProcessWatcher) -> bool:
if self.context.file_downloader.is_multithreaded():
with open('{}/.SRCINFO'.format(project_dir)) as f:
srcinfo = aur.map_srcinfo(f.read())
srcinfo = aur.map_srcinfo(string=f.read(), pkgname=pkgname)
pre_download_files = []
@@ -1639,7 +1639,7 @@ class ArchManager(SoftwareManager):
watcher=context.watcher,
pkgbuild_path='{}/PKGBUILD'.format(context.project_dir)):
context.pkgbuild_edited = True
srcinfo = aur.map_srcinfo(makepkg.gen_srcinfo(context.project_dir))
srcinfo = aur.map_srcinfo(string=makepkg.gen_srcinfo(context.project_dir), pkgname=context.name)
if srcinfo:
context.name = srcinfo['pkgname']
@@ -1656,7 +1656,7 @@ class ArchManager(SoftwareManager):
def _build(self, context: TransactionContext) -> bool:
self._edit_pkgbuild_and_update_context(context)
self._pre_download_source(context.project_dir, context.watcher)
self._pre_download_source(context.name, context.project_dir, context.watcher)
self._update_progress(context, 50)
if not self._handle_aur_package_deps_and_keys(context):
@@ -1691,7 +1691,7 @@ class ArchManager(SoftwareManager):
file_to_install = gen_file[0]
if len(gen_file) > 1:
srcinfo = aur.map_srcinfo(makepkg.gen_srcinfo(context.project_dir))
srcinfo = aur.map_srcinfo(string=makepkg.gen_srcinfo(context.project_dir), pkgname=context.name)
pkgver = '-{}'.format(srcinfo['pkgver']) if srcinfo.get('pkgver') else ''
pkgrel = '-{}'.format(srcinfo['pkgrel']) if srcinfo.get('pkgrel') else ''
arch = '-{}'.format(srcinfo['arch']) if srcinfo.get('arch') else ''
@@ -1762,7 +1762,7 @@ class ArchManager(SoftwareManager):
if context.repository == 'aur':
with open('{}/.SRCINFO'.format(context.project_dir)) as f:
srcinfo = aur.map_srcinfo(f.read())
srcinfo = aur.map_srcinfo(string=f.read(), pkgname=context.name)
pkgs_data = {context.name: self.aur_client.map_update_data(context.name, context.get_version(), srcinfo)}
else:

View File

@@ -0,0 +1,27 @@
pkgbase = bauh
pkgdesc = Graphical interface for managing your applications ( AppImage, Flatpak, Snap, Arch/AUR, Web )
pkgver = 0.9.6
pkgrel = 2
url = https://github.com/vinifmor/bauh
arch = any
license = zlib/libpng
makedepends = git
makedepends = python
makedepends = python-pip
makedepends = python-setuptools
depends = python
depends = python-pyqt5
depends = python-pyqt5-sip
depends = python-requests
depends = python-colorama
depends = python-pyaml
depends = qt5-svg
optdepends = flatpak: required for Flatpak support
optdepends = snapd: required for Snap support
optdepends = python-beautifulsoup4: for Native Web applications support
optdepends = python-lxml: for Native Web applications support
source = https://github.com/vinifmor/bauh/archive/0.9.6.tar.gz
sha512sums = cb1820b8a41dccec746d91d71b7f524c2e3caf6b30b0cd9666598b8ad49302654d9ce9bd1a0a2a9612afebc27ef78a2a94ac10e4e6c183742effe4feeabaa7b2
pkgname = bauh

View File

@@ -0,0 +1,36 @@
pkgbase = mangohud
pkgver = 0.5.1
pkgrel = 3
url = https://github.com/flightlessmango/MangoHud
arch = x86_64
license = MIT
makedepends = meson
makedepends = python-mako
makedepends = glslang
makedepends = libglvnd
makedepends = lib32-libglvnd
makedepends = vulkan-headers
makedepends = vulkan-icd-loader
makedepends = lib32-vulkan-icd-loader
makedepends = libxnvctrl
source = mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz
sha256sums = 3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6
pkgname = mangohud
pkgdesc = A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more
depends = gcc-libs
depends = mangohud-common
optdepends = bash: mangohud helper script
optdepends = libxnvctrl: support for older NVIDIA GPUs
pkgname = lib32-mangohud
pkgdesc = A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more (32-bit)
depends = lib32-gcc-libs
depends = mangohud
depends = mangohud-common
optdepends = lib32-libxnvctrl: support for older NVIDIA GPUs
pkgname = mangohud-common
pkgdesc = Common files for mangohud and lib32-mangohud
optdepends = bash: mangohud helper script

266
tests/gems/arch/test_aur.py Normal file
View File

@@ -0,0 +1,266 @@
import os
from unittest import TestCase
from bauh.gems.arch import aur
FILE_DIR = os.path.dirname(os.path.abspath(__file__))
class AURModuleTest(TestCase):
def test_map_srcinfo__only_one_pkgname(self):
expected_fields = {
'pkgbase': 'bauh',
'pkgname': 'bauh',
'pkgver': '0.9.6',
'pkgrel': '2',
'url': 'https://github.com/vinifmor/bauh',
'arch': 'any',
'license': 'zlib/libpng',
'makedepends': ['git', 'python', 'python-pip', 'python-setuptools'],
'depends': [
'python', 'python-colorama', 'python-pyaml', 'python-pyqt5', 'python-pyqt5-sip', 'python-requests', 'qt5-svg'
],
'optdepends': [
'flatpak: required for Flatpak support',
'python-beautifulsoup4: for Native Web applications support',
'python-lxml: for Native Web applications support',
'snapd: required for Snap support'
],
'source': ['https://github.com/vinifmor/bauh/archive/0.9.6.tar.gz'],
'sha512sums': ['cb1820b8a41dccec746d91d71b7f524c2e3caf6b30b0cd9666598b8ad49302654d9ce9bd1a0a2a9612afebc27ef78a2a94ac10e4e6c183742effe4feeabaa7b2']
}
with open(FILE_DIR + '/resources/bauh_srcinfo') as f:
srcinfo = f.read()
res = aur.map_srcinfo(srcinfo, 'bauh')
for key, val in expected_fields.items():
self.assertIn(key, res, "key '{}' not in res".format(key))
if isinstance(val, list):
val.sort()
res[key].sort()
self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key]))
def test_map_srcinfo__one_name__only_specific_fields(self):
expected_fields = {
'pkgver': '0.9.6',
'pkgrel': '2'
}
with open(FILE_DIR + '/resources/bauh_srcinfo') as f:
srcinfo = f.read()
res = aur.map_srcinfo(srcinfo, 'bauh', fields={*expected_fields.keys()})
self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res)))
for key, val in expected_fields.items():
self.assertIn(key, res, "key '{}' not in res".format(key))
if isinstance(val, list):
val.sort()
res[key].sort()
self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key]))
def test_map_srcinfo__several_pkgnames__pkgname_specified_case_1(self):
expected_fields = {
'pkgbase': 'mangohud',
'pkgname': 'mangohud',
'pkgver': '0.5.1',
'pkgrel': '3',
'pkgdesc': 'A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more',
'source': ['mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz'],
'sha256sums': ['3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6'],
'license': 'MIT',
'arch': 'x86_64',
'url': 'https://github.com/flightlessmango/MangoHud',
'makedepends': [
'glslang', 'libglvnd', 'lib32-libglvnd', 'meson', 'python-mako', 'vulkan-headers', 'vulkan-icd-loader',
'lib32-vulkan-icd-loader', 'libxnvctrl'
],
'depends': ['gcc-libs', 'mangohud-common'],
'optdepends': ['bash: mangohud helper script', 'libxnvctrl: support for older NVIDIA GPUs']
}
with open(FILE_DIR + '/resources/mangohud_srcinfo') as f:
srcinfo = f.read()
res = aur.map_srcinfo(srcinfo, 'mangohud')
self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res)))
for key, val in expected_fields.items():
self.assertIn(key, res, "key '{}' not in res".format(key))
if isinstance(val, list):
val.sort()
if isinstance(res[key], list):
res[key].sort()
self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key]))
def test_map_srcinfo__several_pkgnames__pkgname_specified_case_2(self):
expected_fields = {
'pkgbase': 'mangohud',
'pkgname': 'mangohud-common',
'pkgver': '0.5.1',
'pkgrel': '3',
'pkgdesc': 'Common files for mangohud and lib32-mangohud',
'source': ['mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz'],
'sha256sums': ['3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6'],
'license': 'MIT',
'url': 'https://github.com/flightlessmango/MangoHud',
'arch': 'x86_64',
'makedepends': [
'glslang', 'libglvnd', 'lib32-libglvnd', 'meson', 'python-mako', 'vulkan-headers', 'vulkan-icd-loader',
'lib32-vulkan-icd-loader', 'libxnvctrl'
],
'optdepends': ['bash: mangohud helper script']
}
with open(FILE_DIR + '/resources/mangohud_srcinfo') as f:
srcinfo = f.read()
res = aur.map_srcinfo(srcinfo, 'mangohud-common')
self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res)))
for key, val in expected_fields.items():
self.assertIn(key, res, "key '{}' not in res".format(key))
if isinstance(val, list):
val.sort()
if isinstance(res[key], list):
res[key].sort()
self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key]))
def test_map_srcinfo__several_pkgnames__pkgname_specified_case_3(self):
expected_fields = {
'pkgbase': 'mangohud',
'pkgname': 'lib32-mangohud',
'pkgver': '0.5.1',
'pkgrel': '3',
'pkgdesc': 'A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more (32-bit)',
'source': ['mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz'],
'sha256sums': ['3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6'],
'license': 'MIT',
'url': 'https://github.com/flightlessmango/MangoHud',
'arch': 'x86_64',
'makedepends': [
'glslang', 'libglvnd', 'lib32-libglvnd', 'meson', 'python-mako', 'vulkan-headers', 'vulkan-icd-loader',
'lib32-vulkan-icd-loader', 'libxnvctrl'
],
'depends': ['mangohud', 'mangohud-common', 'lib32-gcc-libs'],
'optdepends': ['lib32-libxnvctrl: support for older NVIDIA GPUs']
}
with open(FILE_DIR + '/resources/mangohud_srcinfo') as f:
srcinfo = f.read()
res = aur.map_srcinfo(srcinfo, 'lib32-mangohud')
self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res)))
for key, val in expected_fields.items():
self.assertIn(key, res, "key '{}' not in res".format(key))
if isinstance(val, list):
val.sort()
if isinstance(res[key], list):
res[key].sort()
self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key]))
def test_map_srcinfo__several_pkgnames__different_pkgname(self):
expected_fields = {
'pkgbase': 'mangohud',
'pkgname': ['lib32-mangohud', 'mangohud', 'mangohud-common'],
'pkgver': '0.5.1',
'pkgrel': '3',
'pkgdesc': [
'A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more (32-bit)',
'Common files for mangohud and lib32-mangohud',
'A Vulkan overlay layer for monitoring FPS, temperatures, CPU/GPU load and more',
],
'source': ['mangohud-0.5.1.tar.gz::https://github.com/flightlessmango/MangoHud/archive/v0.5.1.tar.gz'],
'sha256sums': ['3e91d4fc7369d46763894c13f3315133871dd02705072981770c3cf58e8081c6'],
'license': 'MIT',
'url': 'https://github.com/flightlessmango/MangoHud',
'arch': 'x86_64',
'makedepends': [
'glslang', 'libglvnd', 'lib32-libglvnd', 'meson', 'python-mako', 'vulkan-headers', 'vulkan-icd-loader',
'lib32-vulkan-icd-loader', 'libxnvctrl'
],
'depends': ['mangohud', 'mangohud-common', 'lib32-gcc-libs', 'gcc-libs'],
'optdepends': ['lib32-libxnvctrl: support for older NVIDIA GPUs',
'bash: mangohud helper script',
'libxnvctrl: support for older NVIDIA GPUs']
}
with open(FILE_DIR + '/resources/mangohud_srcinfo') as f:
srcinfo = f.read()
res = aur.map_srcinfo(srcinfo, 'xpto')
self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res)))
for key, val in expected_fields.items():
self.assertIn(key, res, "key '{}' not in res".format(key))
if isinstance(val, list):
val.sort()
if isinstance(res[key], list):
res[key].sort()
self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key]))
def test_map_srcinfo__several_names__pkgname_present__only_specific_fields(self):
expected_fields = {
'pkgver': '0.5.1',
'pkgrel': '3'
}
with open(FILE_DIR + '/resources/mangohud_srcinfo') as f:
srcinfo = f.read()
res = aur.map_srcinfo(srcinfo, 'mangohud-commons', fields={*expected_fields.keys()})
self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res)))
for key, val in expected_fields.items():
self.assertIn(key, res, "key '{}' not in res".format(key))
if isinstance(val, list):
val.sort()
res[key].sort()
self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key]))
def test_map_srcinfo__several_names__pkgname_not_present__only_specific_fields(self):
expected_fields = {
'pkgname': ['mangohud', 'lib32-mangohud', 'mangohud-common'],
'pkgver': '0.5.1'
}
with open(FILE_DIR + '/resources/mangohud_srcinfo') as f:
srcinfo = f.read()
res = aur.map_srcinfo(srcinfo, 'xpto', fields={*expected_fields.keys()})
self.assertEqual(len(expected_fields), len(res), "Expected: {}. Current: {}".format(len(expected_fields), len(res)))
for key, val in expected_fields.items():
self.assertIn(key, res, "key '{}' not in res".format(key))
if isinstance(val, list):
val.sort()
res[key].sort()
self.assertEqual(val, res[key], "expected: {}. current: {}".format(val, res[key]))