From 0ddb695cfbed1edd197bca07dece34c51e1a463f Mon Sep 17 00:00:00 2001 From: William Douglas Date: Tue, 19 Mar 2019 19:00:11 -0700 Subject: [PATCH] Add custom extras subpackage support Enable package builders to create custom extras subpackages (suffixed by -extras). This enables content to be divided more exactly than just the current extras subpackage support and also allows users to specify requirements on other subpackages in the package. It is not intended to replace extras usage as this feature should be used sparingly where content really must be specificly placed. --- README.rst | 22 ++++++++++++++++++++++ autospec/config.py | 33 ++++++++++++++++++++++++++++++++- autospec/files.py | 6 ++++++ autospec/git.py | 1 + autospec/specfiles.py | 6 ++++++ tests/test_files.py | 12 ++++++++++++ tests/test_specfile.py | 24 ++++++++++++++++++++++++ 7 files changed, 103 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 804aab9..cfda6a2 100644 --- a/README.rst +++ b/README.rst @@ -367,6 +367,28 @@ dev_extras functionality to place files used only for development against this software that Autospec does not automatically detect. +$custom_extras + A `toml `_ file with a required 'files' + keypair that has as its value a list of strings that are full paths that + will be put in the $custom-extras subpackage. It can also contain an + optional 'requires' keypair that has as its value a list of strings that + are subpackage names of other subpackages in the package. For example a + foo_extras file containing:: + + files = ['/usr/bin/foo', '/usr/lib64/libfoo.so'] + requires = ['data'] + + will produce a spec file package section for example-foo-extras with the + following content:: + + %package foo-extras + Summary: foo-extras components for the example package. + Group: Default + Requires: example-data = %{version}-%{release} + + %description foo-extras + foo-extras components for the example package. + setuid Each line in this file should contain the full path to a binary in the resulting build that should have the ``setuid`` attribute set with the diff --git a/autospec/config.py b/autospec/config.py index dc13a9d..6ef9ae0 100644 --- a/autospec/config.py +++ b/autospec/config.py @@ -26,12 +26,14 @@ import subprocess import sys import textwrap +import toml + import buildpattern import buildreq import check import license import tarball -from util import call, write_out +from util import call, print_warning, write_out extra_configure = "" extra_configure32 = "" @@ -400,6 +402,17 @@ def read_config_opts(path): transforms.pop(k) +def read_extras_config(path): + """Return parsed extras configurations from path.""" + if not os.path.exists(path): + return None + try: + return toml.load(path) + except Exception as excpt: + print_warning(excpt) + return None + + def rewrite_config_opts(path): """Rewrite options.conf file when an option has changed (verify_required for example).""" config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True) @@ -701,6 +714,24 @@ def parse_config_files(path, bump, filemanager, version): print("extras for : %s." % extra) filemanager.extras += content + for fname in os.listdir(path): + if not re.search('.+_extras$', fname) or fname == "dev_extras": + continue + content = read_extras_config(os.path.join(path, fname)) + if not content: + print_warning(f"Error reading custom extras file: {fname}") + continue + name = fname[:-len("_extras")] + if "files" not in content or type(content['files']) is not list: + print_warning(f"Invalid custom extras file: {fname} invalid or missing files list") + continue + if "requires" in content: + if type(content['requires']) is not list: + print_warning(f"Invalid custom extras file: {fname} invalid requires list") + continue + print(f"{name}-extras for {content['files']}") + filemanager.custom_extras[f"{name}-extras"] = content + content = read_conf_file(os.path.join(path, "dev_extras")) for extra in content: print("dev for : %s." % extra) diff --git a/autospec/files.py b/autospec/files.py index 7a85977..578fef2 100644 --- a/autospec/files.py +++ b/autospec/files.py @@ -39,6 +39,7 @@ class FileManager(object): self.files_blacklist = set() self.excludes = [] self.extras = [] + self.custom_extras = {} self.dev_extras = [] self.setuid = [] self.attrs = {} @@ -178,6 +179,10 @@ class FileManager(object): if filename in self.dev_extras: self.push_package_file(filename, "dev") self.excludes.append(filename) + for k, v in self.custom_extras.items(): + if filename in v['files']: + self.push_package_file(filename, k) + self.excludes.append(filename) if filename in self.setuid: newfn = "%attr(4755, root, root) " + filename @@ -332,3 +337,4 @@ class FileManager(object): specfile.packages = self.packages specfile.excludes = self.excludes specfile.locales = self.locales + specfile.custom_extras = self.custom_extras diff --git a/autospec/git.py b/autospec/git.py index dc5b9bf..3ec24e9 100644 --- a/autospec/git.py +++ b/autospec/git.py @@ -60,6 +60,7 @@ def commit_to_git(path): call("bash -c 'shopt -s failglob; git add -f *.sha256'", check=False, stderr=subprocess.DEVNULL, cwd=path) call("bash -c 'shopt -s failglob; git add -f *.sign'", check=False, stderr=subprocess.DEVNULL, cwd=path) call("bash -c 'shopt -s failglob; git add -f *.pkey'", check=False, stderr=subprocess.DEVNULL, cwd=path) + call("bash -c 'shopt -s failglob; git add -f *_extras'", check=False, stderr=subprocess.DEVNULL, cwd=path) call("git add configure", check=False, stderr=subprocess.DEVNULL, cwd=path) call("git add configure32", check=False, stderr=subprocess.DEVNULL, cwd=path) call("git add configure64", check=False, stderr=subprocess.DEVNULL, cwd=path) diff --git a/autospec/specfiles.py b/autospec/specfiles.py index b95edcc..524993d 100644 --- a/autospec/specfiles.py +++ b/autospec/specfiles.py @@ -80,6 +80,7 @@ class Specfile(object): self.install_prepend = [] self.install_append = [] self.excludes = [] + self.custom_extras = {} self.keyid = "" self.email = "" self.cargo_bin = False @@ -181,6 +182,8 @@ class Specfile(object): for pkg in sorted(self.packages): if pkg == "autostart" and self.no_autostart: continue + if pkg.endswith("-extras"): + continue if pkg in ["ignore", "main", "dev", "active-units", "extras", "lib32", "dev32", "legacypython", "doc", "abi"]: continue @@ -233,6 +236,9 @@ class Specfile(object): deps["python"] = ["python3"] if config.config_opts['dev_requires_extras']: deps["dev"].append("extras") + for k, v in self.custom_extras.items(): + if "requires" in v: + deps[k] = v['requires'] # migration workaround; if we have a python3 or legacypython package # we add an artificial python package diff --git a/tests/test_files.py b/tests/test_files.py index 0fedcb2..c103175 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -143,6 +143,18 @@ class TestFiles(unittest.TestCase): self.fm.push_package_file.assert_has_calls(calls) + def test_push_file_custom_extras(self): + """ + Test push_file to a custom extras package, this excludes the file + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + self.fm.custom_extras = {'test-extras': {'files': ["test"]}} + self.fm.push_file('test') + calls = [call('test', 'test-extras'), call('%exclude test')] + self.fm.push_package_file.assert_has_calls(calls) + + def test_push_file_setuid(self): """ Test push_file with fname in setuid list diff --git a/tests/test_specfile.py b/tests/test_specfile.py index d035ed2..e80b972 100644 --- a/tests/test_specfile.py +++ b/tests/test_specfile.py @@ -219,6 +219,29 @@ class TestSpecfileWrite(unittest.TestCase): "\n"] self.assertEqual(expect, self.WRITES) + def test_write_files_header_custom_extra_requires(self): + """ + test write_files_header with custom extras requires. + """ + self.specfile.packages["data"] = ["file1"] + self.specfile.packages["test-extras"] = ["file2"] + self.specfile.custom_extras = { 'test-extras': { 'requires': ["data"] }} + self.specfile.write_files_header() + expect = ["\n%package data\n", + "Summary: data components for the pkg package.\n", + "Group: Data\n", + "\n%description data\n", + "data components for the pkg package.\n", + "\n", + "\n%package test-extras\n", + "Summary: test-extras components for the pkg package.\n", + "Group: Default\n", + "Requires: pkg-data = %{version}-%{release}\n", + "\n%description test-extras\n", + "test-extras components for the pkg package.\n", + "\n"] + self.assertEqual(expect, self.WRITES) + def test_write_files_header_python_name(self): """ test write_files_header with uppercase letter in Specfile.name, causing @@ -236,6 +259,7 @@ class TestSpecfileWrite(unittest.TestCase): "\n"] self.assertEqual(expect, self.WRITES) + def test_write_files_header_bare(self): """ test write_files_header with no packages