Add custom extras subpackage support

Enable package builders to create custom extras subpackages (suffixed
by -extras). This enables content to be divided more exactly than just
the current extras subpackage support and also allows users to specify
requirements on other subpackages in the package. It is not intended
to replace extras usage as this feature should be used sparingly where
content really must be specificly placed.

README.rst

@@ -367,6 +367,28 @@ dev_extras
   functionality to place files used only for development against this
   software that Autospec does not automatically detect.
 
+$custom_extras
+  A `toml <https://github.com/toml-lang/toml>`_ file with a required 'files'
+  keypair that has as its value a list of strings that are full paths that
+  will be put in the $custom-extras subpackage. It can also contain an
+  optional 'requires' keypair that has as its value a list of strings that
+  are subpackage names of other subpackages in the package. For example a
+  foo_extras file containing::
+
+    files = ['/usr/bin/foo', '/usr/lib64/libfoo.so']
+    requires = ['data']
+
+  will produce a spec file package section for example-foo-extras with the
+  following content::
+
+    %package foo-extras
+    Summary: foo-extras components for the example package.
+    Group: Default
+    Requires: example-data = %{version}-%{release}
+
+    %description foo-extras
+    foo-extras components for the example package.
+
 setuid
   Each line in this file should contain the full path to a binary in the
   resulting build that should have the ``setuid`` attribute set with the

autospec/config.py

@@ -26,12 +26,14 @@ import subprocess
 import sys
 import textwrap
 
+import toml
+
 import buildpattern
 import buildreq
 import check
 import license
 import tarball
-from util import call, write_out
+from util import call, print_warning, write_out
 
 extra_configure = ""
 extra_configure32 = ""
@@ -400,6 +402,17 @@ def read_config_opts(path):
             transforms.pop(k)
 
 
+def read_extras_config(path):
+    """Return parsed extras configurations from path."""
+    if not os.path.exists(path):
+        return None
+    try:
+        return toml.load(path)
+    except Exception as excpt:
+        print_warning(excpt)
+    return None
+
+
 def rewrite_config_opts(path):
     """Rewrite options.conf file when an option has changed (verify_required for example)."""
     config_f = configparser.ConfigParser(interpolation=None, allow_no_value=True)
@@ -701,6 +714,24 @@ def parse_config_files(path, bump, filemanager, version):
         print("extras for  : %s." % extra)
     filemanager.extras += content
 
+    for fname in os.listdir(path):
+        if not re.search('.+_extras$', fname) or fname == "dev_extras":
+            continue
+        content = read_extras_config(os.path.join(path, fname))
+        if not content:
+            print_warning(f"Error reading custom extras file: {fname}")
+            continue
+        name = fname[:-len("_extras")]
+        if "files" not in content or type(content['files']) is not list:
+            print_warning(f"Invalid custom extras file: {fname} invalid or missing files list")
+            continue
+        if "requires" in content:
+            if type(content['requires']) is not list:
+                print_warning(f"Invalid custom extras file: {fname} invalid requires list")
+                continue
+        print(f"{name}-extras for {content['files']}")
+        filemanager.custom_extras[f"{name}-extras"] = content
+
     content = read_conf_file(os.path.join(path, "dev_extras"))
     for extra in content:
         print("dev for     : %s." % extra)

autospec/files.py

@@ -39,6 +39,7 @@ class FileManager(object):
         self.files_blacklist = set()
         self.excludes = []
         self.extras = []
+        self.custom_extras = {}
         self.dev_extras = []
         self.setuid = []
         self.attrs = {}
@@ -178,6 +179,10 @@ class FileManager(object):
         if filename in self.dev_extras:
             self.push_package_file(filename, "dev")
             self.excludes.append(filename)
+        for k, v in self.custom_extras.items():
+            if filename in v['files']:
+                self.push_package_file(filename, k)
+                self.excludes.append(filename)
 
         if filename in self.setuid:
             newfn = "%attr(4755, root, root) " + filename
@@ -332,3 +337,4 @@ class FileManager(object):
         specfile.packages = self.packages
         specfile.excludes = self.excludes
         specfile.locales = self.locales
+        specfile.custom_extras = self.custom_extras

autospec/git.py

@@ -60,6 +60,7 @@ def commit_to_git(path):
     call("bash -c 'shopt -s failglob; git add -f *.sha256'", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("bash -c 'shopt -s failglob; git add -f *.sign'", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("bash -c 'shopt -s failglob; git add -f *.pkey'", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("bash -c 'shopt -s failglob; git add -f *_extras'", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add configure", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add configure32", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add configure64", check=False, stderr=subprocess.DEVNULL, cwd=path)

autospec/specfiles.py

@@ -80,6 +80,7 @@ class Specfile(object):
         self.install_prepend = []
         self.install_append = []
         self.excludes = []
+        self.custom_extras = {}
         self.keyid = ""
         self.email = ""
         self.cargo_bin = False
@@ -181,6 +182,8 @@ class Specfile(object):
         for pkg in sorted(self.packages):
             if pkg == "autostart" and self.no_autostart:
                 continue
+            if pkg.endswith("-extras"):
+                continue
             if pkg in ["ignore", "main", "dev", "active-units", "extras",
                        "lib32", "dev32", "legacypython", "doc", "abi"]:
                 continue
@@ -233,6 +236,9 @@ class Specfile(object):
         deps["python"] = ["python3"]
         if config.config_opts['dev_requires_extras']:
             deps["dev"].append("extras")
+        for k, v in self.custom_extras.items():
+            if "requires" in v:
+                deps[k] = v['requires']
 
         # migration workaround; if we have a python3 or legacypython package
         # we add an artificial python package

tests/test_files.py

@@ -143,6 +143,18 @@ class TestFiles(unittest.TestCase):
         self.fm.push_package_file.assert_has_calls(calls)
 
 
+    def test_push_file_custom_extras(self):
+        """
+        Test push_file to a custom extras package, this excludes the file
+        """
+        self.fm.file_is_locale = MagicMock(return_value=False)
+        self.fm.push_package_file = MagicMock()
+        self.fm.custom_extras = {'test-extras': {'files': ["test"]}}
+        self.fm.push_file('test')
+        calls = [call('test', 'test-extras'), call('%exclude test')]
+        self.fm.push_package_file.assert_has_calls(calls)
+
+
     def test_push_file_setuid(self):
         """
         Test push_file with fname in setuid list

tests/test_specfile.py

@@ -219,6 +219,29 @@ class TestSpecfileWrite(unittest.TestCase):
                   "\n"]
         self.assertEqual(expect, self.WRITES)
 
+    def test_write_files_header_custom_extra_requires(self):
+        """
+        test write_files_header with custom extras requires.
+        """
+        self.specfile.packages["data"] = ["file1"]
+        self.specfile.packages["test-extras"] = ["file2"]
+        self.specfile.custom_extras = { 'test-extras': { 'requires': ["data"] }}
+        self.specfile.write_files_header()
+        expect = ["\n%package data\n",
+                  "Summary: data components for the pkg package.\n",
+                  "Group: Data\n",
+                  "\n%description data\n",
+                  "data components for the pkg package.\n",
+                  "\n",
+                  "\n%package test-extras\n",
+                  "Summary: test-extras components for the pkg package.\n",
+                  "Group: Default\n",
+                  "Requires: pkg-data = %{version}-%{release}\n",
+                  "\n%description test-extras\n",
+                  "test-extras components for the pkg package.\n",
+                  "\n"]
+        self.assertEqual(expect, self.WRITES)
+
     def test_write_files_header_python_name(self):
         """
         test write_files_header with uppercase letter in Specfile.name, causing
@@ -236,6 +259,7 @@ class TestSpecfileWrite(unittest.TestCase):
                   "\n"]
         self.assertEqual(expect, self.WRITES)
 
+
     def test_write_files_header_bare(self):
         """
         test write_files_header with no packages