Release v1.1.2

This release improves automated commit message guesses using the git
shortlog for the updated package. It also adds a --no-prep option that
downloads the sources, does the basic name and version detection, and
puts metadata in the <targetdir>/workingdir directory for human or tool
consumption before exiting without writing to the spec file.

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

Makefile

@@ -1,5 +1,5 @@
 check: autospec/*.py
-	@flake8 --max-line-length=199 --ignore=E402 $^
+	@flake8 --max-line-length=199 --ignore=E722 $^
 
 test_pkg_integrity:
 	PYTHONPATH=${CURDIR}/autospec python3 tests/test_pkg_integrity.py

autospec/autospec.py

@@ -116,6 +116,28 @@ def read_old_metadata():
             archives)
 
 
+def write_prep(workingdir):
+    """
+    Write metadata to the local workingdir when --prep-only is used
+    """
+    if config.urlban:
+        used_url = re.sub(config.urlban, "localhost", tarball.url)
+    else:
+        used_url = tarball.url
+
+    print()
+    print("Exiting after prep due to --prep-only flag")
+    print()
+    print("Results under ./workingdir")
+    print("Source  (./workingdir/{})".format(tarball.tarball_prefix))
+    print("Name    (./workingdir/name)    :", tarball.name)
+    print("Version (./workingdir/version) :", tarball.version)
+    print("URL     (./workingdir/source0) :", used_url)
+    write_out(os.path.join(workingdir, "name"), tarball.name)
+    write_out(os.path.join(workingdir, "version"), tarball.version)
+    write_out(os.path.join(workingdir, "source0"), used_url)
+
+
 def main():
     """
     Main function for autospec
@@ -153,6 +175,9 @@ def main():
                         default=False,
                         help="Search for package signature from source URL and "
                              "attempt to verify package")
+    parser.add_argument("-p", "--prep-only", action="store_true",
+                        default=False,
+                        help="Only perform preparatory work on package")
     parser.add_argument("--non_interactive", action="store_true",
                         default=False,
                         help="Disable interactive mode for package verification")
@@ -175,6 +200,14 @@ def main():
             "-a/--archives or options.conf['package']['archives'] requires an "
             "even number of arguments"))
 
+    if args.prep_only:
+        package(args, url, name, archives, "./workingdir")
+    else:
+        with tempfile.TemporaryDirectory() as workingdir:
+            package(args, url, name, archives, workingdir)
+
+
+def package(args, url, name, archives, workingdir):
     check_requirements(args.git)
     build.setup_workingdir(workingdir)
 
@@ -203,6 +236,10 @@ def main():
     config.parse_config_files(build.download_path, args.bump, filemanager)
     config.parse_existing_spec(build.download_path, tarball.name)
 
+    if args.prep_only:
+        write_prep(workingdir)
+        exit(0)
+
     buildreq.set_build_req()
     buildreq.scan_for_configure(_dir)
     specdescription.scan_for_description(tarball.name, _dir)
@@ -268,5 +305,4 @@ def main():
 
 
 if __name__ == '__main__':
-    with tempfile.TemporaryDirectory() as workingdir:
-        main()
+    main()

autospec/build.py

@@ -33,18 +33,15 @@ success = 0
 round = 0
 must_restart = 0
 base_path = None
-output_path = None
 download_path = None
 uniqueext = ''
 
 
 def setup_workingdir(workingdir):
     global base_path
-    global output_path
     global download_path
     base_path = workingdir
-    output_path = os.path.join(base_path, "output")
-    download_path = os.path.join(output_path, tarball.name)
+    download_path = os.path.join(base_path, tarball.name)
 
 
 def simple_pattern_pkgconfig(line, pattern, pkgconfig):

autospec/commitmessage.py

@@ -32,6 +32,8 @@ import config
 import tarball
 import util
 
+from subprocess import PIPE, run
+
 
 def scan_for_changes(download_path, directory):
     """
@@ -168,6 +170,54 @@ def process_NEWS(newsfile):
     return commitmessage, cves
 
 
+def process_git(giturl, oldversion, newversion):
+    """
+    process_git() checks out a git tree and tries to turn the
+    git history into a commit message
+    """
+
+    oldtag = ""
+    guessed_oldtag = oldversion
+    newtag = ""
+    guessed_newtag = newversion
+
+    if len(giturl) < 1:
+        return ""
+    if oldversion == newversion:
+        return ""
+
+    run(["git", "-C", "results", "clone", giturl, tarball.name])
+    p = run(["git", "-C", "results/" + tarball.name, "tag"], stdout=PIPE)
+    tags = p.stdout.decode('utf-8').split('\n')
+
+    for t in tags:
+        i = t.find(oldversion)
+        if i != -1:
+            guessed_oldtag = t
+        if t == oldversion or t == "v" + oldversion:
+            oldtag = t
+        i = t.find(newversion)
+        if i != -1:
+            guessed_newtag = t
+        if t == newversion or t == "v" + newversion:
+            newtag = t
+
+    if oldtag == "":
+        oldtag = guessed_oldtag
+    if newtag == "":
+        newtag = guessed_newtag
+
+    p = run(["git", "-C", "results/" + tarball.name, "log", oldtag + ".." + newtag], stdout=PIPE)
+    fulllog = p.stdout.decode('utf-8').split('\n')
+    p = run(["git", "-C", "results/" + tarball.name, "shortlog", oldtag + ".." + newtag], stdout=PIPE)
+    shortlog = p.stdout.decode('utf-8').split('\n')
+
+    if len(fulllog) < 15:
+        return fulllog
+    else:
+        return shortlog
+
+
 def guess_commit_message():
     """
     guess_commit_message() parses newsfiles and determines a sane commit
@@ -199,6 +249,10 @@ def guess_commit_message():
     if config.old_version is not None and config.old_version != tarball.version:
         commitmessage.append("{}: Autospec creation for update from version {} to version {}"
                              .format(tarball.name, config.old_version, tarball.version))
+        if tarball.giturl != "":
+            gitmsg = process_git(tarball.giturl, config.old_version, tarball.version)
+            commitmessage.append("")
+            commitmessage.extend(gitmsg)
     else:
         if cves:
             commitmessage.append("{}: Fix for {}"
@@ -224,7 +278,7 @@ def guess_commit_message():
         commitmessage.append("")
 
     util.write_out(os.path.join(build.download_path, "commitmsg"),
-                   "\n".join(commitmessage) + "\n", encode="latin-1")
+                   "\n".join(commitmessage) + "\n")
 
     print("Guessed commit message:")
     try:

autospec/config.py

@@ -276,6 +276,7 @@ def get_metadata_conf():
     metadata['name'] = tarball.name
     metadata['url'] = tarball.url
     metadata['archives'] = ' '.join(tarball.archives)
+    metadata['giturl'] = tarball.giturl
     return metadata
 
 

autospec/specfiles.py

@@ -315,7 +315,7 @@ class Specfile(object):
         self._write("%defattr(-,root,root,-)\n")
         if "main" in self.packages:
             for filename in sorted(self.packages["main"]):
-                self._write("{}\n".format(filename))
+                self._write("{}\n".format(self.quote_filename(filename)))
 
         for pkg in sorted(self.packages):
             if pkg in ["ignore", "main", "locales"]:
@@ -324,7 +324,7 @@ class Specfile(object):
             self._write("\n%files {}\n".format(pkg))
             self._write("%defattr(-,root,root,-)\n")
             for filename in sorted(self.packages[pkg]):
-                self._write("{}\n".format(filename))
+                self._write("{}\n".format(self.quote_filename(filename)))
 
     def write_lang_files(self):
         """
@@ -1162,3 +1162,31 @@ class Specfile(object):
 
     def _write_strip(self, string):
         self.specfile.write_strip(string)
+
+    def quote_filename(self, filename):
+        """
+        Quotes the filename, if necessary. Identifies and skips any RPM directive prefix.
+        """
+        # Characters that require quoting -- only those with special
+        # meaning in specfiles
+        special_chars = set(" \t")
+        # Build up the output as a string
+        quoted = ''
+        # Capture any directive prefix separately from actual filename
+        #                          (1                   )(3 )
+        directive_re = re.compile("(%\w+(\([^\)]*\))?\s+)(.*)")
+        parts = directive_re.match(filename)
+        if parts:
+            # Add prefix to the output
+            quoted += parts.group(1)
+            # Set the filename to the remaining portion
+            filename = parts.group(3)
+
+        # Now check for special characters
+        if any(c in filename for c in special_chars):
+            # Quote the filename
+            quoted += '"{}"'.format(filename)
+        else:
+            # Add the filename as-is
+            quoted += filename
+        return quoted

autospec/tarball.py

@@ -40,6 +40,7 @@ path = ""
 tarball_prefix = ""
 gcov_file = ""
 archives = []
+giturl = ""
 
 
 def get_sha1sum(filename):
@@ -174,6 +175,8 @@ def print_header():
 
 
 def download_tarball(target_dir):
+    global giturl
+
     """
     Download tarball at url (global) to target_dir
 
@@ -193,6 +196,8 @@ def download_tarball(target_dir):
                     config_f["package"].get("url") == url or
                     config_f["package"].get("archives") == " ".join(archives)):
                 target = os.getcwd()
+            if "giturl" in config_f["package"]:
+                giturl = config_f["package"].get("giturl")
 
     if target_dir:
         target = target_dir
@@ -278,6 +283,7 @@ def name_and_version(name_arg, version_arg, filemanager):
     global rawname
     global version
     global url
+    global giturl
 
     tarfile = os.path.basename(url)
 
@@ -323,17 +329,18 @@ def name_and_version(name_arg, version_arg, filemanager):
     if "github.com" in url:
         # define regex accepted for valid packages, important for specific
         # patterns to come before general ones
-        github_patterns = [r"https?://github.com/.*/(.*?)/archive/[v|r]?.*/(.*).tar",
-                           r"https?://github.com/.*/(.*?)/archive/[-a-zA-Z]*-(.*).tar",
-                           r"https?://github.com/.*/(.*?)/archive/[vVrR]?(.*).tar",
-                           r"https?://github.com/.*/(.*?)/releases/download/v.*/(.*).tar"]
+        github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
+                           r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z]*-(.*).tar",
+                           r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",
+                           r"https?://github.com/(.*)/(.*?)/releases/download/v.*/(.*).tar"]
 
         for pattern in github_patterns:
             m = re.search(pattern, url)
             if m:
-                name = m.group(1).strip()
+                name = m.group(2).strip()
                 rawname = name
-                version = convert_version(m.group(2))
+                version = convert_version(m.group(3))
+                giturl = "https://github.com/" + m.group(1).strip() + "/" + name + ".git"
                 break
 
     if "mirrors.kernel.org" in url:
@@ -447,7 +454,7 @@ def prepare_and_extract(extract_cmd):
     """
     shutil.rmtree(os.path.join(build.base_path, name), ignore_errors=True)
     shutil.rmtree(os.path.join(build.base_path, tarball_prefix), ignore_errors=True)
-    os.makedirs("{}".format(build.output_path), exist_ok=True)
+    os.makedirs("{}".format(build.base_path), exist_ok=True)
     call("mkdir -p %s" % build.download_path)
     call(extract_cmd)
 

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 import sys, os
-version = "1.1.0"
+version = "1.1.2"
 
 def readme():
     with open("README.rst") as f:

tests/test_build.py

@@ -23,7 +23,6 @@ class TestBuildpattern(unittest.TestCase):
         build.round = 0
         build.must_restart = 0
         build.base_path = None
-        build.output_path = None
         build.download_path = None
         build.buildreq.buildreqs = set()
         build.config.config_opts['32bit'] = False
@@ -35,9 +34,7 @@ class TestBuildpattern(unittest.TestCase):
         build.tarball.name = "testtarball"
         build.setup_workingdir("test_directory")
         self.assertEqual(build.base_path, "test_directory")
-        self.assertEqual(build.output_path, "test_directory/output")
-        self.assertEqual(build.download_path,
-                         "test_directory/output/testtarball")
+        self.assertEqual(build.download_path, "test_directory/testtarball")
 
     def test_simple_pattern_pkgconfig(self):
         """

tests/test_specfile.py

@@ -343,15 +343,21 @@ class TestSpecfileWrite(unittest.TestCase):
         """
         test write_files base test.
         """
-        self.specfile.packages["main"] = ["mainfile1", "mainfile2", "mainfile3"]
+        self.specfile.packages["main"] = ["mainfile1", "/mainfile2", "/mainfile3",
+                "/mainfile 4", "mainfile\t5", "%foo /mainfile6", "%bar /mainfile 7"]
         self.specfile.packages["ignore"] = ["ignorepkg"]
         self.specfile.packages["other"] = ["other2", "other1"]
         self.specfile.write_files()
+        # Note the special sorting
         expect = ["\n%files\n",
                   "%defattr(-,root,root,-)\n",
+                  '%bar "/mainfile 7"\n',
+                  "%foo /mainfile6\n",
+                  '"/mainfile 4"\n',
+                  "/mainfile2\n",
+                  "/mainfile3\n",
+                  '"mainfile\t5"\n',
                   "mainfile1\n",
-                  "mainfile2\n",
-                  "mainfile3\n",
                   "\n%files other\n",
                   "%defattr(-,root,root,-)\n",
                   "other1\n",