Release v1.1.6

This release deprecates Makefile writing by autospec since per-package
Makefiles are not used by autospec.

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

.gitignore

@@ -3,3 +3,5 @@ __pycache__
 *~
 *.swp
 tags
+.coverage
+htmlcov

.travis.yml

@@ -2,6 +2,10 @@ language: python
 sudo: enabled
 python:
   - "3.6"
+
+install:
+  - pip install -r requirements.txt
+
 # command to run tests
 script:
   - make check

Makefile

@@ -47,4 +47,7 @@ test_autospec:
 	python3 tests/test_autospec.py -c ${CASES}
 
 unittests:
-	PYTHONPATH=${CURDIR}/autospec python3 -m unittest discover -b -s tests -p 'test_*.py'
+	PYTHONPATH=${CURDIR}/autospec coverage run -m unittest discover -b -s tests -p 'test_*.py' && coverage report
+
+coverage:
+	coverage report -m

autospec/autospec.py

@@ -19,6 +19,7 @@
 import argparse
 import sys
 import os
+import shutil
 import re
 import tempfile
 import configparser
@@ -116,6 +117,17 @@ def read_old_metadata():
             archives)
 
 
+def save_build_log(path, iteration):
+    """
+    Save build log to <path>/build.log.round<iteration>
+
+    Must be saved outside of the results/ directory since it gets wiped away on
+    each round.
+    """
+    buildlog = os.path.join(path, "results", "build.log")
+    shutil.copyfile(buildlog, "{}/build.log.round{}".format(path, iteration))
+
+
 def write_prep(workingdir):
     """
     Write metadata to the local workingdir when --prep-only is used
@@ -270,9 +282,20 @@ def package(args, url, name, archives, workingdir):
         filemanager.load_specfile(specfile)
         specfile.write_spec(build.download_path)
         filemanager.newfiles_printed = 0
+        mock_chroot = "/var/lib/mock/clear-{}/root/builddir/build/BUILDROOT/" \
+                      "{}-{}-{}.x86_64".format(build.uniqueext,
+                                               tarball.name,
+                                               tarball.version,
+                                               tarball.release)
+        if filemanager.clean_directories(mock_chroot):
+            # directories added to the blacklist, need to re-run
+            build.must_restart += 1
+
         if build.round > 20 or build.must_restart == 0:
             break
 
+        save_build_log(build.download_path, build.round)
+
     test.check_regression(build.download_path)
 
     if build.success == 0:

autospec/config.py

@@ -158,7 +158,9 @@ simple_pats = [
     (r"Package systemd was not found in the pkg-config search path.", "systemd-dev"),
     (r"Unable to find the requested Boost libraries.", "boost-dev"),
     (r"libproc not found. Please configure without procps", "procps-ng-dev"),
-    (r"configure: error: glib2", "glib-dev")]
+    (r"configure: error: glib2", "glib-dev"),
+    (r"C library 'efivar' not found", "efivar-dev"),
+    (r"Has header \"efi.h\": NO", "gnu-efi-dev")]
 
 # failed_pattern patterns
 # contains patterns for parsing build.log for missing dependencies
@@ -274,7 +276,11 @@ def get_metadata_conf():
     """
     metadata = {}
     metadata['name'] = tarball.name
-    metadata['url'] = tarball.url
+    if urlban:
+        metadata['url'] = re.sub(urlban, "localhost", tarball.url)
+    else:
+        metadata['url'] = tarball.url
+
     metadata['archives'] = ' '.join(tarball.archives)
     metadata['giturl'] = tarball.giturl
     return metadata
@@ -484,8 +490,6 @@ def parse_config_files(path, bump, filemanager):
 
     packages_file = None
 
-    read_config_opts(path)
-
     # Require autospec.conf for additional features
     if os.path.exists(config_file):
         config = configparser.ConfigParser(interpolation=None)
@@ -507,6 +511,9 @@ def parse_config_files(path, bump, filemanager):
 
         urlban = config['autospec'].get('urlban', None)
 
+    # Read values from options.conf (and deprecated files) and rewrite as necessary
+    read_config_opts(path)
+
     if not git_uri:
         print("Warning: Set [autospec][git] upstream template for git support")
     if not license_fetch:

autospec/files.py

@@ -23,6 +23,8 @@ import build
 import tarball
 import config
 import re
+import os
+import util
 from collections import OrderedDict
 # todo package splits
 
@@ -101,6 +103,46 @@ class FileManager(object):
         else:
             return False
 
+    def _clean_dirs(self, root, files):
+        """
+        Do the work to remove the directories from the files list
+        """
+        res = set()
+        removed = False
+
+        directive_re = re.compile("(%\w+(\([^\)]*\))?\s+)(.*)")
+        for f in files:
+            # skip the files with directives at the beginning, including %doc
+            # and %dir directives.
+            # autospec does not currently support adding empty directories to
+            # the file list by prefixing "%dir". Regardless, skip these entries
+            # because if they exist at this point it is intentional (i.e.
+            # support was added).
+            if directive_re.match(f):
+                res.add(f)
+                continue
+
+            if os.path.isdir(os.path.join(root, f.lstrip("/"))):
+                util.print_warning("Removing directory {} from file list".format(f))
+                self.files_blacklist.add(f)
+                removed = True
+            else:
+                res.add(f)
+
+        return (res, removed)
+
+    def clean_directories(self, root):
+        """
+        Remove directories from file list
+        """
+        removed = False
+        for pkg in self.packages:
+            self.packages[pkg], _rem = self._clean_dirs(root, self.packages[pkg])
+            if _rem:
+                removed = True
+
+        return removed
+
     def push_file(self, filename):
         """
         Perform a number of checks against the filename and push the filename

autospec/tarball.py

@@ -441,22 +441,6 @@ def find_extract(tar_path, tarfile):
     return extract_cmd, tar_prefix
 
 
-def write_makefile(archives):
-    """
-    Write the new makefile with url, name, and archives
-    """
-    with open(build.download_path + "/Makefile", "w") as f:
-        f.write("PKG_NAME := " + name + "\n")
-        f.write("URL := " + url + "\n")
-        sep = "ARCHIVES := "
-        for archive in archives:
-            f.write("{}{}".format(sep, archive))
-            sep = " " if sep != " " else " \\\n\t"
-        f.write("\n")
-        f.write("\n")
-        f.write("include ../common/Makefile.common\n")
-
-
 def prepare_and_extract(extract_cmd):
     """
     Prepare the directory and extract the tarball
@@ -532,8 +516,6 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager):
     # Now that the metadata has been collected print the header
     print_header()
     # write out the Makefile with the name, url, and archives we found
-    # DEPRECATED, this will be removed in a future version
-    write_makefile(archives_arg)
     # prepare directory and extract tarball
     prepare_and_extract(extract_cmd)
     # locate or download archives and move them into the right spot

requirements.txt

@@ -2,3 +2,4 @@ flake8>=3.4.0
 pycurl>=7.43.0
 toml>=0.9.0
 mock>=2.0.0
+coverage>=4.4.1

setup.cfg

@@ -2,3 +2,7 @@
 tag_build = 
 [pycodestyle]
 ignore = E501
+
+[coverage:run]
+# omit tests and travis site-packages
+omit = tests/*,*site-packages*,*site.py

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 import sys, os
-version = "1.1.3"
+version = "1.1.6"
 
 def readme():
     with open("README.rst") as f:

tests/test_files.py

@@ -1,5 +1,7 @@
 import unittest
 import files
+import tempfile
+import os
 from unittest.mock import call, MagicMock
 from files import FileManager
 
@@ -184,5 +186,69 @@ class TestFiles(unittest.TestCase):
         self.assertNotIn('test', self.fm.files)
         self.assertNotIn('test', self.fm.files_blacklist)
 
+    def test_clean_directories(self):
+        """
+        Test clean_directories with a directory in the list
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("/directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"], set(["/file1", "/file2"]))
+
+
+    def test_clean_directories_with_dir(self):
+        """
+        Test clean_directories with a %dir directory in the list. This should
+        remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%dir /directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%dir /directory", "/file1", "/file2"]))
+
+
+    def test_clean_directories_with_doc(self):
+        """
+        Test clean_directories with a %doc directive in the list. This should
+        remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%doc /directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%doc /directory", "/file1", "/file2"]))
+
+
 if __name__ == '__main__':
     unittest.main(buffer=True)

tests/test_tarball.py

@@ -168,30 +168,6 @@ class TestTarballVersionName(unittest.TestCase):
         tarball.build_gem_unpack = build_gem_unpack_backup
         tarball.build_untar = build_untar_backup
 
-    def test_write_makefile(self):
-        """
-        Test write_makefile for correct archive format
-        """
-        archives = ['archive1', 'dest1', 'archive2', 'dest2']
-        build.download_path = '.'
-        tarball.name = 'test'
-        tarball.url = 'url'
-        m_open = mock_open()
-        with patch('tarball.open', m_open, create=True):
-            tarball.write_makefile(archives)
-
-        exp_calls = [call().write('PKG_NAME := test\n'),
-                     call().write('URL := url\n'),
-                     call().write('ARCHIVES := archive1'),
-                     call().write(' dest1'),
-                     call().write(' \\\n\tarchive2'),
-                     call().write(' dest2'),
-                     call().write('\n'),
-                     call().write('\n'),
-                     call().write('include ../common/Makefile.common\n')]
-        for m_call in exp_calls:
-            self.assertIn(m_call, m_open.mock_calls)
-
 
 TAR_OUT = 'libjpeg-turbo-1.5.1/\n'                         \
           'libjpeg-turbo-1.5.1/md5/\n'                     \