Release v1.1.3

This release makes some small updates to our README documentation,
improves tarball.py and pkg_integrity.py testing, fixes a bug that
allowed an empty file to remain on failed downloads, and adds a regular
expression for build.log test scanning.

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

README.rst

@@ -68,7 +68,7 @@ Usage: ``python3 autospec.py [options] URL``
 -l, --license-only                              Only scan for license files
 -b, --skip-bump                                 Don't bump release number
 -c CONFIG, --config CONFIG                      Set configuration file to use
--t DIRECTORY --target DIRECTORY                 Set location to create or use
+-t DIRECTORY, --target DIRECTORY                Set location to create or use
 
 
 
@@ -220,6 +220,10 @@ Controlling the build process
     work for the given package. This one line file allows you to override the
     build pattern that ``autospec`` will use. The supported build_pattern types are:
 
+        - R: R language package
+        - cpan: perl language package
+        - ruby: ruby language package
+        - maven: Java language package
         - configure: Traditional ``%configure`` autotools route
         - configure_ac: Like ``configure, but performs ``%reconfigure`` to regenerate ``./configure``
         - autogen: Similar to ``configure_ac`` but uses the existing ``./autogen.sh`` instead of ``%reconfigure``

autospec/count.py

@@ -1104,6 +1104,13 @@ def parse_log(log, pkgname=''):
             total_fail += convert_int(match.group(3))
             continue
 
+        # vim
+        # Executed 9 tests
+        match = re.search(r"Executed ([0-9]+) tests$", line)
+        if match and incheck:
+            total_tests += convert_int(match.group(1))
+            continue
+
         # rubygem-formatador
         #   9 succeeded in 0.00375661 seconds
         match = re.search(r"([0-9]+) succeeded in [0-9]+\.[0-9]+ seconds", line)

autospec/tarball.py

@@ -66,6 +66,7 @@ def really_download(upstream_url, destination):
             c.perform()
         except pycurl.error:
             print_fatal("unable to download {}".format(upstream_url))
+            os.remove(destination)
             exit(1)
         finally:
             c.close()
@@ -284,6 +285,7 @@ def name_and_version(name_arg, version_arg, filemanager):
     global version
     global url
     global giturl
+    global repo
 
     tarfile = os.path.basename(url)
 
@@ -332,15 +334,22 @@ def name_and_version(name_arg, version_arg, filemanager):
         github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
                            r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z]*-(.*).tar",
                            r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",
-                           r"https?://github.com/(.*)/(.*?)/releases/download/v.*/(.*).tar"]
+                           r"https?://github.com/(.*)/(.*?)/releases/download/.*?/(.*).tar",
+                           r"https?://github.com/(.*)/(.*?)/files/.*?/(.*).tar"]
 
         for pattern in github_patterns:
             m = re.search(pattern, url)
             if m:
-                name = m.group(2).strip()
+                repo = m.group(2).strip()
+                if repo not in name:
+                    # Only take the repo name as the package name if it's more descriptive
+                    name = repo
+                elif name != repo:
+                    name = re.sub("release-", '', name)
+                    name = re.sub("\d*$", '', name)
                 rawname = name
                 version = convert_version(m.group(3))
-                giturl = "https://github.com/" + m.group(1).strip() + "/" + name + ".git"
+                giturl = "https://github.com/" + m.group(1).strip() + "/" + repo + ".git"
                 break
 
     if "mirrors.kernel.org" in url:

requirements.txt

@@ -1,3 +1,4 @@
 flake8>=3.4.0
 pycurl>=7.43.0
 toml>=0.9.0
+mock>=2.0.0

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 import sys, os
-version = "1.1.2"
+version = "1.1.3"
 
 def readme():
     with open("README.rst") as f:

tests/test_pkg_integrity.py

@@ -1,13 +1,18 @@
 import os
+import shutil
+import mock
 import unittest
 import tempfile
 import pkg_integrity
 
 
+TESTDIR = os.path.join(os.getcwd(), "tests/testfiles/pkg_integrity")
+
 PACKAGE_URL = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.1.tar.gz"
 XATTR_PKT_URL = "http://pypi.debian.net/xattr/xattr-0.9.1.tar.gz"
 NO_SIGN_PKT_URL = "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz"
 GEM_PKT = "https://rubygems.org/downloads/hoe-debugging-1.2.1.gem"
+NOSIGN_PKT_URL_BAD = "http://gnu.mirrors.pair.com/savannah/savannah/quagga/bad_quagga-1.1.0.tar.gz"
 NOSIGN_PKT_URL = "http://download.savannah.gnu.org/releases/quagga/quagga-1.1.0.tar.gz"
 NOSIGN_SIGN_URL = "http://download.savannah.gnu.org/releases/quagga/quagga-1.1.0.tar.gz.asc"
 PYPI_MD5_ONLY_PKG = "http://pypi.debian.net/tappy/tappy-0.9.2.tar.gz"
@@ -15,36 +20,23 @@ GNOME_SHA256_PKG = "https://download.gnome.org/sources/pygobject/3.24/pygobject-
 KEYID = "EC2392F2EDE74488680DA3CF5F2B4756ED873D23"
 
 
-class TestGPGCli(unittest.TestCase):
+def mock_attempt_to_download(path, dest=None):
+    if dest:
+        shutil.copyfile(os.path.join(TESTDIR, os.path.basename(path)), dest)
+        return 200
 
-    def test_import_export(self):
-        with pkg_integrity.cli_gpg_ctx() as ctx:
-            err, output = ctx.export_key(KEYID)
-            self.assertTrue(err is not None)
-            err, output = ctx.import_key(KEYID)
-            self.assertTrue(err is None)
-            err, output = ctx.export_key(KEYID)
-            self.assertTrue(err is None)
-            self.assertTrue('PGP PUBLIC KEY' in output)
+def mock_head_request(url):
+    bad_sigs = ["http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.1.tar.gz.sig",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.sig",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.asc"]
 
-    def test_import_non_existing_key(self):
-        with pkg_integrity.cli_gpg_ctx() as ctx:
-            keyid = '0' + KEYID[1:]
-            err, output = ctx.import_key(keyid)
-            self.assertTrue(err is not None)
-
-    def test_display_key_info(self):
-        with pkg_integrity.cli_gpg_ctx() as ctx:
-            err, output = ctx.import_key(KEYID)
-            self.assertTrue(err is None)
-            err, output = ctx.export_key(KEYID)
-            with open('key_test.pkey', 'w') as out_key:
-                out_key.write(output)
-            err, output = ctx.display_keyinfo('key_test.pkey')
-            os.remove('key_test.pkey')
-            self.assertTrue('keyid' in output)
+    if url in bad_sigs:
+        return 404
+    return 200
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
+@mock.patch('pkg_integrity.head_request', mock_head_request)
 class TestCheckFn(unittest.TestCase):
 
     def setUp(self):
@@ -53,8 +45,6 @@ class TestCheckFn(unittest.TestCase):
         pkg_integrity.config.rewrite_config_opts = mock_rewrite
         pkg_integrity.config.config_opts['verify_required'] = False
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_check_matching_sign_url(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
@@ -73,6 +63,7 @@ class TestCheckFn(unittest.TestCase):
             self.assertTrue(result)
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
 class TestDomainBasedVerifiers(unittest.TestCase):
 
     def run_test_for_domain(self, Verifier, url):
@@ -85,8 +76,6 @@ class TestDomainBasedVerifiers(unittest.TestCase):
         return None
 
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_pypi(self):
         result = self.run_test_for_domain(pkg_integrity.PyPiVerifier, PYPI_MD5_ONLY_PKG)
         self.assertTrue(result)
@@ -96,6 +85,7 @@ class TestDomainBasedVerifiers(unittest.TestCase):
         self.assertTrue(result)        
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
 class TestGEMShaVerifier(unittest.TestCase):
 
     def setUp(self):
@@ -122,6 +112,8 @@ class TestGEMShaVerifier(unittest.TestCase):
             self.assertEqual(a.exception.code, 1)
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
+@mock.patch('pkg_integrity.head_request', mock_head_request)
 class TestGPGVerifier(unittest.TestCase):
 
     def setUp(self):
@@ -130,8 +122,6 @@ class TestGPGVerifier(unittest.TestCase):
         pkg_integrity.config.rewrite_config_opts = mock_rewrite
         pkg_integrity.config.config_opts['verify_required'] = False
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_from_url(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
@@ -143,12 +133,14 @@ class TestGPGVerifier(unittest.TestCase):
 
     def test_check_quit(self):
         with tempfile.TemporaryDirectory() as tmpd:
-            #with self.assertRaises(SystemExit) as a:
-            pkg_integrity.check(NO_SIGN_PKT_URL, tmpd, interactive=False)
-            #self.assertEqual(a.exception.code, 1)
+            with self.assertRaises(SystemExit) as a:
+                out_file = os.path.join(tmpd, os.path.basename(NOSIGN_PKT_URL_BAD))
+                pkg_integrity.attempt_to_download(NOSIGN_PKT_URL_BAD, out_file)
+                key_file = os.path.join(tmpd, os.path.basename(NOSIGN_PKT_URL_BAD))
+                pkg_integrity.attempt_to_download(NOSIGN_SIGN_URL, key_file + '.asc')
+                result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd)
+                self.assertEqual(a.exception.code, 1)
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_from_disk(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
@@ -158,8 +150,6 @@ class TestGPGVerifier(unittest.TestCase):
             result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key)
             self.assertTrue(result)
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_non_matchingsig(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
@@ -174,35 +164,14 @@ class TestGPGVerifier(unittest.TestCase):
         result = pkg_integrity.from_disk('http://nokey.com/package.tar.gz',
                                          'NonExistentPKG.tar.gz',
                                          'NonExistentKey.asc')
-        self.assertTrue(result is None)
+        self.assertIsNone(result)
 
     def test_result_on_nosign_package(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(NO_SIGN_PKT_URL))
             pkg_integrity.attempt_to_download(NO_SIGN_PKT_URL, out_file)
             result = pkg_integrity.check(NO_SIGN_PKT_URL, tmpd)
-            self.assertTrue(result is None)
-
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
-    def test_pubkey_import(self):
-        def say_yes(_):
-            return True
-        _ = pkg_integrity.InputGetter.get_answer
-        pkg_integrity.InputGetter.get_answer = say_yes
-        keyid = '0' + KEYID[1:]
-        result = pkg_integrity.attempt_key_import(keyid)
-        self.assertTrue(result is False)
-        result = pkg_integrity.attempt_key_import(KEYID)
-        self.assertTrue(result)
-        pkg_integrity.InputGetter.get_answer = _
-        self.removeKey()
-
-    def removeKey(self):
-        key_path = os.path.dirname(os.path.realpath(__file__))
-        key_path = os.path.dirname(key_path) + '/autospec/keyring/{}.pkey'.format(KEYID)
-        if os.path.exists(key_path):
-            os.unlink(key_path)
+            self.assertIsNone(result)
 
 
 class TestInputGetter(unittest.TestCase):
@@ -212,12 +181,14 @@ class TestInputGetter(unittest.TestCase):
     def test_timput(self):
         ig = pkg_integrity.InputGetter(default='N', timeout=2)
         answer = ig.get_answer()
-        self.assertTrue(answer is None)
+        self.assertIsNone(answer)
         ig = pkg_integrity.InputGetter(default='Y', timeout=2)
         answer = ig.get_answer()
-        self.assertTrue(answer is None)
+        self.assertIsNone(answer)
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
+@mock.patch('pkg_integrity.head_request', mock_head_request)
 class TestUtils(unittest.TestCase):
 
     def setUp(self):
@@ -252,23 +223,6 @@ class TestUtils(unittest.TestCase):
         false_name = '/false/name'
         self.assertTrue(pkg_integrity.get_keyid(false_name) is None)
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
-    def test_attempt_to_download(self):
-        fakeURL = "https://download.my.url.com/file.tar.gz"
-        realURLnoFile = "http://pypi.debian.net/alembic/alembic-0.8.8.non-existent.tar.gz"
-        realURL = "http://pypi.debian.net/alembic/alembic-0.8.8.tar.gz"
-
-        tmpf = tempfile.NamedTemporaryFile()
-        fname = tmpf.name
-        tmpf.close()
-
-        self.assertEqual(pkg_integrity.attempt_to_download(fakeURL, fname), None)
-        self.assertEqual(pkg_integrity.attempt_to_download(realURLnoFile, fname), 404)
-        self.assertEqual(pkg_integrity.attempt_to_download(realURL, fname), 200)
-
-        os.unlink(fname)
-
     def test_get_signature_url(self):
 
         url_from_gnu = "http://ftp.gnu.org/pub/gnu/gperf/gperf-3.0.4.tar.gz"

tests/test_tarball.py

@@ -3,6 +3,7 @@ import unittest
 from unittest.mock import patch, Mock, mock_open, call
 import build  # needs to be imported before tarball due to dependencies
 import tarball
+import re
 
 
 class FileManager():
@@ -27,10 +28,17 @@ def test_generator(url, name, version):
         """
         tarball.name = ''
         tarball.version = ''
+        tarball.giturl = ''
         tarball.url = url
         tarball.name_and_version('', '', FileManager())
-        self.assertEqual(tarball.name, name)
-        self.assertEqual(tarball.version, version)
+        self.assertEqual(name, tarball.name)
+        self.assertEqual(version, tarball.version)
+        if re.match("https?://github.com", url) != None:
+            self.assertIsNotNone(tarball.giturl)
+            self.assertNotEqual('', tarball.giturl, "giturl should not be empty")
+            self.assertIsNotNone(
+                    re.match("https://github.com/[^/]+/"+tarball.repo+".git",
+                    tarball.giturl), "%s looks incorrect" % tarball.giturl)
 
     return test_packageurl
 

tests/testfiles/pkg_integrity/pkg-config-0.29.1.tar.gz.asc

@@ -0,0 +1,17 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v2
+
+iQIcBAABCAAGBQJW2ZtDAAoJEAI6RCDH7GkUc0IP/1bH7KEJdUM+lrGM1SOuNHdq
+4VEwDp1II8abbBzHeGEXZ8p4+MwwSOYHFiy+NM1yldZkDXtqAlAqvIuEzc+PtgGd
+vFeNPb9infibNaEDK+zz4fcqJOSab1ZcQ/D3EIJXwKr5nIYP8RuCHu/zstf7o6R0
+/wnGWaAIB1+p9PxvUhMPMbBEQCw/cBzyZ2d7nApHF3b0OH2wM7P8VG8ot4cuglPq
+hzk27ZnrYeUDyUUhMRlL7sZZouJlSy/0OxsBK++tOjE6MiuAZhqtlSW+cFK4L7k/
+q4eLodX7GtF0psSgTRjTk2ozdSIDkB2ccLBN6CzgCcbPrbcz4tVQqaQBcSd0mCl7
+RWAKmSye7p+CY8mIIOjdYm+KaQRmJMKDXs49hMycti22jnu5T2BM6O7MZpiY+cb3
+O2UKUXbVyX/cXKwTYwf4VMddxJKFaqYac+7n5qWbdwBjk9E5OC2ltz94taM1pxZ5
+2jRtfyIb3s+Rj6M5cXI5UChrGqzMK6BmEbyZ0KbHAJ7Y0xvGqwydC6J+RwGIqRlp
+LmW3k1ggpUajoMcgq9KqJgVqo/9f4+6anADHRMNJ93MxR7h5BRQ1/GWSXpOzYsYv
+DnfQPrhc+z20m81qsvfUcBnN/k74yiDVqp3I/HrYGD+f8cXKPpBRESkPAXlUIcu2
+ALHZXEdBGcWdUrAIpJT2
+=SMbV
+-----END PGP SIGNATURE-----

tests/testfiles/pkg_integrity/quagga-1.1.0.tar.gz.asc

@@ -0,0 +1,18 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1
+
+iQJMBAABCAA2BQJYBh/XLxpodHRwczovL3d3dy5qYWttYS5vcmcvfnBhdWwvcGdw
+X3BvbGljeS0xLjEudHh0AAoJEG/lfKjBpK6mNOMP/irlUcU/4NQ6BQQbF7Vc90wo
+hmTH/zGAodnndxSplIGV63BQqmQr5KiSOp6tNQ3OlCIwLFWlr/LiJz+MeThaMtZx
+/mlrZptWIhbEcJzVa6efg8UGIYl2NC+QxFgEyezfCYSEmczd1qE3kiX+5WjcqVPH
+pVjisPwaAYa0FTCsAwBMUKyJr2K3sq5hSJR7DFDVnvH1DJ1xZd9871ZdDpZBHz2z
+GvR+IuZcWbYleH1ArFvT6cMokTpIUMvd33/+Gdpfu9fihzQQn199nN2LFZJzuEGV
+vH6+IZVTmtXb5U/sdtbWGaDv8eFLAWl2NH9VNdlVw5FbWOYRC4YNN39/hBy7s0po
+hvq33ZugC7JqPuje+4W1oF1T/dbRCIBmWUzKsoH8+3z4KJHdotSR0cU+TT+w6SJC
+QhF9TAbjrfbeJs0D0NnZrllDLiLGFgLl5yULzMjRDqKcgqE5+nvYBPXHPUCCPU7+
+59QOkPMsz/kZGV1lRzoUoxlM6V/phJRPU7jit10puiNij0c4peWbjVmTWDei3Xeu
+kHpck5wIAUFzAIXBUpVhYdLl/kxq654Qqthoci869ATZIKH/SOId9Y0K1GE6xO6H
+7vQFUGn3dcdnbgapM+tpmC77EmV4BtzkTjcu2pX6s0qnI4P+M+zXS+7G96xP3zHN
+ZSFq45swEKd6SKbOYxcN
+=mt1F
+-----END PGP SIGNATURE-----