Release v1.1.0

This release greatly improves version and package name detection and
adds over 1900 package URL tests for this feature. Package metadata is
now written to options.conf under the 'package' section. A Makefile is
still being written to, but this is deprecated and will be removed in
the future. The mock command is now detected in a manner compatible with
linux distributions other than Clear Linux.

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+sudo: enabled
 python:
   - "3.6"
 # command to run tests

autospec/autospec.py

@@ -21,6 +21,7 @@ import sys
 import os
 import re
 import tempfile
+import configparser
 
 import build
 import buildpattern
@@ -97,6 +98,24 @@ def load_specfile(specfile):
     test.load_specfile(specfile)
 
 
+def read_old_metadata():
+    if not os.path.exists(os.path.join(os.getcwd(), 'options.conf')):
+        return None, None, []
+
+    config_f = configparser.ConfigParser()
+    config_f.read('options.conf')
+    if "package" not in config_f.sections():
+        return None, None, []
+
+    archives = config_f["package"].get("archives")
+    archives = archives.split() if archives else []
+    print("ARCHIVES {}".format(archives))
+
+    return (config_f["package"].get("name"),
+            config_f["package"].get("url"),
+            archives)
+
+
 def main():
     """
     Main function for autospec
@@ -109,7 +128,7 @@ def main():
                         help="Override the package name")
     parser.add_argument("-v", "--version", action="store", dest="version", default="",
                         help="Override the package version")
-    parser.add_argument("url",
+    parser.add_argument("url", default="", nargs="?",
                         help="tarball URL (e.g."
                              " http://example.com/downloads/mytar.tar.gz)")
     parser.add_argument('-a', "--archives", action="store",
@@ -137,10 +156,24 @@ def main():
     parser.add_argument("--non_interactive", action="store_true",
                         default=False,
                         help="Disable interactive mode for package verification")
+    parser.add_argument("-C", "--cleanup", dest="cleanup", action="store_true",
+                        default=False,
+                        help="Clean up mock chroot after building the package")
     args = parser.parse_args()
-    if len(args.archives) % 2 != 0:
+
+    name, url, archives = read_old_metadata()
+    name = args.name or name
+    url = args.url or url
+    archives = args.archives or archives
+
+    if not url:
         parser.error(argparse.ArgumentTypeError(
-            "-a/--archives requires an even number of arguments"))
+            "the url argument or options.conf['package']['url'] is required"))
+
+    if len(archives) % 2 != 0:
+        parser.error(argparse.ArgumentTypeError(
+            "-a/--archives or options.conf['package']['archives'] requires an "
+            "even number of arguments"))
 
     check_requirements(args.git)
     build.setup_workingdir(workingdir)
@@ -150,7 +183,7 @@ def main():
     # of static analysis on the content of the tarball.
     #
     filemanager = files.FileManager()
-    tarball.process(args.url, args.name, args.version, args.target, args.archives, filemanager)
+    tarball.process(url, name, args.version, args.target, archives, filemanager)
     _dir = tarball.path
 
     if args.license_only:
@@ -175,7 +208,7 @@ def main():
     specdescription.scan_for_description(tarball.name, _dir)
     license.scan_for_licenses(_dir)
     commitmessage.scan_for_changes(build.download_path, _dir)
-    add_sources(build.download_path, args.archives)
+    add_sources(build.download_path, archives)
     test.scan_for_tests(_dir)
 
     #
@@ -191,12 +224,12 @@ def main():
 
     if args.integrity:
         interactive_mode = not args.non_interactive
-        pkg_integrity.check(args.url, build.download_path, interactive=interactive_mode)
+        pkg_integrity.check(url, build.download_path, interactive=interactive_mode)
         pkg_integrity.load_specfile(specfile)
 
     specfile.write_spec(build.download_path)
     while 1:
-        build.package(filemanager)
+        build.package(filemanager, args.cleanup)
         filemanager.load_specfile(specfile)
         specfile.write_spec(build.download_path)
         filemanager.newfiles_printed = 0

autospec/build.py

@@ -23,8 +23,8 @@ import buildreq
 import re
 import tarball
 import os
-import grp
 import shutil
+import subprocess
 
 import config
 import util
@@ -35,7 +35,7 @@ must_restart = 0
 base_path = None
 output_path = None
 download_path = None
-mock_cmd = '/usr/bin/mock'
+uniqueext = ''
 
 
 def setup_workingdir(workingdir):
@@ -164,29 +164,73 @@ def parse_build_results(filename, returncode, filemanager):
             success = 1
 
 
-def set_mock():
-    global mock_cmd
-    # get group list of current user
-    user_grps = [grp.getgrgid(g).gr_name for g in os.getgroups()]
-    if os.path.exists('/usr/bin/mock'):
-        if 'mock' not in user_grps:
-            mock_cmd = 'sudo /usr/bin/mock'
+def reserve_path(path):
+    try:
+        subprocess.check_output(['sudo', 'mkdir', path], stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as err:
+        out = err.output.decode('utf-8')
+        return "File exists" not in out
+
+    return True
 
 
-def package(filemanager):
+def get_uniqueext(dirn, dist, name):
+    """
+    Find a unique name to create mock chroot without reusing an old one
+    """
+    # Default to tarball name
+    resultsdir = os.path.join(dirn, "{}-{}".format(dist, name))
+    if reserve_path(resultsdir):
+        return name
+
+    # Find a unique extension by checking if it exists in /var/lib/mock
+    # Increment the pathname until an unused path is found
+    resultsdir += "-1"
+    seq = 1
+    while not reserve_path(resultsdir):
+        seq += 1
+        resultsdir = resultsdir.replace("-{}".format(seq - 1), "-{}".format(seq))
+
+    return "{}-{}".format(name, seq)
+
+
+def get_mock_cmd():
+    # Some distributions (e.g. Fedora) use consolehelper to run mock,
+    # while others (e.g. Clear Linux) expect the user run it via sudo.
+    if os.path.basename(os.path.realpath('/usr/bin/mock')) == 'consolehelper':
+        return '/usr/bin/mock'
+    return 'sudo /usr/bin/mock'
+
+
+def package(filemanager, cleanup=False):
     global round
+    global uniqueext
     round = round + 1
-    set_mock()
+    mock_cmd = get_mock_cmd()
     print("Building package " + tarball.name + " round", round)
-    # call(mock_cmd + " -q -r clear --scrub=cache")
-    # call(mock_cmd + " -q -r clear --scrub=all")
+
+    # determine uniqueext only once
+    if cleanup:
+        uniqueext = uniqueext or get_uniqueext("/var/lib/mock", "clear", tarball.name)
+        cleanup_flag = "--cleanup-after"
+    else:
+        uniqueext = tarball.name
+        cleanup_flag = "--no-cleanup-after"
+
+    print("{} mock chroot at /var/lib/mock/clear-{}".format(tarball.name, uniqueext))
+
     shutil.rmtree('{}/results'.format(download_path), ignore_errors=True)
     os.makedirs('{}/results'.format(download_path))
-    util.call(mock_cmd + " -r clear --buildsrpm --sources=./ --spec={0}.spec --uniqueext={0} --result=results/ --no-cleanup-after".format(tarball.name),
+    util.call("{} -r clear --buildsrpm --sources=./ --spec={}.spec "
+              "--uniqueext={} --result=results/ {}"
+              .format(mock_cmd, tarball.name, uniqueext, cleanup_flag),
               logfile="%s/mock_srpm.log" % download_path, cwd=download_path)
+
     util.call("rm -f results/build.log", cwd=download_path)
     srcrpm = "results/%s-%s-%s.src.rpm" % (tarball.name, tarball.version, tarball.release)
-    returncode = util.call(mock_cmd + " -r clear  --result=results/ %s --enable-plugin=ccache  --uniqueext=%s --no-cleanup-after" % (srcrpm, tarball.name),
+    returncode = util.call("{} -r clear  --result=results/ {} "
+                           "--enable-plugin=ccache  --uniqueext={} {}"
+                           .format(mock_cmd, srcrpm, uniqueext, cleanup_flag),
                            logfile="%s/mock_build.log" % download_path, check=False, cwd=download_path)
     # sanity check the build log
     if not os.path.exists(download_path + "/results/build.log"):

autospec/config.py

@@ -268,11 +268,27 @@ failed_pats = [
      r"available", 0, 'maven')]
 
 
+def get_metadata_conf():
+    """
+    gather package metadata from the tarball module
+    """
+    metadata = {}
+    metadata['name'] = tarball.name
+    metadata['url'] = tarball.url
+    metadata['archives'] = ' '.join(tarball.archives)
+    return metadata
+
+
 def create_conf(path):
     """
     Create options.conf file. Use deprecated configuration files or defaults to populate
     """
     config_f = configparser.ConfigParser(allow_no_value=True)
+
+    # first the metadata
+    config_f['package'] = get_metadata_conf()
+
+    # next the options
     config_f['autospec'] = {}
     for fname, comment in sorted(config_options.items()):
         config_f.set('autospec', '# {}'.format(comment))
@@ -326,6 +342,7 @@ def rewrite_config_opts(path):
     Rewrite options.conf file when an option has changed (verify_required for example)
     """
     config_f = configparser.ConfigParser(allow_no_value=True)
+    config_f['package'] = get_metadata_conf()
     config_f['autospec'] = {}
 
     # Populate missing configuration options

autospec/keyring/B3C7CE210DE76DFC.pkey

@@ -0,0 +1,86 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFL9JHIBEADDKL+vc3MHctXCWxG0IcuLPm4i+WO9jH6MdF8p2bokaL5lilcJ
+o2i0RHImP83FpqCDj/hE7AeB3Ix3Oq3tIe6lpCz468OVbLivErX/yL8rYUhC+yba
+zlSJcBipTIezD3pZI/vpFyCIC8mSEI4PJGiLotioNJWQBStQuuPqMKo4cd6cRWx6
+WovQMKa85rQPvv+wYCseBqU6xBx1xSgpkjDJnAH7We5G6T8TYVM2gqXtii/JYyUC
+8E7Z2BBdSGkq1Dma1W6paOw7VOtLu6YbAH9v/VUVx0UdzwiyXHQ8qLKbxeXkex9d
+zoe+oy8wt3bvVHirxU42IlJJVLY4NcUYhr/xOq/YqT+3sWAqgdVM/WnzY3bPiMnQ
+XqQ3UZ0LKXXsF2+9u2qM6FTiAHIMLtH95IlOQo1qXmDl1HRJTNq3B41UduLDRUhP
+uGBaXbOtb6ql4nV3/BTxcHumisMh5ALhqXHRNOdn+mEjvT2HT6pp07ZfVhFwM4U0
+K78f52S+NppiS0cWtn9ibq+UmqZyvw9HMOHYt51lSUOueg/sOvsxrG1gjvcUMlJw
+nCgS3b+QUmZYo+pl/l13fiWPQpteVGZqKPMBBaQ4MCrwHgvikN8uppKDw8fFGs44
+LgbCsDoQA8wKxNzMEMTvuhFFgCiOxMCzR0XKxlcKr8CiHZjYwovJLFfLvwARAQAB
+tCVUaW5nUGluZyA8dGluZ3BpbmdAZmVkb3JhcHJvamVjdC5vcmc+iQIcBBABCAAG
+BQJXCDvTAAoJEJl85gUEYfxXa6IP/jiLYg/YD73KOSs50K5TjdZq667SDtUEKo7d
+n6RSewoVHCNqWlDoI5njVBmmzXNAQsf25SSJ7IGwxMDEoc/u+Ft/bxX3rZoOBGF+
+CjH4Z76hlKHVUI9X4Zj+j6dZoImxFQLxFcGxNrN0hZedM4YFVBoJX/8D0foLRlnG
+wBmOXRBO1P1EBQU/vKBnFeYsrL5jNOLqh6CNcUochPxHQSNfj4y02mVmA824Xcb/
+d1IPCvshOw/hIyniMxpyE+2IgFFZHKDeqR6AidV/0y6BNOUoz4MOwzizn5QDuoBq
+pW4plMdTwJ/CqcZ/Hu9IGNC+tRkhvG8NvvN21K5Vxy1915znIoRkF/zQmSqOYucq
+/7uaAU+0ey4ESRU0BT0REYW1Q8BGFOtCKgUsZ0XupRo+ix9MkgbBaeYYDY72gLef
+9DRW1+ARo+okvblYI9155SS3mSwu9N/G6QDJ+YHFtLICG5xqZ8+gm9qWMiKBLD/u
+5n+28lsqf1SLjZH9fifmszKneJbO3iZWo79j7BWNFhemlZuam79y0AyNZzQRD+ya
+YlP+t3w9FyBt2hkynr75KBDJZj1K4yLcxYR9cUuhfzTiBpzadDOUrMkC3+t7NMfm
+dKB6esxIDtd7M0Xq6gYI3Go1kdIa6qY1mk2SNbfzp8JKLdzL+f15/PzjNXDhg2ia
+Kan4G6bTiQI+BBMBAgAoBQJS/SRyAhsDBQkJZgGABgsJCAcDAgYVCAIJCgsEFgID
+AQIeAQIXgAAKCRCzx84hDedt/GPYD/9fz6uJ1pEJKUKb2o8b1mQk9Qa+rXv7sPm9
+x+KnBrjSkwdq3s17OGbEGA8kUSCvxN7+auesq4UfE8bCllB5FhiGwETtWCDUJFc1
+bzJAmABAoXskCrUJgn/zDw/MDAtO0uGAhnz1aw1BU17hHORRDOVPqqApE0eSg5gx
+N8jjdx3yVvTaanaU7aZzWCqQSkCTda6Hjks0cAljGuPwN7n3ZVzTXO1E1f5DQedG
+eMxBTqtorhOW9bxBOUvbJnfHZsMOfvB3gcQKCuRFkxVj051Ua9+u+d70uZuXelFD
+agfIAa7SVDy0BxjqSoUYfQJEtdzPvGHVowM6sDJZfM4NrXW4tfZ1opgMklER3Inh
+uJSsSU5pJ/gpjWShUR6zn/zfNZ5A2N99TgNmkSfvweCJrQFT9iWGR/Y8k0hav2cg
+OJVtu26gXnqNHBQSOXk2wKzl/EAa1BhToIQhdpXXFVwyEzQ57aDFNBzhAflw2k27
+y6X61BqoycxIjjeWE8BGRAWHh3lndkkyTxXghhz37/cyjvjWs7vZovRp+w0nMVKb
+TFH9MiVXXuOYnTGpnGD90GFAtuPWgIMEPVE9a1lNHTjwgIloUYVQEoZFCakJnihc
+q5Sqay7sF8dlgyCgnthfzYAJHmSXNEWVvzCEFeuAffcibO9pEcmTH0PO3Nz0w79E
+Ao2VvV/r9LQmUGF0cmljayBHcmlmZmlzIDx0aW5ncGluZ0B0aW5ncGluZy5zZT6J
+AhwEEAEIAAYFAljFXSwACgkQmXzmBQRh/Fd9bw//eZs+FsIXDIRTMd1HP8tXd9WW
+2rYhFPkdS9Gb4wdsG8hsj16OnOgbkaVxkIE/61YFy8VnnPSQvuqIC6pMD29Eb8m1
+3b9hGO4TbP3sQSjqktyrSzBjIaF8Y2pL1B48uuwoglvPRIUbH5YFJfRFS3aGYRYU
+hGgPDpZIDXJYLH8tV3vPGFACNqa6JJdWMSr55bV5MEy4TbTCIxXMjKF3OLpqgQYr
+xOhIDmdSgJeaD31NnQDXvoK6RidJ1LK6Mw2ct9k0iMG6wZi8ENafL52WtT8SSHxl
+15TUZVDyrnmrFe41nnOjKs8BkihcBEfsllnvtM/1qDqFDmBuBUzGeL/gojJ3g6P/
+4VSvKkP5ntXY4BH4iCNyqtsBA5qw+UO9TrzNDyICmipJM/xoZ2lYmPmDPHdRY4gv
+ObcHMg8TO2P0Z/3g/xriXD3dFgWibUHd41H7wHy7/GEqN0Kzt/yfH9xkxOXJhXrC
+MxGf2Imp8BiXjaJ+28iUU1CpKPx3dYq3rIjB/7P85yo3DnYoJadwwQcrrm36Q2qn
+WRbKbm5fKJsJQBe0hZ8zC3excKU3XhCAP1YvXsPa25MEVgrP4el13Ddh82h+Lm2z
+d3rOzMZwJ/1uJtsYD+PXoQVpl+GotKjHfg9PLL7MOjde8pEZr+QdliJyyVUxyUZq
+/IP2/nY/GOni4rspxw6JAjwEEwEIACcFAlfNKA4CGwMFCQlmAYAFCwkIBwIGFQgJ
+CgsCBBYCAwECHgECF4AACgkQs8fOIQ3nbfwcQw/2Lhf3qFhgh3Y/9Anh/nkvWLbu
+B7P2LeQ+KVyxA9NKWwKL33Cvn3ofG4H1axe0xCaTFNdYIeEDEAIGkZa3nszchWdr
+uPwV6JKKVKNdq3X9wSvdoX+rdkmREsZzXREtH/jlzgQRkKy1L/zhDloP3gNLvQh/
+7kPmWih7KFsWd1iLRbsUjBIgTjCzwQRML3lUfsz17qp20BRPMlgJIqxLN0ImjOKR
+QZfzmwypmeTRYvgNrrhe1b/Lkciy9N3qFxo0FFAXfsxRmbUdu5h0hhTAkz2rC79G
+xfK2txMJh9kUu8AR3p3nUPDtWn1W8vATuElZc+QyGlFBGUgrayeyQXtvZNo/6C6t
+3YJFViRUmA/OaJD6wyF1/7T7niW9wUczlZiybMNhuXxTaqGGUE9hKaDN9dIFmbT+
+1IO+pLh02pBp4W2fQy20mprUnsVQyqcPTTiTysuAWzzy3rdjT80/92y0MNBwILWx
+nsNToCN+Lbr07UiOV5Tgu5/SemvgVDl4leofzB/qr3S3RUtTeQrRrEfB9BF51mSZ
+epCrut2wHPvjVFr2UyMq2sspr+6wmcfq+bsdduAKt8prE046WPt2+YSMt3sVoT+p
+Li61jN8SVFnaKm+q0xmyRc+A7eLQvvYI2+BoVvFOjjWHqINfXwUW3MVzzyo04ZZS
+6F42zJpL5YTOCnCf3bkCDQRS/SRyARAAsdRSVgStUTkbvqx2udlHNPiZHK+/dQNd
+CLcMr+n7ANEU+pqKG+rPEb3rfDUB+kmVUaohst3fIJcZ3nU99mcR1kaCwvD3Oj50
+IVVwd0UmaBPVVZvc/u5xzhl1wo9GtDrIMhkhy3qxPezL5TJZH9hLDk4+oq2WQ7yR
++BG5W5NVswuCuRAua+xS/+LHRy2w1hFEd8Hpgu7NJ/SLrB8DQYk0ksaWoP3avfun
+lcqAN6KbKklN8JUqn5t1P9BtzBuCZMA5kt5yuIZpqpwmpHV2WqimQxmY32s7U7ju
+UT+9ynzR0jX7K3jl2N5XV+yctanR6f4QaeghV4R7pXeGcli/jurjEXDUHX3f2/FQ
+Q3t89i2MNwB54YwJ5sCao354R96+4mzNvhwLa4/CeUMHv9vECTkd1H09frWFvCNG
+6ZwXrH84JvVadrdyM5GhnV0DrAfSnXCkVeL4oquFT0tR3bb6SwMcazcRYLiJWAJC
+zCcMLmmivF/iv5IAJt6ufrcbMUf4w5g2haqo9GJ30+7PnM20pJL1wxuvLE1a9XNr
+u9YvENF10kqy99P4//v/wjfZj+qCOnWEQyucAhWmMHEHuU3f24iQ1l7F6JTgtAh5
+8tU03zb2ivm9GJLGrO2i8qRznOcX1SSNXqL8JAaVwEEAxSaFpWbnV11L4zxbBMrf
+Y3di1JeX2XEAEQEAAYkCJQQYAQIADwUCUv0kcgIbDAUJCWYBgAAKCRCzx84hDedt
+/H30D/9q1BN2RI+f5K7aYA4rf7dEIpOu+A1CqoEw0tQp20B3S6oHsiHm1DRbScxL
+byPO0RlWe11agPjTVkAKFuJeGBjxBq3QdrGtymB35UWxWycDjlU7goVcpDq8kph6
+TKn7J4HMad+8SEVF1+mo3YeAv164VsCqCAPQetB0gaQ6KkRg5awwv3ch4C4d/k1f
+KmxUIDoSYiCeFysSyajwSImYKo6HoNNGXQbv1HvPco2sXKkp6U6DU8VPTuRPb2UQ
+eqqGRU0oQvdvp+38ICjmtBQ9BFsMJSYIxOrxtx8G94r9cpnWY5HJbqXSfNH3mbJn
+mkvBL9NhBxWsXfzQMd0nUIa1Mn+KNXOhHiI4YDUNoxNeYUi6F9b2w+KahrajblW9
+tyQ1ysXJV0C5ldOeIO0gI4zOR4ZuJq4s5PmdvV5bYGIvnHpJG6XTn76UR7ohm06E
+ydBwlTnZ2vtfPUwp7CgUGImW2TQ6a7tL+JndQ2edX/9V1ovtZVTKloRkfgBUMMmK
+ms+89YCcrnu2BqSRTfKdx6Jj0XMhn+pGdTIz5/E7CWrPP8nD3OpWikTDMXWfTPe7
+/jWSDnbXqu+PKZbleBTH2ddJPKdWDUTCUGPbPhElpM5A8wUfpsrjalgTAVOQnNgK
+AECYfQx9fE1jabBZ/y5XEcbsY37M/BiCPDGwf7tIXJK6u3l8eA==
+=oA+6
+-----END PGP PUBLIC KEY BLOCK-----

autospec/tarball.py

@@ -24,6 +24,7 @@ import re
 import shutil
 import subprocess
 import pycurl
+import configparser
 
 import build
 import buildpattern
@@ -38,6 +39,7 @@ url = ""
 path = ""
 tarball_prefix = ""
 gcov_file = ""
+archives = []
 
 
 def get_sha1sum(filename):
@@ -174,11 +176,28 @@ def print_header():
 def download_tarball(target_dir):
     """
     Download tarball at url (global) to target_dir
-    """
-    global gcov_file
 
+    priority for target directory:
+    - target_dir set from args
+    - current directory if options.conf['package'] exists and
+      any of the options match what has been detected.
+    - curdir/name
+    """
     tarfile = os.path.basename(url)
-    build.download_path = target_dir if target_dir else os.path.join(os.getcwd(), name)
+    target = os.path.join(os.getcwd(), name)
+    if os.path.exists(os.path.join(os.getcwd(), 'options.conf')):
+        config_f = configparser.ConfigParser()
+        config_f.read('options.conf')
+        if "package" in config_f.sections():
+            if (config_f["package"].get("name") == name or
+                    config_f["package"].get("url") == url or
+                    config_f["package"].get("archives") == " ".join(archives)):
+                target = os.getcwd()
+
+    if target_dir:
+        target = target_dir
+
+    build.download_path = target
     call("mkdir -p {}".format(build.download_path))
 
     # locate the tarball locally or download
@@ -189,20 +208,66 @@ def convert_version(ver_str):
     """
     Remove disallowed characters from the version
     """
-    suffix = ''
-    # remove architecture string
-    ver_str = ver_str.replace('x86_64', '')
-    ver_str = ver_str.replace(name, '')
+    # banned substrings. It is better to remove these here instead of filtering
+    # them out with expensive regular expressions
+    banned_subs = ["x86.64", "source", "src", "all", "bin", "release", "rh",
+                   "ga", ".ce", "lcms", "onig", "linux", "gc", "sdk", "orig",
+                   "jurko"]
+
+    # package names may be modified in the version string by adding "lib" for
+    # example. Remove these from the name before trying to remove the name from
+    # the version
+    name_mods = ["lib", "core", "pom", "opa-"]
+
+    # enforce lower-case strings to make them easier to standardize
+    ver_str = ver_str.lower()
+    # remove the package name from the version string
+    ver_str = ver_str.replace(name.lower(), '')
+    # handle modified name substrings in the version string
+    for mod in name_mods:
+        ver_str = ver_str.replace(name.replace(mod, ""), "")
+
+    # replace illegal characters
     ver_str = ver_str.strip().replace('-', '.').replace('_', '.')
-    ver_str_pat = r'[0-9\.]+(beta|pre|b|alpha|sdist)([\.0-9]*)$'
-    match = re.search(ver_str_pat, ver_str)
-    # it is important the first group matches, not just the second
-    if match and match.group(1):
-        # match.group(2) is an empty string if it was not matched
-        suffix = match.group(1) + match.group(2)
-        ver_str = re.sub(r'{}$'.format(suffix), '', ver_str)
-    ver_str = ''.join(c for c in ver_str if c.isdigit() or c == '.')
-    return '{}.{}'.format(ver_str.strip('.'), suffix) if suffix else ver_str.strip('.')
+
+    # remove banned substrings
+    for sub in banned_subs:
+        ver_str = ver_str.replace(sub, "")
+
+    # remove consecutive '.' characters
+    while ".." in ver_str:
+        ver_str = ver_str.replace("..", ".")
+
+    return ver_str.strip(".")
+
+
+def detect_build_from_url(url):
+    """
+    Detect build patterns and build requirements from the patterns detected
+    in the url.
+    """
+    # R package
+    if "cran.r-project.org" in url or "cran.rstudio.com" in url:
+        buildpattern.set_build_pattern("R", 10)
+        buildreq.add_buildreq("clr-R-helpers")
+
+    # python
+    if "pypi.python.org" in url or "pypi.debian.net" in url:
+        buildpattern.set_build_pattern("distutils23", 10)
+        buildreq.add_buildreq("python3-dev")
+        buildreq.add_buildreq("python-dev")
+
+    # cpan
+    if ".cpan.org/" in url or ".metacpan.org/" in url:
+        buildpattern.set_build_pattern("cpan", 10)
+
+    # ruby
+    if "rubygems.org/" in url:
+        buildpattern.set_build_pattern("ruby", 10)
+
+    # maven
+    if ".maven." in url:
+        buildpattern.set_build_pattern("maven", 10)
 
 
 def name_and_version(name_arg, version_arg, filemanager):
@@ -230,14 +295,12 @@ def name_and_version(name_arg, version_arg, filemanager):
 
     # it is important for the more specific patterns to come first
     pattern_options = [
-        r"(.*?)[\-_](v*[0-9]+[a-zalpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.orig\.tar",
-        r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.src\.(tgz|tar|zip)",
-        r"(.*?)[\-_](v*[0-9]+[alpha\+_sbpfourcesigedsvstableP0-9\.\-\~]*)\.(tgz|tar|zip)",
-        r"(.*?)[\-_](v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~]*)(-.*?)?\.tar",
+        # handle font packages with names ending in -nnndpi
+        r"(.*-[0-9]+dpi)[-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
+        r"(.*?)[-_][vs]?([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.(tgz|tar|zip)",
     ]
     for pattern in pattern_options:
-        p = re.compile(pattern)
-        m = p.search(tarfile)
+        m = re.search(pattern, tarfile)
         if m:
             name = m.group(1).strip()
             version = convert_version(m.group(2))
@@ -245,96 +308,80 @@ def name_and_version(name_arg, version_arg, filemanager):
 
     rawname = name
     # R package
-    if url.find("cran.r-project.org") > 0 or url.find("cran.rstudio.com") > 0:
-        buildpattern.set_build_pattern("R", 10)
+    if "cran.r-project.org" in url or "cran.rstudio.com" in url:
         filemanager.want_dev_split = False
-        buildreq.add_buildreq("clr-R-helpers")
-        p = re.compile(r"([A-Za-z0-9.]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz")
-        m = p.search(tarfile)
+        m = re.search(r"([A-Za-z0-9.]+)_(v*[0-9]+[a-zA-Z0-9\+_\.\~\-]*)\.tar\.gz",
+                      tarfile)
         if m:
             name = "R-" + m.group(1).strip()
             rawname = m.group(1).strip()
-            version = m.group(2).strip().replace('-', '.')
+            version = convert_version(m.group(2))
 
-    if url.find("pypi.python.org") > 0:
-        buildpattern.set_build_pattern("distutils23", 10)
-        url = "http://pypi.debian.net/" + name + "/" + tarfile
-        buildreq.add_buildreq("python3-dev")
-        buildreq.add_buildreq("python-dev")
-
-    if url.find("pypi.debian.net") > 0:
-        buildpattern.set_build_pattern("distutils23", 10)
-        buildreq.add_buildreq("python3-dev")
-        buildreq.add_buildreq("python-dev")
-
-    if url.find(".cpan.org/CPAN/") > 0:
-        buildpattern.set_build_pattern("cpan", 10)
-        if name:
-            name = "perl-" + name
-    if url.find(".metacpan.org/") > 0:
-        buildpattern.set_build_pattern("cpan", 10)
-        if name:
-            name = "perl-" + name
+    if ".cpan.org/" in url or ".metacpan.org/" in url and name:
+        name = "perl-" + name
 
     if "github.com" in url:
         # define regex accepted for valid packages, important for specific
         # patterns to come before general ones
-        github_patterns = [r"https?://github.com/.*/(.*?)/archive/(.*)-final.tar",
-                           r"https?://github.com/.*/.*/archive/[0-9a-fA-F]{1,40}\/(.*)\-(.*).tar",
-                           r"https?://github.com/.*/(.*?)/archive/v?(.*).orig.tar",
-                           r"https?://github.com/.*/(.*?)/archive/(.*).zip",
-                           r"https?://github.com/.*/(.*?)/archive/v?(.*).tar",
+        github_patterns = [r"https?://github.com/.*/(.*?)/archive/[v|r]?.*/(.*).tar",
+                           r"https?://github.com/.*/(.*?)/archive/[-a-zA-Z]*-(.*).tar",
+                           r"https?://github.com/.*/(.*?)/archive/[vVrR]?(.*).tar",
                            r"https?://github.com/.*/(.*?)/releases/download/v.*/(.*).tar"]
 
         for pattern in github_patterns:
-            p = re.compile(pattern)
-            m = p.search(url)
+            m = re.search(pattern, url)
             if m:
                 name = m.group(1).strip()
                 rawname = name
                 version = convert_version(m.group(2))
                 break
 
-    if url.find("bitbucket.org") > 0:
-        p = re.compile(r"https?://bitbucket.org/.*/(.*?)/.*/([.0-9a-zA-Z_-]*?).tar")
-        m = p.search(url)
+    if "mirrors.kernel.org" in url:
+        m = re.search(r".*/sourceware/(.*?)/releases/(.*?).tgz", url)
         if m:
             name = m.group(1).strip()
             version = convert_version(m.group(2))
-        else:
-            version = "1"
+
+    if "sourceforge.net" in url:
+        scf_pats = [r"projects/.*/files/(.*?)/(.*?)/[^-]*(-src)?.tar.gz",
+                    r"downloads.sourceforge.net/.*/([a-zA-Z]+)([-0-9\.]*)(-src)?.tar.gz"]
+        for pat in scf_pats:
+            m = re.search(pat, url)
+            if m:
+                name = m.group(1).strip()
+                version = convert_version(m.group(2))
+                break
+
+    if "bitbucket.org" in url:
+        bitbucket_pats = [r"/.*/(.*?)/.*/.*v([-\.0-9a-zA-Z_]*?).(tar|zip)",
+                          r"/.*/(.*?)/.*/([-\.0-9a-zA-Z_]*?).(tar|zip)"]
+        for pat in bitbucket_pats:
+            m = re.search(pat, url)
+            version = 1
+            if m:
+                name = m.group(1).strip()
+                version = convert_version(m.group(2))
+                break
 
     # ruby
-    if url.find("rubygems.org/") > 0:
-        buildpattern.set_build_pattern("ruby", 10)
-        p = re.compile(r"(.*?)[\-_](v*[0-9]+[alpha\+_spbfourcesigedsvstableP0-9\.\-\~]*)\.gem")
-        m = p.search(tarfile)
+    if "rubygems.org/" in url:
+        m = re.search(r"(.*?)[\-_](v*[0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.gem", tarfile)
         if m:
             name = "rubygem-" + m.group(1).strip()
-            # remove release candidate tag
+            # remove release candidate tag from the package name
+            # https://rubygems.org/downloads/ruby-rc4-0.1.5.gem
             b = name.find("-rc")
             if b > 0:
                 name = name[:b]
             rawname = m.group(1).strip()
-            version = m.group(2).strip()
-            b = version.find("-")
-            if b >= 0:
-                version = version[:b]
+            version = convert_version(m.group(2))
 
     # maven
-    if url.find("maven.org") > 0:
-        buildpattern.set_build_pattern("maven", 10)
-
-    if not name:
-        split = url.split('/')
-        if len(split) > 3 and split[-2] in ('archive', 'tarball'):
-            name = split[-3]
-            version = split[-1]
-            version = version.lstrip('v')
-            # remove extension
-            version = version.rsplit('.', 1)[0]
-            if version.endswith('.tar'):
-                version = version.replace('.tar', '')
+    if ".maven." in url:
+        m = re.search(r"/maven.*/org/.*/(.*?)/.*/.*[\-_]([0-9]+[a-zA-Z0-9\+_\.\-\~]*)\.jar", url)
+        if m:
+            name = m.group(1).strip()
+            version = convert_version(m.group(2))
 
     # override name and version from commandline
     name = name_arg if name_arg else name
@@ -435,7 +482,7 @@ def process_archives(archives):
         write_upstream(sha1, os.path.basename(archive), mode="a")
 
 
-def process(url_arg, name_arg, ver_arg, target, archives, filemanager):
+def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager):
     """
     Download and process the tarball at url_arg
     """
@@ -444,14 +491,20 @@ def process(url_arg, name_arg, ver_arg, target, archives, filemanager):
     global version
     global path
     global tarball_prefix
+    global archives
     url = url_arg
     name = name_arg
     version = ver_arg
+    archives = archives_arg
     tarfile = os.path.basename(url_arg)
-    # set gcov file information
-    set_gcov()
     # determine name and version of package
     name_and_version(name_arg, ver_arg, filemanager)
+    # determine build pattern and build requirements from url
+    detect_build_from_url(url)
+    # set gcov file information, must be done after name is set since the gcov
+    # name is created by adding ".gcov" to the package name (if a gcov file
+    # exists)
+    set_gcov()
     # download the tarball to tar_path
     tar_path = download_tarball(target)
     # write the sha of the upstream tarfile to the "upstream" file
@@ -463,11 +516,12 @@ def process(url_arg, name_arg, ver_arg, target, archives, filemanager):
     # Now that the metadata has been collected print the header
     print_header()
     # write out the Makefile with the name, url, and archives we found
-    write_makefile(archives)
+    # DEPRECATED, this will be removed in a future version
+    write_makefile(archives_arg)
     # prepare directory and extract tarball
     prepare_and_extract(extract_cmd)
     # locate or download archives and move them into the right spot
-    process_archives(archives)
+    process_archives(archives_arg)
 
 
 def load_specfile(specfile):

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 import sys, os
-version = "1.0.2"
+version = "1.1.0"
 
 def readme():
     with open("README.rst") as f:

tests/test_build.py

@@ -1,4 +1,6 @@
 import unittest
+import tempfile
+import os
 from unittest.mock import patch, mock_open
 import build
 import files
@@ -23,7 +25,6 @@ class TestBuildpattern(unittest.TestCase):
         build.base_path = None
         build.output_path = None
         build.download_path = None
-        build.mock_cmd = '/usr/bin/mock'
         build.buildreq.buildreqs = set()
         build.config.config_opts['32bit'] = False
 
@@ -316,65 +317,64 @@ class TestBuildpattern(unittest.TestCase):
         self.assertIn('testpkg-python', build.buildreq.buildreqs)
         self.assertEqual(build.must_restart, 1)
 
-    def test_set_mock(self):
+    def test_get_mock_cmd_without_consolehelper(self):
         """
-        Test set_mock when user is not in the mock group
+        Test get_mock_cmd when /usr/bin/mock doesn't point to consolehelper
         """
-        class mock_getgrgid(object):
-            def __init__(self, g):
-                self.gr_name = 'not_mock'
+        def mock_realpath(path):
+            return path
 
-        def mock_getgroups():
-            return ['gr1', 'gr2', 'gr3']
+        realpath_backup = build.os.path.realpath
 
-        def mock_exists(path):
-            return True
+        build.os.path.realpath = mock_realpath
 
-        getgrgid_backup = build.grp.getgrgid
-        getgroups_backup = build.os.getgroups
-        exists_backup = build.os.path.exists
+        mock_cmd = build.get_mock_cmd()
 
-        build.grp.getgrgid = mock_getgrgid
-        build.os.getgroups = mock_getgroups
-        build.os.path.exists = mock_exists
+        build.os.path.realpath = realpath_backup
 
-        build.set_mock()
+        self.assertEqual(mock_cmd, 'sudo /usr/bin/mock')
 
-        build.grp.getgrgid = getgrgid_backup
-        build.os.getgroups = getgroups_backup
-        build.os.path.exists = exists_backup
-
-        self.assertEqual(build.mock_cmd, 'sudo /usr/bin/mock')
-
-    def test_set_mock_user_in_mock_group(self):
+    def test_get_mock_cmd_with_consolehelper(self):
         """
-        Test set_mock when user is in the mock group
+        Test get_mock_cmd when /usr/bin/mock points to consolehelper
         """
-        class mock_getgrgid(object):
-            def __init__(self, g):
-                self.gr_name = 'mock'
+        def mock_realpath(path):
+            return '/usr/bin/consolehelper'
 
-        def mock_getgroups():
-            return ['gr1', 'gr2', 'gr3']
+        realpath_backup = build.os.path.realpath
 
-        def mock_exists(path):
-            return True
+        build.os.path.realpath = mock_realpath
 
-        getgrgid_backup = build.grp.getgrgid
-        getgroups_backup = build.os.getgroups
-        exists_backup = build.os.path.exists
+        mock_cmd = build.get_mock_cmd()
 
-        build.grp.getgrgid = mock_getgrgid
-        build.os.getgroups = mock_getgroups
-        build.os.path.exists = mock_exists
+        build.os.path.realpath = realpath_backup
 
-        build.set_mock()
+        self.assertEqual(mock_cmd, '/usr/bin/mock')
 
-        build.grp.getgrgid = getgrgid_backup
-        build.os.getgroups = getgroups_backup
-        build.os.path.exists = exists_backup
+    def test_get_uniqueext_first(self):
+        """
+        Test get_uniqueext() with no collisions
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            self.assertEqual(build.get_uniqueext(tmpd, "test", "pkg"), "pkg")
+
+    def test_get_uniqueext_second(self):
+        """
+        Test get_uniqueext() with one collision
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "test-pkg"))
+            self.assertEqual(build.get_uniqueext(tmpd, "test", "pkg"), "pkg-1")
+
+    def test_get_uniqueext_third(self):
+        """
+        Test get_uniqueext() with two collisions
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "test-pkg"))
+            os.mkdir(os.path.join(tmpd, "test-pkg-1"))
+            self.assertEqual(build.get_uniqueext(tmpd, "test", "pkg"), "pkg-2")
 
-        self.assertEqual(build.mock_cmd, '/usr/bin/mock')
 
 
 if __name__ == '__main__':

tests/test_pkg_integrity.py

@@ -85,6 +85,8 @@ class TestDomainBasedVerifiers(unittest.TestCase):
         return None
 
 
+    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
+                     "Skipping this test on Travis CI.")
     def test_pypi(self):
         result = self.run_test_for_domain(pkg_integrity.PyPiVerifier, PYPI_MD5_ONLY_PKG)
         self.assertTrue(result)
@@ -250,6 +252,8 @@ class TestUtils(unittest.TestCase):
         false_name = '/false/name'
         self.assertTrue(pkg_integrity.get_keyid(false_name) is None)
 
+    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
+                     "Skipping this test on Travis CI.")
     def test_attempt_to_download(self):
         fakeURL = "https://download.my.url.com/file.tar.gz"
         realURLnoFile = "http://pypi.debian.net/alembic/alembic-0.8.8.non-existent.tar.gz"