Release v1.1.8

* Add catkin build pattern recognition
* Add configuration file for cmake_srcdir
* Add failed_commands pattern for ldap-dev
* Update README.rst

Signed-off-by: Matthew Johnson <matthew.johnson@intel.com>

.gitignore

@@ -3,3 +3,5 @@ __pycache__
 *~
 *.swp
 tags
+.coverage
+htmlcov

.travis.yml

@@ -2,6 +2,10 @@ language: python
 sudo: enabled
 python:
   - "3.6"
+
+install:
+  - pip install -r requirements.txt
+
 # command to run tests
 script:
   - make check

Makefile

@@ -1,5 +1,5 @@
 check: autospec/*.py
-	@flake8 --max-line-length=199 --ignore=E402 $^
+	@flake8 --max-line-length=199 --ignore=E722 $^
 
 test_pkg_integrity:
 	PYTHONPATH=${CURDIR}/autospec python3 tests/test_pkg_integrity.py
@@ -47,4 +47,7 @@ test_autospec:
 	python3 tests/test_autospec.py -c ${CASES}
 
 unittests:
-	PYTHONPATH=${CURDIR}/autospec python3 -m unittest discover -b -s tests -p 'test_*.py'
+	PYTHONPATH=${CURDIR}/autospec coverage run -m unittest discover -b -s tests -p 'test_*.py' && coverage report
+
+coverage:
+	coverage report -m

README.rst

@@ -5,16 +5,18 @@
 Autospec
 ========
 
-autospec is a tool to assist in the automated creation and maintainence
-of RPM packaging. It will continuously run updated builds based on new
-information discovered from build failures, until it has a complete and
-valid .spec file. The tool makes use of mock to achieve this.
+autospec is a tool to assist in the automated creation and maintenance of RPM
+packaging. It will continuously run updated builds based on new information
+discovered from build failures until it has a complete and valid .spec file. The
+tool makes use of mock to achieve this.
+
+.. contents:: Table of Contents
 
 License
 =======
 autospec is available under the terms of the GPL, version 3.0
 
-Copyright (C) 2015 Intel Corporation
+Copyright (C) 2017 Intel Corporation
 
 
 Configuration of autospec
@@ -30,45 +32,62 @@ Example ``autospec.conf`` file::
     license_fetch = http://yourhost/hash.php
     license_show = http://yourhost/showone.php?hash=%(HASH)s
     packages_file = file:///path/to/package_list_file
+    yum_conf = file:///path/to/yum.conf
     upstream = http://yourhost/tarballs/%(HASH)s/%(NAME)s
 
+git
+  The upstream git repository URL base
 
-**git**
-    The upstream git repository URL base
+license_fetch
+  Optional URL to use for scanning license files
 
-**license_fetch**
-    Optional URL to use for scanning license files
+license_show
+  Optional URL to interact with online license checker
 
-**license_show**
-    Optional URL to interact with online license checker
+packages_file
+  Optional path to add autodetected runtime requirement checking
 
-**packages_file**
-    Optional path to add autodetected runtime requirement checking against
+yum_conf
+  Optional path to yum configuration
 
-**yum_conf**
-    Optional path to yum configuration
-
-**upstream**
-    Base URL for stored upstream tarballs
+upstream
+  Base URL for stored upstream tarballs
 
 Synopsis
 ========
 
-Usage: ``python3 autospec.py [options] URL``
+.. code-block:: bash
 
+  usage: autospec.py [-h] [-g] [-n NAME] [-v VERSION]
+                     [-a [ARCHIVES [ARCHIVES ...]]] [-l] [-b] [-c CONFIG]
+                     [-t TARGET] [-i] [-p] [--non_interactive] [-C]
+                     [url]
 
--h, --help                                      show help message and exit
--g, --skip-git                                  Don't commit result to git
--n NAME, --name NAME                            Override the package name
--a ARCHIVES, --archives ARCHIVES
-                                                tarball URLs for additional source archives and a
-                                                location for the sources to be extacted to (e.g.
-                                                http://example.com/downloads/dependency.tar.gz
-                                                /directory/relative/to/extract/root )
--l, --license-only                              Only scan for license files
--b, --skip-bump                                 Don't bump release number
--c CONFIG, --config CONFIG                      Set configuration file to use
--t DIRECTORY --target DIRECTORY                 Set location to create or use
+    url                 (required) tarball URL (e.g.
+                        http://example.com/downloads/mytar.tar.gz)
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -g, --skip-git        Don't commit result to git
+  -n NAME, --name NAME  Override the package name
+  -v VERSION, --version VERSION
+                        Override the package version
+  -a ARCHIVES, --archives ARCHIVES
+                        tarball URLs for additional source archives and a
+                        location for the sources to be extacted to (e.g.
+                        http://example.com/downloads/dependency.tar.gz
+                        /directory/relative/to/extract/root )
+  -l, --license-only    Only scan for license files
+  -b, --skip-bump       Don't bump release number
+  -c CONFIG, --config CONFIG
+                        Set configuration file to use
+  -t TARGET, --target TARGET
+                        Target location to create or reuse
+  -i, --integrity       Search for package signature from source URL and
+                        attempt to verify package
+  -p, --prep-only       Only perform preparatory work on package
+  --non_interactive     Disable interactive mode for package verification
+  -C, --cleanup         Clean up mock chroot after building the package
 
 
 
@@ -77,27 +96,26 @@ Requirements
 
 In order to run correctly, ``autospec`` requires the following components:
 
- * python3
- * correctly configured mock
+* python3
+* correctly configured mock
 
-If ``autospec`` is not configured to use a license server, then you will
-need a ``common/licenses`` file -  which should be an up to date list of
-licenses to facilitate automatic license detection during the scan of a
-tarball. For correctness, license names should be in the SPDX identifier
-format. Each line in the file constitutes a license definition, for example::
+If ``autospec`` is not configured to use a license server, then it will use the
+``autospec/license_hashes`` file -  which is a list of licenses to facilitate
+automatic license detection during the scan of a tarball. For correctness,
+license names should be in the SPDX identifier format. Each line in the file
+constitutes a license definition, for example::
 
-    750b9d9cc986bfc80b47c9672c48ca615cac0c87 | BSD-3-Clause
-    175e59be229a5bedc6be93e958a970385bb04a62 | Apache-2.0
-    794a893e510ca5c15c9c97a609ce47b0df74fc1a | BSD-2-Clause
+    750b9d9cc986bfc80b47c9672c48ca615cac0c87, BSD-3-Clause
+    175e59be229a5bedc6be93e958a970385bb04a62, Apache-2.0
+    794a893e510ca5c15c9c97a609ce47b0df74fc1a, BSD-2-Clause
 
 
 Control files
 ==============
-
-It is possible to influence precisely how autospec will behave in order to
-gain fine control over the build itself. These files may be used to alter
-the default behaviour of the configure routine, to blacklist build dependencies
-from being automatically added, and such.
+It is possible to influence precisely how autospec will behave in order to gain
+fine control over the build itself. These files may be used to alter the default
+behaviour of the configure routine, to blacklist build dependencies from being
+automatically added, and such.
 
 These files are expected to live in same directory that the resulting ``.spec``
 will live.
@@ -105,315 +123,274 @@ will live.
 Common files
 ------------
 
-**release**
+release
+  This file contains the current release number that will be used in the
+  ``.spec``. This is also bumped and generated on existing and new packages,
+  respectively. This results in less manual work via automatic management.
 
-    This file contains the current release number that will be used in the
-    ``.spec``. This is also bumped and generated on existing and new packages,
-    respectively. This results in less manual work via automatic management.
-
-**$package.license**
-
-    In certain cases, the package license may not be automatically discovered.
-    In this instance, ``autospec`` will exit with an error. Update this file
-    to contain the valid SPDX identifier for any license(s) for the package,
-    replacing ``$package`` in the filename with the actual package name.
+$package.license
+  In certain cases, the package license may not be automatically discovered.  In
+  this instance, ``autospec`` will exit with an error. Update this file to
+  contain the valid SPDX identifier for any license(s) for the package,
+  replacing ``$package`` in the filename with the actual package name.
 
 Controlling dependencies
 -------------------------
 
-**buildreq_add**
+buildreq_add
+  Each line in the file provides the name of a package to add as a build
+  dependency to the ``.spec``.
 
-    Each line in the file provides the name of a package to add
-    as a build dependency to the ``.spec``.
+pkgconfig_add
+  Each line in the file is assumed to be a pkgconfig() build dependency.  Add
+  the pkg-config names here, as ``autospec`` will automatically transform the
+  names into their ``pkgconfig($name)`` style when generating the ``.spec``.
 
-**pkgconfig_add**
+requires_add
+  Each line in the file provides the name of a package to add as a runtime
+  dependency to the ``.spec``.
 
-    Each line in the file is assumed to be a pkgconfig() build dependency.
-    Add the pkg-config names here, as ``autospec`` will automatically transform
-    the names into their ``pkgconfig($name)`` style when generating the ``.spec``.
+buildreq_ban
+  Each line in the file is a build dependency that under no circumstance should
+  be automatically added to the build dependencies. This is useful to block
+  automatic configuration routines adding undesired functionality, or to omit
+  any automatically discovered dependencies during tarball scanning.
 
-**requires_add**
+pkgconfig_ban
+  Each line in this file is a pkgconfig() build dependency that should not be
+  added automatically to the build, much the same as ``buildreq_ban``.  As with
+  ``pkgconfig_add``, these names are automatically transformed by ``autospec``
+  into their correct ``pkgconfig($name))`` style.
 
-    Each line in the file provides the name of a package to add
-    as a runtime dependency to the ``.spec``.
+requires_ban
+  Each line in the file is a runtime dependency that under no circumstance
+  should be automatically added to the runtime dependencies. This is useful to
+  block automatic configuration routines adding undesired functionality, or to
+  omit any automatically discovered dependencies during tarball scanning.
 
-**buildreq_ban**
+.. note::
 
-    Each line in the file is a build dependency that under no circumstance
-    should be automatically added to the build dependencies. This is useful
-    to block automatic configuration routines adding undesired functionality,
-    or to omit any automatically discovered dependencies during tarball scanning.
-
-**pkgconfig_ban**
-
-    Each line in this file is a pkgconfig() build dependency that should not
-    be added automatically to the build, much the same as ``buildreq_ban``.
-    As with ``pkgconfig_add``, these names are automatically transformed by
-    ``autospec`` into their correct ``pkgconfig($name))`` style.
-
-**requires_ban**
-
-    Each line in the file is a runtime dependency that under no circumstance
-    should be automatically added to the runtime dependencies. This is useful
-    to block automatic configuration routines adding undesired functionality,
-    or to omit any automatically discovered dependencies during tarball scanning.
-
-Run time requirements are not assumed to be build time requirements.
-If a package has the same build and run time requirement it must be added
-to both buildreq_add and requires_add.
+  Run time requirements are not assumed to be build time requirement
+  If a package has the same build and run time requirement it must be added
+  to both buildreq_add and requires_add.
 
 Controlling the build process
 ------------------------------
 
-**configure**
+configure
+  This file contains configuration flags to pass to the ``%configure`` macro for
+  autotools based tarballs. As an example, adding ``--disable-static`` to
+  ``./configure`` for an autootools based tarball would result in ``%configure
+  --disable-static`` being emitted in the ``.spec``.
 
-    This file contains configuration flags to pass to the ``%configure``
-    macro for autotools based tarballs. As an example, adding ``--disable-static``
-    to ``./configure`` for an autootools based tarball would result in
-    ``%configure --disable-static`` being emitted in the ``.spec``.
+cmake_args
+  This file contains arguments that should be passed to the ``%cmake`` macro for
+  CMake based tarballs. As an example, adding ``-DUSE_LIB64=ON`` to
+  ``./cmake_args`` would result in ``%cmake -DUSE_LIB64=ON`` being emitted in
+  the ``.spec``.
 
-**cmake_args**
+make_args
+  The contents of this file are appended to the ``make`` invocation. This may be
+  useful for passing arguments to ``make``, i.e. ``make TOOLDIR=/usr``
 
-    This file contains arguments that should be passed to the ``%cmake``
-    macro for CMake based tarballs. As an example, adding ``-DUSE_LIB64=ON`` to
-    ``./cmake_args`` would result in ``%cmake -DUSE_LIB64=ON`` being emitted
-    in the ``.spec``.
+make_install_args
+  Much like ``make_args``, this will pass arguments to the ``make install``
+  macro in the ``.spec``
 
-**broken_parallel_build**
+make_install_append
+  Additional actions that should take place after the ``make install`` step has
+  completed. This will be placed in the resulting ``.spec``, and is used for
+  situations where fine-grained control is required.
 
-    This option is set in the ``options.conf`` file described below. If this
-    option is set, then parallelisation will be disabled in the build.
-    This usually means that ``%{?_smp_mflags}`` will not be passed to ``make``
+install_macro
+  The contents of this file be used instead of the automatically detected
+  ``install`` routine, i.e. use this if ``%make_install`` is insufficient.
 
-**make_args**
+subdir
+  Not all packages have their ``Makefile``'s available in the root of the
+  tarball.  An example of this may be cross-platform projects that split
+  Makefile's into the ``unix`` subdirectory. Set the name in this file and the
+  ``.spec`` will emit the correct ``pushd`` and ``popd`` lines to utilise these
+  directories for each step in the build.
 
-    The contents of this file are appended to the ``make`` invocation. This
-    may be useful for passing arguments to ``make``, i.e. ``make TOOLDIR=/usr``
+cmake_srcdir
+ The contents of this file are a path the source directory in which to run cmake
+ for non-standard packages.
 
-**make_install_args**
+build_pattern
+  In certain situations, the automatically detected build pattern may not work
+  for the given package. This one line file allows you to override the build
+  pattern that ``autospec`` will use. The supported build_pattern types are:
 
-    Much like ``make_args``, this will pass arguments to the ``make install``
-    macro in the ``.spec``
+  * R: R language package
+  * cpan: perl language package
+  * ruby: ruby language package
+  * maven: Java language package
+  * configure: Traditional ``%configure`` autotools route
+  * configure_ac: Like ``configure, but performs ``%reconfigure`` to regenerate
+    ``./configure``
+  * autogen: Similar to ``configure_ac`` but uses the existing ``./autogen.sh``
+    instead of ``%reconfigure``
+  * distutils: Only build the Pythonic package with Python 2
+  * distutils3: Only build the Pythonic package with Python 3
+  * distutils23: Build the Pythonic package using both Python 2 and Python 3
 
-**make_install_append**
+series
+  This file contains a list of patches to apply during the build, using the
+  ``%patch`` macro. As such it is affected by ``-p1`` style modifiers.
 
-    Additional actions that should take place after the ``make install`` step
-    has completed. This will be placed in the resulting ``.spec``, and is used
-    for situations where fine-grained control is required.
-
-**install_macro**
-
-    The contents of this file be used instead of the automatically detected
-    ``install`` routine, i.e. use this if ``%make_install`` is insufficient.
-
-**subdir**
-
-    Not all packages have their ``Makefile``'s available in the root of the tarball.
-    An example of this may be cross-platform projects that split Makefile's into
-    the ``unix`` subdirectory. Set the name in this file and the ``.spec`` will
-    emit the correct ``pushd`` and ``popd`` lines to utilise these directories
-    for each step in the build.
-
-**build_pattern**
-
-    In certain situations, the automatically detected build pattern may not
-    work for the given package. This one line file allows you to override the
-    build pattern that ``autospec`` will use. The supported build_pattern types are:
-
-        - configure: Traditional ``%configure`` autotools route
-        - configure_ac: Like ``configure, but performs ``%reconfigure`` to regenerate ``./configure``
-        - autogen: Similar to ``configure_ac`` but uses the existing ``./autogen.sh`` instead of ``%reconfigure``
-        - distutils: Only build the Pythonic package with Python 2
-        - distutils3: Only build the Pythonic package with Python 3
-        - distutils23: Build the Pythonic package using both Python 2 and Python 3
-
-**series**
-
-    This file contains a list of patches to apply during the build, using the ``%patch``
-    macro. As such it is affected by ``-p1`` style modifiers.
-
-**golang_libpath**
-
-    When building go packages, the go import path will be guessed automatically
-    (e.g. building ``https://github.com/go-yaml/yaml/`` would get
-    ``github.com/go-yaml/yaml``). While this is handy, it's not always correct
-    (in the previous example, the correct import path should be
-    ``gopkg.in/yaml.v2``). This could be easily fixed by placing
-    ``gopkg.in/yaml.v`` in this file, changing where the go bits will be placed.
+golang_libpath
+  When building go packages, the go import path will be guessed automatically
+  (e.g. building ``https://github.com/go-yaml/yaml/`` would get
+  ``github.com/go-yaml/yaml``). While this is handy, it's not always correct (in
+  the previous example, the correct import path should be ``gopkg.in/yaml.v2``).
+  This could be easily fixed by placing ``gopkg.in/yaml.v`` in this file,
+  changing where the go bits will be placed.
 
 Controlling files and subpackages
 ---------------------------------
 
-**excludes**
+excludes
+  This file is used to generate ``%exclude`` lines in the ``.spec``. This is
+  useful for omitting files from being included in the resulting package.  Each
+  line in the file should be a full path name.
 
-    This file is used to generate ``%exclude`` lines in the ``.spec``. This
-    is useful for omitting files from being included in the resulting package.
-    Each line in the file should be a full path name.
+extras
+  Each line in the file should be a full path within the resulting package, that
+  you wish to be placed into an automatic ``-extras`` subpackage. This allows
+  one to keep the main package slim and split out optional functionality or
+  files.
 
-**keepstatic**
+setuid
+  Each line in this file should contain the full path to a binary in the
+  resulting build that should have the ``setuid`` attribute set with the
+  ``%attr`` macro.
 
-    This option is set in the ``options.conf`` file described below. If this
-    option is set, then ``%define keepstatic 1`` is emitted in the ``.spec``.
-    As a result, any static archive (``.a``) files will not be removed by rpmbuild.
-
-**extras**
-
-    Each line in the file should be a full path within the resulting package,
-    that you wish to be placed into an automatic ``-extras`` subpackage. This
-    allows one to keep the main package slim and split out optional functionality
-    or files.
-
-**no_autostart**
-
-    This option is set in the ``options.conf`` file described below. If this
-    option is set the autostart subpackage (which contains all files matching
-    /usr/lib/systemd/system/*.target.wants/) will not be required by the base package.
-
-**setuid**
-
-    Each line in this file should contain the full path to a binary in the resulting
-    build that should have the ``setuid`` attribute set with the ``%attr`` macro.
-
-**attrs**
-
-    Each line in this file should be a full ``%attr`` macro line that will be included
-    in the ``.spec`` to have fine-grained control over the permissions and ownership
-    of files in the package.
+attrs
+  Each line in this file should be a full ``%attr`` macro line that will be
+  included in the ``.spec`` to have fine-grained control over the permissions
+  and ownership of files in the package.
 
 
 Controlling test suites
 -----------------------
-
 By default, ``autospec`` will attempt to detect potential test suites that
 can be run in the ``%check`` portion of the ``.spec``.
 
-**skip_test_suite**
-
-    If this file exists, ``autospec`` will not emit any ``%check`` functionality.
-    This file has been deprecated and will be removed during an autospec run and
-    replaced with a ``skip_tests`` option in ``options.conf``.
-
-**unit_tests_must_pass**
-
-    This file is automatically created upon successful completion of a package build.
-    This allows one to identify regressions in test failures when updating or
-    altering a package.
-    ``autospec`` will fail a package that does not pass it's test suite if this file
-    exists.
-
-**make_check_command**
-
-    Override or set the command to use in the ``%check`` portion of the ``.spec``.
-    This may be useful when a package uses a custom test suite, or requires
-    additional work/parameters, to work correctly.
-
-**allow_test_failures**
-
-    This option is set in the ``options.conf`` file described below. If this
-    option is set it will allow test failures, and will still emit the
-    ``%check`` code in a way that allows the build to continue.
+make_check_command
+  Override or set the command to use in the ``%check`` portion of the ``.spec``.
+  This may be useful when a package uses a custom test suite, or requires
+  additional work/parameters, to work correctly.
 
 
-Controlling flags and optimisation
+Controlling flags and optimization
 ----------------------------------
-
 Further control of the build can be achieved through the use of the
-``options.conf`` file. If this file does not exist it is created by autospec.
-Autospec generates this file based on the presence of deprecated 'file-exists'
-files, then removes the deprecated files.
+``options.conf`` file. If this file does not exist it is created by autospec
+with default values. If certain deprecated configuration files exists autospec
+will use the value indicated by those files and remove them.
 
 The options that can be set in ``options.conf`` are as follows:
 
-**asneeded**
+asneeded
+  If this is option set, the ``.spec`` will disable the LD_AS_NEEDED variable.
+  Supporting binutils (such as found in Clear Linux Project for Intel
+  Architecture) will then revert to their normal behaviour, instead of enforcing
+  ``-Wl,-as-needed`` in the most correct sense.
 
-    If this is option set, the ``.spec`` will disable the LD_AS_NEEDED variable.
-    Supporting binutils (such as found in Clear Linux Project for Intel Architecture)
-    will then revert to their normal behaviour, instead of enforcing ``-Wl,-as-needed``
-    in the most correct sense.
+optimize_size
+  If this option is set, the ``CFLAGS/LDFLAGS`` will be extended to build the
+  package optimized for *size*, and not for *speed*. Use this when size is more
+  critical than performance.
 
-**optimize_size**
+funroll-loops
+  If this option is set, the ``CFLAGS/LDFLAGS`` will be extended to build the
+  package optimized for *speed*. In short this where speed is of paramount
+  importance, and will use ``-03`` by default.
 
-    If this option is set, the ``CFLAGS/LDFLAGS`` will be extended to build
-    the package optimised for *size*, and not for *speed*. Use this when
-    size is more critical than performance.
+insecure_build
+  If this option is set, the ``CFLAGS/LDFLAGS`` will be **replaced**, using the
+  smallest ``-02`` based generic flags possible. This is useful for operating
+  systems employing heavy optimizations or full RELRO by default.
 
-**funroll-loops**
+pgo
+  If this option is set, the ``CFLAGS/CXXFLAGS`` will be extended to build the
+  package with profile-guided optimization data. It will add ``-O3``,
+  ``-fprofile-use``, ``-fprofile-correction`` and ``-fprofile-dir=pgo``.
 
-    If this option is set, the ``CFLAGS/LDFLAGS`` will be extended to build
-    the package optimised for *speed*. In short this where speed is of
-    paramount importance, and will use ``-03`` by default.
+use_lto
+  If this option is set, link time optimization is enabled for the build.
 
-**insecure_build**
+use_avx2
+  If this option is set, a second set of libraries, for AVX2, is built.
 
-    If this option is set, the ``CFLAGS/LDFLAGS`` will be **replaced**, using
-    the smallest ``-02`` based generic flags possible. This is useful for
-    operating systems employing heavy optimisations or full RELRO by default.
+fast-math
+  If this option is set, -ffast-math is passed to the compiler.
 
-**pgo**
+broken_c++
+  If this option is set, flags are extended with -std=gnu++98.
 
-    If this option is set, the ``CFLAGS/CXXFLAGS`` will be extended to build
-    the package with profile-guided optimization data. It will add ``-O3``,
-    ``-fprofile-use``, ``-fprofile-correction`` and ``-fprofile-dir=pgo``.
+allow_test_failures
+  If this option is set it will allow test failures, and will still emit the
+  ``%check`` code in a way that allows the build to continue.
 
-**use_lto**
+skip_tests
+  If this option is set the test suite will not be run.
 
-    If this option is set, link time optimization is enabled for the build.
+no_autostart
+  If this option is set the autostart subpackage (which contains all files
+  matching /usr/lib/systemd/system/\*.target.wants/) will not be required by the
+  base package.
 
-**use_avx2**
+conservative_flags
+  If this option is set autospec will set conservative build flags
 
-    If this option is set, a second set of libraries, for AVX2, is built.
+use_clang
+  If this option is set autospec will utilize clang. This unsets the
+  funroll-loops optimization if it is set.
 
-**fast-math**
+keepstatic
+  If this option is set, then ``%define keepstatic 1`` is emitted in the
+  ``.spec``.  As a result, any static archive (``.a``) files will not be removed
+  by rpmbuild.
 
-    If this option is set, -ffast-math is passed to the compiler.
+32bit
+  This option will trigger the creation of 32-bit libraries for a 32-bit build.
 
-**broken_c++**
+nostrip
+  This option will suppress the stripping of the created binaries.
 
-    If this option is set, flags are extended with -std=gnu++98.
+verify_required
+  This option will make package verification required for the build. This option
+  is automatically set if package verification is ever successful, but can be
+  turned off manually.
 
-**allow_test_failures**
+security_sensitive
+  This options sets flags for security-sensitive builds.
 
-    If this option is set it will allow test failures, and will still emit the
-    ``%check`` code in a way that allows the build to continue.
-
-**no_autostart**
-
-    If this option is set the autostart subpackage (which contains all files matching
-    /usr/lib/systemd/system/*.target.wants/) will not be required by the base package.
-
-**conservative_flags**
-
-    If this option is set autospec will set conservative build flags
-
-**use_clang**
-
-    If this option is set autospec will utilize clang. This unsets the
-    funroll-loops optimization if it is set.
-
-**keepstatic**
-
-    If this option is set, then ``%define keepstatic 1`` is emitted in the ``.spec``.
-    As a result, any static archive (``.a``) files will not be removed by rpmbuild.
-
-**32bit**
-
-    This option will trigger the creation of 32-bit libraries for a 32-bit
-    build.
+so_to_lib
+  This option causes package ``.so`` files to be added to the ``lib`` subpackage
+  instead of the ``dev`` subpackage.
 
+autoupdate
+  This option indicates that the package is trusted enough to be automatically
+  update to its newest available version when set to ``true``. This flag is
+  intended to be used by tools running autospec automatically.
 
 Name and version resolution
 ===========================
 
-``autospec`` will attempt to use a number of patterns to determine the name
-and version of the package by examining the URL. For most tarballs this is
-simple, if they are of the format ``$name-$version.tar.$compression``.
+``autospec`` will attempt to use a number of patterns to determine the name and
+version of the package by examining the URL. For most tarballs this is simple,
+if they are of the format ``$name-$version.tar.$compression``.
 
-For websites such as ``bitbucket`` or ``GitHub``, using ``get$`` and ``v$.tar.*``
-style links, the project name itself is used from the URL and the version is
-determined by stripping down the tag.
+For websites such as ``bitbucket`` or ``GitHub``, using ``get$`` and
+``v$.tar.*`` style links, the project name itself is used from the URL and the
+version is determined by stripping down the tag.
 
 CPAN Perl packages, R packages, and rubygems.org rubygems are automatically
-prefixed with their language name: ``perl-``, ``R-`` and ``rubygem-`` respectively.
+prefixed with their language name: ``perl-``, ``R-`` and ``rubygem-``
+respectively.
 
 When these automated detections are not desirable, it is possible to override
 these with the ``--name`` flag when invoking ``autospec``
@@ -428,14 +405,14 @@ talks HTTP.
 
 This URL should accept ``POST`` requests with the following keys:
 
-**hash**
-    Contains the SHA-1 hash of the potential license file being checked.
+hash
+  Contains the SHA-1 hash of the potential license file being checked.
 
-**package**
-    The name of the package being examined
+package
+  The name of the package being examined
 
-**text**
-    The contents of the potential license file
+text
+  The contents of the potential license file
 
 Implementations return a *plain text* response with the SPDX identifier
 of the license, if known. An empty response is assumed to mean that this
@@ -443,4 +420,4 @@ license is unknown, in which case ``autospec`` will emit the ``license_show``
 URL. The implementation should show the now-stored license file via a
 web page, and enable a human to make a decision on the license. This is
 then stored internally, allowing future requests to automatically know
-the license type when this hash is encounted again. 
+the license type when this hash is encountered again.

autospec/autospec.py

@@ -19,6 +19,7 @@
 import argparse
 import sys
 import os
+import shutil
 import re
 import tempfile
 import configparser
@@ -116,6 +117,39 @@ def read_old_metadata():
             archives)
 
 
+def save_build_log(path, iteration):
+    """
+    Save build log to <path>/build.log.round<iteration>
+
+    Must be saved outside of the results/ directory since it gets wiped away on
+    each round.
+    """
+    buildlog = os.path.join(path, "results", "build.log")
+    shutil.copyfile(buildlog, "{}/build.log.round{}".format(path, iteration))
+
+
+def write_prep(workingdir):
+    """
+    Write metadata to the local workingdir when --prep-only is used
+    """
+    if config.urlban:
+        used_url = re.sub(config.urlban, "localhost", tarball.url)
+    else:
+        used_url = tarball.url
+
+    print()
+    print("Exiting after prep due to --prep-only flag")
+    print()
+    print("Results under ./workingdir")
+    print("Source  (./workingdir/{})".format(tarball.tarball_prefix))
+    print("Name    (./workingdir/name)    :", tarball.name)
+    print("Version (./workingdir/version) :", tarball.version)
+    print("URL     (./workingdir/source0) :", used_url)
+    write_out(os.path.join(workingdir, "name"), tarball.name)
+    write_out(os.path.join(workingdir, "version"), tarball.version)
+    write_out(os.path.join(workingdir, "source0"), used_url)
+
+
 def main():
     """
     Main function for autospec
@@ -153,6 +187,9 @@ def main():
                         default=False,
                         help="Search for package signature from source URL and "
                              "attempt to verify package")
+    parser.add_argument("-p", "--prep-only", action="store_true",
+                        default=False,
+                        help="Only perform preparatory work on package")
     parser.add_argument("--non_interactive", action="store_true",
                         default=False,
                         help="Disable interactive mode for package verification")
@@ -175,6 +212,14 @@ def main():
             "-a/--archives or options.conf['package']['archives'] requires an "
             "even number of arguments"))
 
+    if args.prep_only:
+        package(args, url, name, archives, "./workingdir")
+    else:
+        with tempfile.TemporaryDirectory() as workingdir:
+            package(args, url, name, archives, workingdir)
+
+
+def package(args, url, name, archives, workingdir):
     check_requirements(args.git)
     build.setup_workingdir(workingdir)
 
@@ -203,6 +248,10 @@ def main():
     config.parse_config_files(build.download_path, args.bump, filemanager)
     config.parse_existing_spec(build.download_path, tarball.name)
 
+    if args.prep_only:
+        write_prep(workingdir)
+        exit(0)
+
     buildreq.set_build_req()
     buildreq.scan_for_configure(_dir)
     specdescription.scan_for_description(tarball.name, _dir)
@@ -233,9 +282,20 @@ def main():
         filemanager.load_specfile(specfile)
         specfile.write_spec(build.download_path)
         filemanager.newfiles_printed = 0
+        mock_chroot = "/var/lib/mock/clear-{}/root/builddir/build/BUILDROOT/" \
+                      "{}-{}-{}.x86_64".format(build.uniqueext,
+                                               tarball.name,
+                                               tarball.version,
+                                               tarball.release)
+        if filemanager.clean_directories(mock_chroot):
+            # directories added to the blacklist, need to re-run
+            build.must_restart += 1
+
         if build.round > 20 or build.must_restart == 0:
             break
 
+        save_build_log(build.download_path, build.round)
+
     test.check_regression(build.download_path)
 
     if build.success == 0:
@@ -268,5 +328,4 @@ def main():
 
 
 if __name__ == '__main__':
-    with tempfile.TemporaryDirectory() as workingdir:
-        main()
+    main()

autospec/build.py

@@ -33,18 +33,15 @@ success = 0
 round = 0
 must_restart = 0
 base_path = None
-output_path = None
 download_path = None
 uniqueext = ''
 
 
 def setup_workingdir(workingdir):
     global base_path
-    global output_path
     global download_path
     base_path = workingdir
-    output_path = os.path.join(base_path, "output")
-    download_path = os.path.join(output_path, tarball.name)
+    download_path = os.path.join(base_path, tarball.name)
 
 
 def simple_pattern_pkgconfig(line, pattern, pkgconfig):
@@ -104,6 +101,9 @@ def failed_pattern(line, pattern, verbose, buildtool=None):
                 must_restart += buildreq.add_buildreq(config.maven_jars[s])
             else:
                 must_restart += buildreq.add_buildreq('jdk-%s' % s)
+        elif buildtool == 'catkin':
+            must_restart += buildreq.add_pkgconfig_buildreq(s)
+
     except:
         if verbose > 0:
             print("Unknown pattern match: ", s)

autospec/buildreq.py

@@ -33,6 +33,7 @@ import subprocess
 banned_requires = set()
 buildreqs = set()
 requires = set()
+extra_cmake = set()
 verbose = False
 cargo_bin = False
 banned_buildreqs = set(["llvm-devel",
@@ -486,6 +487,30 @@ def scan_for_configure(dirn):
             add_buildreq("cmake")
             buildpattern.set_build_pattern("cmake", default_score)
 
+            with open(os.path.join(dirpath, "CMakeLists.txt"), "r", encoding="latin-1") as f:
+                lines = f.readlines()
+
+                pat = re.compile(r"^find_package.*\(.*(catkin)(?: REQUIRED *)?(?:COMPONENTS (?P<comp>.*))?\)$")
+                for line in lines:
+                    match = pat.search(line)
+                    if match:
+                        # include catkin's required components
+                        comp = match.group("comp")
+                        if comp:
+                            for curr in comp.split(" "):
+                                add_pkgconfig_buildreq(curr)
+
+                        for curr in ["catkin", "catkin_pkg", "empy", "googletest"]:
+                            add_buildreq(curr)
+
+                        # catkin find_package() function will always rely on CMAKE_PREFIX_PATH
+                        # make sure we keep it consistent with CMAKE_INSTALL_PREFIX otherwise
+                        # it'll never be able to find its modules
+                        extra_cmake.add("-DCMAKE_PREFIX_PATH=/usr")
+                        extra_cmake.add("-DCATKIN_BUILD_BINARY_PACKAGE=ON")
+                        extra_cmake.add("-DSETUPTOOLS_DEB_LAYOUT=OFF")
+                        break
+
         if "configure" in files and os.access(dirpath + '/configure', os.X_OK):
             buildpattern.set_build_pattern("configure", default_score)
 
@@ -560,3 +585,4 @@ def load_specfile(specfile):
     specfile.buildreqs = buildreqs
     specfile.requires = requires
     specfile.cargo_bin = cargo_bin
+    specfile.extra_cmake += " " + " ".join(extra_cmake)

autospec/commitmessage.py

@@ -32,6 +32,8 @@ import config
 import tarball
 import util
 
+from subprocess import PIPE, run
+
 
 def scan_for_changes(download_path, directory):
     """
@@ -168,6 +170,54 @@ def process_NEWS(newsfile):
     return commitmessage, cves
 
 
+def process_git(giturl, oldversion, newversion):
+    """
+    process_git() checks out a git tree and tries to turn the
+    git history into a commit message
+    """
+
+    oldtag = ""
+    guessed_oldtag = oldversion
+    newtag = ""
+    guessed_newtag = newversion
+
+    if len(giturl) < 1:
+        return ""
+    if oldversion == newversion:
+        return ""
+
+    run(["git", "-C", "results", "clone", giturl, tarball.name])
+    p = run(["git", "-C", "results/" + tarball.name, "tag"], stdout=PIPE)
+    tags = p.stdout.decode('utf-8').split('\n')
+
+    for t in tags:
+        i = t.find(oldversion)
+        if i != -1:
+            guessed_oldtag = t
+        if t == oldversion or t == "v" + oldversion:
+            oldtag = t
+        i = t.find(newversion)
+        if i != -1:
+            guessed_newtag = t
+        if t == newversion or t == "v" + newversion:
+            newtag = t
+
+    if oldtag == "":
+        oldtag = guessed_oldtag
+    if newtag == "":
+        newtag = guessed_newtag
+
+    p = run(["git", "-C", "results/" + tarball.name, "log", oldtag + ".." + newtag], stdout=PIPE)
+    fulllog = p.stdout.decode('utf-8').split('\n')
+    p = run(["git", "-C", "results/" + tarball.name, "shortlog", oldtag + ".." + newtag], stdout=PIPE)
+    shortlog = p.stdout.decode('utf-8').split('\n')
+
+    if len(fulllog) < 15:
+        return fulllog
+    else:
+        return shortlog
+
+
 def guess_commit_message():
     """
     guess_commit_message() parses newsfiles and determines a sane commit
@@ -199,6 +249,10 @@ def guess_commit_message():
     if config.old_version is not None and config.old_version != tarball.version:
         commitmessage.append("{}: Autospec creation for update from version {} to version {}"
                              .format(tarball.name, config.old_version, tarball.version))
+        if tarball.giturl != "":
+            gitmsg = process_git(tarball.giturl, config.old_version, tarball.version)
+            commitmessage.append("")
+            commitmessage.extend(gitmsg)
     else:
         if cves:
             commitmessage.append("{}: Fix for {}"
@@ -224,7 +278,7 @@ def guess_commit_message():
         commitmessage.append("")
 
     util.write_out(os.path.join(build.download_path, "commitmsg"),
-                   "\n".join(commitmessage) + "\n", encode="latin-1")
+                   "\n".join(commitmessage) + "\n")
 
     print("Guessed commit message:")
     try:

autospec/config.py

@@ -42,6 +42,7 @@ urlban = ""
 extra_make = ""
 extra_make_install = ""
 extra_cmake = ""
+cmake_srcdir = ""
 prep_append = []
 subdir = ""
 install_macro = "%make_install"
@@ -107,7 +108,9 @@ config_options = {
     "nostrip": "disable stripping binaries",
     "verify_required": "require package verification for build",
     "security_sensitive": "set flags for security-sensitive builds",
-    "so_to_lib": "add .so files to the lib package instead of dev"}
+    "so_to_lib": "add .so files to the lib package instead of dev",
+    "autoupdate": "this package is trusted enough to automatically update "
+                  "(used by other tools)"}
 
 # simple_pattern_pkgconfig patterns
 # contains patterns for parsing build.log for missing dependencies
@@ -158,7 +161,9 @@ simple_pats = [
     (r"Package systemd was not found in the pkg-config search path.", "systemd-dev"),
     (r"Unable to find the requested Boost libraries.", "boost-dev"),
     (r"libproc not found. Please configure without procps", "procps-ng-dev"),
-    (r"configure: error: glib2", "glib-dev")]
+    (r"configure: error: glib2", "glib-dev"),
+    (r"C library 'efivar' not found", "efivar-dev"),
+    (r"Has header \"efi.h\": NO", "gnu-efi-dev")]
 
 # failed_pattern patterns
 # contains patterns for parsing build.log for missing dependencies
@@ -265,7 +270,8 @@ failed_pats = [
     (r"\[ERROR\] .* Cannot access central \(.*\) in offline mode and the artifact "
      r".*:(.*):[jar|pom]+:.* has not been downloaded from it before.*", 0, 'maven'),
     (r"\[WARNING\] The POM for .*:(.*):[jar|pom]+:.* is missing, no dependency information "
-     r"available", 0, 'maven')]
+     r"available", 0, 'maven'),
+    (r"^.*Could not find a package configuration file provided by \"(.*)\" with.*$", 0, 'catkin')]
 
 
 def get_metadata_conf():
@@ -274,8 +280,13 @@ def get_metadata_conf():
     """
     metadata = {}
     metadata['name'] = tarball.name
-    metadata['url'] = tarball.url
+    if urlban:
+        metadata['url'] = re.sub(urlban, "localhost", tarball.url)
+    else:
+        metadata['url'] = tarball.url
+
     metadata['archives'] = ' '.join(tarball.archives)
+    metadata['giturl'] = tarball.giturl
     return metadata
 
 
@@ -472,6 +483,7 @@ def parse_config_files(path, bump, filemanager):
     global extra_make
     global extra_make_install
     global extra_cmake
+    global cmake_srcdir
     global prep_append
     global subdir
     global install_macro
@@ -483,8 +495,6 @@ def parse_config_files(path, bump, filemanager):
 
     packages_file = None
 
-    read_config_opts(path)
-
     # Require autospec.conf for additional features
     if os.path.exists(config_file):
         config = configparser.ConfigParser(interpolation=None)
@@ -506,6 +516,9 @@ def parse_config_files(path, bump, filemanager):
 
         urlban = config['autospec'].get('urlban', None)
 
+    # Read values from options.conf (and deprecated files) and rewrite as necessary
+    read_config_opts(path)
+
     if not git_uri:
         print("Warning: Set [autospec][git] upstream template for git support")
     if not license_fetch:
@@ -663,6 +676,10 @@ def parse_config_files(path, bump, filemanager):
     if content and content[0]:
         extra_cmake = content[0]
 
+    content = read_conf_file(os.path.join(path, "cmake_srcdir"))
+    if content and content[0]:
+        cmake_srcdir = content[0]
+
     content = read_conf_file(os.path.join(path, "subdir"))
     if content and content[0]:
         subdir = content[0]
@@ -715,6 +732,7 @@ def load_specfile(specfile):
     specfile.extra_make = extra_make
     specfile.extra_make_install = extra_make_install
     specfile.extra_cmake = extra_cmake
+    specfile.cmake_srcdir = cmake_srcdir or specfile.cmake_srcdir
     specfile.prep_append = prep_append
     specfile.subdir = subdir
     specfile.install_macro = install_macro

autospec/count.py

@@ -211,7 +211,7 @@ def parse_log(log, pkgname=''):
 
         # apr
         # testatomic          :  SUCCESS
-        if re.search(r"\:  SUCCESS$", line) and incheck:
+        if re.search(r":  SUCCESS$", line) and incheck:
             counted_pass += 1
             continue
 
@@ -283,7 +283,7 @@ def parse_log(log, pkgname=''):
             total_fail += convert_int(match.group(1))
             continue
 
-        match = re.search(r"/== ([0-9]+) failed, ([0-9]+) passed, ([0-9]+) xfailed in ", line)
+        match = re.search(r"== ([0-9]+) failed, ([0-9]+) passed, ([0-9]+) xfailed in ", line)
         if match and incheck:
             total_pass += convert_int(match.group(2))
             total_fail += convert_int(match.group(1))
@@ -967,13 +967,6 @@ def parse_log(log, pkgname=''):
             counted_pass += 1
             continue
 
-        # apr-util
-        # :  SUCCESS
-        match = re.search(r":  SUCCESS$", line)
-        if match and incheck:
-            counted_pass += 1
-            continue
-
         # bash
         # < Failed 126 of 1378 Unicode tests
         match = re.search(r"^[<,>] Failed ([0-9]+) of ([0-9]+)", line)
@@ -1015,18 +1008,6 @@ def parse_log(log, pkgname=''):
             counted_pass += 1
             continue
 
-        # *** zlib test OK ***
-        # *** zlib 64-bit test OK ***
-        match = re.search(r"\*\*\* zlib .*test OK \*\*\*", line)
-        if match and incheck:
-            counted_pass += 1
-            continue
-
-        match = re.search(r"\*\*\* zlib .*test [A-Z!O][A-Z!K] \*\*\*", line)
-        if match and incheck:
-            counted_fail += 1
-            continue
-
         # LVM2
         # valgrind pool awareness ... fail
         # dfa matching ... fail
@@ -1104,6 +1085,13 @@ def parse_log(log, pkgname=''):
             total_fail += convert_int(match.group(3))
             continue
 
+        # vim
+        # Executed 9 tests
+        match = re.search(r"Executed ([0-9]+) tests$", line)
+        if match and incheck:
+            total_tests += convert_int(match.group(1))
+            continue
+
         # rubygem-formatador
         #   9 succeeded in 0.00375661 seconds
         match = re.search(r"([0-9]+) succeeded in [0-9]+\.[0-9]+ seconds", line)
@@ -1288,16 +1276,6 @@ def parse_log(log, pkgname=''):
             total_skip += 1
             continue
 
-        match = re.search(r"^Testing .+\ +\*FAILED\*$", line)
-        if match and incheck:
-            total_fail += 1
-            continue
-
-        match = re.search(r"^Verifying .+\ +\*FAILED\*$", line)
-        if match and incheck:
-            total_fail += 1
-            continue
-
         # libconfig
         # 3 tests; 3 passed, 0 failed
         match = re.search(r"^([0-9]+) tests; ([0-9]+) passed\, ([0-9]+) failed", line)

autospec/failed_commands

@@ -505,3 +505,4 @@ libxml/xmlversion.h, libxml2-dev
 g-ir-scanner, gobject-introspection
 get_mempolicy, numactl-dev
 pcap_dump, libpcap-dev
+ldap.h, openldap-dev

autospec/files.py

@@ -23,6 +23,8 @@ import build
 import tarball
 import config
 import re
+import os
+import util
 from collections import OrderedDict
 # todo package splits
 
@@ -101,6 +103,46 @@ class FileManager(object):
         else:
             return False
 
+    def _clean_dirs(self, root, files):
+        """
+        Do the work to remove the directories from the files list
+        """
+        res = set()
+        removed = False
+
+        directive_re = re.compile("(%\w+(\([^\)]*\))?\s+)(.*)")
+        for f in files:
+            # skip the files with directives at the beginning, including %doc
+            # and %dir directives.
+            # autospec does not currently support adding empty directories to
+            # the file list by prefixing "%dir". Regardless, skip these entries
+            # because if they exist at this point it is intentional (i.e.
+            # support was added).
+            if directive_re.match(f):
+                res.add(f)
+                continue
+
+            if os.path.isdir(os.path.join(root, f.lstrip("/"))):
+                util.print_warning("Removing directory {} from file list".format(f))
+                self.files_blacklist.add(f)
+                removed = True
+            else:
+                res.add(f)
+
+        return (res, removed)
+
+    def clean_directories(self, root):
+        """
+        Remove directories from file list
+        """
+        removed = False
+        for pkg in self.packages:
+            self.packages[pkg], _rem = self._clean_dirs(root, self.packages[pkg])
+            if _rem:
+                removed = True
+
+        return removed
+
     def push_file(self, filename):
         """
         Perform a number of checks against the filename and push the filename

autospec/license.py

@@ -75,8 +75,8 @@ def license_from_copying_hash(copying):
         try:
             c.perform()
         except Exception as excep:
-            print_fatal("Failed to fetch license from " + config.license_fetch,
-                        excep)
+            print_fatal("Failed to fetch license from {}: {}"
+                        .format(config.license_fetch, excep))
             c.close()
             sys.exit(1)
 

autospec/specfiles.py

@@ -70,6 +70,7 @@ class Specfile(object):
         self.install_macro = "%make_install"
         self.disable_static = "--disable-static"
         self.extra_cmake = ""
+        self.cmake_srcdir = ".."
         self.make_install_append = []
         self.excludes = []
         self.keyid = ""
@@ -983,7 +984,7 @@ class Specfile(object):
         self._write_strip("mkdir clr-build")
         self._write_strip("pushd clr-build")
         self.write_variables()
-        self._write_strip("cmake .. -G \"Unix Makefiles\" "
+        self._write_strip("cmake " + self.cmake_srcdir + " -G \"Unix Makefiles\" "
                           "-DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS:BOOL=ON "
                           "-DLIB_INSTALL_DIR:PATH=/usr/lib64 "
                           "-DCMAKE_AR=/usr/bin/gcc-ar "
@@ -1009,7 +1010,7 @@ class Specfile(object):
             self._write_strip('export PKG_CONFIG_PATH="/usr/lib32/pkgconfig"')
             self._write_strip('export CFLAGS="$CFLAGS -m32"')
             self._write_strip('export CXXFLAGS="$CXXFLAGS -m32"')
-            self._write_strip('cmake .. -G "Unix Makefiles" '
+            self._write_strip('cmake " + self.cmake_srcdir + " -G "Unix Makefiles" '
                               "-DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS:BOOL=ON "
                               "-DLIB_INSTALL_DIR:PATH=/usr/lib32 "
                               "-DCMAKE_AR=/usr/bin/gcc-ar "
@@ -1027,7 +1028,7 @@ class Specfile(object):
             self.need_avx2_flags = saved_avx2flags
             self._write_strip('export CFLAGS="$CFLAGS -march=haswell"')
             self._write_strip('export CXXFLAGS="$CXXFLAGS -march=haswell"')
-            self._write_strip('cmake .. -G "Unix Makefiles" '
+            self._write_strip('cmake " + self.cmake_srcdir + " -G "Unix Makefiles" '
                               "-DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS:BOOL=ON "
                               "-DLIB_INSTALL_DIR:PATH=/usr/lib/haswell "
                               "-DCMAKE_AR=/usr/bin/gcc-ar "

autospec/tarball.py

@@ -40,6 +40,7 @@ path = ""
 tarball_prefix = ""
 gcov_file = ""
 archives = []
+giturl = ""
 
 
 def get_sha1sum(filename):
@@ -65,6 +66,7 @@ def really_download(upstream_url, destination):
             c.perform()
         except pycurl.error:
             print_fatal("unable to download {}".format(upstream_url))
+            os.remove(destination)
             exit(1)
         finally:
             c.close()
@@ -174,6 +176,8 @@ def print_header():
 
 
 def download_tarball(target_dir):
+    global giturl
+
     """
     Download tarball at url (global) to target_dir
 
@@ -193,6 +197,8 @@ def download_tarball(target_dir):
                     config_f["package"].get("url") == url or
                     config_f["package"].get("archives") == " ".join(archives)):
                 target = os.getcwd()
+            if "giturl" in config_f["package"]:
+                giturl = config_f["package"].get("giturl")
 
     if target_dir:
         target = target_dir
@@ -278,6 +284,8 @@ def name_and_version(name_arg, version_arg, filemanager):
     global rawname
     global version
     global url
+    global giturl
+    global repo
 
     tarfile = os.path.basename(url)
 
@@ -323,17 +331,25 @@ def name_and_version(name_arg, version_arg, filemanager):
     if "github.com" in url:
         # define regex accepted for valid packages, important for specific
         # patterns to come before general ones
-        github_patterns = [r"https?://github.com/.*/(.*?)/archive/[v|r]?.*/(.*).tar",
-                           r"https?://github.com/.*/(.*?)/archive/[-a-zA-Z]*-(.*).tar",
-                           r"https?://github.com/.*/(.*?)/archive/[vVrR]?(.*).tar",
-                           r"https?://github.com/.*/(.*?)/releases/download/v.*/(.*).tar"]
+        github_patterns = [r"https?://github.com/(.*)/(.*?)/archive/[v|r]?.*/(.*).tar",
+                           r"https?://github.com/(.*)/(.*?)/archive/[-a-zA-Z]*-(.*).tar",
+                           r"https?://github.com/(.*)/(.*?)/archive/[vVrR]?(.*).tar",
+                           r"https?://github.com/(.*)/(.*?)/releases/download/.*?/(.*).tar",
+                           r"https?://github.com/(.*)/(.*?)/files/.*?/(.*).tar"]
 
         for pattern in github_patterns:
             m = re.search(pattern, url)
             if m:
-                name = m.group(1).strip()
+                repo = m.group(2).strip()
+                if repo not in name:
+                    # Only take the repo name as the package name if it's more descriptive
+                    name = repo
+                elif name != repo:
+                    name = re.sub("release-", '', name)
+                    name = re.sub("\d*$", '', name)
                 rawname = name
-                version = convert_version(m.group(2))
+                version = convert_version(m.group(3))
+                giturl = "https://github.com/" + m.group(1).strip() + "/" + repo + ".git"
                 break
 
     if "mirrors.kernel.org" in url:
@@ -425,29 +441,13 @@ def find_extract(tar_path, tarfile):
     return extract_cmd, tar_prefix
 
 
-def write_makefile(archives):
-    """
-    Write the new makefile with url, name, and archives
-    """
-    with open(build.download_path + "/Makefile", "w") as f:
-        f.write("PKG_NAME := " + name + "\n")
-        f.write("URL := " + url + "\n")
-        sep = "ARCHIVES := "
-        for archive in archives:
-            f.write("{}{}".format(sep, archive))
-            sep = " " if sep != " " else " \\\n\t"
-        f.write("\n")
-        f.write("\n")
-        f.write("include ../common/Makefile.common\n")
-
-
 def prepare_and_extract(extract_cmd):
     """
     Prepare the directory and extract the tarball
     """
     shutil.rmtree(os.path.join(build.base_path, name), ignore_errors=True)
     shutil.rmtree(os.path.join(build.base_path, tarball_prefix), ignore_errors=True)
-    os.makedirs("{}".format(build.output_path), exist_ok=True)
+    os.makedirs("{}".format(build.base_path), exist_ok=True)
     call("mkdir -p %s" % build.download_path)
     call(extract_cmd)
 
@@ -516,8 +516,6 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager):
     # Now that the metadata has been collected print the header
     print_header()
     # write out the Makefile with the name, url, and archives we found
-    # DEPRECATED, this will be removed in a future version
-    write_makefile(archives_arg)
     # prepare directory and extract tarball
     prepare_and_extract(extract_cmd)
     # locate or download archives and move them into the right spot

requirements.txt

@@ -1,3 +1,5 @@
 flake8>=3.4.0
 pycurl>=7.43.0
 toml>=0.9.0
+mock>=2.0.0
+coverage>=4.4.1

setup.cfg

@@ -2,3 +2,7 @@
 tag_build = 
 [pycodestyle]
 ignore = E501
+
+[coverage:run]
+# omit tests and travis site-packages
+omit = tests/*,*site-packages*,*site.py

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 import sys, os
-version = "1.1.1"
+version = "1.1.8"
 
 def readme():
     with open("README.rst") as f:

tests/test_build.py

@@ -23,7 +23,6 @@ class TestBuildpattern(unittest.TestCase):
         build.round = 0
         build.must_restart = 0
         build.base_path = None
-        build.output_path = None
         build.download_path = None
         build.buildreq.buildreqs = set()
         build.config.config_opts['32bit'] = False
@@ -35,9 +34,7 @@ class TestBuildpattern(unittest.TestCase):
         build.tarball.name = "testtarball"
         build.setup_workingdir("test_directory")
         self.assertEqual(build.base_path, "test_directory")
-        self.assertEqual(build.output_path, "test_directory/output")
-        self.assertEqual(build.download_path,
-                         "test_directory/output/testtarball")
+        self.assertEqual(build.download_path, "test_directory/testtarball")
 
     def test_simple_pattern_pkgconfig(self):
         """
@@ -239,7 +236,7 @@ class TestBuildpattern(unittest.TestCase):
         self.assertIn('jdk-apache-parent', build.buildreq.buildreqs)
         self.assertEqual(build.must_restart, 1)
 
-    def test_parse_build_resultsi_pkgconfig(self):
+    def test_parse_build_results_pkgconfig(self):
         """
         Test parse_build_results with a test log indicating failure due to a
         missing qmake package (pkgconfig error)
@@ -317,6 +314,40 @@ class TestBuildpattern(unittest.TestCase):
         self.assertIn('testpkg-python', build.buildreq.buildreqs)
         self.assertEqual(build.must_restart, 1)
 
+    def test_parse_build_results_files(self):
+        """
+        Test parse_build_results with a test log indicating files are missing
+        """
+        def mock_util_call(cmd):
+            del cmd
+
+        build.config.setup_patterns()
+        call_backup = build.util.call
+        build.util.call = mock_util_call
+        fm = files.FileManager()
+
+        open_name = 'build.open'
+        content = 'line 1\n' \
+                  'Installed (but unpackaged) file(s) found:\n' \
+                  '/usr/testdir/file\n' \
+                  '/usr/testdir/file1\n' \
+                  '/usr/testdir/file2\n' \
+                  'RPM build errors\n' \
+                  'errors here\n'
+        m_open = mock_open(read_data=content)
+
+        with patch(open_name, m_open, create=True):
+            build.parse_build_results('testname', 0, fm)
+
+        build.util.call = call_backup
+
+        self.assertEqual(fm.files,
+                         ['/usr/testdir/file',
+                          '/usr/testdir/file1',
+                          '/usr/testdir/file2'])
+        # one for each file added
+        self.assertEqual(build.must_restart, 3)
+
     def test_get_mock_cmd_without_consolehelper(self):
         """
         Test get_mock_cmd when /usr/bin/mock doesn't point to consolehelper

tests/test_count.py

@@ -36,6 +36,8 @@ pats = [
      [332, 281, 38, 0, 13, 0, 0, 0, 0, 0]),
     ('===================== 5 failed, 318 passed in 1.06 seconds =====================',
      [323, 318, 5, 0, 0, 0, 0, 0, 0, 0]),
+    ('===================== 5 failed, 9 passed, 7 xfailed in 1.06 seconds ============',
+     [21, 9, 5, 7, 0, 0, 0, 0, 0, 0]),
     ('============= 1628 passed, 72 skipped, 4 xfailed in 146.26 seconds =============',
      [1704, 1628, 0, 4, 72, 0, 0, 0, 0, 0]),
     ('=============== 119 passed, 2 skipped, 54 error in 2.19 seconds ================',
@@ -87,8 +89,9 @@ pats = [
     ('# of expected passes\t1144\n'
      '# of expected failures\t57\n'
      '# of untested testcases\t1\n'
-     '# of unsupported tests\t12',
-     [1213, 1144, 0, 57, 12, 0, 0, 0, 0, 0]),
+     '# of unsupported tests\t12\n'
+     '# of unexpected failures\t1\n',
+     [1214, 1144, 1, 57, 12, 0, 0, 0, 0, 0]),
     # ccache
     ('PASSED: 448 assertions, 88 tests, 10 suites',
      [88, 88, 0, 0, 0, 0, 0, 0, 0, 0]),
@@ -179,6 +182,13 @@ pats = [
      'Failed with core:   0\n'
      'Unknown status:     0',
      [13042, 13036, 6, 0, 0, 1, 1, 0, 0, 0]),
+    # nss
+    ('cert.sh: #101: Import chain-2-serverCA-ec CA -t u,u,u for localhost.localdomain (ext.)  - FAILED\n'
+     'Passed:             13036\n'
+     'Failed:             6\n'
+     'Failed with core:   0\n'
+     'Unknown status:     0',
+     [13042, 13036, 6, 0, 0, 0, 0, 1, 0, 0]),
     # rsync
     ('     34 passed\n'
      '     5 skipped',
@@ -191,6 +201,12 @@ pats = [
      [30, 0, 6, 6, 18, 0, 0, 0, 0, 0]),
     ('FAILED (failures=1, errors=499, skipped=48)',
      [548, 0, 1, 499, 48, 0, 0, 0, 0, 0]),
+    ('FAILED (failures=1, errors=499)',
+     [500, 0, 1, 499, 0, 0, 0, 0, 0, 0]),
+    ('FAILED (failures=1)',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('FAILED (errors=1)',
+     [1, 0, 0, 1, 0, 0, 0, 0, 0, 0]),
     ('OK (KNOWNFAIL=5, SKIP=15)',
      [20, 0, 0, 5, 15, 0, 0, 0, 0, 0]),
     # qpid-python
@@ -304,6 +320,9 @@ pats = [
     # rubygem-ansi
     ('Executed 12 tests with 7 passing, 5 errors.',
      [12, 7, 5, 0, 0, 0, 0, 0, 0, 0]),
+    # vim
+    ('Executed 12 tests',
+     [12, 12, 0, 0, 0, 0, 0, 0, 0, 0]),
     # rubygem-formatador
     ('  9 succeeded in 0.00375661 seconds',
      [9, 9, 0, 0, 0, 0, 0, 0, 0, 0]),
@@ -372,8 +391,12 @@ pats = [
     # hdf5
     ('Testing h5repack h5repack_szip.h5 -f dset_szip:GZIP=1                  -SKIP-',
      [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
+    ('Verifying h5repack h5repack_szip.h5 -f dset_szip:GZIP=1                -SKIP-',
+     [1, 0, 0, 0, 1, 0, 0, 0, 0, 0]),
     ('Verifying h5dump output -f GZIP=1 -m 1024                             *FAILED*',
      [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
+    ('Testing h5dump output -f GZIP=1 -m 1024                               *FAILED*',
+     [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]),
     ('Testing h5repack --metadata_block_size=8192                            PASSED',
      [1, 1, 0, 0, 0, 0, 0, 0, 0, 0]),
     ('Verifying h5diff output h5repack_layout.h5 out-meta_long.h5repack_layo PASSED',

tests/test_files.py

@@ -1,5 +1,7 @@
 import unittest
 import files
+import tempfile
+import os
 from unittest.mock import call, MagicMock
 from files import FileManager
 
@@ -184,5 +186,69 @@ class TestFiles(unittest.TestCase):
         self.assertNotIn('test', self.fm.files)
         self.assertNotIn('test', self.fm.files_blacklist)
 
+    def test_clean_directories(self):
+        """
+        Test clean_directories with a directory in the list
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("/directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"], set(["/file1", "/file2"]))
+
+
+    def test_clean_directories_with_dir(self):
+        """
+        Test clean_directories with a %dir directory in the list. This should
+        remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%dir /directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%dir /directory", "/file1", "/file2"]))
+
+
+    def test_clean_directories_with_doc(self):
+        """
+        Test clean_directories with a %doc directive in the list. This should
+        remain.
+        """
+        with tempfile.TemporaryDirectory() as tmpd:
+            os.mkdir(os.path.join(tmpd, "directory"))
+            with open(os.path.join(tmpd, "file1"), "w") as f:
+                f.write(" ")
+
+            with open(os.path.join(tmpd, "file2"), "w") as f:
+                f.write(" ")
+
+            self.fm.packages["main"] = set()
+            self.fm.packages["main"].add("%doc /directory")
+            self.fm.packages["main"].add("/file1")
+            self.fm.packages["main"].add("/file2")
+            self.fm.clean_directories(tmpd)
+            self.assertEqual(self.fm.packages["main"],
+                             set(["%doc /directory", "/file1", "/file2"]))
+
+
 if __name__ == '__main__':
     unittest.main(buffer=True)

tests/test_license.py

@@ -65,6 +65,30 @@ class TestLicense(unittest.TestCase):
 
         self.assertIn('GPL-3.0', license.licenses)
 
+    def test_license_from_copying_hash_no_license_show(self):
+        """
+        Test license_from_copying_hash with invalid hash and no license_show
+        set
+        """
+        # Calls out to tarball.get_sha1sum to get the hash of the license
+        # we might as well test that is returning what it should because it
+        # doesn't call any external resources, it just calculates the hash.
+        open_name = 'tarball.open'
+        with open('tests/COPYING_TEST', 'rb') as copyingf:
+            content = copyingf.read()
+
+        bkup_hash = license.config.license_hashes[license.tarball.get_sha1sum('tests/COPYING_TEST')]
+        # remove the hash from license_hashes
+        del(license.config.license_hashes[license.tarball.get_sha1sum('tests/COPYING_TEST')])
+        license.config.license_show = "license.show.url"
+        m_open = mock_open(read_data=content)
+        with patch(open_name, m_open, create=True):
+            license.license_from_copying_hash('copying.txt')
+
+        # restore the hash
+        license.config.license_hashes[license.tarball.get_sha1sum('tests/COPYING_TEST')] = bkup_hash
+        self.assertEquals(license.licenses, [])
+
     def test_license_from_copying_hash_bad_license(self):
         """
         Test license_from_copying_hash with invalid license file
@@ -83,6 +107,50 @@ class TestLicense(unittest.TestCase):
 
         self.assertEquals(license.licenses, [])
 
+    @patch('pycurl.Curl')
+    def test_license_from_copying_hash_license_server_excep(self, mock_pycurl_curl):
+        """
+        Test license_from_copying_hash with license server when pycurl raises
+        an exception.
+        """
+        class MockCurl():
+            URL = None
+            WRITEDATA = None
+            POSTFIELDS = None
+            def setopt(_, __, ___):
+                pass
+
+            def perform(_):
+                raise Exception('Test Exception')
+
+            def close(_):
+                pass
+
+        # set the mock curl
+        license.pycurl.Curl = MockCurl
+
+        license.config.license_fetch = 'license.server.url'
+        with open('tests/COPYING_TEST', 'rb') as copyingf:
+            content = copyingf.read()
+
+        # Calls out to tarball.get_sha1sum to get the hash of the license
+        # we might as well test that is returning what it should because it
+        # doesn't call any external resources, it just calculates the hash.
+        # Also patch the open in license.py
+        m_open = mock_open(read_data=content)
+        with patch('tarball.open', m_open, create=True):
+            with patch('license.open', m_open, create=True):
+                # let's check that the proper thing is being printed as well
+                out = StringIO()
+                with redirect_stdout(out):
+                    with self.assertRaises(SystemExit):
+                        license.license_from_copying_hash('copying.txt')
+
+        self.assertIn('Failed to fetch license from ', out.getvalue())
+
+        # unset the manual mock
+        license.pycurl.Curl = pycurl.Curl
+
     @patch('pycurl.Curl')
     def test_license_from_copying_hash_license_server(self, mock_pycurl_curl):
         """
@@ -97,12 +165,10 @@ class TestLicense(unittest.TestCase):
                 return 'GPL-3.0'.encode('utf-8')
 
         # set the mocks
-        mock_pycurl_curl.return_value = MagicMock()
         license.BytesIO = MockBytesIO
 
         license.config.license_fetch = 'license.server.url'
         with open('tests/COPYING_TEST', 'rb') as copyingf:
-            # note the replace corrupting the file contents
             content = copyingf.read()
 
         # Calls out to tarball.get_sha1sum to get the hash of the license

tests/test_pkg_integrity.py

@@ -1,13 +1,18 @@
 import os
+import shutil
+import mock
 import unittest
 import tempfile
 import pkg_integrity
 
 
+TESTDIR = os.path.join(os.getcwd(), "tests/testfiles/pkg_integrity")
+
 PACKAGE_URL = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.1.tar.gz"
 XATTR_PKT_URL = "http://pypi.debian.net/xattr/xattr-0.9.1.tar.gz"
 NO_SIGN_PKT_URL = "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz"
 GEM_PKT = "https://rubygems.org/downloads/hoe-debugging-1.2.1.gem"
+NOSIGN_PKT_URL_BAD = "http://gnu.mirrors.pair.com/savannah/savannah/quagga/bad_quagga-1.1.0.tar.gz"
 NOSIGN_PKT_URL = "http://download.savannah.gnu.org/releases/quagga/quagga-1.1.0.tar.gz"
 NOSIGN_SIGN_URL = "http://download.savannah.gnu.org/releases/quagga/quagga-1.1.0.tar.gz.asc"
 PYPI_MD5_ONLY_PKG = "http://pypi.debian.net/tappy/tappy-0.9.2.tar.gz"
@@ -15,36 +20,23 @@ GNOME_SHA256_PKG = "https://download.gnome.org/sources/pygobject/3.24/pygobject-
 KEYID = "EC2392F2EDE74488680DA3CF5F2B4756ED873D23"
 
 
-class TestGPGCli(unittest.TestCase):
+def mock_attempt_to_download(path, dest=None):
+    if dest:
+        shutil.copyfile(os.path.join(TESTDIR, os.path.basename(path)), dest)
+        return 200
 
-    def test_import_export(self):
-        with pkg_integrity.cli_gpg_ctx() as ctx:
-            err, output = ctx.export_key(KEYID)
-            self.assertTrue(err is not None)
-            err, output = ctx.import_key(KEYID)
-            self.assertTrue(err is None)
-            err, output = ctx.export_key(KEYID)
-            self.assertTrue(err is None)
-            self.assertTrue('PGP PUBLIC KEY' in output)
+def mock_head_request(url):
+    bad_sigs = ["http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.1.tar.gz.sig",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.sig",
+                "http://www.ferzkopp.net/Software/SDL_gfx-2.0/SDL_gfx-2.0.25.tar.gz.asc"]
 
-    def test_import_non_existing_key(self):
-        with pkg_integrity.cli_gpg_ctx() as ctx:
-            keyid = '0' + KEYID[1:]
-            err, output = ctx.import_key(keyid)
-            self.assertTrue(err is not None)
-
-    def test_display_key_info(self):
-        with pkg_integrity.cli_gpg_ctx() as ctx:
-            err, output = ctx.import_key(KEYID)
-            self.assertTrue(err is None)
-            err, output = ctx.export_key(KEYID)
-            with open('key_test.pkey', 'w') as out_key:
-                out_key.write(output)
-            err, output = ctx.display_keyinfo('key_test.pkey')
-            os.remove('key_test.pkey')
-            self.assertTrue('keyid' in output)
+    if url in bad_sigs:
+        return 404
+    return 200
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
+@mock.patch('pkg_integrity.head_request', mock_head_request)
 class TestCheckFn(unittest.TestCase):
 
     def setUp(self):
@@ -53,8 +45,6 @@ class TestCheckFn(unittest.TestCase):
         pkg_integrity.config.rewrite_config_opts = mock_rewrite
         pkg_integrity.config.config_opts['verify_required'] = False
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_check_matching_sign_url(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
@@ -73,6 +63,7 @@ class TestCheckFn(unittest.TestCase):
             self.assertTrue(result)
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
 class TestDomainBasedVerifiers(unittest.TestCase):
 
     def run_test_for_domain(self, Verifier, url):
@@ -85,8 +76,6 @@ class TestDomainBasedVerifiers(unittest.TestCase):
         return None
 
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_pypi(self):
         result = self.run_test_for_domain(pkg_integrity.PyPiVerifier, PYPI_MD5_ONLY_PKG)
         self.assertTrue(result)
@@ -96,6 +85,7 @@ class TestDomainBasedVerifiers(unittest.TestCase):
         self.assertTrue(result)        
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
 class TestGEMShaVerifier(unittest.TestCase):
 
     def setUp(self):
@@ -122,6 +112,8 @@ class TestGEMShaVerifier(unittest.TestCase):
             self.assertEqual(a.exception.code, 1)
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
+@mock.patch('pkg_integrity.head_request', mock_head_request)
 class TestGPGVerifier(unittest.TestCase):
 
     def setUp(self):
@@ -130,8 +122,6 @@ class TestGPGVerifier(unittest.TestCase):
         pkg_integrity.config.rewrite_config_opts = mock_rewrite
         pkg_integrity.config.config_opts['verify_required'] = False
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_from_url(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
@@ -143,12 +133,14 @@ class TestGPGVerifier(unittest.TestCase):
 
     def test_check_quit(self):
         with tempfile.TemporaryDirectory() as tmpd:
-            #with self.assertRaises(SystemExit) as a:
-            pkg_integrity.check(NO_SIGN_PKT_URL, tmpd, interactive=False)
-            #self.assertEqual(a.exception.code, 1)
+            with self.assertRaises(SystemExit) as a:
+                out_file = os.path.join(tmpd, os.path.basename(NOSIGN_PKT_URL_BAD))
+                pkg_integrity.attempt_to_download(NOSIGN_PKT_URL_BAD, out_file)
+                key_file = os.path.join(tmpd, os.path.basename(NOSIGN_PKT_URL_BAD))
+                pkg_integrity.attempt_to_download(NOSIGN_SIGN_URL, key_file + '.asc')
+                result = pkg_integrity.check(NOSIGN_PKT_URL_BAD, tmpd)
+                self.assertEqual(a.exception.code, 1)
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_from_disk(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
@@ -158,8 +150,6 @@ class TestGPGVerifier(unittest.TestCase):
             result = pkg_integrity.from_disk(PACKAGE_URL, out_file, out_key)
             self.assertTrue(result)
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
     def test_non_matchingsig(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(PACKAGE_URL))
@@ -174,35 +164,14 @@ class TestGPGVerifier(unittest.TestCase):
         result = pkg_integrity.from_disk('http://nokey.com/package.tar.gz',
                                          'NonExistentPKG.tar.gz',
                                          'NonExistentKey.asc')
-        self.assertTrue(result is None)
+        self.assertIsNone(result)
 
     def test_result_on_nosign_package(self):
         with tempfile.TemporaryDirectory() as tmpd:
             out_file = os.path.join(tmpd, os.path.basename(NO_SIGN_PKT_URL))
             pkg_integrity.attempt_to_download(NO_SIGN_PKT_URL, out_file)
             result = pkg_integrity.check(NO_SIGN_PKT_URL, tmpd)
-            self.assertTrue(result is None)
-
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
-    def test_pubkey_import(self):
-        def say_yes(_):
-            return True
-        _ = pkg_integrity.InputGetter.get_answer
-        pkg_integrity.InputGetter.get_answer = say_yes
-        keyid = '0' + KEYID[1:]
-        result = pkg_integrity.attempt_key_import(keyid)
-        self.assertTrue(result is False)
-        result = pkg_integrity.attempt_key_import(KEYID)
-        self.assertTrue(result)
-        pkg_integrity.InputGetter.get_answer = _
-        self.removeKey()
-
-    def removeKey(self):
-        key_path = os.path.dirname(os.path.realpath(__file__))
-        key_path = os.path.dirname(key_path) + '/autospec/keyring/{}.pkey'.format(KEYID)
-        if os.path.exists(key_path):
-            os.unlink(key_path)
+            self.assertIsNone(result)
 
 
 class TestInputGetter(unittest.TestCase):
@@ -212,12 +181,14 @@ class TestInputGetter(unittest.TestCase):
     def test_timput(self):
         ig = pkg_integrity.InputGetter(default='N', timeout=2)
         answer = ig.get_answer()
-        self.assertTrue(answer is None)
+        self.assertIsNone(answer)
         ig = pkg_integrity.InputGetter(default='Y', timeout=2)
         answer = ig.get_answer()
-        self.assertTrue(answer is None)
+        self.assertIsNone(answer)
 
 
+@mock.patch('pkg_integrity.attempt_to_download', mock_attempt_to_download)
+@mock.patch('pkg_integrity.head_request', mock_head_request)
 class TestUtils(unittest.TestCase):
 
     def setUp(self):
@@ -252,23 +223,6 @@ class TestUtils(unittest.TestCase):
         false_name = '/false/name'
         self.assertTrue(pkg_integrity.get_keyid(false_name) is None)
 
-    @unittest.skipIf("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true",
-                     "Skipping this test on Travis CI.")
-    def test_attempt_to_download(self):
-        fakeURL = "https://download.my.url.com/file.tar.gz"
-        realURLnoFile = "http://pypi.debian.net/alembic/alembic-0.8.8.non-existent.tar.gz"
-        realURL = "http://pypi.debian.net/alembic/alembic-0.8.8.tar.gz"
-
-        tmpf = tempfile.NamedTemporaryFile()
-        fname = tmpf.name
-        tmpf.close()
-
-        self.assertEqual(pkg_integrity.attempt_to_download(fakeURL, fname), None)
-        self.assertEqual(pkg_integrity.attempt_to_download(realURLnoFile, fname), 404)
-        self.assertEqual(pkg_integrity.attempt_to_download(realURL, fname), 200)
-
-        os.unlink(fname)
-
     def test_get_signature_url(self):
 
         url_from_gnu = "http://ftp.gnu.org/pub/gnu/gperf/gperf-3.0.4.tar.gz"

tests/test_tarball.py

@@ -3,6 +3,7 @@ import unittest
 from unittest.mock import patch, Mock, mock_open, call
 import build  # needs to be imported before tarball due to dependencies
 import tarball
+import re
 
 
 class FileManager():
@@ -27,10 +28,17 @@ def test_generator(url, name, version):
         """
         tarball.name = ''
         tarball.version = ''
+        tarball.giturl = ''
         tarball.url = url
         tarball.name_and_version('', '', FileManager())
-        self.assertEqual(tarball.name, name)
-        self.assertEqual(tarball.version, version)
+        self.assertEqual(name, tarball.name)
+        self.assertEqual(version, tarball.version)
+        if re.match("https?://github.com", url) != None:
+            self.assertIsNotNone(tarball.giturl)
+            self.assertNotEqual('', tarball.giturl, "giturl should not be empty")
+            self.assertIsNotNone(
+                    re.match("https://github.com/[^/]+/"+tarball.repo+".git",
+                    tarball.giturl), "%s looks incorrect" % tarball.giturl)
 
     return test_packageurl
 
@@ -160,30 +168,6 @@ class TestTarballVersionName(unittest.TestCase):
         tarball.build_gem_unpack = build_gem_unpack_backup
         tarball.build_untar = build_untar_backup
 
-    def test_write_makefile(self):
-        """
-        Test write_makefile for correct archive format
-        """
-        archives = ['archive1', 'dest1', 'archive2', 'dest2']
-        build.download_path = '.'
-        tarball.name = 'test'
-        tarball.url = 'url'
-        m_open = mock_open()
-        with patch('tarball.open', m_open, create=True):
-            tarball.write_makefile(archives)
-
-        exp_calls = [call().write('PKG_NAME := test\n'),
-                     call().write('URL := url\n'),
-                     call().write('ARCHIVES := archive1'),
-                     call().write(' dest1'),
-                     call().write(' \\\n\tarchive2'),
-                     call().write(' dest2'),
-                     call().write('\n'),
-                     call().write('\n'),
-                     call().write('include ../common/Makefile.common\n')]
-        for m_call in exp_calls:
-            self.assertIn(m_call, m_open.mock_calls)
-
 
 TAR_OUT = 'libjpeg-turbo-1.5.1/\n'                         \
           'libjpeg-turbo-1.5.1/md5/\n'                     \

tests/testfiles/pkg_integrity/pkg-config-0.29.1.tar.gz.asc

@@ -0,0 +1,17 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v2
+
+iQIcBAABCAAGBQJW2ZtDAAoJEAI6RCDH7GkUc0IP/1bH7KEJdUM+lrGM1SOuNHdq
+4VEwDp1II8abbBzHeGEXZ8p4+MwwSOYHFiy+NM1yldZkDXtqAlAqvIuEzc+PtgGd
+vFeNPb9infibNaEDK+zz4fcqJOSab1ZcQ/D3EIJXwKr5nIYP8RuCHu/zstf7o6R0
+/wnGWaAIB1+p9PxvUhMPMbBEQCw/cBzyZ2d7nApHF3b0OH2wM7P8VG8ot4cuglPq
+hzk27ZnrYeUDyUUhMRlL7sZZouJlSy/0OxsBK++tOjE6MiuAZhqtlSW+cFK4L7k/
+q4eLodX7GtF0psSgTRjTk2ozdSIDkB2ccLBN6CzgCcbPrbcz4tVQqaQBcSd0mCl7
+RWAKmSye7p+CY8mIIOjdYm+KaQRmJMKDXs49hMycti22jnu5T2BM6O7MZpiY+cb3
+O2UKUXbVyX/cXKwTYwf4VMddxJKFaqYac+7n5qWbdwBjk9E5OC2ltz94taM1pxZ5
+2jRtfyIb3s+Rj6M5cXI5UChrGqzMK6BmEbyZ0KbHAJ7Y0xvGqwydC6J+RwGIqRlp
+LmW3k1ggpUajoMcgq9KqJgVqo/9f4+6anADHRMNJ93MxR7h5BRQ1/GWSXpOzYsYv
+DnfQPrhc+z20m81qsvfUcBnN/k74yiDVqp3I/HrYGD+f8cXKPpBRESkPAXlUIcu2
+ALHZXEdBGcWdUrAIpJT2
+=SMbV
+-----END PGP SIGNATURE-----

tests/testfiles/pkg_integrity/quagga-1.1.0.tar.gz.asc

@@ -0,0 +1,18 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1
+
+iQJMBAABCAA2BQJYBh/XLxpodHRwczovL3d3dy5qYWttYS5vcmcvfnBhdWwvcGdw
+X3BvbGljeS0xLjEudHh0AAoJEG/lfKjBpK6mNOMP/irlUcU/4NQ6BQQbF7Vc90wo
+hmTH/zGAodnndxSplIGV63BQqmQr5KiSOp6tNQ3OlCIwLFWlr/LiJz+MeThaMtZx
+/mlrZptWIhbEcJzVa6efg8UGIYl2NC+QxFgEyezfCYSEmczd1qE3kiX+5WjcqVPH
+pVjisPwaAYa0FTCsAwBMUKyJr2K3sq5hSJR7DFDVnvH1DJ1xZd9871ZdDpZBHz2z
+GvR+IuZcWbYleH1ArFvT6cMokTpIUMvd33/+Gdpfu9fihzQQn199nN2LFZJzuEGV
+vH6+IZVTmtXb5U/sdtbWGaDv8eFLAWl2NH9VNdlVw5FbWOYRC4YNN39/hBy7s0po
+hvq33ZugC7JqPuje+4W1oF1T/dbRCIBmWUzKsoH8+3z4KJHdotSR0cU+TT+w6SJC
+QhF9TAbjrfbeJs0D0NnZrllDLiLGFgLl5yULzMjRDqKcgqE5+nvYBPXHPUCCPU7+
+59QOkPMsz/kZGV1lRzoUoxlM6V/phJRPU7jit10puiNij0c4peWbjVmTWDei3Xeu
+kHpck5wIAUFzAIXBUpVhYdLl/kxq654Qqthoci869ATZIKH/SOId9Y0K1GE6xO6H
+7vQFUGn3dcdnbgapM+tpmC77EmV4BtzkTjcu2pX6s0qnI4P+M+zXS+7G96xP3zHN
+ZSFq45swEKd6SKbOYxcN
+=mt1F
+-----END PGP SIGNATURE-----