From d497f7e2cbc33ea97d6c99831f2b6ce27ddf3371 Mon Sep 17 00:00:00 2001 From: Matthew Johnson Date: Thu, 9 Mar 2017 08:21:33 -0800 Subject: [PATCH] files: general cleanup of module The files module was being treated like a class, in that it was necessary to keep state throughout the entire autospec run. Convert the files module to a class and make some other general clean ups. This changed how the files module was instantiated and called by the main autospec module and other modules such as tarball and config. Test the files module with unit tests and add the target to the makefile. --- Makefile | 3 + autospec/autospec.py | 13 +- autospec/build.py | 10 +- autospec/config.py | 11 +- autospec/files.py | 496 +++++++++++++++++------------------------- autospec/tarball.py | 5 +- tests/test_files.py | 188 ++++++++++++++++ tests/test_tarball.py | 5 +- 8 files changed, 419 insertions(+), 312 deletions(-) create mode 100644 tests/test_files.py diff --git a/Makefile b/Makefile index 861c1ca..d85d04d 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ test_abireport: test_commitmessage: PYTHONPATH=`pwd`/autospec python3 tests/test_commitmessage.py +test_files: + PYTHONPATH=`pwd`/autospec python3 tests/test_files.py + test_autospec: python3 tests/test_autospec.py -c ${CASES} diff --git a/autospec/autospec.py b/autospec/autospec.py index 1ac1fc1..3fefa9c 100644 --- a/autospec/autospec.py +++ b/autospec/autospec.py @@ -89,7 +89,6 @@ def load_specfile(specfile): tarball.load_specfile(specfile) specdescription.load_specfile(specfile) license.load_specfile(specfile) - files.load_specfile(specfile) buildreq.load_specfile(specfile) buildpattern.load_specfile(specfile) test.load_specfile(specfile) @@ -142,7 +141,8 @@ def main(workingdir): # First, download the tarball, extract it and then do a set # of static analysis on the content of the tarball. # - tarball.name_and_version(args.url, args.name) + filemanager = files.FileManager() + tarball.name_and_version(args.url, args.name, filemanager) tarball.download_tarball(args.url, args.name, args.archives, args.target) _dir = tarball.path @@ -160,7 +160,7 @@ def main(workingdir): config.setup_patterns() config.config_file = args.config - config.parse_config_files(build.download_path, args.bump) + config.parse_config_files(build.download_path, args.bump, filemanager) config.parse_existing_spec(build.download_path, tarball.name) buildreq.set_build_req() @@ -177,6 +177,7 @@ def main(workingdir): # package builds # specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release) + filemanager.load_specfile(specfile) load_specfile(specfile) print("\n") @@ -188,10 +189,10 @@ def main(workingdir): specfile.write_spec(build.download_path) while 1: - build.package() - files.load_specfile(specfile) + build.package(filemanager) + filemanager.load_specfile(specfile) specfile.write_spec(build.download_path) - files.newfiles_printed = 0 + filemanager.newfiles_printed = 0 if build.round > 20 or build.must_restart == 0: break diff --git a/autospec/build.py b/autospec/build.py index e34fabf..76b8141 100644 --- a/autospec/build.py +++ b/autospec/build.py @@ -118,7 +118,7 @@ def failed_pattern(line, pattern, verbose, buildtool=None): print("Unknown pattern match: ", s) -def parse_build_results(filename, returncode): +def parse_build_results(filename, returncode, filemanager): global must_restart global success buildreq.verbose = 1 @@ -157,7 +157,7 @@ def parse_build_results(filename, returncode): if "Installed (but unpackaged) file(s) found:" in line: infiles = 1 elif infiles == 1 and "not matching the package arch" not in line: - files.push_file(line.strip()) + filemanager.push_file(line.strip()) if line.startswith("Sorry: TabError: inconsistent use of tabs and spaces in indentation"): print(line) @@ -166,7 +166,7 @@ def parse_build_results(filename, returncode): if "File not found: /builddir/build/BUILDROOT/" in line: left = "File not found: /builddir/build/BUILDROOT/%s-%s-%s.x86_64/" % (tarball.name, tarball.version, tarball.release) missing_file = "/" + line.split(left)[1].strip() - files.remove_file(missing_file) + filemanager.remove_file(missing_file) if line.startswith("Executing(%clean") and returncode == 0: print("RPM build successful") @@ -182,7 +182,7 @@ def set_mock(): mock_cmd = 'sudo /usr/bin/mock' -def package(): +def package(filemanager): global round round = round + 1 set_mock() @@ -197,4 +197,4 @@ def package(): srcrpm = "results/%s-%s-%s.src.rpm" % (tarball.name, tarball.version, tarball.release) returncode = util.call(mock_cmd + " -r clear --result=results/ %s --enable-plugin=ccache --uniqueext=%s --no-cleanup-after" % (srcrpm, tarball.name), logfile="%s/mock_build.log" % download_path, check=False, cwd=download_path) - parse_build_results(download_path + "/results/build.log", returncode) + parse_build_results(download_path + "/results/build.log", returncode, filemanager) diff --git a/autospec/config.py b/autospec/config.py index 56b39ee..5dbab32 100644 --- a/autospec/config.py +++ b/autospec/config.py @@ -22,7 +22,6 @@ import buildpattern import build import buildreq -import files import license import os import sys @@ -373,7 +372,7 @@ def parse_existing_spec(path, name): cves.append(patch.upper().split(".PATCH")[0]) -def parse_config_files(path, bump): +def parse_config_files(path, bump, filemanager): global extra_configure global extra_configure32 global config_files @@ -496,17 +495,17 @@ def parse_config_files(path, bump): content = read_conf_file(os.path.join(path, "excludes")) for exclude in content: print("%%exclude for: %s." % exclude) - files.excludes += content + filemanager.excludes += content content = read_conf_file(os.path.join(path, "extras")) for extra in content: print("extras for: %s." % extra) - files.extras += content + filemanager.extras += content content = read_conf_file(os.path.join(path, "setuid")) for suid in content: print("setuid for: %s." % suid) - files.setuid += content + filemanager.setuid += content content = read_conf_file(os.path.join(path, "attrs")) for line in content: @@ -514,7 +513,7 @@ def parse_config_files(path, bump): attr = [a.strip() for a in attr] filename = attr.pop() print("attr for: %s." % filename) - files.attrs[filename] = attr + filemanager.attrs[filename] = attr patches += read_conf_file(os.path.join(path, "series")) pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches] diff --git a/autospec/files.py b/autospec/files.py index bdba216..9ce4a40 100644 --- a/autospec/files.py +++ b/autospec/files.py @@ -20,321 +20,235 @@ # import build -from collections import OrderedDict -import re import tarball -import buildreq -import config +import re +from collections import OrderedDict # todo package splits -# per sub-package file list for spec purposes -packages = OrderedDict() -# global file list to weed out dupes -files = [] -files_blacklist = [] -excludes = [] -extras = [] -setuid = [] -attrs = {} -locales = [] +class FileManager(object): + """ + Files class handles spec file %files section management + """ + def __init__(self): + self.packages = OrderedDict() # per sub-package file list for spec purposes + self.files = [] # global file list to weed out dupes + self.files_blacklist = set() + self.excludes = [] + self.extras = [] + self.setuid = [] + self.attrs = {} + self.locales = [] + self.newfiles_printed = False + # Do we need ALL include files in a dev package, even if they're not in + # /usr/include? Yes in the general case, but for example for R + # packages, the answer is No. + self.want_dev_split = True -newfiles_printed = 0 + def push_package_file(self, filename, package="main"): + """ + Add found %file and indicate to build module that we must restart the + build. + """ + if package not in self.packages: + self.packages[package] = set() -# -# Do we need ALL include files in a dev package, even if they're not in /usr/include? -# Yes in the general case, but for example for R packages, -# the answer is No. -want_dev_split = 1 + self.packages[package].add(filename) + build.must_restart += 1 + if not self.newfiles_printed: + print(" New %files content found") + self.newfiles_printed = True + def file_pat_match(self, filename, pattern, package, replacement="", prefix=""): + """ + Search for pattern in filename, if pattern matches push package file. + If that file is also in the excludes list, prepend "%exclude " before + pushing the filename. + Returns True if a file was pushed, False otherwise. + """ + if not replacement: + replacement = prefix + filename -def push_package_file(filename, package="main"): - global packages - global newfiles_printed + pat = re.compile(pattern) + match = pat.search(filename) + if match: + if filename in self.excludes: + self.push_package_file("%exclude " + filename, package) + return True - if (package not in packages): - packages[package] = set() - packages[package].add(filename) - build.must_restart = build.must_restart + 1 - if newfiles_printed == 0: - print(" New %files content found") - newfiles_printed = 1 - - -def file_pat_match(filename, pattern, package, replacement="", prefix=""): - if replacement == "": - replacement = prefix + filename - - pat = re.compile(pattern) - match = pat.search(filename) - if match: - if filename in excludes: - push_package_file("%exclude " + filename, package) + self.push_package_file(replacement, package) return True - push_package_file(replacement, package) - return True - else: - return False + else: + return False + def file_is_locale(self, filename): + """ + If a file is a locale, appends to self.locales and returns True, + returns False otherwise + """ + pat = re.compile(r"^/usr/share/locale/.*/(.*)\.mo") + match = pat.search(filename) + if match: + lang = match.group(1) + if lang not in self.locales: + self.locales.append(lang) + print(" New locale:", lang) -def file_is_locale(filename): - pat = re.compile("^/usr/share/locale/.*/(.*)\.mo") - match = pat.search(filename) - if match: - l = match.group(1) - add_lang(l) - return True - else: - return False + return True + else: + return False + def push_file(self, filename): + """ + Perform a number of checks against the filename and push the filename + if appropriate. + """ + if filename in self.files or filename in self.files_blacklist: + return -def push_file(filename): - global files - global files_blacklist - global extras - global setuid - global attrs - for file in files: - if file == filename: - return 0 + self.files.append(filename) + if self.file_is_locale(filename): + return - if filename in files_blacklist: - return 0 - files.append(filename) + # autostart + part = re.compile(r"^/usr/lib/systemd/system/.+\.target\.wants/.+") + if part.search(filename) and 'update-triggers.target.wants' not in filename: + self.push_package_file(filename, "autostart") + self.excludes.append(filename) - if file_is_locale(filename): - return + # extras + if filename in self.extras: + self.push_package_file(filename, "extras") + self.excludes.append(filename) - # autostart - part = re.compile("^/usr/lib/systemd/system/.+\.target\.wants/.+") - if part.search(filename) and 'update-triggers.target.wants' not in filename: - push_package_file(filename, "autostart") - excludes.append(filename) + if filename in self.setuid: + newfn = "%attr(4755, root, root) " + filename + self.push_package_file(newfn, "setuid") + self.excludes.append(filename) - # extras - if filename in extras: - push_package_file(filename, "extras") - excludes.append(filename) + if filename in self.attrs: + newfn = "{0}({1}) {2}".format(self.attrs[filename][0], + ','.join(self.attrs[filename][1:3]), + filename) + self.push_package_file(newfn, "attr") + self.excludes.append(filename) - if filename in setuid: - fn = "%attr(4755, root, root) " + filename - push_package_file(fn, "setuid") - excludes.append(filename) + if self.want_dev_split and self.file_pat_match(filename, r"^/usr/.*/include/.*\.h$", "dev"): + return - if filename in attrs: - fn = "%s(%s,%s,%s) %s" % (attrs[filename][0], attrs[filename][1], attrs[filename][2], - attrs[filename][3], filename) - push_package_file(fn, "attr") - excludes.append(filename) + patterns = [ + # Patterns for matching files, format is a tuple as follows: + # (, , , ) + # order matters! + (r"^/usr/share/omf", "main", "/usr/share/omf/*"), + (r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"), + (r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"), + (r"^/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib32"), + (r"^/usr/lib64/lib(asm|dw|elf)-[0-9.]+\.so", "lib"), + (r"^/usr/lib32/lib(asm|dw|elf)-[0-9.]+\.so", "lib32"), + (r"^/usr/lib64/avx2/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"), + (r"^/usr/lib64/gobject-introspection/", "lib"), + (r"^/usr/libexec/", "bin"), + (r"^/usr/bin/", "bin"), + (r"^/usr/sbin/", "bin"), + (r"^/sbin/", "bin"), + (r"^/bin/", "bin"), + (r"^/usr/lib/python.*/", "python", "/usr/lib/python*/*"), + (r"^/usr/lib64/python.*/", "python", "/usr/lib64/python*/*"), + (r"^/usr/share/gir-[0-9\.]+/[a-zA-Z0-9\.\_\-\+]*\.gir", "dev", "/usr/share/gir-1.0/*.gir"), + (r"^/usr/share/cmake/", "data", "/usr/share/cmake/*"), + (r"^/usr/share/cmake-3.1/", "data", "/usr/share/cmake-3.1/*"), + (r"^/usr/share/cmake-3.7/", "data", "/usr/share/cmake-3.7/*"), + (r"^/usr/share/cmake-3.6/", "data", "/usr/share/cmake-3.6/*"), + (r"^/usr/share/girepository-1\.0/.*\.typelib\$", "dev", "/usr/share/girepository-1.0/*.typelib"), + (r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hxx", "dev", "/usr/include/*.hxx"), + (r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hpp", "dev", "/usr/include/*.hpp"), + (r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h\+\+", "dev", "/usr/include/*.h\+\+"), + (r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h", "dev", "/usr/include/*.h"), + (r"^/usr/include/", "dev"), + (r"^/usr/lib64/girepository-1.0/", "dev"), + (r"^/usr/share/cmake/", "dev"), + (r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"), + (r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"), + (r"^/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev32"), + (r"^/usr/lib64/avx2/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"), + (r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib/*.a"), + (r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib64/*.a"), + (r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev32", "/usr/lib32/*.a"), + (r"^/usr/lib/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev"), + (r"^/usr/lib64/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev"), + (r"^/usr/lib32/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev32"), + (r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal/*.ac"), + (r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal/*.m4"), + (r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal-1.*/*.ac"), + (r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal-1.*/*.m4"), + (r"^/usr/share/doc/" + re.escape(tarball.name) + "/", "doc", "%doc /usr/share/doc/" + re.escape(tarball.name) + "/*"), + (r"^/usr/share/gtk-doc/html", "doc"), + (r"^/usr/share/info/", "doc", "%doc /usr/share/info/*"), + (r"^/usr/share/man/man0", "doc", "%doc /usr/share/man/man0/*"), + (r"^/usr/share/man/man1", "doc", "%doc /usr/share/man/man1/*"), + (r"^/usr/share/man/man2", "doc", "%doc /usr/share/man/man2/*"), + (r"^/usr/share/man/man3", "doc", "%doc /usr/share/man/man3/*"), + (r"^/usr/share/man/man4", "doc", "%doc /usr/share/man/man4/*"), + (r"^/usr/share/man/man5", "doc", "%doc /usr/share/man/man5/*"), + (r"^/usr/share/man/man6", "doc", "%doc /usr/share/man/man6/*"), + (r"^/usr/share/man/man7", "doc", "%doc /usr/share/man/man7/*"), + (r"^/usr/share/man/man8", "doc", "%doc /usr/share/man/man8/*"), + (r"^/usr/share/man/man9", "doc", "%doc /usr/share/man/man9/*"), + (r"^/etc/systemd/system/.*\.wants/", "active-units"), + # now a set of catch-all rules + (r"^/etc/", "config", "", "%config "), + (r"^/usr/etc/", "config", "", "%config "), + (r"^/lib/systemd", "config"), + (r"^/usr/lib/systemd", "config"), + (r"^/usr/lib/udev/rules.d", "config"), + (r"^/usr/lib/modules-load.d", "config"), + (r"^/usr/lib/tmpfiles.d", "config"), + (r"^/usr/lib/sysusers.d", "config"), + (r"^/usr/lib/sysctl.d", "config"), + (r"^/usr/share/", "data"), + # finally move any dynamically loadable plugins (not + # perl/python/ruby/etc.. extensions) into lib package + (r"^/usr/lib/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"), + (r"^/usr/lib64/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"), + (r"^/usr/lib32/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib32"), + # locale data gets picked up via file_is_locale + (r"^/usr/share/locale/", "ignore")] - if file_pat_match(filename, r"^/usr/share/omf", "main", "/usr/share/omf/*"): - return + for pat_args in patterns: + if self.file_pat_match(filename, *pat_args): + return - if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"): - return - if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"): - return - if file_pat_match(filename, r"^/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib32"): - return + if filename in self.excludes: + self.push_package_file("%exclude " + filename) + return - # Workarounds for some elfutils shared libraries ending with .so - if file_pat_match(filename, r"^/usr/lib64/lib(asm|dw|elf)-[0-9.]+\.so", "lib"): - return - if file_pat_match(filename, r"^/usr/lib32/lib(asm|dw|elf)-[0-9.]+\.so", "lib32"): - return + self.push_package_file(filename) - if file_pat_match(filename, r"^/usr/lib64/avx2/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"): - return - if file_pat_match(filename, r"^/usr/lib64/gobject-introspection/", "lib"): - return - if file_pat_match(filename, r"^/usr/libexec/", "bin"): - return - if file_pat_match(filename, r"^/usr/bin/", "bin"): - return - if file_pat_match(filename, r"^/usr/sbin/", "bin"): - return - if file_pat_match(filename, r"^/sbin/", "bin"): - return - if file_pat_match(filename, r"^/bin/", "bin"): - return - if file_pat_match(filename, r"^/bin/", "bin"): - return + def remove_file(self, filename): + """ + Remove filename from local file list + """ + hit = False - if file_pat_match(filename, r"^/usr/lib/python3.*/", "python", "/usr/lib/python3*/*"): - return - if file_pat_match(filename, r"^/usr/lib/python2.*/", "python", "/usr/lib/python2*/*"): - return - if file_pat_match(filename, r"^/usr/lib64/python.*/", "python", "/usr/lib64/python*/*"): - return - if file_pat_match(filename, r"^/usr/share/gir-[0-9\.]+/[a-zA-Z0-9\.\_\-\+]*\.gir", "dev", "/usr/share/gir-1.0/*.gir"): - print("HIT GIR\n") - return - if file_pat_match(filename, r"^/usr/share/cmake/", "data", "/usr/share/cmake/*"): - return - if file_pat_match(filename, r"^/usr/share/cmake-3.1/", "data", "/usr/share/cmake-3.1/*"): - return - if file_pat_match(filename, r"^/usr/share/cmake-3.7/", "data", "/usr/share/cmake-3.7/*"): - return - if file_pat_match(filename, r"^/usr/share/cmake-3.6/", "data", "/usr/share/cmake-3.6/*"): - return - if file_pat_match(filename, r"^/usr/share/girepository-1\.0/.*\.typelib\$", "dev", "/usr/share/girepository-1.0/*.typelib"): - return - - if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hxx", "dev", "/usr/include/*.hxx"): - return - if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hpp", "dev", "/usr/include/*.hpp"): - return - if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h\+\+", "dev", "/usr/include/*.h\+\+"): - return - if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h", "dev", "/usr/include/*.h"): - return - if file_pat_match(filename, r"^/usr/include/", "dev"): - return - if file_pat_match(filename, r"^/usr/lib64/girepository-1.0/", "dev"): - return - if file_pat_match(filename, r"^/usr/share/cmake/", "dev"): - return - if want_dev_split > 0 and file_pat_match(filename, r"^/usr/.*/include/.*\.h$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev32"): - return - if file_pat_match(filename, r"^/usr/lib64/avx2/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib/*.a"): - return - if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib64/*.a"): - return - if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev32", "/usr/lib32/*.a"): - return - if file_pat_match(filename, r"^/usr/lib/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib64/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib32/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev32"): - return - if file_pat_match(filename, r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal/*.ac"): - return - if file_pat_match(filename, r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal/*.m4"): - return - if file_pat_match(filename, r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal-1.*/*.ac"): - return - if file_pat_match(filename, r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal-1.*/*.m4"): - return - - if file_pat_match(filename, r"^/usr/share/doc/" + re.escape(tarball.name) + "/", "doc", "%doc /usr/share/doc/" + re.escape(tarball.name) + "/*"): - return - if file_pat_match(filename, r"^/usr/share/gtk-doc/html", "doc"): - return - if file_pat_match(filename, r"^/usr/share/info/", "doc", "%doc /usr/share/info/*"): - return - - if file_pat_match(filename, r"^/usr/share/man/man0", "doc", "%doc /usr/share/man/man0/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man1", "doc", "%doc /usr/share/man/man1/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man2", "doc", "%doc /usr/share/man/man2/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man3", "doc", "%doc /usr/share/man/man3/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man4", "doc", "%doc /usr/share/man/man4/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man5", "doc", "%doc /usr/share/man/man5/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man6", "doc", "%doc /usr/share/man/man6/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man7", "doc", "%doc /usr/share/man/man7/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man8", "doc", "%doc /usr/share/man/man8/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man9", "doc", "%doc /usr/share/man/man9/*"): - return - - if file_pat_match(filename, r"^/etc/systemd/system/.*\.wants/", "active-units"): - return - - # now a set of catch-all rules - if file_pat_match(filename, r"^/etc/", "config", "", "%config "): - return - if file_pat_match(filename, r"^/usr/etc/", "config", "", "%config "): - return - if file_pat_match(filename, r"^/lib/systemd", "config"): - return - if file_pat_match(filename, r"^/usr/lib/systemd", "config"): - return - if file_pat_match(filename, r"^/usr/lib/udev/rules.d", "config"): - return - if file_pat_match(filename, r"^/usr/lib/modules-load.d", "config"): - return - if file_pat_match(filename, r"^/usr/lib/tmpfiles.d", "config"): - return - if file_pat_match(filename, r"^/usr/lib/sysusers.d", "config"): - return - if file_pat_match(filename, r"^/usr/lib/sysctl.d", "config"): - return - - if file_pat_match(filename, r"^/usr/share/", "data"): - return - - # finally move any dynamically loadable plugins (not - # perl/python/ruby/etc.. extensions) into lib package - if file_pat_match(filename, r"^/usr/lib/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"): - return - if file_pat_match(filename, r"^/usr/lib64/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"): - return - if file_pat_match(filename, r"^/usr/lib32/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib32"): - return - - # locale data gets picked up via find_lang - if file_pat_match(filename, r"^/usr/share/locale/", "ignore"): - return - - if filename in excludes: - push_package_file("%exclude " + filename) - return - - push_package_file(filename) - - -def remove_file(file): - global files - global packages - global files_blacklist - hit = False - - if file in files: - files.remove(file) - print("File no longer present: %s" % file) - hit = True - for pkg in packages: - if file in packages[pkg]: - packages[pkg].remove(file) - print("File no longer present in %s: %s" % (pkg, file)) + if filename in self.files: + self.files.remove(filename) + print("File no longer present: {}".format(filename)) hit = True - if hit: - if file not in files_blacklist: - files_blacklist.append(file) - build.must_restart = build.must_restart + 1 - - -def add_lang(lang): - global locales - global packages - if lang in locales: - return - locales.append(lang) - print(" New locale:", lang) - - if "locales" in packages: - return - packages["locales"] = [] - - -def load_specfile(specfile): - specfile.packages = packages - specfile.excludes = excludes - specfile.locales = locales + for pkg in self.packages: + if filename in self.packages[pkg]: + self.packages[pkg].remove(filename) + print("File no longer present in {}: {}".format(pkg, filename)) + hit = True + if hit: + self.files_blacklist.add(filename) + build.must_restart += 1 + def load_specfile(self, specfile): + """ + Load a specfile instance with relevant information to be written to the + spec file. + """ + specfile.packages = self.packages + specfile.excludes = self.excludes + specfile.locales = self.locales diff --git a/autospec/tarball.py b/autospec/tarball.py index 6fab430..5045a2f 100644 --- a/autospec/tarball.py +++ b/autospec/tarball.py @@ -20,7 +20,6 @@ import build import buildpattern import buildreq -import files import config import glob import hashlib @@ -235,7 +234,7 @@ def convert_version(version): return version.strip('.') + suffix -def name_and_version(url_argument, name_argument): +def name_and_version(url_argument, name_argument, filemanager): global name global rawname global version @@ -262,7 +261,7 @@ def name_and_version(url_argument, name_argument): # R package if url_argument.find("cran.r-project.org") > 0 or url_argument.find("cran.rstudio.com") > 0: buildpattern.set_build_pattern("R", 10) - files.want_dev_split = 0 + filemanager.want_dev_split = False buildreq.add_buildreq("clr-R-helpers") p = re.compile(r"([A-Za-z0-9.]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz") m = p.search(tarfile) diff --git a/tests/test_files.py b/tests/test_files.py new file mode 100644 index 0000000..bbe8dbf --- /dev/null +++ b/tests/test_files.py @@ -0,0 +1,188 @@ +import unittest +import files +from unittest.mock import call, MagicMock +from files import FileManager + + +def mock_return(retval): + """ + Simple mock method to set return value of a function + """ + def mock_fn(_): + return retval + + return mock_fn + + +class TestFiles(unittest.TestCase): + + def setUp(self): + self.fm = FileManager() + + def test_push_package_file(self): + """ + Test push_package_file with no package name specified (package name + should default to 'main' + """ + self.assertFalse(self.fm.newfiles_printed) + self.fm.push_package_file('test-fn') + self.assertEqual(self.fm.packages['main'], set(['test-fn'])) + self.assertTrue(self.fm.newfiles_printed) + + def test_push_package_file_dev(self): + """ + Test push_package_file with dev package specified + """ + self.fm.push_package_file('test-fn', 'dev') + self.assertEqual(self.fm.packages['dev'], set(['test-fn'])) + self.assertTrue(self.fm.newfiles_printed) + + def test_file_pat_match(self): + """ + Test file_pat_match with good match and no replacement or prefix + specified. + """ + self.fm.push_package_file = MagicMock() + self.assertTrue(self.fm.file_pat_match('test-fn', r'test-fn', 'main')) + self.fm.push_package_file.assert_called_with('test-fn', 'main') + + def test_file_pat_match_exclude(self): + """ + Test file_pat_match with good match and filename in excludes list. + """ + self.fm.push_package_file = MagicMock() + self.fm.excludes.append('test-fn') + self.assertTrue(self.fm.file_pat_match('test-fn', r'test-fn', 'main')) + self.fm.push_package_file.assert_called_with('%exclude test-fn', 'main') + + def test_file_pat_match_replacement(self): + """ + Test file_pat_match with replacement provided + """ + self.fm.push_package_file = MagicMock() + self.assertTrue(self.fm.file_pat_match('test-fn', r'test-fn', 'main', 'testfn')) + self.fm.push_package_file.assert_called_with('testfn', 'main') + + def test_file_pat_match_no_match(self): + """ + Test file_pat_match with no match + """ + self.fm.push_package_file = MagicMock() + self.assertFalse(self.fm.file_pat_match('test-fn', r'testfn', 'main')) + self.fm.push_package_file.assert_not_called() + + def test_file_is_locale(self): + """ + Test file_is_locale with locale filename not present in locale list + """ + self.assertEqual(self.fm.locales, []) + self.assertTrue(self.fm.file_is_locale('/usr/share/locale/a/loc.mo')) + self.assertEqual(self.fm.locales, ['loc']) + + def test_file_is_locale_non_locale(self): + """ + Test file_is_locale with non-locale filename + """ + self.assertFalse(self.fm.file_is_locale('test-fn')) + self.assertEqual(self.fm.locales, []) + + def test_file_is_locale_present(self): + """ + Test file_is_locale with locale present in locale list + """ + self.fm.locales.append('loc') + self.assertEqual(self.fm.locales, ['loc']) + self.assertTrue(self.fm.file_is_locale('/usr/share/locale/a/loc.mo')) + self.assertEqual(self.fm.locales, ['loc']) + + def test_push_file_autostart(self): + """ + Test push_file to autostart package, this excludes the file. + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + autostart = '/usr/lib/systemd/system/some.target.wants/some' + self.fm.push_file(autostart) + calls = [call(autostart, 'autostart'), call('%exclude ' + autostart, 'config')] + self.fm.push_package_file.assert_has_calls(calls) + + def test_push_file_extras(self): + """ + Test push_file to extras package, this excludes the file + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + self.fm.extras.append('test') + self.fm.push_file('test') + calls = [call('test', 'extras'), call('%exclude test')] + self.fm.push_package_file.assert_has_calls(calls) + + + def test_push_file_setuid(self): + """ + Test push_file with fname in setuid list + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + self.fm.setuid.append('test') + self.fm.push_file('test') + calls = [call('%attr(4755, root, root) test', 'setuid'), call('%exclude test')] + self.fm.push_package_file.assert_has_calls(calls) + + + def test_push_file_match(self): + """ + Test push_file with match in pattern list + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + self.fm.push_file('/usr/bin/test') + self.fm.push_package_file.assert_called_once_with('/usr/bin/test', 'bin') + + def test_push_file_match_tarball_name_dependency(self): + """ + Test push_file with match in the list on the single item that is + dependent on the tarball name. + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + files.tarball.name = 'testball' + self.fm.push_file('/usr/share/doc/testball/') + self.fm.push_package_file.assert_called_once_with('%doc /usr/share/doc/testball/*', 'doc') + + def test_push_file_no_match(self): + """ + Test push_file with no pattern match on the file name. Should just push + the unmodified filename once. + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + self.fm.push_file('doesntmatcha thing') + self.fm.push_package_file.assert_called_once_with('doesntmatcha thing') + + def test_remove_file(self): + """ + Test remove_file with filename in files list and main package + """ + self.fm.files.append('test') + self.fm.packages['main'] = ['test'] + self.assertIn('test', self.fm.files) + self.assertNotIn('test', self.fm.files_blacklist) + self.assertIn('test', self.fm.packages['main']) + self.fm.remove_file('test') + self.assertNotIn('test', self.fm.files) + self.assertNotIn('test', self.fm.packages['main']) + self.assertIn('test', self.fm.files_blacklist) + + def test_remove_file_not_present(self): + """ + Test remove_file with filename not in files list. + """ + self.assertNotIn('test', self.fm.files) + self.assertNotIn('test', self.fm.files_blacklist) + self.fm.remove_file('test') + self.assertNotIn('test', self.fm.files) + self.assertNotIn('test', self.fm.files_blacklist) + +if __name__ == '__main__': + unittest.main(buffer=True) diff --git a/tests/test_tarball.py b/tests/test_tarball.py index 2bc9e54..5fc9fd6 100644 --- a/tests/test_tarball.py +++ b/tests/test_tarball.py @@ -17,6 +17,9 @@ class TestTarballVersionName(unittest.TestCase): Does not use self.assertEquals but appends to a list so all failures are reported instead of just the first one. """ + class FileManager(): + want_dev_split = False + errors = [] with open('tests/packageurls', 'r') as pkgurls: for urlline in pkgurls.read().split('\n'): @@ -26,7 +29,7 @@ class TestTarballVersionName(unittest.TestCase): tarball.name = '' tarball.version = '' (url, name, version) = urlline.split(',') - tarball.name_and_version(url, '') + tarball.name_and_version(url, '', FileManager()) if tarball.name != name: errors.append("name: '{}' != '{}' for url: {}" .format(tarball.name, name, url))