diff --git a/Makefile b/Makefile index 861c1ca..d85d04d 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ test_abireport: test_commitmessage: PYTHONPATH=`pwd`/autospec python3 tests/test_commitmessage.py +test_files: + PYTHONPATH=`pwd`/autospec python3 tests/test_files.py + test_autospec: python3 tests/test_autospec.py -c ${CASES} diff --git a/autospec/autospec.py b/autospec/autospec.py index 1ac1fc1..3fefa9c 100644 --- a/autospec/autospec.py +++ b/autospec/autospec.py @@ -89,7 +89,6 @@ def load_specfile(specfile): tarball.load_specfile(specfile) specdescription.load_specfile(specfile) license.load_specfile(specfile) - files.load_specfile(specfile) buildreq.load_specfile(specfile) buildpattern.load_specfile(specfile) test.load_specfile(specfile) @@ -142,7 +141,8 @@ def main(workingdir): # First, download the tarball, extract it and then do a set # of static analysis on the content of the tarball. # - tarball.name_and_version(args.url, args.name) + filemanager = files.FileManager() + tarball.name_and_version(args.url, args.name, filemanager) tarball.download_tarball(args.url, args.name, args.archives, args.target) _dir = tarball.path @@ -160,7 +160,7 @@ def main(workingdir): config.setup_patterns() config.config_file = args.config - config.parse_config_files(build.download_path, args.bump) + config.parse_config_files(build.download_path, args.bump, filemanager) config.parse_existing_spec(build.download_path, tarball.name) buildreq.set_build_req() @@ -177,6 +177,7 @@ def main(workingdir): # package builds # specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release) + filemanager.load_specfile(specfile) load_specfile(specfile) print("\n") @@ -188,10 +189,10 @@ def main(workingdir): specfile.write_spec(build.download_path) while 1: - build.package() - files.load_specfile(specfile) + build.package(filemanager) + filemanager.load_specfile(specfile) specfile.write_spec(build.download_path) - files.newfiles_printed = 0 + filemanager.newfiles_printed = 0 if build.round > 20 or build.must_restart == 0: break diff --git a/autospec/build.py b/autospec/build.py index e34fabf..76b8141 100644 --- a/autospec/build.py +++ b/autospec/build.py @@ -118,7 +118,7 @@ def failed_pattern(line, pattern, verbose, buildtool=None): print("Unknown pattern match: ", s) -def parse_build_results(filename, returncode): +def parse_build_results(filename, returncode, filemanager): global must_restart global success buildreq.verbose = 1 @@ -157,7 +157,7 @@ def parse_build_results(filename, returncode): if "Installed (but unpackaged) file(s) found:" in line: infiles = 1 elif infiles == 1 and "not matching the package arch" not in line: - files.push_file(line.strip()) + filemanager.push_file(line.strip()) if line.startswith("Sorry: TabError: inconsistent use of tabs and spaces in indentation"): print(line) @@ -166,7 +166,7 @@ def parse_build_results(filename, returncode): if "File not found: /builddir/build/BUILDROOT/" in line: left = "File not found: /builddir/build/BUILDROOT/%s-%s-%s.x86_64/" % (tarball.name, tarball.version, tarball.release) missing_file = "/" + line.split(left)[1].strip() - files.remove_file(missing_file) + filemanager.remove_file(missing_file) if line.startswith("Executing(%clean") and returncode == 0: print("RPM build successful") @@ -182,7 +182,7 @@ def set_mock(): mock_cmd = 'sudo /usr/bin/mock' -def package(): +def package(filemanager): global round round = round + 1 set_mock() @@ -197,4 +197,4 @@ def package(): srcrpm = "results/%s-%s-%s.src.rpm" % (tarball.name, tarball.version, tarball.release) returncode = util.call(mock_cmd + " -r clear --result=results/ %s --enable-plugin=ccache --uniqueext=%s --no-cleanup-after" % (srcrpm, tarball.name), logfile="%s/mock_build.log" % download_path, check=False, cwd=download_path) - parse_build_results(download_path + "/results/build.log", returncode) + parse_build_results(download_path + "/results/build.log", returncode, filemanager) diff --git a/autospec/config.py b/autospec/config.py index 56b39ee..5dbab32 100644 --- a/autospec/config.py +++ b/autospec/config.py @@ -22,7 +22,6 @@ import buildpattern import build import buildreq -import files import license import os import sys @@ -373,7 +372,7 @@ def parse_existing_spec(path, name): cves.append(patch.upper().split(".PATCH")[0]) -def parse_config_files(path, bump): +def parse_config_files(path, bump, filemanager): global extra_configure global extra_configure32 global config_files @@ -496,17 +495,17 @@ def parse_config_files(path, bump): content = read_conf_file(os.path.join(path, "excludes")) for exclude in content: print("%%exclude for: %s." % exclude) - files.excludes += content + filemanager.excludes += content content = read_conf_file(os.path.join(path, "extras")) for extra in content: print("extras for: %s." % extra) - files.extras += content + filemanager.extras += content content = read_conf_file(os.path.join(path, "setuid")) for suid in content: print("setuid for: %s." % suid) - files.setuid += content + filemanager.setuid += content content = read_conf_file(os.path.join(path, "attrs")) for line in content: @@ -514,7 +513,7 @@ def parse_config_files(path, bump): attr = [a.strip() for a in attr] filename = attr.pop() print("attr for: %s." % filename) - files.attrs[filename] = attr + filemanager.attrs[filename] = attr patches += read_conf_file(os.path.join(path, "series")) pfiles = [("%s/%s" % (path, x.split(" ")[0])) for x in patches] diff --git a/autospec/files.py b/autospec/files.py index bdba216..9ce4a40 100644 --- a/autospec/files.py +++ b/autospec/files.py @@ -20,321 +20,235 @@ # import build -from collections import OrderedDict -import re import tarball -import buildreq -import config +import re +from collections import OrderedDict # todo package splits -# per sub-package file list for spec purposes -packages = OrderedDict() -# global file list to weed out dupes -files = [] -files_blacklist = [] -excludes = [] -extras = [] -setuid = [] -attrs = {} -locales = [] +class FileManager(object): + """ + Files class handles spec file %files section management + """ + def __init__(self): + self.packages = OrderedDict() # per sub-package file list for spec purposes + self.files = [] # global file list to weed out dupes + self.files_blacklist = set() + self.excludes = [] + self.extras = [] + self.setuid = [] + self.attrs = {} + self.locales = [] + self.newfiles_printed = False + # Do we need ALL include files in a dev package, even if they're not in + # /usr/include? Yes in the general case, but for example for R + # packages, the answer is No. + self.want_dev_split = True -newfiles_printed = 0 + def push_package_file(self, filename, package="main"): + """ + Add found %file and indicate to build module that we must restart the + build. + """ + if package not in self.packages: + self.packages[package] = set() -# -# Do we need ALL include files in a dev package, even if they're not in /usr/include? -# Yes in the general case, but for example for R packages, -# the answer is No. -want_dev_split = 1 + self.packages[package].add(filename) + build.must_restart += 1 + if not self.newfiles_printed: + print(" New %files content found") + self.newfiles_printed = True + def file_pat_match(self, filename, pattern, package, replacement="", prefix=""): + """ + Search for pattern in filename, if pattern matches push package file. + If that file is also in the excludes list, prepend "%exclude " before + pushing the filename. + Returns True if a file was pushed, False otherwise. + """ + if not replacement: + replacement = prefix + filename -def push_package_file(filename, package="main"): - global packages - global newfiles_printed + pat = re.compile(pattern) + match = pat.search(filename) + if match: + if filename in self.excludes: + self.push_package_file("%exclude " + filename, package) + return True - if (package not in packages): - packages[package] = set() - packages[package].add(filename) - build.must_restart = build.must_restart + 1 - if newfiles_printed == 0: - print(" New %files content found") - newfiles_printed = 1 - - -def file_pat_match(filename, pattern, package, replacement="", prefix=""): - if replacement == "": - replacement = prefix + filename - - pat = re.compile(pattern) - match = pat.search(filename) - if match: - if filename in excludes: - push_package_file("%exclude " + filename, package) + self.push_package_file(replacement, package) return True - push_package_file(replacement, package) - return True - else: - return False + else: + return False + def file_is_locale(self, filename): + """ + If a file is a locale, appends to self.locales and returns True, + returns False otherwise + """ + pat = re.compile(r"^/usr/share/locale/.*/(.*)\.mo") + match = pat.search(filename) + if match: + lang = match.group(1) + if lang not in self.locales: + self.locales.append(lang) + print(" New locale:", lang) -def file_is_locale(filename): - pat = re.compile("^/usr/share/locale/.*/(.*)\.mo") - match = pat.search(filename) - if match: - l = match.group(1) - add_lang(l) - return True - else: - return False + return True + else: + return False + def push_file(self, filename): + """ + Perform a number of checks against the filename and push the filename + if appropriate. + """ + if filename in self.files or filename in self.files_blacklist: + return -def push_file(filename): - global files - global files_blacklist - global extras - global setuid - global attrs - for file in files: - if file == filename: - return 0 + self.files.append(filename) + if self.file_is_locale(filename): + return - if filename in files_blacklist: - return 0 - files.append(filename) + # autostart + part = re.compile(r"^/usr/lib/systemd/system/.+\.target\.wants/.+") + if part.search(filename) and 'update-triggers.target.wants' not in filename: + self.push_package_file(filename, "autostart") + self.excludes.append(filename) - if file_is_locale(filename): - return + # extras + if filename in self.extras: + self.push_package_file(filename, "extras") + self.excludes.append(filename) - # autostart - part = re.compile("^/usr/lib/systemd/system/.+\.target\.wants/.+") - if part.search(filename) and 'update-triggers.target.wants' not in filename: - push_package_file(filename, "autostart") - excludes.append(filename) + if filename in self.setuid: + newfn = "%attr(4755, root, root) " + filename + self.push_package_file(newfn, "setuid") + self.excludes.append(filename) - # extras - if filename in extras: - push_package_file(filename, "extras") - excludes.append(filename) + if filename in self.attrs: + newfn = "{0}({1}) {2}".format(self.attrs[filename][0], + ','.join(self.attrs[filename][1:3]), + filename) + self.push_package_file(newfn, "attr") + self.excludes.append(filename) - if filename in setuid: - fn = "%attr(4755, root, root) " + filename - push_package_file(fn, "setuid") - excludes.append(filename) + if self.want_dev_split and self.file_pat_match(filename, r"^/usr/.*/include/.*\.h$", "dev"): + return - if filename in attrs: - fn = "%s(%s,%s,%s) %s" % (attrs[filename][0], attrs[filename][1], attrs[filename][2], - attrs[filename][3], filename) - push_package_file(fn, "attr") - excludes.append(filename) + patterns = [ + # Patterns for matching files, format is a tuple as follows: + # (, , , ) + # order matters! + (r"^/usr/share/omf", "main", "/usr/share/omf/*"), + (r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"), + (r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"), + (r"^/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib32"), + (r"^/usr/lib64/lib(asm|dw|elf)-[0-9.]+\.so", "lib"), + (r"^/usr/lib32/lib(asm|dw|elf)-[0-9.]+\.so", "lib32"), + (r"^/usr/lib64/avx2/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"), + (r"^/usr/lib64/gobject-introspection/", "lib"), + (r"^/usr/libexec/", "bin"), + (r"^/usr/bin/", "bin"), + (r"^/usr/sbin/", "bin"), + (r"^/sbin/", "bin"), + (r"^/bin/", "bin"), + (r"^/usr/lib/python.*/", "python", "/usr/lib/python*/*"), + (r"^/usr/lib64/python.*/", "python", "/usr/lib64/python*/*"), + (r"^/usr/share/gir-[0-9\.]+/[a-zA-Z0-9\.\_\-\+]*\.gir", "dev", "/usr/share/gir-1.0/*.gir"), + (r"^/usr/share/cmake/", "data", "/usr/share/cmake/*"), + (r"^/usr/share/cmake-3.1/", "data", "/usr/share/cmake-3.1/*"), + (r"^/usr/share/cmake-3.7/", "data", "/usr/share/cmake-3.7/*"), + (r"^/usr/share/cmake-3.6/", "data", "/usr/share/cmake-3.6/*"), + (r"^/usr/share/girepository-1\.0/.*\.typelib\$", "dev", "/usr/share/girepository-1.0/*.typelib"), + (r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hxx", "dev", "/usr/include/*.hxx"), + (r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hpp", "dev", "/usr/include/*.hpp"), + (r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h\+\+", "dev", "/usr/include/*.h\+\+"), + (r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h", "dev", "/usr/include/*.h"), + (r"^/usr/include/", "dev"), + (r"^/usr/lib64/girepository-1.0/", "dev"), + (r"^/usr/share/cmake/", "dev"), + (r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"), + (r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"), + (r"^/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev32"), + (r"^/usr/lib64/avx2/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"), + (r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib/*.a"), + (r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib64/*.a"), + (r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev32", "/usr/lib32/*.a"), + (r"^/usr/lib/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev"), + (r"^/usr/lib64/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev"), + (r"^/usr/lib32/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev32"), + (r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal/*.ac"), + (r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal/*.m4"), + (r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal-1.*/*.ac"), + (r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal-1.*/*.m4"), + (r"^/usr/share/doc/" + re.escape(tarball.name) + "/", "doc", "%doc /usr/share/doc/" + re.escape(tarball.name) + "/*"), + (r"^/usr/share/gtk-doc/html", "doc"), + (r"^/usr/share/info/", "doc", "%doc /usr/share/info/*"), + (r"^/usr/share/man/man0", "doc", "%doc /usr/share/man/man0/*"), + (r"^/usr/share/man/man1", "doc", "%doc /usr/share/man/man1/*"), + (r"^/usr/share/man/man2", "doc", "%doc /usr/share/man/man2/*"), + (r"^/usr/share/man/man3", "doc", "%doc /usr/share/man/man3/*"), + (r"^/usr/share/man/man4", "doc", "%doc /usr/share/man/man4/*"), + (r"^/usr/share/man/man5", "doc", "%doc /usr/share/man/man5/*"), + (r"^/usr/share/man/man6", "doc", "%doc /usr/share/man/man6/*"), + (r"^/usr/share/man/man7", "doc", "%doc /usr/share/man/man7/*"), + (r"^/usr/share/man/man8", "doc", "%doc /usr/share/man/man8/*"), + (r"^/usr/share/man/man9", "doc", "%doc /usr/share/man/man9/*"), + (r"^/etc/systemd/system/.*\.wants/", "active-units"), + # now a set of catch-all rules + (r"^/etc/", "config", "", "%config "), + (r"^/usr/etc/", "config", "", "%config "), + (r"^/lib/systemd", "config"), + (r"^/usr/lib/systemd", "config"), + (r"^/usr/lib/udev/rules.d", "config"), + (r"^/usr/lib/modules-load.d", "config"), + (r"^/usr/lib/tmpfiles.d", "config"), + (r"^/usr/lib/sysusers.d", "config"), + (r"^/usr/lib/sysctl.d", "config"), + (r"^/usr/share/", "data"), + # finally move any dynamically loadable plugins (not + # perl/python/ruby/etc.. extensions) into lib package + (r"^/usr/lib/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"), + (r"^/usr/lib64/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"), + (r"^/usr/lib32/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib32"), + # locale data gets picked up via file_is_locale + (r"^/usr/share/locale/", "ignore")] - if file_pat_match(filename, r"^/usr/share/omf", "main", "/usr/share/omf/*"): - return + for pat_args in patterns: + if self.file_pat_match(filename, *pat_args): + return - if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"): - return - if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"): - return - if file_pat_match(filename, r"^/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib32"): - return + if filename in self.excludes: + self.push_package_file("%exclude " + filename) + return - # Workarounds for some elfutils shared libraries ending with .so - if file_pat_match(filename, r"^/usr/lib64/lib(asm|dw|elf)-[0-9.]+\.so", "lib"): - return - if file_pat_match(filename, r"^/usr/lib32/lib(asm|dw|elf)-[0-9.]+\.so", "lib32"): - return + self.push_package_file(filename) - if file_pat_match(filename, r"^/usr/lib64/avx2/[a-zA-Z0-9\.\_\-\+]*\.so\.", "lib"): - return - if file_pat_match(filename, r"^/usr/lib64/gobject-introspection/", "lib"): - return - if file_pat_match(filename, r"^/usr/libexec/", "bin"): - return - if file_pat_match(filename, r"^/usr/bin/", "bin"): - return - if file_pat_match(filename, r"^/usr/sbin/", "bin"): - return - if file_pat_match(filename, r"^/sbin/", "bin"): - return - if file_pat_match(filename, r"^/bin/", "bin"): - return - if file_pat_match(filename, r"^/bin/", "bin"): - return + def remove_file(self, filename): + """ + Remove filename from local file list + """ + hit = False - if file_pat_match(filename, r"^/usr/lib/python3.*/", "python", "/usr/lib/python3*/*"): - return - if file_pat_match(filename, r"^/usr/lib/python2.*/", "python", "/usr/lib/python2*/*"): - return - if file_pat_match(filename, r"^/usr/lib64/python.*/", "python", "/usr/lib64/python*/*"): - return - if file_pat_match(filename, r"^/usr/share/gir-[0-9\.]+/[a-zA-Z0-9\.\_\-\+]*\.gir", "dev", "/usr/share/gir-1.0/*.gir"): - print("HIT GIR\n") - return - if file_pat_match(filename, r"^/usr/share/cmake/", "data", "/usr/share/cmake/*"): - return - if file_pat_match(filename, r"^/usr/share/cmake-3.1/", "data", "/usr/share/cmake-3.1/*"): - return - if file_pat_match(filename, r"^/usr/share/cmake-3.7/", "data", "/usr/share/cmake-3.7/*"): - return - if file_pat_match(filename, r"^/usr/share/cmake-3.6/", "data", "/usr/share/cmake-3.6/*"): - return - if file_pat_match(filename, r"^/usr/share/girepository-1\.0/.*\.typelib\$", "dev", "/usr/share/girepository-1.0/*.typelib"): - return - - if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hxx", "dev", "/usr/include/*.hxx"): - return - if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.hpp", "dev", "/usr/include/*.hpp"): - return - if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h\+\+", "dev", "/usr/include/*.h\+\+"): - return - if file_pat_match(filename, r"^/usr/include/[a-zA-Z0-9\.\_\-\+]*\.h", "dev", "/usr/include/*.h"): - return - if file_pat_match(filename, r"^/usr/include/", "dev"): - return - if file_pat_match(filename, r"^/usr/lib64/girepository-1.0/", "dev"): - return - if file_pat_match(filename, r"^/usr/share/cmake/", "dev"): - return - if want_dev_split > 0 and file_pat_match(filename, r"^/usr/.*/include/.*\.h$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib32/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev32"): - return - if file_pat_match(filename, r"^/usr/lib64/avx2/[a-zA-Z0-9\.\_\-\+]*\.so$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib/*.a"): - return - if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev", "/usr/lib64/*.a"): - return - if file_pat_match(filename, r"^/usr/lib64/[a-zA-Z0-9\.\_\-\+]*\.a$", "dev32", "/usr/lib32/*.a"): - return - if file_pat_match(filename, r"^/usr/lib/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib64/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev"): - return - if file_pat_match(filename, r"^/usr/lib32/pkgconfig/[a-zA-Z0-9\.\_\-\+]*\.pc$", "dev32"): - return - if file_pat_match(filename, r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal/*.ac"): - return - if file_pat_match(filename, r"^/usr/share/aclocal/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal/*.m4"): - return - if file_pat_match(filename, r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.ac$", "dev", "/usr/share/aclocal-1.*/*.ac"): - return - if file_pat_match(filename, r"^/usr/share/aclocal-1.[0-9]+/[a-zA-Z0-9\.\_\-\+]*\.m4$", "dev", "/usr/share/aclocal-1.*/*.m4"): - return - - if file_pat_match(filename, r"^/usr/share/doc/" + re.escape(tarball.name) + "/", "doc", "%doc /usr/share/doc/" + re.escape(tarball.name) + "/*"): - return - if file_pat_match(filename, r"^/usr/share/gtk-doc/html", "doc"): - return - if file_pat_match(filename, r"^/usr/share/info/", "doc", "%doc /usr/share/info/*"): - return - - if file_pat_match(filename, r"^/usr/share/man/man0", "doc", "%doc /usr/share/man/man0/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man1", "doc", "%doc /usr/share/man/man1/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man2", "doc", "%doc /usr/share/man/man2/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man3", "doc", "%doc /usr/share/man/man3/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man4", "doc", "%doc /usr/share/man/man4/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man5", "doc", "%doc /usr/share/man/man5/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man6", "doc", "%doc /usr/share/man/man6/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man7", "doc", "%doc /usr/share/man/man7/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man8", "doc", "%doc /usr/share/man/man8/*"): - return - if file_pat_match(filename, r"^/usr/share/man/man9", "doc", "%doc /usr/share/man/man9/*"): - return - - if file_pat_match(filename, r"^/etc/systemd/system/.*\.wants/", "active-units"): - return - - # now a set of catch-all rules - if file_pat_match(filename, r"^/etc/", "config", "", "%config "): - return - if file_pat_match(filename, r"^/usr/etc/", "config", "", "%config "): - return - if file_pat_match(filename, r"^/lib/systemd", "config"): - return - if file_pat_match(filename, r"^/usr/lib/systemd", "config"): - return - if file_pat_match(filename, r"^/usr/lib/udev/rules.d", "config"): - return - if file_pat_match(filename, r"^/usr/lib/modules-load.d", "config"): - return - if file_pat_match(filename, r"^/usr/lib/tmpfiles.d", "config"): - return - if file_pat_match(filename, r"^/usr/lib/sysusers.d", "config"): - return - if file_pat_match(filename, r"^/usr/lib/sysctl.d", "config"): - return - - if file_pat_match(filename, r"^/usr/share/", "data"): - return - - # finally move any dynamically loadable plugins (not - # perl/python/ruby/etc.. extensions) into lib package - if file_pat_match(filename, r"^/usr/lib/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"): - return - if file_pat_match(filename, r"^/usr/lib64/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib"): - return - if file_pat_match(filename, r"^/usr/lib32/.*/[a-zA-Z0-9\.\_\-\+]*\.so", "lib32"): - return - - # locale data gets picked up via find_lang - if file_pat_match(filename, r"^/usr/share/locale/", "ignore"): - return - - if filename in excludes: - push_package_file("%exclude " + filename) - return - - push_package_file(filename) - - -def remove_file(file): - global files - global packages - global files_blacklist - hit = False - - if file in files: - files.remove(file) - print("File no longer present: %s" % file) - hit = True - for pkg in packages: - if file in packages[pkg]: - packages[pkg].remove(file) - print("File no longer present in %s: %s" % (pkg, file)) + if filename in self.files: + self.files.remove(filename) + print("File no longer present: {}".format(filename)) hit = True - if hit: - if file not in files_blacklist: - files_blacklist.append(file) - build.must_restart = build.must_restart + 1 - - -def add_lang(lang): - global locales - global packages - if lang in locales: - return - locales.append(lang) - print(" New locale:", lang) - - if "locales" in packages: - return - packages["locales"] = [] - - -def load_specfile(specfile): - specfile.packages = packages - specfile.excludes = excludes - specfile.locales = locales + for pkg in self.packages: + if filename in self.packages[pkg]: + self.packages[pkg].remove(filename) + print("File no longer present in {}: {}".format(pkg, filename)) + hit = True + if hit: + self.files_blacklist.add(filename) + build.must_restart += 1 + def load_specfile(self, specfile): + """ + Load a specfile instance with relevant information to be written to the + spec file. + """ + specfile.packages = self.packages + specfile.excludes = self.excludes + specfile.locales = self.locales diff --git a/autospec/tarball.py b/autospec/tarball.py index 6fab430..5045a2f 100644 --- a/autospec/tarball.py +++ b/autospec/tarball.py @@ -20,7 +20,6 @@ import build import buildpattern import buildreq -import files import config import glob import hashlib @@ -235,7 +234,7 @@ def convert_version(version): return version.strip('.') + suffix -def name_and_version(url_argument, name_argument): +def name_and_version(url_argument, name_argument, filemanager): global name global rawname global version @@ -262,7 +261,7 @@ def name_and_version(url_argument, name_argument): # R package if url_argument.find("cran.r-project.org") > 0 or url_argument.find("cran.rstudio.com") > 0: buildpattern.set_build_pattern("R", 10) - files.want_dev_split = 0 + filemanager.want_dev_split = False buildreq.add_buildreq("clr-R-helpers") p = re.compile(r"([A-Za-z0-9.]+)_(v*[0-9]+[\+_spbfourcesigedsvstableP0-9\.\~\-]*)\.tar\.gz") m = p.search(tarfile) diff --git a/tests/test_files.py b/tests/test_files.py new file mode 100644 index 0000000..bbe8dbf --- /dev/null +++ b/tests/test_files.py @@ -0,0 +1,188 @@ +import unittest +import files +from unittest.mock import call, MagicMock +from files import FileManager + + +def mock_return(retval): + """ + Simple mock method to set return value of a function + """ + def mock_fn(_): + return retval + + return mock_fn + + +class TestFiles(unittest.TestCase): + + def setUp(self): + self.fm = FileManager() + + def test_push_package_file(self): + """ + Test push_package_file with no package name specified (package name + should default to 'main' + """ + self.assertFalse(self.fm.newfiles_printed) + self.fm.push_package_file('test-fn') + self.assertEqual(self.fm.packages['main'], set(['test-fn'])) + self.assertTrue(self.fm.newfiles_printed) + + def test_push_package_file_dev(self): + """ + Test push_package_file with dev package specified + """ + self.fm.push_package_file('test-fn', 'dev') + self.assertEqual(self.fm.packages['dev'], set(['test-fn'])) + self.assertTrue(self.fm.newfiles_printed) + + def test_file_pat_match(self): + """ + Test file_pat_match with good match and no replacement or prefix + specified. + """ + self.fm.push_package_file = MagicMock() + self.assertTrue(self.fm.file_pat_match('test-fn', r'test-fn', 'main')) + self.fm.push_package_file.assert_called_with('test-fn', 'main') + + def test_file_pat_match_exclude(self): + """ + Test file_pat_match with good match and filename in excludes list. + """ + self.fm.push_package_file = MagicMock() + self.fm.excludes.append('test-fn') + self.assertTrue(self.fm.file_pat_match('test-fn', r'test-fn', 'main')) + self.fm.push_package_file.assert_called_with('%exclude test-fn', 'main') + + def test_file_pat_match_replacement(self): + """ + Test file_pat_match with replacement provided + """ + self.fm.push_package_file = MagicMock() + self.assertTrue(self.fm.file_pat_match('test-fn', r'test-fn', 'main', 'testfn')) + self.fm.push_package_file.assert_called_with('testfn', 'main') + + def test_file_pat_match_no_match(self): + """ + Test file_pat_match with no match + """ + self.fm.push_package_file = MagicMock() + self.assertFalse(self.fm.file_pat_match('test-fn', r'testfn', 'main')) + self.fm.push_package_file.assert_not_called() + + def test_file_is_locale(self): + """ + Test file_is_locale with locale filename not present in locale list + """ + self.assertEqual(self.fm.locales, []) + self.assertTrue(self.fm.file_is_locale('/usr/share/locale/a/loc.mo')) + self.assertEqual(self.fm.locales, ['loc']) + + def test_file_is_locale_non_locale(self): + """ + Test file_is_locale with non-locale filename + """ + self.assertFalse(self.fm.file_is_locale('test-fn')) + self.assertEqual(self.fm.locales, []) + + def test_file_is_locale_present(self): + """ + Test file_is_locale with locale present in locale list + """ + self.fm.locales.append('loc') + self.assertEqual(self.fm.locales, ['loc']) + self.assertTrue(self.fm.file_is_locale('/usr/share/locale/a/loc.mo')) + self.assertEqual(self.fm.locales, ['loc']) + + def test_push_file_autostart(self): + """ + Test push_file to autostart package, this excludes the file. + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + autostart = '/usr/lib/systemd/system/some.target.wants/some' + self.fm.push_file(autostart) + calls = [call(autostart, 'autostart'), call('%exclude ' + autostart, 'config')] + self.fm.push_package_file.assert_has_calls(calls) + + def test_push_file_extras(self): + """ + Test push_file to extras package, this excludes the file + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + self.fm.extras.append('test') + self.fm.push_file('test') + calls = [call('test', 'extras'), call('%exclude test')] + self.fm.push_package_file.assert_has_calls(calls) + + + def test_push_file_setuid(self): + """ + Test push_file with fname in setuid list + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + self.fm.setuid.append('test') + self.fm.push_file('test') + calls = [call('%attr(4755, root, root) test', 'setuid'), call('%exclude test')] + self.fm.push_package_file.assert_has_calls(calls) + + + def test_push_file_match(self): + """ + Test push_file with match in pattern list + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + self.fm.push_file('/usr/bin/test') + self.fm.push_package_file.assert_called_once_with('/usr/bin/test', 'bin') + + def test_push_file_match_tarball_name_dependency(self): + """ + Test push_file with match in the list on the single item that is + dependent on the tarball name. + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + files.tarball.name = 'testball' + self.fm.push_file('/usr/share/doc/testball/') + self.fm.push_package_file.assert_called_once_with('%doc /usr/share/doc/testball/*', 'doc') + + def test_push_file_no_match(self): + """ + Test push_file with no pattern match on the file name. Should just push + the unmodified filename once. + """ + self.fm.file_is_locale = MagicMock(return_value=False) + self.fm.push_package_file = MagicMock() + self.fm.push_file('doesntmatcha thing') + self.fm.push_package_file.assert_called_once_with('doesntmatcha thing') + + def test_remove_file(self): + """ + Test remove_file with filename in files list and main package + """ + self.fm.files.append('test') + self.fm.packages['main'] = ['test'] + self.assertIn('test', self.fm.files) + self.assertNotIn('test', self.fm.files_blacklist) + self.assertIn('test', self.fm.packages['main']) + self.fm.remove_file('test') + self.assertNotIn('test', self.fm.files) + self.assertNotIn('test', self.fm.packages['main']) + self.assertIn('test', self.fm.files_blacklist) + + def test_remove_file_not_present(self): + """ + Test remove_file with filename not in files list. + """ + self.assertNotIn('test', self.fm.files) + self.assertNotIn('test', self.fm.files_blacklist) + self.fm.remove_file('test') + self.assertNotIn('test', self.fm.files) + self.assertNotIn('test', self.fm.files_blacklist) + +if __name__ == '__main__': + unittest.main(buffer=True) diff --git a/tests/test_tarball.py b/tests/test_tarball.py index 2bc9e54..5fc9fd6 100644 --- a/tests/test_tarball.py +++ b/tests/test_tarball.py @@ -17,6 +17,9 @@ class TestTarballVersionName(unittest.TestCase): Does not use self.assertEquals but appends to a list so all failures are reported instead of just the first one. """ + class FileManager(): + want_dev_split = False + errors = [] with open('tests/packageurls', 'r') as pkgurls: for urlline in pkgurls.read().split('\n'): @@ -26,7 +29,7 @@ class TestTarballVersionName(unittest.TestCase): tarball.name = '' tarball.version = '' (url, name, version) = urlline.split(',') - tarball.name_and_version(url, '') + tarball.name_and_version(url, '', FileManager()) if tarball.name != name: errors.append("name: '{}' != '{}' for url: {}" .format(tarball.name, name, url))