From 2a181a2ddbdf47153a601bcd102a4c92b2912802 Mon Sep 17 00:00:00 2001 From: William Douglas Date: Thu, 26 Mar 2020 10:34:12 -0700 Subject: [PATCH] Refactor buildreq into a class The buildreq module had a large number of globals being touched across many modules that would import it. This made changes to the buildreq fragile. It also made testing fragile due to certain globals needing to be reset in order to properly test a change. This change moves the global state (and functions that needed to act on that global state) into a Requirements class. The goal of this work is to better track what content can be updated by a particular function and load data in a sensible order and have it owned by a sensible component. --- autospec/autospec.py | 23 +- autospec/build.py | 55 +- autospec/buildreq.py | 1387 +++++++++++++++--------------- autospec/check.py | 13 +- autospec/config.py | 53 +- autospec/specfiles.py | 37 +- autospec/tarball.py | 7 - tests/test_build.py | 116 ++- tests/test_buildreq.py | 213 +++-- tests/test_check.py | 25 +- tests/test_infile_update_spec.py | 3 +- tests/test_specfile.py | 18 +- 12 files changed, 977 insertions(+), 973 deletions(-) diff --git a/autospec/autospec.py b/autospec/autospec.py index 1791c94..8444b0d 100644 --- a/autospec/autospec.py +++ b/autospec/autospec.py @@ -90,7 +90,6 @@ def load_specfile(conf, specfile): tarball.load_specfile(specfile) specdescription.load_specfile(specfile, conf.custom_desc, conf.custom_summ) license.load_specfile(specfile) - buildreq.load_specfile(specfile) buildpattern.load_specfile(specfile) check.load_specfile(specfile) @@ -267,7 +266,9 @@ def package(args, url, name, archives, workingdir, infile_dict): conf.setup_patterns() conf.config_file = args.config - conf.parse_config_files(build.download_path, args.bump, filemanager, tarball.version) + requirements = buildreq.Requirements(tarball.url) + requirements.set_build_req() + conf.parse_config_files(build.download_path, args.bump, filemanager, tarball.version, requirements) conf.setup_patterns(conf.failed_pattern_dir) conf.parse_existing_spec(build.download_path, tarball.name) @@ -275,21 +276,25 @@ def package(args, url, name, archives, workingdir, infile_dict): write_prep(conf, workingdir) exit(0) - buildreq.set_build_req() - buildreq.scan_for_configure(_dir, tarball.name, build.download_path, conf) + requirements.scan_for_configure(_dir, tarball.name, build.download_path, conf) specdescription.scan_for_description(tarball.name, _dir, conf.license_translations, conf.license_blacklist) # Start one directory higher so we scan *all* versions for licenses license.scan_for_licenses(os.path.dirname(_dir), conf) commitmessage.scan_for_changes(build.download_path, _dir, conf.transforms) add_sources(build.download_path, archives) - check.scan_for_tests(_dir, conf) + check.scan_for_tests(_dir, conf, requirements) # # Now, we have enough to write out a specfile, and try to build it. # We will then analyze the build result and learn information until the # package builds # - specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release, conf) + specfile = specfiles.Specfile(tarball.url, + tarball.version, + tarball.name, + tarball.release, + conf, + requirements) filemanager.load_specfile(specfile) load_specfile(conf, specfile) @@ -308,7 +313,7 @@ def package(args, url, name, archives, workingdir, infile_dict): specfile.write_spec(build.download_path) while 1: - build.package(filemanager, args.mock_config, args.mock_opts, conf, args.cleanup) + build.package(filemanager, args.mock_config, args.mock_opts, conf, requirements, args.cleanup) filemanager.load_specfile(specfile) specfile.write_spec(build.download_path) filemanager.newfiles_printed = 0 @@ -329,7 +334,7 @@ def package(args, url, name, archives, workingdir, infile_dict): check.check_regression(build.download_path, conf.config_opts['skip_tests']) if build.success == 0: - conf.create_buildreq_cache(build.download_path, tarball.version) + conf.create_buildreq_cache(build.download_path, tarball.version, requirements.buildreqs_cache) print_fatal("Build failed, aborting") sys.exit(1) elif os.path.isfile("README.clear"): @@ -353,7 +358,7 @@ def package(args, url, name, archives, workingdir, infile_dict): logcheck(build.download_path) commitmessage.guess_commit_message(pkg_integrity.IMPORTED, conf) - conf.create_buildreq_cache(build.download_path, tarball.version) + conf.create_buildreq_cache(build.download_path, tarball.version, requirements.buildreqs_cache) if args.git: git.commit_to_git(build.download_path, conf) diff --git a/autospec/build.py b/autospec/build.py index 0579a9f..ffc9a4b 100644 --- a/autospec/build.py +++ b/autospec/build.py @@ -23,7 +23,6 @@ import os import re import shutil -import buildreq import tarball import util @@ -44,22 +43,22 @@ def setup_workingdir(workingdir): download_path = os.path.join(base_path, tarball.name) -def simple_pattern_pkgconfig(line, pattern, pkgconfig, conf32): +def simple_pattern_pkgconfig(line, pattern, pkgconfig, conf32, requirements): """Check for pkgconfig patterns and restart build as needed.""" global must_restart pat = re.compile(pattern) match = pat.search(line) if match: - must_restart += buildreq.add_pkgconfig_buildreq(pkgconfig, conf32, cache=True) + must_restart += requirements.add_pkgconfig_buildreq(pkgconfig, conf32, cache=True) -def simple_pattern(line, pattern, req): +def simple_pattern(line, pattern, req, requirements): """Check for simple patterns and restart the build as needed.""" global must_restart pat = re.compile(pattern) match = pat.search(line) if match: - must_restart += buildreq.add_buildreq(req, cache=True) + must_restart += requirements.add_buildreq(req, cache=True) def cleanup_req(s: str) -> str: @@ -96,7 +95,7 @@ def cleanup_req(s: str) -> str: return s -def failed_pattern(line, config, pattern, verbose, buildtool=None): +def failed_pattern(line, config, requirements, pattern, verbose, buildtool=None): """Check against failed patterns to restart build as needed.""" global must_restart global warned_about @@ -116,29 +115,29 @@ def failed_pattern(line, config, pattern, verbose, buildtool=None): if not buildtool: req = config.failed_commands[s] if req: - must_restart += buildreq.add_buildreq(req, cache=True) + must_restart += requirements.add_buildreq(req, cache=True) elif buildtool == 'pkgconfig': - must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) + must_restart += requirements.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) elif buildtool == 'R': - if buildreq.add_buildreq("R-" + s, cache=True) > 0: + if requirements.add_buildreq("R-" + s, cache=True) > 0: must_restart += 1 - buildreq.add_requires("R-" + s, config.os_packages) + requirements.add_requires("R-" + s, config.os_packages) elif buildtool == 'perl': s = s.replace('inc::', '') - must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True) + must_restart += requirements.add_buildreq('perl(%s)' % s, cache=True) elif buildtool == 'pypi': s = util.translate(s) if not s: return - must_restart += buildreq.add_buildreq(util.translate('%s-python' % s), cache=True) + must_restart += requirements.add_buildreq(util.translate('%s-python' % s), cache=True) elif buildtool == 'ruby': if s in config.gems: - must_restart += buildreq.add_buildreq(config.gems[s], cache=True) + must_restart += requirements.add_buildreq(config.gems[s], cache=True) else: - must_restart += buildreq.add_buildreq('rubygem-%s' % s, cache=True) + must_restart += requirements.add_buildreq('rubygem-%s' % s, cache=True) elif buildtool == 'ruby table': if s in config.gems: - must_restart += buildreq.add_buildreq(config.gems[s], cache=True) + must_restart += requirements.add_buildreq(config.gems[s], cache=True) else: print("Unknown ruby gem match", s) elif buildtool == 'maven' or buildtool == 'gradle': @@ -149,10 +148,10 @@ def failed_pattern(line, config, pattern, verbose, buildtool=None): # Hyphens are disallowed for version strings, so use dots instead ver = match.group(2).replace('-', '.') mvn_provide = f'mvn({name}) = {ver}' - must_restart += buildreq.add_buildreq(mvn_provide, cache=True) + must_restart += requirements.add_buildreq(mvn_provide, cache=True) elif s in config.maven_jars: # Overrides for dependencies with custom grouping - must_restart += buildreq.add_buildreq(config.maven_jars[s], cache=True) + must_restart += requirements.add_buildreq(config.maven_jars[s], cache=True) elif group_count == 3: org = match.group(1) name = match.group(2) @@ -161,13 +160,13 @@ def failed_pattern(line, config, pattern, verbose, buildtool=None): mvn_provide = f'mvn({org}:{name}:pom) = {ver}' else: mvn_provide = f'mvn({org}:{name}:jar) = {ver}' - must_restart += buildreq.add_buildreq(mvn_provide, cache=True) + must_restart += requirements.add_buildreq(mvn_provide, cache=True) else: # Fallback to mvn-ARTIFACTID package name - must_restart += buildreq.add_buildreq('mvn-%s' % s, cache=True) + must_restart += requirements.add_buildreq('mvn-%s' % s, cache=True) elif buildtool == 'catkin': - must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) - must_restart += buildreq.add_buildreq(s, cache=True) + must_restart += requirements.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) + must_restart += requirements.add_buildreq(s, cache=True) except Exception: if s not in warned_about and s[:2] != '--': @@ -206,11 +205,11 @@ def check_for_warning_pattern(line): util.print_warning("Build log contains: {}".format(pat)) -def parse_build_results(filename, returncode, filemanager, config): +def parse_build_results(filename, returncode, filemanager, config, requirements): """Handle build log contents.""" global must_restart global success - buildreq.verbose = 1 + requirements.verbose = 1 must_restart = 0 infiles = 0 @@ -221,13 +220,13 @@ def parse_build_results(filename, returncode, filemanager, config): for line in loglines: for pat in config.pkgconfig_pats: - simple_pattern_pkgconfig(line, *pat, config.config_opts.get('32bit')) + simple_pattern_pkgconfig(line, *pat, config.config_opts.get('32bit'), requirements) for pat in config.simple_pats: - simple_pattern(line, *pat) + simple_pattern(line, *pat, requirements) for pat in config.failed_pats: - failed_pattern(line, config, *pat) + failed_pattern(line, config, requirements, *pat) check_for_warning_pattern(line) @@ -277,7 +276,7 @@ def get_mock_cmd(): return 'sudo /usr/bin/mock' -def package(filemanager, mockconfig, mockopts, config, cleanup=False): +def package(filemanager, mockconfig, mockopts, config, requirements, cleanup=False): """Run main package build routine.""" global round global uniqueext @@ -343,7 +342,7 @@ def package(filemanager, mockconfig, mockopts, config, cleanup=False): is_clean = parse_buildroot_log(download_path + "/results/root.log", ret) if is_clean: - parse_build_results(download_path + "/results/build.log", ret, filemanager, config) + parse_build_results(download_path + "/results/build.log", ret, filemanager, config, requirements) if filemanager.has_banned: util.print_fatal("Content in banned paths found, aborting build") exit(1) diff --git a/autospec/buildreq.py b/autospec/buildreq.py index 9e8f90b..f283f2b 100644 --- a/autospec/buildreq.py +++ b/autospec/buildreq.py @@ -30,147 +30,61 @@ import specdescription import toml import util -banned_requires = set(["futures", - "configparser"]) -buildreqs = set() -buildreqs_cache = set() -requires = set() -extra_cmake = set() -extra_cmake_openmpi = set() -verbose = False -cargo_bin = False -pypi_provides = None -pypi_requires = set() -banned_buildreqs = set(["llvm-devel", - "gcj", - "pkgconfig(dnl)", - "pkgconfig(hal)", - "tslib-0.0", - "pkgconfig(parallels-sdk)", - "oslo-python", - "libxml2No-python", - "futures", - "configparser"]) -autoreconf_reqs = ["gettext-bin", - "automake-dev", - "automake", - "m4", - "libtool", - "libtool-dev", - "pkg-config-dev"] + +def is_qmake_pro(f): + """Test if file extension is pro and not hidden.""" + return f.endswith(".pro") and not f.startswith(".") -def add_buildreq(req, cache=False): - """Add req to the global buildreqs set if req is not banned.""" - global buildreqs - global buildreqs_cache - new = True +def get_python_build_version_from_classifier(filename): + """Detect if setup should use distutils3 only. - req.strip() + Uses "Programming Language :: Python :: [2,3] :: Only" classifiers in the + setup.py file. Defaults to distutils3 if no such classifiers are found. + """ + with util.open_auto(filename) as setup_file: + data = setup_file.read() - if req in banned_buildreqs: - return False - if req in buildreqs: - new = False - if verbose and new: - print(" Adding buildreq:", req) + if "Programming Language :: Python :: 3 :: Only" in data: + return "distutils3" - buildreqs.add(req) - if cache and new: - buildreqs_cache.add(req) - return new + return "distutils3" -def add_requires(req, packages, override=False): - """Add req to the global requires set if it is present in buildreqs and packages and is not banned.""" - global buildreqs - global requires - new = True - req = req.strip() - if req in requires: - new = False - if req in banned_requires: - return False +def clean_python_req(req, add_python=True): + """Strip version information from req.""" + if req.find("#") == 0: + return "" + ret = req.rstrip("\n\r").strip() + i = ret.find(";") + if i > 0: + ret = ret[:i] + i = ret.find("<") + if i > 0: + ret = ret[:i] + i = ret.find("\n") + if i > 0: + ret = ret[:i] + i = ret.find(">") + if i > 0: + ret = ret[:i] + i = ret.find("=") + if i > 0: + ret = ret[:i] + i = ret.find("#") + if i > 0: + ret = ret[:i] + i = ret.find("!") + if i > 0: + ret = ret[:i] - # Try dashes instead of underscores as some ecosystems are inconsistent in their naming - req2 = req.replace("_", "-") - if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires: - # Since this is done for python add a buildreq just in case (might not be correct though) - buildreqs.add(req2) - requires.add(req2) - return True - - # Try reversing the case of the first letter as some ecosystems are inconsistent in their naming - if len(req) > 1: - if req[0].isupper(): - req2 = req[0].lower() + req[1:] - else: - req2 = req[0].upper() + req[1:] - if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires: - # Since this is done for python add a buildreq just in case (might not be correct though) - buildreqs.add(req2) - requires.add(req2) - return True - - if req not in buildreqs and req not in packages and not override: - if req: - print("requirement '{}' not found in buildreqs or os_packages, skipping".format(req)) - return False - if new: - # print("Adding requirement:", req) - requires.add(req) - return new - - -def add_pkgconfig_buildreq(preq, conf32, cache=False): - """Format preq as pkgconfig req and add to buildreqs.""" - if conf32: - req = "pkgconfig(32" + preq + ")" - add_buildreq(req, cache) - req = "pkgconfig(" + preq + ")" - return add_buildreq(req, cache) - - -def configure_ac_line(line, conf32): - """Parse configure_ac line and add appropriate buildreqs.""" - # print("----\n", line, "\n----") - # ignore comments - if line.startswith('#'): - return - - pat_reqs = [(r"AC_CHECK_FUNC\([tgetent]", ["ncurses-devel"]), - ("PROG_INTLTOOL", ["intltool"]), - ("GETTEXT_PACKAGE", ["gettext", "perl(XML::Parser)"]), - ("AM_GLIB_GNU_GETTEXT", ["gettext", "perl(XML::Parser)"]), - ("GTK_DOC_CHECK", ["gtk-doc", "gtk-doc-dev", "libxslt-bin", "docbook-xml"]), - ("AC_PROG_SED", ["sed"]), - ("AC_PROG_GREP", ["grep"])] - - for pat, reqs in pat_reqs: - if pat in line: - for req in reqs: - add_buildreq(req) - - line = line.strip() - - # XFCE uses an equivalent to PKG_CHECK_MODULES, handle them both the same - for style in [r"PKG_CHECK_MODULES\((.*?)\)", r"XDT_CHECK_PACKAGE\((.*?)\)"]: - match = re.search(style, line) - L = [] - if match: - L = match.group(1).split(",") - if len(L) > 1: - rqlist = L[1].strip() - for req in parse_modules_list(rqlist): - add_pkgconfig_buildreq(req, conf32) - - # PKG_CHECK_EXISTS(MODULES, action-if-found, action-if-not-found) - match = re.search(r"PKG_CHECK_EXISTS\((.*?)\)", line) - if match: - L = match.group(1).split(",") - rqlist = L[0].strip() - for req in parse_modules_list(rqlist): - add_pkgconfig_buildreq(req, conf32) + ret = ret.strip() + # is ret actually a valid (non-empty) string? + if ret and add_python: + ret = ret.strip() + # use the dictionary to translate funky names to our current pgk names + ret = util.translate(ret) + return ret def is_number(num_str): @@ -190,6 +104,39 @@ def is_version(num_str): return False +def parse_modules_list(modules_string, is_cmake=False): + """Parse the modules_string for the list of modules, stripping out the version requirements.""" + if is_cmake: + modules = [m for m in re.split(r'\s*([><]?=|\${?[^}]*}?)\s*', modules_string)] + modules = filter(None, modules) + else: + modules = [m.strip('[]') for m in modules_string.split()] + res = [] + next_is_ver = False + for mod in modules: + if next_is_ver: + next_is_ver = False + continue + + if any(s in mod for s in ['<', '>', '=']): + next_is_ver = True + continue + + if is_number(mod): + continue + + if is_version(mod): + continue + + if mod.startswith('$'): + continue + + if len(mod) >= 2: + res.append(mod) + + return res + + def parse_go_mod(path): """Parse go.mod file for build requirements. @@ -234,83 +181,6 @@ def parse_go_mod(path): return reqs -def parse_modules_list(modules_string, is_cmake=False): - """Parse the modules_string for the list of modules, stripping out the version requirements.""" - if is_cmake: - modules = [m for m in re.split(r'\s*([><]?=|\${?[^}]*}?)\s*', modules_string)] - modules = filter(None, modules) - else: - modules = [m.strip('[]') for m in modules_string.split()] - res = [] - next_is_ver = False - for mod in modules: - if next_is_ver: - next_is_ver = False - continue - - if any(s in mod for s in ['<', '>', '=']): - next_is_ver = True - continue - - if is_number(mod): - continue - - if is_version(mod): - continue - - if mod.startswith('$'): - continue - - if len(mod) >= 2: - res.append(mod) - - return res - - -def parse_configure_ac(filename, conf32): - """Parse the configure.ac file for build requirements.""" - buf = "" - depth = 0 - # print("Configure parse: ", filename) - buildpattern.set_build_pattern("configure_ac", 1) - f = util.open_auto(filename, "r") - while 1: - c = f.read(1) - if not c: - break - if c == "(": - depth += 1 - if c == ")" and depth > 0: - depth -= 1 - if c != "\n": - buf += c - if c == "\n" and depth == 0: - configure_ac_line(buf, conf32) - buf = "" - - configure_ac_line(buf, conf32) - f.close() - - -def parse_cargo_toml(filename, packages): - """Update build requirements using Cargo.toml. - - Set the build requirements for building rust programs using cargo. - """ - global cargo_bin - buildpattern.set_build_pattern("cargo", 1) - add_buildreq("rustc") - with util.open_auto(filename, "r") as ctoml: - cargo = toml.loads(ctoml.read()) - if cargo.get("bin") or os.path.exists(os.path.join(os.path.dirname(filename), "src/main.rs")): - cargo_bin = True - if not cargo.get("dependencies"): - return - for cdep in cargo["dependencies"]: - if add_buildreq(cdep): - add_requires(cdep, packages) - - def _get_desc_field(field, desc): """Get a field value from an R package DESCRIPTION file. @@ -400,541 +270,642 @@ def _get_r_provides(): return set(provides) -def parse_r_description(filename, packages): - """Update build/runtime requirements according to the R package description.""" - deps = [] - with util.open_auto(filename, "r") as desc: - content = desc.read() - deps = _get_desc_field("Depends", content) - deps.extend(_get_desc_field("Imports", content)) - deps.extend(_get_desc_field("LinkingTo", content)) - r_provides = _get_r_provides() - for dep in deps: - if dep == 'R': - continue - if dep in r_provides: - continue - pkg = 'R-' + dep - if pkg in packages: - add_buildreq(pkg) - add_requires(pkg, packages) - else: - print("CRAN package '{}' not found in os_packages, skipping".format(pkg)) +class Requirements(object): + """Handle package build and runtime requiremnts.""" + def __init__(self, url): + """Initialize Default requirements settings.""" + self.banned_requires = set(["futures", + "configparser"]) + self.buildreqs = set() + self.buildreqs_cache = set() + self.requires = set() + self.extra_cmake = set() + self.extra_cmake_openmpi = set() + self.verbose = False + self.cargo_bin = False + self.pypi_provides = None + self.pypi_requires = set() + self.banned_buildreqs = set(["llvm-devel", + "gcj", + "pkgconfig(dnl)", + "pkgconfig(hal)", + "tslib-0.0", + "pkgconfig(parallels-sdk)", + "oslo-python", + "libxml2No-python", + "futures", + "configparser"]) + self.autoreconf_reqs = ["gettext-bin", + "automake-dev", + "automake", + "m4", + "libtool", + "libtool-dev", + "pkg-config-dev"] + if "gnome.org" in url: + self.add_buildreq("buildreq-gnome") + if "kde.org" in url or "https://github.com/KDE" in url: + self.add_buildreq("buildreq-kde") -def set_build_req(): - """Add build requirements based on the buildpattern pattern.""" - if buildpattern.default_pattern == "maven": - maven_reqs = ["apache-maven", - "openjdk-dev", - "mvn-aether-core", - "mvn-aopalliance", - "mvn-cdi-api", - "mvn-commons-cli", - "mvn-commons-codec", - "mvn-commons-io", - "mvn-commons-lang", - "mvn-commons-lang3", - "mvn-commons-logging", - "mvn-guice", - "mvn-guava", - "mvn-httpcomponents-client", - "mvn-httpcomponents-core", - "mvn-jsoup", - "mvn-jsr305", - "mvn-wagon", - "mvn-sisu", - "mvn-plexus-cipher", - "mvn-plexus-classworlds", - "mvn-plexus-containers", - "mvn-plexus-interpolation", - "mvn-sonatype-plexus-sec-dispatcher", - "mvn-plexus-utils", - "mvn-slf4j"] - for req in maven_reqs: - add_buildreq(req) + def add_buildreq(self, req, cache=False): + """Add req to the global buildreqs set if req is not banned.""" + new = True + req.strip() + if req in self.banned_buildreqs: + return False + if req in self.buildreqs: + new = False + if self.verbose and new: + print(" Adding buildreq:", req) - if buildpattern.default_pattern == "ruby": - add_buildreq("ruby") - add_buildreq("rubygem-rdoc") - if buildpattern.default_pattern == "cargo": - add_buildreq("rustc") + self.buildreqs.add(req) + if cache and new: + self.buildreqs_cache.add(req) + return new + def add_requires(self, req, packages, override=False): + """Add req to the global requires set if it is present in buildreqs and packages and is not banned.""" + new = True + req = req.strip() + if req in self.requires: + new = False + if req in self.banned_requires: + return False -def rakefile(filename, gems): - """Scan Rakefile for build requirements.""" - with util.open_auto(filename, "r") as f: - lines = f.readlines() + # Try dashes instead of underscores as some ecosystems are inconsistent in their naming + req2 = req.replace("_", "-") + if req not in self.buildreqs and req2 in packages and req2 not in self.requires and req2 not in self.banned_requires: + # Since this is done for python add a buildreq just in case (might not be correct though) + self.buildreqs.add(req2) + self.requires.add(req2) + return True - pat = re.compile(r"^require '(.*)'$") - for line in lines: - match = pat.search(line) - if match: - s = match.group(1) - if s != "rubygems" and s in gems: - print("Rakefile-dep: " + gems[s]) - add_buildreq(gems[s]) + # Try reversing the case of the first letter as some ecosystems are inconsistent in their naming + if len(req) > 1: + if req[0].isupper(): + req2 = req[0].lower() + req[1:] else: - print("Rakefile-new: rubygem-" + s) + req2 = req[0].upper() + req[1:] + if req not in self.buildreqs and req2 in packages and req2 not in self.requires and req2 not in self.banned_requires: + # Since this is done for python add a buildreq just in case (might not be correct though) + self.buildreqs.add(req2) + self.requires.add(req2) + return True + if req not in self.buildreqs and req not in packages and not override: + if req: + print("requirement '{}' not found in buildreqs or os_packages, skipping".format(req)) + return False + if new: + # print("Adding requirement:", req) + self.requires.add(req) + return new -def parse_cmake(filename, cmake_modules, conf32): - """Scan a .cmake or CMakeLists.txt file for what's it's actually looking for.""" - findpackage = re.compile(r"^[^#]*find_package\((\w+)\b.*\)", re.I) - pkgconfig = re.compile(r"^[^#]*pkg_check_modules\s*\(\w+ (.*)\)", re.I) - pkg_search_modifiers = {'REQUIRED', 'QUIET', 'NO_CMAKE_PATH', - 'NO_CMAKE_ENVIRONMENT_PATH', 'IMPORTED_TARGET'} - extractword = re.compile(r'(?:"([^"]+)"|(\S+))(.*)') + def add_pkgconfig_buildreq(self, preq, conf32, cache=False): + """Format preq as pkgconfig req and add to buildreqs.""" + if conf32: + req = "pkgconfig(32" + preq + ")" + self.add_buildreq(req, cache) + req = "pkgconfig(" + preq + ")" + return self.add_buildreq(req, cache) - with util.open_auto(filename, "r") as f: - lines = f.readlines() - for line in lines: - match = findpackage.search(line) + def configure_ac_line(self, line, conf32): + """Parse configure_ac line and add appropriate buildreqs.""" + # print("----\n", line, "\n----") + # ignore comments + if line.startswith('#'): + return + + pat_reqs = [(r"AC_CHECK_FUNC\([tgetent]", ["ncurses-devel"]), + ("PROG_INTLTOOL", ["intltool"]), + ("GETTEXT_PACKAGE", ["gettext", "perl(XML::Parser)"]), + ("AM_GLIB_GNU_GETTEXT", ["gettext", "perl(XML::Parser)"]), + ("GTK_DOC_CHECK", ["gtk-doc", "gtk-doc-dev", "libxslt-bin", "docbook-xml"]), + ("AC_PROG_SED", ["sed"]), + ("AC_PROG_GREP", ["grep"])] + + for pat, reqs in pat_reqs: + if pat in line: + for req in reqs: + self.add_buildreq(req) + + line = line.strip() + + # XFCE uses an equivalent to PKG_CHECK_MODULES, handle them both the same + for style in [r"PKG_CHECK_MODULES\((.*?)\)", r"XDT_CHECK_PACKAGE\((.*?)\)"]: + match = re.search(style, line) + L = [] + if match: + L = match.group(1).split(",") + if len(L) > 1: + rqlist = L[1].strip() + for req in parse_modules_list(rqlist): + self.add_pkgconfig_buildreq(req, conf32) + + # PKG_CHECK_EXISTS(MODULES, action-if-found, action-if-not-found) + match = re.search(r"PKG_CHECK_EXISTS\((.*?)\)", line) if match: - module = match.group(1) - try: - pkg = cmake_modules[module] - add_buildreq(pkg) - except Exception: - pass + L = match.group(1).split(",") + rqlist = L[0].strip() + for req in parse_modules_list(rqlist): + self.add_pkgconfig_buildreq(req, conf32) - match = pkgconfig.search(line) - if match: - rest = match.group(1) - while rest: - wordmatch = extractword.search(rest) - if not wordmatch: - break - rest = wordmatch.group(3) - if wordmatch.group(2) in pkg_search_modifiers: - continue - # Only one of the two groups can match at a time - module = wordmatch.group(1) - if not module: - module = wordmatch.group(2) - # We have a match, so strip out any version info - for m in parse_modules_list(module, is_cmake=True): - add_pkgconfig_buildreq(m, conf32) + def parse_configure_ac(self, filename, conf32): + """Parse the configure.ac file for build requirements.""" + buf = "" + depth = 0 + # print("Configure parse: ", filename) + buildpattern.set_build_pattern("configure_ac", 1) + f = util.open_auto(filename, "r") + while 1: + c = f.read(1) + if not c: + break + if c == "(": + depth += 1 + if c == ")" and depth > 0: + depth -= 1 + if c != "\n": + buf += c + if c == "\n" and depth == 0: + self.configure_ac_line(buf, conf32) + buf = "" + self.configure_ac_line(buf, conf32) + f.close() + def parse_cargo_toml(self, filename, packages): + """Update build requirements using Cargo.toml. -def qmake_profile(filename, qt_modules): - """Scan .pro file for build requirements.""" - with util.open_auto(filename, "r") as f: - lines = f.readlines() + Set the build requirements for building rust programs using cargo. + """ + buildpattern.set_build_pattern("cargo", 1) + self.add_buildreq("rustc") + with util.open_auto(filename, "r") as ctoml: + cargo = toml.loads(ctoml.read()) + if cargo.get("bin") or os.path.exists(os.path.join(os.path.dirname(filename), "src/main.rs")): + self.cargo_bin = True + if not cargo.get("dependencies"): + return + for cdep in cargo["dependencies"]: + if self.add_buildreq(cdep): + self.add_requires(cdep, packages) - pat = re.compile(r"(QT|QT_PRIVATE|QT_FOR_CONFIG).*=\s*(.*)\s*") - for line in lines: - match = pat.search(line) - if not match: - continue - s = match.group(2) - for module in s.split(): - module = re.sub('-private$', '', module) - try: - pc = qt_modules[module] - add_buildreq('pkgconfig({})'.format(pc)) - except Exception: - pass - - -def clean_python_req(req, add_python=True): - """Strip version information from req.""" - if req.find("#") == 0: - return "" - ret = req.rstrip("\n\r").strip() - i = ret.find(";") - if i > 0: - ret = ret[:i] - i = ret.find("<") - if i > 0: - ret = ret[:i] - i = ret.find("\n") - if i > 0: - ret = ret[:i] - i = ret.find(">") - if i > 0: - ret = ret[:i] - i = ret.find("=") - if i > 0: - ret = ret[:i] - i = ret.find("#") - if i > 0: - ret = ret[:i] - i = ret.find("!") - if i > 0: - ret = ret[:i] - - ret = ret.strip() - # is ret actually a valid (non-empty) string? - if ret and add_python: - ret = ret.strip() - # use the dictionary to translate funky names to our current pgk names - ret = util.translate(ret) - return ret - - -def grab_python_requirements(descfile, packages): - """Add python requirements from requirements.txt file.""" - if "/demo/" in descfile: - return - if "/doc/" in descfile: - return - if "/docs/" in descfile: - return - if "/example/" in descfile: - return - if "/test/" in descfile: - return - if "/tests/" in descfile: - return - - with util.open_auto(descfile, "r") as f: - lines = f.readlines() - - for line in lines: - # don't add the test section - if clean_python_req(line) == '[test]': - break - if clean_python_req(line) == '[testing]': - break - if clean_python_req(line) == '[dev]': - break - if clean_python_req(line) == '[doc]': - break - if clean_python_req(line) == '[docs]': - break - if 'pytest' in line: - continue - if clean_python_req(line) == 'mock': - continue - add_requires(clean_python_req(line), packages) - - -def get_python_build_version_from_classifier(filename): - """Detect if setup should use distutils3 only. - - Uses "Programming Language :: Python :: [2,3] :: Only" classifiers in the - setup.py file. Defaults to distutils3 if no such classifiers are found. - """ - with util.open_auto(filename) as setup_file: - data = setup_file.read() - - if "Programming Language :: Python :: 3 :: Only" in data: - return "distutils3" - - return "distutils3" - - -def add_setup_py_requires(filename, packages): - """Detect build requirements listed in setup.py in the install_requires and setup_requires lists. - - Handles the following patterns: - install_requires='one' - install_requires=['one', 'two', 'three'] - install_requires=['one', - 'two', - 'three'] - setup_requires=[ - 'one>=2.1', # >=2.1 is removed - 'two', - 'three' - ] - setuptools.setup( - setup_requires=['one', 'two'], - ...) - setuptools.setup(setup_requires=['one', 'two'], ...) - - Does not evaluate variables for security purposes - """ - multiline = False - with util.open_auto(filename) as f: - lines = f.readlines() - - for line in lines: - if "install_requires" in line or "setup_requires" in line: - req = "install_requires" in line - # find the value for *_requires - line = line.split("=", 1) - if len(line) == 2: - line = line[1].strip() - else: - # skip because this could be a conditionally extended list - # we only want to automatically detect the core packages + def parse_r_description(self, filename, packages): + """Update build/runtime requirements according to the R package description.""" + deps = [] + with util.open_auto(filename, "r") as desc: + content = desc.read() + deps = _get_desc_field("Depends", content) + deps.extend(_get_desc_field("Imports", content)) + deps.extend(_get_desc_field("LinkingTo", content)) + r_provides = _get_r_provides() + for dep in deps: + if dep == 'R': continue - - # easy, one-line case - if line.startswith("[") and "]" in line: - # remove the leading [ and split off everthing after the ] - line = line[1:].split("]")[0] - for item in line.split(','): - item = item.strip() - try: - # eval the string and add requirements - dep = clean_python_req(ast.literal_eval(item), False) - add_buildreq(dep) - if req: - add_requires(dep, packages) - - except Exception: - # do not fail, the line contained a variable and - # had to be skipped - pass - + if dep in r_provides: continue - - # more complicated, multi-line list. - # this sets the py_dep_string with the current line, which - # is the beginning of a multi-line list. - elif line.startswith("["): - multiline = True - line = line.lstrip("[") - - # if the line doesn't start with '[' it is the case where - # there is (should be) a single dependency as a string + pkg = 'R-' + dep + if pkg in packages: + self.add_buildreq(pkg) + self.add_requires(pkg, packages) else: - line = line.strip() + print("CRAN package '{}' not found in os_packages, skipping".format(pkg)) + + def set_build_req(self): + """Add build requirements based on the buildpattern pattern.""" + if buildpattern.default_pattern == "maven": + maven_reqs = ["apache-maven", + "openjdk-dev", + "mvn-aether-core", + "mvn-aopalliance", + "mvn-cdi-api", + "mvn-commons-cli", + "mvn-commons-codec", + "mvn-commons-io", + "mvn-commons-lang", + "mvn-commons-lang3", + "mvn-commons-logging", + "mvn-guice", + "mvn-guava", + "mvn-httpcomponents-client", + "mvn-httpcomponents-core", + "mvn-jsoup", + "mvn-jsr305", + "mvn-wagon", + "mvn-sisu", + "mvn-plexus-cipher", + "mvn-plexus-classworlds", + "mvn-plexus-containers", + "mvn-plexus-interpolation", + "mvn-sonatype-plexus-sec-dispatcher", + "mvn-plexus-utils", + "mvn-slf4j"] + for req in maven_reqs: + self.add_buildreq(req) + + if buildpattern.default_pattern == "ruby": + self.add_buildreq("ruby") + self.add_buildreq("rubygem-rdoc") + if buildpattern.default_pattern == "cargo": + self.add_buildreq("rustc") + + def rakefile(self, filename, gems): + """Scan Rakefile for build requirements.""" + with util.open_auto(filename, "r") as f: + lines = f.readlines() + + pat = re.compile(r"^require '(.*)'$") + for line in lines: + match = pat.search(line) + if match: + s = match.group(1) + if s != "rubygems" and s in gems: + print("Rakefile-dep: " + gems[s]) + self.add_buildreq(gems[s]) + else: + print("Rakefile-new: rubygem-" + s) + + def parse_cmake(self, filename, cmake_modules, conf32): + """Scan a .cmake or CMakeLists.txt file for what's it's actually looking for.""" + findpackage = re.compile(r"^[^#]*find_package\((\w+)\b.*\)", re.I) + pkgconfig = re.compile(r"^[^#]*pkg_check_modules\s*\(\w+ (.*)\)", re.I) + pkg_search_modifiers = {'REQUIRED', 'QUIET', 'NO_CMAKE_PATH', + 'NO_CMAKE_ENVIRONMENT_PATH', 'IMPORTED_TARGET'} + extractword = re.compile(r'(?:"([^"]+)"|(\S+))(.*)') + + with util.open_auto(filename, "r") as f: + lines = f.readlines() + for line in lines: + match = findpackage.search(line) + if match: + module = match.group(1) try: - dep = clean_python_req(ast.literal_eval(line), False) - add_buildreq(dep) - if req: - add_requires(dep, packages) - + pkg = cmake_modules[module] + self.add_buildreq(pkg) except Exception: - # Do not fail, just keep looking pass + match = pkgconfig.search(line) + if match: + rest = match.group(1) + while rest: + wordmatch = extractword.search(rest) + if not wordmatch: + break + rest = wordmatch.group(3) + if wordmatch.group(2) in pkg_search_modifiers: + continue + # Only one of the two groups can match at a time + module = wordmatch.group(1) + if not module: + module = wordmatch.group(2) + # We have a match, so strip out any version info + for m in parse_modules_list(module, is_cmake=True): + self.add_pkgconfig_buildreq(m, conf32) + + def qmake_profile(self, filename, qt_modules): + """Scan .pro file for build requirements.""" + with util.open_auto(filename, "r") as f: + lines = f.readlines() + + pat = re.compile(r"(QT|QT_PRIVATE|QT_FOR_CONFIG).*=\s*(.*)\s*") + for line in lines: + match = pat.search(line) + if not match: + continue + s = match.group(2) + for module in s.split(): + module = re.sub('-private$', '', module) + try: + pc = qt_modules[module] + self.add_buildreq('pkgconfig({})'.format(pc)) + except Exception: + pass + + def grab_python_requirements(self, descfile, packages): + """Add python requirements from requirements.txt file.""" + if "/demo/" in descfile: + return + if "/doc/" in descfile: + return + if "/docs/" in descfile: + return + if "/example/" in descfile: + return + if "/test/" in descfile: + return + if "/tests/" in descfile: + return + + with util.open_auto(descfile, "r") as f: + lines = f.readlines() + + for line in lines: + # don't add the test section + if clean_python_req(line) == '[test]': + break + if clean_python_req(line) == '[testing]': + break + if clean_python_req(line) == '[dev]': + break + if clean_python_req(line) == '[doc]': + break + if clean_python_req(line) == '[docs]': + break + if 'pytest' in line: + continue + if clean_python_req(line) == 'mock': + continue + self.add_requires(clean_python_req(line), packages) + + def add_setup_py_requires(self, filename, packages): + """Detect build requirements listed in setup.py in the install_requires and setup_requires lists. + + Handles the following patterns: + install_requires='one' + install_requires=['one', 'two', 'three'] + install_requires=['one', + 'two', + 'three'] + setup_requires=[ + 'one>=2.1', # >=2.1 is removed + 'two', + 'three' + ] + setuptools.setup( + setup_requires=['one', 'two'], + ...) + setuptools.setup(setup_requires=['one', 'two'], ...) + + Does not evaluate variables for security purposes + """ + multiline = False + with util.open_auto(filename) as f: + lines = f.readlines() + + for line in lines: + if "install_requires" in line or "setup_requires" in line: + req = "install_requires" in line + # find the value for *_requires + line = line.split("=", 1) + if len(line) == 2: + line = line[1].strip() + else: + # skip because this could be a conditionally extended list + # we only want to automatically detect the core packages + continue + + # easy, one-line case + if line.startswith("[") and "]" in line: + # remove the leading [ and split off everthing after the ] + line = line[1:].split("]")[0] + for item in line.split(','): + item = item.strip() + try: + # eval the string and add requirements + dep = clean_python_req(ast.literal_eval(item), False) + self.add_buildreq(dep) + if req: + self.add_requires(dep, packages) + + except Exception: + # do not fail, the line contained a variable and + # had to be skipped + pass + + continue + + # more complicated, multi-line list. + # this sets the py_dep_string with the current line, which + # is the beginning of a multi-line list. + elif line.startswith("["): + multiline = True + line = line.lstrip("[") + + # if the line doesn't start with '[' it is the case where + # there is (should be) a single dependency as a string + else: + line = line.strip() + try: + dep = clean_python_req(ast.literal_eval(line), False) + self.add_buildreq(dep) + if req: + self.add_requires(dep, packages) + + except Exception: + # Do not fail, just keep looking + pass + + continue + + # if multiline was set above when a multi-line list was + # detected, for each line until the end bracket is found attempt to + # add the line as a buildreq + if multiline: + # if end bracket found, reset the flag + if "]" in line: + multiline = False + line = line.split("]")[0] + + try: + dep = ast.literal_eval(line.split('#')[0].strip(' ,\n')) + dep = clean_python_req(dep) + self.add_buildreq(dep) + if req: + self.add_requires(dep, packages) + + except Exception: + # do not fail, the line contained a variable and had to + # be skipped + pass + + def parse_catkin_deps(self, cmakelists_file, conf32): + """Determine requirements for catkin packages.""" + f = util.open_auto(cmakelists_file, "r") + lines = f.readlines() + pat = re.compile(r"^find_package.*\(.*(catkin)(?: REQUIRED *)?(?:COMPONENTS (?P.*))?\)$") + catkin = False + for line in lines: + match = pat.search(line) + if not match: continue - # if multiline was set above when a multi-line list was - # detected, for each line until the end bracket is found attempt to - # add the line as a buildreq - if multiline: - # if end bracket found, reset the flag - if "]" in line: - multiline = False - line = line.split("]")[0] + # include catkin's required components + comp = match.group("comp") + if comp: + for curr in comp.split(" "): + self.add_pkgconfig_buildreq(curr, conf32) - try: - dep = ast.literal_eval(line.split('#')[0].strip(' ,\n')) - dep = clean_python_req(dep) - add_buildreq(dep) - if req: - add_requires(dep, packages) + catkin = True - except Exception: - # do not fail, the line contained a variable and had to - # be skipped - pass + # catkin find_package() function will always rely on CMAKE_PREFIX_PATH + # make sure we keep it consistent with CMAKE_INSTALL_PREFIX otherwise + # it'll never be able to find its modules + if catkin: + for curr in ["catkin", "catkin_pkg", "empy", "googletest"]: + self.add_buildreq(curr) + self.extra_cmake.add("-DCMAKE_PREFIX_PATH=/usr") + self.extra_cmake.add("-DCATKIN_BUILD_BINARY_PACKAGE=ON") + self.extra_cmake.add("-DSETUPTOOLS_DEB_LAYOUT=OFF") -def parse_catkin_deps(cmakelists_file, conf32): - """Determine requirements for catkin packages.""" - f = util.open_auto(cmakelists_file, "r") - lines = f.readlines() - pat = re.compile(r"^find_package.*\(.*(catkin)(?: REQUIRED *)?(?:COMPONENTS (?P.*))?\)$") - catkin = False + def scan_for_configure(self, dirn, tname, dlpath, config): + """Scan the package directory for build files to determine build pattern.""" + if buildpattern.default_pattern == "distutils36": + self.add_buildreq("buildreq-distutils36") + elif buildpattern.default_pattern == "distutils3": + self.add_buildreq("buildreq-distutils3") + elif buildpattern.default_pattern == "golang": + self.add_buildreq("buildreq-golang") + elif buildpattern.default_pattern == "cmake": + self.add_buildreq("buildreq-cmake") + elif buildpattern.default_pattern == "configure": + self.add_buildreq("buildreq-configure") + elif buildpattern.default_pattern == "qmake": + self.add_buildreq("buildreq-qmake") + elif buildpattern.default_pattern == "cpan": + self.add_buildreq("buildreq-cpan") + elif buildpattern.default_pattern == "scons": + self.add_buildreq("buildreq-scons") + elif buildpattern.default_pattern == "R": + self.add_buildreq("buildreq-R") + self.parse_r_description(os.path.join(dirn, "DESCRIPTION"), config.os_packages) + elif buildpattern.default_pattern == "phpize": + self.add_buildreq("buildreq-php") + elif buildpattern.default_pattern == "nginx": + self.add_buildreq("buildreq-nginx") - for line in lines: - match = pat.search(line) + count = 0 + for dirpath, _, files in os.walk(dirn): + default_score = 2 if dirpath == dirn else 1 - if not match: - continue - - # include catkin's required components - comp = match.group("comp") - if comp: - for curr in comp.split(" "): - add_pkgconfig_buildreq(curr, conf32) - - catkin = True - - # catkin find_package() function will always rely on CMAKE_PREFIX_PATH - # make sure we keep it consistent with CMAKE_INSTALL_PREFIX otherwise - # it'll never be able to find its modules - if catkin: - for curr in ["catkin", "catkin_pkg", "empy", "googletest"]: - add_buildreq(curr) - - extra_cmake.add("-DCMAKE_PREFIX_PATH=/usr") - extra_cmake.add("-DCATKIN_BUILD_BINARY_PACKAGE=ON") - extra_cmake.add("-DSETUPTOOLS_DEB_LAYOUT=OFF") - - -def is_qmake_pro(f): - """Test if file extension is pro and not hidden.""" - return f.endswith(".pro") and not f.startswith(".") - - -def scan_for_configure(dirn, tname, dlpath, config): - """Scan the package directory for build files to determine build pattern.""" - global pypi_provides - global pypi_requires - if buildpattern.default_pattern == "distutils36": - add_buildreq("buildreq-distutils36") - elif buildpattern.default_pattern == "distutils3": - add_buildreq("buildreq-distutils3") - elif buildpattern.default_pattern == "golang": - add_buildreq("buildreq-golang") - elif buildpattern.default_pattern == "cmake": - add_buildreq("buildreq-cmake") - elif buildpattern.default_pattern == "configure": - add_buildreq("buildreq-configure") - elif buildpattern.default_pattern == "qmake": - add_buildreq("buildreq-qmake") - elif buildpattern.default_pattern == "cpan": - add_buildreq("buildreq-cpan") - elif buildpattern.default_pattern == "scons": - add_buildreq("buildreq-scons") - elif buildpattern.default_pattern == "R": - add_buildreq("buildreq-R") - parse_r_description(os.path.join(dirn, "DESCRIPTION"), config.os_packages) - elif buildpattern.default_pattern == "phpize": - add_buildreq("buildreq-php") - elif buildpattern.default_pattern == "nginx": - add_buildreq("buildreq-nginx") - - count = 0 - for dirpath, _, files in os.walk(dirn): - default_score = 2 if dirpath == dirn else 1 - - if any(f.endswith(".go") for f in files): - add_buildreq("buildreq-golang") - buildpattern.set_build_pattern("golang", default_score) - - if "go.mod" in files: - if "Makefile" not in files: - # Go packages usually have make build systems so far - # so only use go directly if we can't find a Makefile + if any(f.endswith(".go") for f in files): + self.add_buildreq("buildreq-golang") buildpattern.set_build_pattern("golang", default_score) - add_buildreq("buildreq-golang") - if buildpattern.default_pattern == "golang-mod" or buildpattern.default_pattern == "godep": - config.set_gopath = False - mod_path = os.path.join(dirpath, "go.mod") - reqs = parse_go_mod(mod_path) - for req in reqs: - # req[0] is a SCM url segment in the form, repo/XXX/dependency-name - # req[1] is the version of the dependency - pkg = "go-" + req[0].replace("/", "-") - add_buildreq(pkg) - if buildpattern.default_pattern == "godep": - add_requires(pkg, config.os_packages) - if "CMakeLists.txt" in files and "configure.ac" not in files: - add_buildreq("buildreq-cmake") - buildpattern.set_build_pattern("cmake", default_score) + if "go.mod" in files: + if "Makefile" not in files: + # Go packages usually have make build systems so far + # so only use go directly if we can't find a Makefile + buildpattern.set_build_pattern("golang", default_score) + self.add_buildreq("buildreq-golang") + if buildpattern.default_pattern == "golang-mod" or buildpattern.default_pattern == "godep": + config.set_gopath = False + mod_path = os.path.join(dirpath, "go.mod") + reqs = parse_go_mod(mod_path) + for req in reqs: + # req[0] is a SCM url segment in the form, repo/XXX/dependency-name + # req[1] is the version of the dependency + pkg = "go-" + req[0].replace("/", "-") + self.add_buildreq(pkg) + if buildpattern.default_pattern == "godep": + self.add_requires(pkg, config.os_packages) - srcdir = os.path.abspath(os.path.join(dirn, "clr-build", config.cmake_srcdir or "..")) - if os.path.samefile(dirpath, srcdir): - parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt"), config.config_opts.get('32bit')) - - if "configure" in files and os.access(dirpath + '/configure', os.X_OK): - buildpattern.set_build_pattern("configure", default_score) - elif any(is_qmake_pro(f) for f in files): - add_buildreq("buildreq-qmake") - buildpattern.set_build_pattern("qmake", default_score) - - if "requires.txt" in files: - grab_python_requirements(dirpath + '/requires.txt', config.os_packages) - - if "setup.py" in files: - add_buildreq("buildreq-distutils3") - add_setup_py_requires(dirpath + '/setup.py', config.os_packages) - python_pattern = get_python_build_version_from_classifier(dirpath + '/setup.py') - buildpattern.set_build_pattern(python_pattern, default_score) - - if "Makefile.PL" in files or "Build.PL" in files: - buildpattern.set_build_pattern("cpan", default_score) - add_buildreq("buildreq-cpan") - - if "SConstruct" in files: - add_buildreq("buildreq-scons") - buildpattern.set_build_pattern("scons", default_score) - - if "requirements.txt" in files: - grab_python_requirements(dirpath + '/requirements.txt', config.os_packages) - - if "meson.build" in files: - add_buildreq("buildreq-meson") - buildpattern.set_build_pattern("meson", default_score) - - if "build.xml" in files: - add_buildreq("apache-ant") - buildpattern.set_build_pattern("ant", default_score) - - for name in files: - if name.lower() == "cargo.toml" and dirpath == dirn: - parse_cargo_toml(os.path.join(dirpath, name), config.os_packages) - if name.lower().startswith("configure."): - parse_configure_ac(os.path.join(dirpath, name), config.config_opts.get('32bit')) - if name.lower().startswith("rakefile") and buildpattern.default_pattern == "ruby": - rakefile(os.path.join(dirpath, name), config.gems) - if name.endswith(".pro") and buildpattern.default_pattern == "qmake": - qmake_profile(os.path.join(dirpath, name), config.qt_modules) - if name.lower() == "makefile": - buildpattern.set_build_pattern("make", default_score) - if name.lower() == "autogen.sh": - buildpattern.set_build_pattern("autogen", default_score) - if name.lower() == "cmakelists.txt": + if "CMakeLists.txt" in files and "configure.ac" not in files: + self.add_buildreq("buildreq-cmake") buildpattern.set_build_pattern("cmake", default_score) - if (name.lower() == "cmakelists.txt" or name.endswith(".cmake")) \ - and buildpattern.default_pattern == "cmake": - parse_cmake(os.path.join(dirpath, name), config.cmake_modules, config.config_opts.get('32bit')) - can_reconf = os.path.exists(os.path.join(dirn, "configure.ac")) - if not can_reconf: - can_reconf = os.path.exists(os.path.join(dirn, "configure.in")) + srcdir = os.path.abspath(os.path.join(dirn, "clr-build", config.cmake_srcdir or "..")) + if os.path.samefile(dirpath, srcdir): + self.parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt"), config.config_opts.get('32bit')) - if can_reconf and config.autoreconf: - print("Patches touch configure.*, adding autoreconf stage") - for breq in autoreconf_reqs: - add_buildreq(breq) - else: - config.autoreconf = False + if "configure" in files and os.access(dirpath + '/configure', os.X_OK): + buildpattern.set_build_pattern("configure", default_score) + elif any(is_qmake_pro(f) for f in files): + self.add_buildreq("buildreq-qmake") + buildpattern.set_build_pattern("qmake", default_score) - if buildpattern.default_pattern == "distutils3": - # First look for a local override - pypi_json = "" - pypi_file = os.path.join(dlpath, "pypi.json") - if os.path.isfile(pypi_file): - with open(pypi_file, "r") as pfile: - pypi_json = pfile.read() + if "requires.txt" in files: + self.grab_python_requirements(dirpath + '/requires.txt', config.os_packages) + + if "setup.py" in files: + self.add_buildreq("buildreq-distutils3") + self.add_setup_py_requires(dirpath + '/setup.py', config.os_packages) + python_pattern = get_python_build_version_from_classifier(dirpath + '/setup.py') + buildpattern.set_build_pattern(python_pattern, default_score) + + if "Makefile.PL" in files or "Build.PL" in files: + buildpattern.set_build_pattern("cpan", default_score) + self.add_buildreq("buildreq-cpan") + + if "SConstruct" in files: + self.add_buildreq("buildreq-scons") + buildpattern.set_build_pattern("scons", default_score) + + if "requirements.txt" in files: + self.grab_python_requirements(dirpath + '/requirements.txt', config.os_packages) + + if "meson.build" in files: + self.add_buildreq("buildreq-meson") + buildpattern.set_build_pattern("meson", default_score) + + if "build.xml" in files: + self.add_buildreq("apache-ant") + buildpattern.set_build_pattern("ant", default_score) + + for name in files: + if name.lower() == "cargo.toml" and dirpath == dirn: + self.parse_cargo_toml(os.path.join(dirpath, name), config.os_packages) + if name.lower().startswith("configure."): + self.parse_configure_ac(os.path.join(dirpath, name), config.config_opts.get('32bit')) + if name.lower().startswith("rakefile") and buildpattern.default_pattern == "ruby": + self.rakefile(os.path.join(dirpath, name), config.gems) + if name.endswith(".pro") and buildpattern.default_pattern == "qmake": + self.qmake_profile(os.path.join(dirpath, name), config.qt_modules) + if name.lower() == "makefile": + buildpattern.set_build_pattern("make", default_score) + if name.lower() == "autogen.sh": + buildpattern.set_build_pattern("autogen", default_score) + if name.lower() == "cmakelists.txt": + buildpattern.set_build_pattern("cmake", default_score) + if (name.lower() == "cmakelists.txt" or name.endswith(".cmake")) \ + and buildpattern.default_pattern == "cmake": + self.parse_cmake(os.path.join(dirpath, name), config.cmake_modules, config.config_opts.get('32bit')) + + can_reconf = os.path.exists(os.path.join(dirn, "configure.ac")) + if not can_reconf: + can_reconf = os.path.exists(os.path.join(dirn, "configure.in")) + if can_reconf and config.autoreconf: + print("Patches touch configure.*, adding autoreconf stage") + for breq in self.autoreconf_reqs: + self.add_buildreq(breq) else: - # Try and grab the pypi details for the package - if config.alias: - tname = config.alias - pypi_name = pypidata.get_pypi_name(tname) - pypi_json = pypidata.get_pypi_metadata(pypi_name) - if pypi_json: - try: - package_pypi = json.loads(pypi_json) - except json.JSONDecodeError: - package_pypi = {} - if package_pypi.get("name"): - pypi_provides = package_pypi["name"] - if package_pypi.get("requires"): - pypi_requires = set(package_pypi["requires"]) - if package_pypi.get("license"): - # The license field is freeform, might be worth looking at though - print(f"Pypi says the license is: {package_pypi['license']}") - if package_pypi.get("summary"): - specdescription.assign_summary(package_pypi["summary"], 4) + config.autoreconf = False - print("Buildreqs : ", end="") - for lic in sorted(buildreqs): - if count > 4: - count = 0 - print("\nBuildreqs : ", end="") - count = count + 1 - print(lic + " ", end="") - print("") + if buildpattern.default_pattern == "distutils3": + # First look for a local override + pypi_json = "" + pypi_file = os.path.join(dlpath, "pypi.json") + if os.path.isfile(pypi_file): + with open(pypi_file, "r") as pfile: + pypi_json = pfile.read() + else: + # Try and grab the pypi details for the package + if config.alias: + tname = config.alias + pypi_name = pypidata.get_pypi_name(tname) + pypi_json = pypidata.get_pypi_metadata(pypi_name) + if pypi_json: + try: + package_pypi = json.loads(pypi_json) + except json.JSONDecodeError: + package_pypi = {} + if package_pypi.get("name"): + self.pypi_provides = package_pypi["name"] + if package_pypi.get("requires"): + self.pypi_requires = set(package_pypi["requires"]) + if package_pypi.get("license"): + # The license field is freeform, might be worth looking at though + print(f"Pypi says the license is: {package_pypi['license']}") + if package_pypi.get("summary"): + specdescription.assign_summary(package_pypi["summary"], 4) - -def load_specfile(specfile): - """Load specfile object with necessary buildreq data.""" - specfile.buildreqs = buildreqs - specfile.requires = requires - specfile.cargo_bin = cargo_bin - specfile.extra_cmake += " " + " ".join(extra_cmake) - specfile.pypi_provides = pypi_provides - specfile.pypi_requires = sorted(pypi_requires) - specfile.extra_cmake_openmpi += " " + " ".join(extra_cmake_openmpi) + print("Buildreqs : ", end="") + for lic in sorted(self.buildreqs): + if count > 4: + count = 0 + print("\nBuildreqs : ", end="") + count = count + 1 + print(lic + " ", end="") + print("") diff --git a/autospec/check.py b/autospec/check.py index a1d355f..3da2c16 100644 --- a/autospec/check.py +++ b/autospec/check.py @@ -23,7 +23,6 @@ import os import re import buildpattern -import buildreq import count import tarball import util @@ -55,7 +54,7 @@ def check_regression(pkg_dir, skip_tests): util.write_out(os.path.join(pkg_dir, "testresults"), res_str) -def scan_for_tests(src_dir, config): +def scan_for_tests(src_dir, config, requirements): """Scan source directory for test files and set tests_config accordingly.""" global tests_config @@ -166,11 +165,11 @@ def scan_for_tests(src_dir, config): break if "tox.ini" in files: - buildreq.add_buildreq("tox") - buildreq.add_buildreq("pytest") - buildreq.add_buildreq("virtualenv") - buildreq.add_buildreq("pluggy") - buildreq.add_buildreq("py-python") + requirements.add_buildreq("tox") + requirements.add_buildreq("pytest") + requirements.add_buildreq("virtualenv") + requirements.add_buildreq("pluggy") + requirements.add_buildreq("py-python") def load_specfile(specfile): diff --git a/autospec/config.py b/autospec/config.py index 9db9370..6639b58 100644 --- a/autospec/config.py +++ b/autospec/config.py @@ -28,7 +28,6 @@ import textwrap from collections import OrderedDict import buildpattern -import buildreq import check import license import tarball @@ -394,11 +393,11 @@ class Config(object): os.remove("skip_test_suite") write_config(config_f, path) - def create_buildreq_cache(self, path, version): + def create_buildreq_cache(self, path, version, buildreqs_cache): """Make the buildreq_cache file.""" content = self.read_conf_file(os.path.join(path, "buildreq_cache")) # don't create an empty cache file - if len(buildreq.buildreqs_cache) < 1: + if len(buildreqs_cache) < 1: try: # file was possibly added to git so we should clean it up os.unlink(content) @@ -406,9 +405,9 @@ class Config(object): pass return if not content: - pkgs = sorted(buildreq.buildreqs_cache) + pkgs = sorted(buildreqs_cache) else: - pkgs = sorted(set(content[1:]).union(buildreq.buildreqs_cache)) + pkgs = sorted(set(content[1:]).union(buildreqs_cache)) with open(os.path.join(path, 'buildreq_cache'), "w") as cachefile: cachefile.write("\n".join([version] + pkgs)) self.config_files.add('buildreq_cache') @@ -604,7 +603,7 @@ class Config(object): write_out(filename, wrapper.fill(description) + "\n") - def parse_config_files(self, path, bump, filemanager, version): + def parse_config_files(self, path, bump, filemanager, version, requirements): """Parse the various configuration files that may exist in the package directory.""" packages_file = None @@ -701,35 +700,35 @@ class Config(object): content = self.read_conf_file(os.path.join(path, "buildreq_ban")) for banned in content: print("Banning build requirement: %s." % banned) - buildreq.banned_buildreqs.add(banned) - buildreq.buildreqs.discard(banned) - buildreq.buildreqs_cache.discard(banned) + requirements.banned_buildreqs.add(banned) + requirements.buildreqs.discard(banned) + requirements.buildreqs_cache.discard(banned) content = self.read_conf_file(os.path.join(path, "pkgconfig_ban")) for banned in content: banned = "pkgconfig(%s)" % banned print("Banning build requirement: %s." % banned) - buildreq.banned_buildreqs.add(banned) - buildreq.buildreqs.discard(banned) - buildreq.buildreqs_cache.discard(banned) + requirements.banned_buildreqs.add(banned) + requirements.buildreqs.discard(banned) + requirements.buildreqs_cache.discard(banned) content = self.read_conf_file(os.path.join(path, "requires_ban")) for banned in content: print("Banning runtime requirement: %s." % banned) - buildreq.banned_requires.add(banned) - buildreq.requires.discard(banned) + requirements.banned_requires.add(banned) + requirements.requires.discard(banned) content = self.read_conf_file(os.path.join(path, "buildreq_add")) for extra in content: print("Adding additional build requirement: %s." % extra) - buildreq.add_buildreq(extra) + requirements.add_buildreq(extra) cache_file = os.path.join(path, "buildreq_cache") content = self.read_conf_file(cache_file) if content and content[0] == version: for extra in content[1:]: print("Adding additional build (cache) requirement: %s." % extra) - buildreq.add_buildreq(extra) + requirements.add_buildreq(extra) else: try: os.unlink(cache_file) @@ -742,12 +741,12 @@ class Config(object): for extra in content: extra = "pkgconfig(%s)" % extra print("Adding additional build requirement: %s." % extra) - buildreq.add_buildreq(extra) + requirements.add_buildreq(extra) content = self.read_conf_file(os.path.join(path, "requires_add")) for extra in content: print("Adding additional runtime requirement: %s." % extra) - buildreq.add_requires(extra, self.os_packages, override=True) + requirements.add_requires(extra, self.os_packages, override=True) content = self.read_conf_file(os.path.join(path, "excludes")) for exclude in content: @@ -898,20 +897,20 @@ class Config(object): if self.config_opts['use_clang']: self.config_opts['funroll-loops'] = False - buildreq.add_buildreq("llvm") + requirements.add_buildreq("llvm") if self.config_opts['32bit']: - buildreq.add_buildreq("glibc-libc32") - buildreq.add_buildreq("glibc-dev32") - buildreq.add_buildreq("gcc-dev32") - buildreq.add_buildreq("gcc-libgcc32") - buildreq.add_buildreq("gcc-libstdc++32") + requirements.add_buildreq("glibc-libc32") + requirements.add_buildreq("glibc-dev32") + requirements.add_buildreq("gcc-dev32") + requirements.add_buildreq("gcc-libgcc32") + requirements.add_buildreq("gcc-libstdc++32") if self.config_opts['openmpi']: - buildreq.add_buildreq("openmpi-dev") - buildreq.add_buildreq("modules") + requirements.add_buildreq("openmpi-dev") + requirements.add_buildreq("modules") # MPI testsuites generally require "openssh" - buildreq.add_buildreq("openssh") + requirements.add_buildreq("openssh") self.prep_prepend = self.read_script_file(os.path.join(path, "prep_prepend")) if os.path.isfile(os.path.join(path, "prep_append")): diff --git a/autospec/specfiles.py b/autospec/specfiles.py index b5f333a..6d7e609 100644 --- a/autospec/specfiles.py +++ b/autospec/specfiles.py @@ -25,7 +25,6 @@ import time import types from collections import OrderedDict -import buildreq import tarball from util import _file_write from util import open_auto @@ -34,13 +33,14 @@ from util import open_auto class Specfile(object): """Holds data and methods needed to write the spec file.""" - def __init__(self, url, version, name, release, config): + def __init__(self, url, version, name, release, config, requirements): """Add default information for specfile template.""" self.url = url self.version = version self.name = name self.release = release self.config = config + self.requirements = requirements self.specfile = None self.sources = {"unit": [], "gcov": [], "tmpfile": [], "archive": [], "destination": [], "godep": []} self.source_index = {} @@ -49,10 +49,6 @@ class Specfile(object): self.licenses = [] self.license_files = [] self.packages = OrderedDict() - self.requires = set() - self.buildreqs = [] - self.pypi_provides = None - self.pypi_requires = [] self.default_desc = "" self.locales = [] self.default_pattern = "" @@ -70,7 +66,8 @@ class Specfile(object): self.custom_extras = {} self.keyid = "" self.email = "" - self.cargo_bin = False + self.extra_cmake = config.extra_cmake + " " + " ".join(requirements.extra_cmake) + self.extra_cmake_openmpi = config.extra_cmake_openmpi + " " + " ".join(requirements.extra_cmake_openmpi) def write_spec(self, path): """Write spec file.""" @@ -186,16 +183,16 @@ class Specfile(object): "staticdev32"]: continue # honor requires_ban for manual overrides - if "{}-{}".format(self.name, pkg) in buildreq.banned_requires: + if "{}-{}".format(self.name, pkg) in self.requirements.banned_requires: continue self._write("Requires: {}-{} = %{{version}}-%{{release}}\n".format(self.name, pkg)) - for pkg in sorted(self.requires): + for pkg in sorted(self.requirements.requires): self._write("Requires: {}\n".format(pkg)) def write_buildreq(self): """Write build requirements.""" - for req in sorted(self.buildreqs): + for req in sorted(self.requirements.buildreqs): self._write("BuildRequires : {}\n".format(req)) def write_strip_command(self): @@ -297,9 +294,9 @@ class Specfile(object): if pkg == "python3": self._write("Requires: python3-core\n") - if self.pypi_provides: - self._write(f"Provides: pypi({self.pypi_provides})\n") - for req in self.pypi_requires: + if self.requirements.pypi_provides: + self._write(f"Provides: pypi({self.requirements.pypi_provides})\n") + for req in self.requirements.pypi_requires: self._write(f"Requires: pypi({req})\n") if pkg == "perl": @@ -411,7 +408,7 @@ class Specfile(object): '-DCMAKE_INSTALL_LIBDIR=$MPI_LIB -DCMAKE_INSTALL_INCLUDEDIR=$MPI_INCLUDE -DLIB_INSTALL_DIR=$MPI_LIB \\\n' \ '-DBUILD_SHARED_LIBS:BOOL=ON -DLIB_SUFFIX=64 \\\n' \ '-DCMAKE_AR=/usr/bin/gcc-ar -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_RANLIB=/usr/bin/gcc-ranlib \\\n' - self._write_strip("{} {} {}".format(cmake_string, self.config.cmake_srcdir, self.config.extra_cmake_openmpi)) + self._write_strip("{} {} {}".format(cmake_string, self.config.cmake_srcdir, self.extra_cmake_openmpi)) def write_prep(self, ruby_pattern=False): """Write prep section to spec file.""" @@ -1491,7 +1488,7 @@ class Specfile(object): def write_cmake_pattern(self): """Write cmake pattern to spec file.""" - if self.config.extra_make == "" or self.config.extra_cmake == " ": + if self.config.extra_make == "" or self.extra_cmake == " ": self.config.extra_make = "VERBOSE=1" self.write_prep() self.write_lang_c(export_epoch=True) @@ -1502,7 +1499,7 @@ class Specfile(object): self._write_strip("mkdir -p clr-build") self._write_strip("pushd clr-build") self.write_variables() - self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.config.extra_cmake)) + self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake)) self.write_profile_payload("cmake") @@ -1519,7 +1516,7 @@ class Specfile(object): self.need_avx2_flags = saved_avx2flags self._write_strip('export CFLAGS="$CFLAGS -march=haswell -m64"') self._write_strip('export CXXFLAGS="$CXXFLAGS -march=haswell -m64"') - self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.config.extra_cmake)) + self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake)) self.write_make_line() self._write_strip("popd") @@ -1533,7 +1530,7 @@ class Specfile(object): self.need_avx512_flags = saved_avx512flags self._write_strip('export CFLAGS="$CFLAGS -march=skylake-avx512 -m64 "') self._write_strip('export CXXFLAGS="$CXXFLAGS -march=skylake-avx512 -m64 "') - self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.config.extra_cmake)) + self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake)) self.write_make_line() self._write_strip("popd") @@ -1546,7 +1543,7 @@ class Specfile(object): self._write_strip("%cmake -DLIB_INSTALL_DIR:PATH=/usr/lib32 " "-DCMAKE_INSTALL_LIBDIR=/usr/lib32 " "-DLIB_SUFFIX=32 " - "{} {} ".format(self.config.cmake_srcdir, self.config.extra_cmake)) + "{} {} ".format(self.config.cmake_srcdir, self.extra_cmake)) self.write_make_line() self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("popd") @@ -1636,7 +1633,7 @@ class Specfile(object): self._write_strip("\n") self._write_strip("%install") self.write_install_prepend() - if self.cargo_bin: + if self.requirements.cargo_bin: self._write_strip("cargo install --frozen --root /") self._write_strip("cargo clean") self._write_strip("install -d -p %{buildroot}" + src_dir) diff --git a/autospec/tarball.py b/autospec/tarball.py index b426314..e8c8506 100644 --- a/autospec/tarball.py +++ b/autospec/tarball.py @@ -26,7 +26,6 @@ from collections import OrderedDict import build import buildpattern -import buildreq import download from util import call, do_regex, get_sha1sum, print_fatal, write_out @@ -359,12 +358,6 @@ def name_and_version(name_arg, version_arg, filemanager, config): if not giturl: giturl = "https://github.com/" + match.group(1).strip() + "/" + repo + ".git" - if "gnome.org" in url: - buildreq.add_buildreq("buildreq-gnome") - - if "kde.org" in url or "https://github.com/KDE" in url: - buildreq.add_buildreq("buildreq-kde") - # SQLite tarballs use 7 digit versions, e.g 3290000 = 3.29.0, 3081002 = 3.8.10.2 if "sqlite.org" in url: major = version[0] diff --git a/tests/test_build.py b/tests/test_build.py index d466afd..6a59083 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -3,6 +3,7 @@ import tempfile import os from unittest.mock import patch, mock_open, MagicMock import build +import buildreq import config import files @@ -18,7 +19,6 @@ class TestBuildpattern(unittest.TestCase): build.must_restart = 0 build.base_path = None build.download_path = None - build.buildreq.buildreqs = set() def test_setup_workingdir(self): """ @@ -33,34 +33,40 @@ class TestBuildpattern(unittest.TestCase): """ Test simple_pattern_pkgconfig with match """ + reqs = buildreq.Requirements("") build.simple_pattern_pkgconfig('line to test for testpkg.xyz', r'testpkg.xyz', 'testpkg', - False) - self.assertIn('pkgconfig(testpkg)', build.buildreq.buildreqs) + False, + reqs) + self.assertIn('pkgconfig(testpkg)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_simple_pattern_pkgconfig_32bit(self): """ Test simple_pattern_pkgconfig with match and 32bit option set """ + reqs = buildreq.Requirements("") build.simple_pattern_pkgconfig('line to test for testpkg.zyx', r'testpkg.zyx', 'testpkgz', - True) - self.assertIn('pkgconfig(32testpkgz)', build.buildreq.buildreqs) - self.assertIn('pkgconfig(testpkgz)', build.buildreq.buildreqs) + True, + reqs) + self.assertIn('pkgconfig(32testpkgz)', reqs.buildreqs) + self.assertIn('pkgconfig(testpkgz)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_simple_pattern_pkgconfig_no_match(self): """ Test simple_pattern_pkgconfig with no match, nothing should be modified """ + reqs = buildreq.Requirements("") build.simple_pattern_pkgconfig('line to test for somepkg.xyz', r'testpkg.xyz', 'testpkg', - False) - self.assertEqual(build.buildreq.buildreqs, set()) + False, + reqs) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_simple_pattern(self): @@ -69,20 +75,24 @@ class TestBuildpattern(unittest.TestCase): simple_pattern and simple_pattern_pkgconfig is the string that is added to buildreq.buildreqs. """ + reqs = buildreq.Requirements("") build.simple_pattern('line to test for testpkg.xyz', r'testpkg.xyz', - 'testpkg') - self.assertIn('testpkg', build.buildreq.buildreqs) + 'testpkg', + reqs) + self.assertIn('testpkg', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_simple_pattern_no_match(self): """ Test simple_pattern with no match, nothing should be modified """ + reqs = buildreq.Requirements("") build.simple_pattern('line to test for somepkg.xyz', r'testpkg.xyz', - 'testpkg') - self.assertEqual(build.buildreq.buildreqs, set()) + 'testpkg', + reqs) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_failed_pattern_no_match(self): @@ -90,8 +100,9 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with no match """ conf = config.Config() - build.failed_pattern('line to test for failure: somepkg', conf, r'(test)', 0) - self.assertEqual(build.buildreq.buildreqs, set()) + reqs = buildreq.Requirements("") + build.failed_pattern('line to test for failure: somepkg', conf, reqs, r'(test)', 0) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_failed_pattern_no_buildtool(self): @@ -100,8 +111,9 @@ class TestBuildpattern(unittest.TestCase): match in failed_commands. """ conf = config.Config() - build.failed_pattern('line to test for failure: testpkg', conf, r'(test)', 0) - self.assertEqual(build.buildreq.buildreqs, set()) + reqs = buildreq.Requirements("") + build.failed_pattern('line to test for failure: testpkg', conf, reqs, r'(test)', 0) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_failed_pattern_no_buildtool_match(self): @@ -109,9 +121,10 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool unset and match in failed_commands """ conf = config.Config() + reqs = buildreq.Requirements("") conf.setup_patterns() - build.failed_pattern('line to test for failure: lex', conf, r'(lex)', 0) - self.assertIn('flex', build.buildreq.buildreqs) + build.failed_pattern('line to test for failure: lex', conf, reqs, r'(lex)', 0) + self.assertIn('flex', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_pkgconfig(self): @@ -119,12 +132,14 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to pkgconfig """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.xyz', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='pkgconfig') - self.assertIn('pkgconfig(testpkg)', build.buildreq.buildreqs) + self.assertIn('pkgconfig(testpkg)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_R(self): @@ -133,13 +148,15 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.r', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='R') - self.assertIn('R-testpkg', build.buildreq.buildreqs) - self.assertIn('R-testpkg', build.buildreq.requires) + self.assertIn('R-testpkg', reqs.buildreqs) + self.assertIn('R-testpkg', reqs.requires) self.assertEqual(build.must_restart, 1) def test_failed_pattern_perl(self): @@ -147,12 +164,14 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to perl """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.pl', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='perl') - self.assertIn('perl(testpkg)', build.buildreq.buildreqs) + self.assertIn('perl(testpkg)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_pypi(self): @@ -160,12 +179,14 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to pypi """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.py', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='pypi') - self.assertIn('testpkg-python', build.buildreq.buildreqs) + self.assertIn('testpkg-python', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_ruby(self): @@ -174,12 +195,14 @@ class TestBuildpattern(unittest.TestCase): config.gems, it should just prepend 'rubygem-' to the package name. """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.rb', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='ruby') - self.assertIn('rubygem-testpkg', build.buildreq.buildreqs) + self.assertIn('rubygem-testpkg', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_ruby_gem_match(self): @@ -190,12 +213,14 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: test/unit', conf, + reqs, r'(test/unit)', 0, # verbose=0 buildtool='ruby') - self.assertIn('rubygem-test-unit', build.buildreq.buildreqs) + self.assertIn('rubygem-test-unit', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_ruby_table(self): @@ -205,12 +230,14 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: test/unit', conf, + reqs, r'(test/unit)', 0, # verbose=0 buildtool='ruby table') - self.assertIn('rubygem-test-unit', build.buildreq.buildreqs) + self.assertIn('rubygem-test-unit', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_ruby_table_no_match(self): @@ -219,12 +246,14 @@ class TestBuildpattern(unittest.TestCase): config.gems. This should not modify anything. """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='ruby table') - self.assertEqual(build.buildreq.buildreqs, set()) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_failed_pattern_maven(self): @@ -233,12 +262,14 @@ class TestBuildpattern(unittest.TestCase): config.maven_jars, it should just prepend 'mvn-' to the package name. """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='maven') - self.assertIn('mvn-testpkg', build.buildreq.buildreqs) + self.assertIn('mvn-testpkg', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_maven_match(self): @@ -249,12 +280,14 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: aether', conf, + reqs, r'(aether)', 0, # verbose=0 buildtool='maven') - self.assertIn('mvn-aether-core', build.buildreq.buildreqs) + self.assertIn('mvn-aether-core', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_parse_buildroot_log_fail(self): @@ -337,6 +370,7 @@ class TestBuildpattern(unittest.TestCase): conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") conf.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call @@ -347,12 +381,12 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) build.util.call = call_backup - self.assertIn('pkgconfig(Qt)', build.buildreq.buildreqs) - self.assertIn('pkgconfig(32Qt)', build.buildreq.buildreqs) + self.assertIn('pkgconfig(Qt)', reqs.buildreqs) + self.assertIn('pkgconfig(32Qt)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_parse_build_results_simple_pats(self): @@ -365,6 +399,7 @@ class TestBuildpattern(unittest.TestCase): conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") call_backup = build.util.call build.util.call = mock_util_call fm = files.FileManager(conf) @@ -374,11 +409,11 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) build.util.call = call_backup - self.assertIn('httpd-dev', build.buildreq.buildreqs) + self.assertIn('httpd-dev', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_parse_build_results_failed_pats(self): @@ -388,6 +423,7 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") call_backup = build.util.call open_auto_backup = build.util.open_auto build.util.call = MagicMock(return_value=None) @@ -398,11 +434,11 @@ class TestBuildpattern(unittest.TestCase): for error in builderrors: if not error.startswith('#'): input, output = error.strip('\n').split('|') - build.buildreq.buildreqs = set() + reqs.buildreqs = set() build.util.open_auto = mock_open(read_data=input) - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) - self.assertIn(output, build.buildreq.buildreqs) + self.assertIn(output, reqs.buildreqs) self.assertGreater(build.must_restart, 0) # Restoring functions @@ -418,6 +454,7 @@ class TestBuildpattern(unittest.TestCase): conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") call_backup = build.util.call build.util.call = mock_util_call fm = files.FileManager(conf) @@ -433,7 +470,7 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) build.util.call = call_backup @@ -453,6 +490,7 @@ class TestBuildpattern(unittest.TestCase): conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") call_backup = build.util.call build.util.call = mock_util_call fm = files.FileManager(conf) @@ -470,7 +508,7 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) build.util.call = call_backup diff --git a/tests/test_buildreq.py b/tests/test_buildreq.py index 09d23e3..c4458fe 100644 --- a/tests/test_buildreq.py +++ b/tests/test_buildreq.py @@ -10,116 +10,105 @@ import config class TestBuildreq(unittest.TestCase): - @classmethod - def setUpClass(self): - """ - Class setup method to configure necessary modules - """ - buildreq.banned_buildreqs.add('bannedreq') - def setUp(self): """ Test setup method to reset the buildreq module """ - buildreq.banned_requires = set() - buildreq.buildreqs = set() - buildreq.requires = set() - buildreq.verbose = False - buildreq.cargo_bin = False - buildreq.buildpattern.pattern_strength = 0 + self.reqs = buildreq.Requirements("") + self.reqs.banned_buildreqs.add('bannedreq') def test_add_buildreq(self): """ Test add_buildreq with unbanned new req. Follow up by asserting that trying to add the same req a second time results in a False return. """ - self.assertTrue(buildreq.add_buildreq('testreq')) - self.assertIn('testreq', buildreq.buildreqs) - self.assertFalse(buildreq.add_buildreq('testreq')) + self.assertTrue(self.reqs.add_buildreq('testreq')) + self.assertIn('testreq', self.reqs.buildreqs) + self.assertFalse(self.reqs.add_buildreq('testreq')) def test_add_buildreq_banned(self): """ Test add_buildreq with banned new req """ - self.assertFalse(buildreq.add_buildreq('bannedreq')) - self.assertNotIn('bannedreq', buildreq.buildreqs) + self.assertFalse(self.reqs.add_buildreq('bannedreq')) + self.assertNotIn('bannedreq', self.reqs.buildreqs) def test_add_requires(self): """ Test add_requires with unbanned new req already present in buildreqs but not yet present in requires """ - buildreq.add_buildreq('testreq') - self.assertTrue(buildreq.add_requires('testreq', ['testreq'])) - self.assertIn('testreq', buildreq.requires) + self.reqs.add_buildreq('testreq') + self.assertTrue(self.reqs.add_requires('testreq', ['testreq'])) + self.assertIn('testreq', self.reqs.requires) def test_add_requires_not_in_buildreqs(self): """ Test add_requires with unbanned new req not present in buildreqs. """ - self.assertFalse(buildreq.add_requires('testreq', [])) - self.assertNotIn('testreq', buildreq.requires) + self.assertFalse(self.reqs.add_requires('testreq', [])) + self.assertNotIn('testreq', self.reqs.requires) def test_add_pkgconfig_buildreq(self): """ Test add_pkgconfig_buildreq with config_opts['32bit'] set to False """ - self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', False)) - self.assertIn('pkgconfig(testreq)', buildreq.buildreqs) + self.assertTrue(self.reqs.add_pkgconfig_buildreq('testreq', False)) + self.assertIn('pkgconfig(testreq)', self.reqs.buildreqs) def test_add_pkgconfig_buildreq_32bit(self): """ Test add_pkgconfig_buildreq with config_opts['32bit'] set to True """ - self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', True)) - self.assertIn('pkgconfig(testreq)', buildreq.buildreqs) - self.assertIn('pkgconfig(32testreq)', buildreq.buildreqs) + self.assertTrue(self.reqs.add_pkgconfig_buildreq('testreq', True)) + self.assertIn('pkgconfig(testreq)', self.reqs.buildreqs) + self.assertIn('pkgconfig(32testreq)', self.reqs.buildreqs) def test_configure_ac_line(self): """ Test configure_ac_line with standard pattern """ - buildreq.configure_ac_line('AC_CHECK_FUNC\([tgetent])', False) - self.assertIn('ncurses-devel', buildreq.buildreqs) + self.reqs.configure_ac_line('AC_CHECK_FUNC\([tgetent])', False) + self.assertIn('ncurses-devel', self.reqs.buildreqs) def test_configure_ac_line_comment(self): """ Test configure_ac_line with commented line """ - buildreq.configure_ac_line('# AC_CHECK_FUNC\([tgetent])', False) - self.assertEqual(buildreq.buildreqs, set()) + self.reqs.configure_ac_line('# AC_CHECK_FUNC\([tgetent])', False) + self.assertEqual(self.reqs.buildreqs, set()) def test_configure_ac_line_pkg_check_modules(self): """ Test the somewhat complicated logic of configure_ac_line check for the PKG_CHECK_MODULES\((.*?)\) line. """ - buildreq.configure_ac_line( + self.reqs.configure_ac_line( 'PKG_CHECK_MODULES(prefix, ' '[module > 2 module2 < 2], ' 'action-if-found, action-if-not-found)', False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(module)', 'pkgconfig(module2)'])) def test_configure_ac_line_xdt_check_package(self): """ Test configure_ac_line for the XFCE version of PKG_CHECK_MODULES """ - buildreq.configure_ac_line( + self.reqs.configure_ac_line( 'XDT_CHECK_PACKAGE(prefix, ' '[module = 2 module2 > 9], ' 'action-if-found, action-if-not-found)', False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(module)', 'pkgconfig(module2)'])) def test_configure_ac_line_pkg_check_exists(self): """ Test configure_ac_line for the PKG_CHECK_EXISTS macro """ - buildreq.configure_ac_line('PKG_CHECK_EXISTS([module1 > 1 module2], ' + self.reqs.configure_ac_line('PKG_CHECK_EXISTS([module1 > 1 module2], ' 'action-if-found, ' 'action-if-not-found)', False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(module1)', 'pkgconfig(module2)'])) def test_parse_configure_ac(self): @@ -127,6 +116,7 @@ class TestBuildreq(unittest.TestCase): Test parse_configure_ac with changing () depths and package requirements """ + buildreq.buildpattern.pattern_strength = 0 open_name = 'buildreq.util.open_auto' content = 'AC_CHECK_FUNC([tgetent])\n' \ 'XDT_CHECK_PACKAGE(prefix, ' \ @@ -139,10 +129,10 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_configure_ac(os.path.join(tmpd, 'fname'), False) + self.reqs.parse_configure_ac(os.path.join(tmpd, 'fname'), False) self.assertEqual(buildreq.buildpattern.default_pattern, 'configure_ac') - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['gettext', 'perl(XML::Parser)', 'pkgconfig(module2)', @@ -200,15 +190,16 @@ class TestBuildreq(unittest.TestCase): open_name = 'buildreq.util.open_auto' content = 'does not matter, let us mock' m_open = mock_open(read_data=content) + buildreq.buildpattern.pattern_strength = 0 with patch(open_name, m_open, create=True): - buildreq.parse_cargo_toml('filename', ['dep1', 'dep2', 'dep3']) + self.reqs.parse_cargo_toml('filename', ['dep1', 'dep2', 'dep3']) buildreq.os.path.exists = exists_backup buildreq.toml.loads = loads_backup - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['rustc', 'dep1', 'dep2', 'dep3'])) - self.assertTrue(buildreq.cargo_bin) + self.assertTrue(self.reqs.cargo_bin) self.assertEqual(buildreq.buildpattern.default_pattern, 'cargo') def test_set_build_req_maven(self): @@ -218,8 +209,8 @@ class TestBuildreq(unittest.TestCase): case the overall package list changes in the future. """ buildreq.buildpattern.default_pattern = 'maven' - buildreq.set_build_req() - self.assertIn('apache-maven', buildreq.buildreqs) + self.reqs.set_build_req() + self.assertIn('apache-maven', self.reqs.buildreqs) def test_set_build_req_ruby(self): """ @@ -228,8 +219,8 @@ class TestBuildreq(unittest.TestCase): case the overall package list changes in the future. """ buildreq.buildpattern.default_pattern = 'ruby' - buildreq.set_build_req() - self.assertIn('ruby', buildreq.buildreqs) + self.reqs.set_build_req() + self.assertIn('ruby', self.reqs.buildreqs) def test_set_build_req_cargo(self): """ @@ -238,8 +229,8 @@ class TestBuildreq(unittest.TestCase): case the overall package list changes in the future. """ buildreq.buildpattern.default_pattern = 'cargo' - buildreq.set_build_req() - self.assertIn('rustc', buildreq.buildreqs) + self.reqs.set_build_req() + self.assertIn('rustc', self.reqs.buildreqs) def test_rakefile(self): """ @@ -251,9 +242,9 @@ class TestBuildreq(unittest.TestCase): content = "line1\nrequire 'bundler/gem_tasks'\nline3\nrequire 'nope'" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.rakefile('filename', conf.gems) + self.reqs.rakefile('filename', conf.gems) - self.assertEqual(buildreq.buildreqs, set(['rubygem-rubygems-tasks'])) + self.assertEqual(self.reqs.buildreqs, set(['rubygem-rubygems-tasks'])) def test_clean_python_req(self): """ @@ -282,32 +273,32 @@ class TestBuildreq(unittest.TestCase): Test grab_python_requirements with a reasonable requirements file """ # buildreqs must include the requires also - buildreq.buildreqs = set(['req1', 'req2', 'req7']) + self.reqs.buildreqs = set(['req1', 'req2', 'req7']) open_name = 'buildreq.util.open_auto' content = 'req1 <= 1.2.3\n' \ 'req2 >= 1.55\n' \ 'req7 == 3.3.3\n' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3']) + self.reqs.grab_python_requirements('filename', ['req1', 'req2', 'req3']) - self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2', 'req7'])) def test_grab_python_requirements_strange_file(self): """ Test grab_python_requirements with a poorly written file """ # buildreqs must include the requires also - buildreq.buildreqs = set(['req1', 'req2', 'req7']) + self.reqs.buildreqs = set(['req1', 'req2', 'req7']) open_name = 'buildreq.util.open_auto' content = ' req1 <= 1.2.3\n ' \ 'req2 >= 1.55 \n' \ ' req7 == 3.3.3\n ' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3']) + self.reqs.grab_python_requirements('filename', ['req1', 'req2', 'req3']) - self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2', 'req7'])) def test_add_setup_py_requires(self): """ @@ -319,10 +310,10 @@ class TestBuildreq(unittest.TestCase): "setup_requires=['req2']" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) - self.assertEqual(buildreq.requires, set(['req1'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2'])) + self.assertEqual(self.reqs.requires, set(['req1'])) def test_add_setup_py_requires_multiline(self): """ @@ -334,10 +325,10 @@ class TestBuildreq(unittest.TestCase): "'req3']\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3'])) - self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2', 'req3'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2', 'req3'])) def test_add_setup_py_requires_multiline_formatted(self): """ @@ -352,10 +343,10 @@ class TestBuildreq(unittest.TestCase): "]\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3'])) - self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2', 'req3'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2', 'req3'])) def test_add_setup_py_requires_multiline_variable(self): """ @@ -370,10 +361,10 @@ class TestBuildreq(unittest.TestCase): "]\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) - self.assertEqual(buildreq.requires, set(['req1', 'req2'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2'])) def test_add_setup_py_requires_variable(self): """ @@ -383,10 +374,10 @@ class TestBuildreq(unittest.TestCase): content = "install_requires=[reqname, 'req1', 'req2']\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) - self.assertEqual(buildreq.requires, set(['req1', 'req2'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2'])) def test_add_setup_py_requires_single_variable(self): """ @@ -396,10 +387,10 @@ class TestBuildreq(unittest.TestCase): content = "install_requires=reqname" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', []) + self.reqs.add_setup_py_requires('filename', []) - self.assertEqual(buildreq.buildreqs, set()) - self.assertEqual(buildreq.requires, set()) + self.assertEqual(self.reqs.buildreqs, set()) + self.assertEqual(self.reqs.requires, set()) def test_setup_py3_version_classifier(self): """ @@ -432,9 +423,9 @@ class TestBuildreq(unittest.TestCase): open(os.path.join(tmpd, 'SConstruct'), 'w').close() open(os.path.join(tmpd, 'meson.build'), 'w').close() - buildreq.scan_for_configure(tmpd, "", "", conf) + self.reqs.scan_for_configure(tmpd, "", "", conf) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['buildreq-golang', 'buildreq-cmake', 'buildreq-scons', 'buildreq-distutils3', 'buildreq-meson'])) def test_scan_for_configure_pypi(self): @@ -443,6 +434,7 @@ class TestBuildreq(unittest.TestCase): pattern to test pypi metadata handling. """ conf = config.Config() + buildreq.buildpattern.pattern_strength = 0 orig_summary = buildreq.specdescription.default_summary orig_sscore = buildreq.specdescription.default_summary_score orig_pypi_name = buildreq.pypidata.get_pypi_name @@ -459,7 +451,7 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: os.mkdir(os.path.join(tmpd, 'subdir')) open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close() - buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) + self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) ssummary = buildreq.specdescription.default_summary buildreq.specdescription.default_summary = orig_summary @@ -467,8 +459,8 @@ class TestBuildreq(unittest.TestCase): buildreq.pypidata.get_pypi_name = orig_pypi_name buildreq.pypidata.get_pypi_metadata = orig_pypi_meta - self.assertEqual(buildreq.pypi_provides, name) - self.assertEqual(buildreq.pypi_requires, set(requires)) + self.assertEqual(self.reqs.pypi_provides, name) + self.assertEqual(self.reqs.pypi_requires, set(requires)) self.assertEqual(ssummary, summary) def test_scan_for_configure_pypi_override(self): @@ -477,6 +469,7 @@ class TestBuildreq(unittest.TestCase): pattern to test pypi metadata file override handling. """ conf = config.Config() + buildreq.buildpattern.pattern_strength = 0 open_name = 'buildreq.open' orig_summary = buildreq.specdescription.default_summary orig_sscore = buildreq.specdescription.default_summary_score @@ -492,14 +485,14 @@ class TestBuildreq(unittest.TestCase): open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close() open(os.path.join(tmpd, 'pypi.json'), 'w').close() with patch(open_name, m_open, create=True): - buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) + self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) ssummary = buildreq.specdescription.default_summary buildreq.specdescription.default_summary = orig_summary buildreq.specdescription.default_summary_score = orig_sscore - self.assertEqual(buildreq.pypi_provides, name) - self.assertEqual(buildreq.pypi_requires, set(requires)) + self.assertEqual(self.reqs.pypi_provides, name) + self.assertEqual(self.reqs.pypi_requires, set(requires)) self.assertEqual(ssummary, summary) def test_parse_cmake_pkg_check_modules(self): @@ -513,9 +506,9 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)'])) def test_parse_cmake_pkg_check_modules_whitespace(self): @@ -529,9 +522,9 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)'])) def test_parse_cmake_pkg_check_modules_in_a_comment(self): @@ -548,9 +541,9 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set([])) def test_parse_cmake_pkg_check_modules_variables(self): @@ -564,9 +557,9 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(libavcodec)', 'pkgconfig(libavutil)'])) def test_parse_cmake_find_package(self): @@ -587,9 +580,9 @@ find_package(different_name) with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['valid', 'another_name'])) def test_r_desc_field_begin(self): @@ -695,8 +688,8 @@ find_package(different_name) content = 'Depends: pkg1' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertTrue('R-pkg1' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertTrue('R-pkg1' in self.reqs.buildreqs) def test_parse_r_desc_imports(self): """Test parsing of an R description Imports field.""" @@ -705,8 +698,8 @@ find_package(different_name) content = 'Imports: pkg2' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertTrue('R-pkg2' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertTrue('R-pkg2' in self.reqs.buildreqs) def test_parse_r_desc_linkingto(self): """Test parsing of an R description LinkingTo field.""" @@ -715,8 +708,8 @@ find_package(different_name) content = 'LinkingTo: pkg3' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertTrue('R-pkg3' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertTrue('R-pkg3' in self.reqs.buildreqs) def test_parse_r_desc_multiple(self): """Test parsing of an R description file that captures multiple fields.""" @@ -736,13 +729,13 @@ find_package(different_name) ] m_open = mock_open(read_data='\n'.join(content)) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertFalse('R-foo' in buildreq.buildreqs) - self.assertFalse('R-bar' in buildreq.buildreqs) - self.assertTrue('R-pkg1' in buildreq.buildreqs) - self.assertTrue('R-pkg2' in buildreq.buildreqs) - self.assertTrue('R-pkg3' in buildreq.buildreqs) - self.assertTrue('R-pkg4' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertFalse('R-foo' in self.reqs.buildreqs) + self.assertFalse('R-bar' in self.reqs.buildreqs) + self.assertTrue('R-pkg1' in self.reqs.buildreqs) + self.assertTrue('R-pkg2' in self.reqs.buildreqs) + self.assertTrue('R-pkg3' in self.reqs.buildreqs) + self.assertTrue('R-pkg4' in self.reqs.buildreqs) def test_parse_r_desc_not_in_os(self): """Test parsing of an R description file with some non-OS packages.""" @@ -756,10 +749,10 @@ find_package(different_name) ] m_open = mock_open(read_data='\n'.join(content)) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertTrue('R-pkg1' in buildreq.buildreqs) - self.assertFalse('R-pkg2' in buildreq.buildreqs) - self.assertFalse('R-pkg3' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertTrue('R-pkg1' in self.reqs.buildreqs) + self.assertFalse('R-pkg2' in self.reqs.buildreqs) + self.assertFalse('R-pkg3' in self.reqs.buildreqs) if __name__ == '__main__': diff --git a/tests/test_check.py b/tests/test_check.py index f8925f6..ba153a0 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -3,6 +3,7 @@ import tempfile import unittest from unittest.mock import mock_open, patch +import buildreq import check import config @@ -28,7 +29,6 @@ class TestTest(unittest.TestCase): def setUp(self): check.tests_config = '' check.tarball.name = '' - check.buildreq.buildreqs = set() check.buildpattern.default_pattern = "make" def test_check_regression(self): @@ -88,6 +88,7 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with makecheck suite """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.in']) @@ -95,7 +96,7 @@ class TestTest(unittest.TestCase): m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "configure" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -106,13 +107,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with makecheck suite via Makefile.am """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.am']) m_open = mock_open() with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "configure_ac" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -123,11 +125,12 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with perlcheck suite """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.PL']) check.buildpattern.default_pattern = "cpan" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test') @@ -136,6 +139,7 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with perlcheck suite via Makefile.in """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.in']) @@ -143,7 +147,7 @@ class TestTest(unittest.TestCase): m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "cpan" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -153,6 +157,7 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with setup.py suite """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['setup.py']) @@ -160,7 +165,7 @@ class TestTest(unittest.TestCase): m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "distutils3" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -172,6 +177,7 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with cmake suite """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['CMakeLists.txt']) @@ -179,7 +185,7 @@ class TestTest(unittest.TestCase): m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "cmake" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -191,12 +197,13 @@ class TestTest(unittest.TestCase): Test scan_for_tests with tox.ini in the files list, should add several build requirements """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['tox.ini']) - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup - self.assertEqual(check.buildreq.buildreqs, + self.assertEqual(reqs.buildreqs, set(['tox', 'pytest', 'virtualenv', diff --git a/tests/test_infile_update_spec.py b/tests/test_infile_update_spec.py index b10d2a2..a1a60a2 100644 --- a/tests/test_infile_update_spec.py +++ b/tests/test_infile_update_spec.py @@ -1,5 +1,6 @@ import unittest +import buildreq import config import infile_update_spec import specfiles @@ -9,7 +10,7 @@ class TestUpdateSpecfile(unittest.TestCase): def setUp(self): # url, version, name, release url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz" - self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1', config.Config()) + self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1', config.Config(), buildreq.Requirements(url)) self.bb_dict = { "DEPENDS": "ncurses gettext-native", diff --git a/tests/test_specfile.py b/tests/test_specfile.py index 9868bc7..2d51ca9 100644 --- a/tests/test_specfile.py +++ b/tests/test_specfile.py @@ -1,5 +1,6 @@ import unittest import unittest.mock +import buildreq import config import specfiles @@ -15,7 +16,8 @@ class TestSpecfileWrite(unittest.TestCase): conf = config.Config() conf.config_opts['dev_requires_extras'] = False url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz" - self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2', conf) + reqs = buildreq.Requirements(url) + self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2', conf, reqs) def mock_write(string): self.WRITES.append(string) @@ -100,8 +102,8 @@ class TestSpecfileWrite(unittest.TestCase): self.specfile.packages["autostart"] = ["autostart"] self.specfile.packages["bin"] = [] self.specfile.packages["lib"] = ["package.so"] - self.specfile.requires.add("pkg1") - self.specfile.requires.add("pkg2") + self.specfile.requirements.requires.add("pkg1") + self.specfile.requirements.requires.add("pkg2") self.specfile.config.config_opts['no_autostart'] = True self.specfile.write_main_subpackage_requires() expect = ["Requires: pkg-bin = %{version}-%{release}\n", @@ -122,8 +124,8 @@ class TestSpecfileWrite(unittest.TestCase): self.specfile.packages["ignore"] = [] self.specfile.packages["dev"] = [] self.specfile.packages["active-units"] = [] - self.specfile.requires.add("pkg1") - self.specfile.requires.add("pkg2") + self.specfile.requirements.requires.add("pkg1") + self.specfile.requirements.requires.add("pkg2") self.specfile.write_main_subpackage_requires() expect = ["Requires: pkg-autostart = %{version}-%{release}\n", "Requires: pkg-bin = %{version}-%{release}\n", @@ -143,7 +145,7 @@ class TestSpecfileWrite(unittest.TestCase): """ test write_buildreq with unsorted list of build requirements. """ - self.specfile.buildreqs = ["python", "ruby", "go"] + self.specfile.requirements.buildreqs = ["python", "ruby", "go"] self.specfile.write_buildreq() expect = ["BuildRequires : go\n", "BuildRequires : python\n", @@ -195,8 +197,8 @@ class TestSpecfileWrite(unittest.TestCase): self.specfile.packages["python"] = ["pyfile1", "pyfile2"] self.specfile.packages["dev"] = ["dev1", "dev2"] self.specfile.packages["pack"] = ["packf1"] - self.specfile.requires = ["pep8", "pylint", "pycurl"] - self.specfile.buildreqs = ["pep8", "pycurl"] + self.specfile.requirements.requires = ["pep8", "pylint", "pycurl"] + self.specfile.requirements.buildreqs = ["pep8", "pycurl"] self.specfile.write_files_header() expect = ["\n%package data\n", "Summary: data components for the pkg package.\n",