diff --git a/autospec/autospec.py b/autospec/autospec.py index 1791c94..8444b0d 100644 --- a/autospec/autospec.py +++ b/autospec/autospec.py @@ -90,7 +90,6 @@ def load_specfile(conf, specfile): tarball.load_specfile(specfile) specdescription.load_specfile(specfile, conf.custom_desc, conf.custom_summ) license.load_specfile(specfile) - buildreq.load_specfile(specfile) buildpattern.load_specfile(specfile) check.load_specfile(specfile) @@ -267,7 +266,9 @@ def package(args, url, name, archives, workingdir, infile_dict): conf.setup_patterns() conf.config_file = args.config - conf.parse_config_files(build.download_path, args.bump, filemanager, tarball.version) + requirements = buildreq.Requirements(tarball.url) + requirements.set_build_req() + conf.parse_config_files(build.download_path, args.bump, filemanager, tarball.version, requirements) conf.setup_patterns(conf.failed_pattern_dir) conf.parse_existing_spec(build.download_path, tarball.name) @@ -275,21 +276,25 @@ def package(args, url, name, archives, workingdir, infile_dict): write_prep(conf, workingdir) exit(0) - buildreq.set_build_req() - buildreq.scan_for_configure(_dir, tarball.name, build.download_path, conf) + requirements.scan_for_configure(_dir, tarball.name, build.download_path, conf) specdescription.scan_for_description(tarball.name, _dir, conf.license_translations, conf.license_blacklist) # Start one directory higher so we scan *all* versions for licenses license.scan_for_licenses(os.path.dirname(_dir), conf) commitmessage.scan_for_changes(build.download_path, _dir, conf.transforms) add_sources(build.download_path, archives) - check.scan_for_tests(_dir, conf) + check.scan_for_tests(_dir, conf, requirements) # # Now, we have enough to write out a specfile, and try to build it. # We will then analyze the build result and learn information until the # package builds # - specfile = specfiles.Specfile(tarball.url, tarball.version, tarball.name, tarball.release, conf) + specfile = specfiles.Specfile(tarball.url, + tarball.version, + tarball.name, + tarball.release, + conf, + requirements) filemanager.load_specfile(specfile) load_specfile(conf, specfile) @@ -308,7 +313,7 @@ def package(args, url, name, archives, workingdir, infile_dict): specfile.write_spec(build.download_path) while 1: - build.package(filemanager, args.mock_config, args.mock_opts, conf, args.cleanup) + build.package(filemanager, args.mock_config, args.mock_opts, conf, requirements, args.cleanup) filemanager.load_specfile(specfile) specfile.write_spec(build.download_path) filemanager.newfiles_printed = 0 @@ -329,7 +334,7 @@ def package(args, url, name, archives, workingdir, infile_dict): check.check_regression(build.download_path, conf.config_opts['skip_tests']) if build.success == 0: - conf.create_buildreq_cache(build.download_path, tarball.version) + conf.create_buildreq_cache(build.download_path, tarball.version, requirements.buildreqs_cache) print_fatal("Build failed, aborting") sys.exit(1) elif os.path.isfile("README.clear"): @@ -353,7 +358,7 @@ def package(args, url, name, archives, workingdir, infile_dict): logcheck(build.download_path) commitmessage.guess_commit_message(pkg_integrity.IMPORTED, conf) - conf.create_buildreq_cache(build.download_path, tarball.version) + conf.create_buildreq_cache(build.download_path, tarball.version, requirements.buildreqs_cache) if args.git: git.commit_to_git(build.download_path, conf) diff --git a/autospec/build.py b/autospec/build.py index 0579a9f..ffc9a4b 100644 --- a/autospec/build.py +++ b/autospec/build.py @@ -23,7 +23,6 @@ import os import re import shutil -import buildreq import tarball import util @@ -44,22 +43,22 @@ def setup_workingdir(workingdir): download_path = os.path.join(base_path, tarball.name) -def simple_pattern_pkgconfig(line, pattern, pkgconfig, conf32): +def simple_pattern_pkgconfig(line, pattern, pkgconfig, conf32, requirements): """Check for pkgconfig patterns and restart build as needed.""" global must_restart pat = re.compile(pattern) match = pat.search(line) if match: - must_restart += buildreq.add_pkgconfig_buildreq(pkgconfig, conf32, cache=True) + must_restart += requirements.add_pkgconfig_buildreq(pkgconfig, conf32, cache=True) -def simple_pattern(line, pattern, req): +def simple_pattern(line, pattern, req, requirements): """Check for simple patterns and restart the build as needed.""" global must_restart pat = re.compile(pattern) match = pat.search(line) if match: - must_restart += buildreq.add_buildreq(req, cache=True) + must_restart += requirements.add_buildreq(req, cache=True) def cleanup_req(s: str) -> str: @@ -96,7 +95,7 @@ def cleanup_req(s: str) -> str: return s -def failed_pattern(line, config, pattern, verbose, buildtool=None): +def failed_pattern(line, config, requirements, pattern, verbose, buildtool=None): """Check against failed patterns to restart build as needed.""" global must_restart global warned_about @@ -116,29 +115,29 @@ def failed_pattern(line, config, pattern, verbose, buildtool=None): if not buildtool: req = config.failed_commands[s] if req: - must_restart += buildreq.add_buildreq(req, cache=True) + must_restart += requirements.add_buildreq(req, cache=True) elif buildtool == 'pkgconfig': - must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) + must_restart += requirements.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) elif buildtool == 'R': - if buildreq.add_buildreq("R-" + s, cache=True) > 0: + if requirements.add_buildreq("R-" + s, cache=True) > 0: must_restart += 1 - buildreq.add_requires("R-" + s, config.os_packages) + requirements.add_requires("R-" + s, config.os_packages) elif buildtool == 'perl': s = s.replace('inc::', '') - must_restart += buildreq.add_buildreq('perl(%s)' % s, cache=True) + must_restart += requirements.add_buildreq('perl(%s)' % s, cache=True) elif buildtool == 'pypi': s = util.translate(s) if not s: return - must_restart += buildreq.add_buildreq(util.translate('%s-python' % s), cache=True) + must_restart += requirements.add_buildreq(util.translate('%s-python' % s), cache=True) elif buildtool == 'ruby': if s in config.gems: - must_restart += buildreq.add_buildreq(config.gems[s], cache=True) + must_restart += requirements.add_buildreq(config.gems[s], cache=True) else: - must_restart += buildreq.add_buildreq('rubygem-%s' % s, cache=True) + must_restart += requirements.add_buildreq('rubygem-%s' % s, cache=True) elif buildtool == 'ruby table': if s in config.gems: - must_restart += buildreq.add_buildreq(config.gems[s], cache=True) + must_restart += requirements.add_buildreq(config.gems[s], cache=True) else: print("Unknown ruby gem match", s) elif buildtool == 'maven' or buildtool == 'gradle': @@ -149,10 +148,10 @@ def failed_pattern(line, config, pattern, verbose, buildtool=None): # Hyphens are disallowed for version strings, so use dots instead ver = match.group(2).replace('-', '.') mvn_provide = f'mvn({name}) = {ver}' - must_restart += buildreq.add_buildreq(mvn_provide, cache=True) + must_restart += requirements.add_buildreq(mvn_provide, cache=True) elif s in config.maven_jars: # Overrides for dependencies with custom grouping - must_restart += buildreq.add_buildreq(config.maven_jars[s], cache=True) + must_restart += requirements.add_buildreq(config.maven_jars[s], cache=True) elif group_count == 3: org = match.group(1) name = match.group(2) @@ -161,13 +160,13 @@ def failed_pattern(line, config, pattern, verbose, buildtool=None): mvn_provide = f'mvn({org}:{name}:pom) = {ver}' else: mvn_provide = f'mvn({org}:{name}:jar) = {ver}' - must_restart += buildreq.add_buildreq(mvn_provide, cache=True) + must_restart += requirements.add_buildreq(mvn_provide, cache=True) else: # Fallback to mvn-ARTIFACTID package name - must_restart += buildreq.add_buildreq('mvn-%s' % s, cache=True) + must_restart += requirements.add_buildreq('mvn-%s' % s, cache=True) elif buildtool == 'catkin': - must_restart += buildreq.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) - must_restart += buildreq.add_buildreq(s, cache=True) + must_restart += requirements.add_pkgconfig_buildreq(s, config.config_opts.get('32bit'), cache=True) + must_restart += requirements.add_buildreq(s, cache=True) except Exception: if s not in warned_about and s[:2] != '--': @@ -206,11 +205,11 @@ def check_for_warning_pattern(line): util.print_warning("Build log contains: {}".format(pat)) -def parse_build_results(filename, returncode, filemanager, config): +def parse_build_results(filename, returncode, filemanager, config, requirements): """Handle build log contents.""" global must_restart global success - buildreq.verbose = 1 + requirements.verbose = 1 must_restart = 0 infiles = 0 @@ -221,13 +220,13 @@ def parse_build_results(filename, returncode, filemanager, config): for line in loglines: for pat in config.pkgconfig_pats: - simple_pattern_pkgconfig(line, *pat, config.config_opts.get('32bit')) + simple_pattern_pkgconfig(line, *pat, config.config_opts.get('32bit'), requirements) for pat in config.simple_pats: - simple_pattern(line, *pat) + simple_pattern(line, *pat, requirements) for pat in config.failed_pats: - failed_pattern(line, config, *pat) + failed_pattern(line, config, requirements, *pat) check_for_warning_pattern(line) @@ -277,7 +276,7 @@ def get_mock_cmd(): return 'sudo /usr/bin/mock' -def package(filemanager, mockconfig, mockopts, config, cleanup=False): +def package(filemanager, mockconfig, mockopts, config, requirements, cleanup=False): """Run main package build routine.""" global round global uniqueext @@ -343,7 +342,7 @@ def package(filemanager, mockconfig, mockopts, config, cleanup=False): is_clean = parse_buildroot_log(download_path + "/results/root.log", ret) if is_clean: - parse_build_results(download_path + "/results/build.log", ret, filemanager, config) + parse_build_results(download_path + "/results/build.log", ret, filemanager, config, requirements) if filemanager.has_banned: util.print_fatal("Content in banned paths found, aborting build") exit(1) diff --git a/autospec/buildreq.py b/autospec/buildreq.py index 9e8f90b..f283f2b 100644 --- a/autospec/buildreq.py +++ b/autospec/buildreq.py @@ -30,147 +30,61 @@ import specdescription import toml import util -banned_requires = set(["futures", - "configparser"]) -buildreqs = set() -buildreqs_cache = set() -requires = set() -extra_cmake = set() -extra_cmake_openmpi = set() -verbose = False -cargo_bin = False -pypi_provides = None -pypi_requires = set() -banned_buildreqs = set(["llvm-devel", - "gcj", - "pkgconfig(dnl)", - "pkgconfig(hal)", - "tslib-0.0", - "pkgconfig(parallels-sdk)", - "oslo-python", - "libxml2No-python", - "futures", - "configparser"]) -autoreconf_reqs = ["gettext-bin", - "automake-dev", - "automake", - "m4", - "libtool", - "libtool-dev", - "pkg-config-dev"] + +def is_qmake_pro(f): + """Test if file extension is pro and not hidden.""" + return f.endswith(".pro") and not f.startswith(".") -def add_buildreq(req, cache=False): - """Add req to the global buildreqs set if req is not banned.""" - global buildreqs - global buildreqs_cache - new = True +def get_python_build_version_from_classifier(filename): + """Detect if setup should use distutils3 only. - req.strip() + Uses "Programming Language :: Python :: [2,3] :: Only" classifiers in the + setup.py file. Defaults to distutils3 if no such classifiers are found. + """ + with util.open_auto(filename) as setup_file: + data = setup_file.read() - if req in banned_buildreqs: - return False - if req in buildreqs: - new = False - if verbose and new: - print(" Adding buildreq:", req) + if "Programming Language :: Python :: 3 :: Only" in data: + return "distutils3" - buildreqs.add(req) - if cache and new: - buildreqs_cache.add(req) - return new + return "distutils3" -def add_requires(req, packages, override=False): - """Add req to the global requires set if it is present in buildreqs and packages and is not banned.""" - global buildreqs - global requires - new = True - req = req.strip() - if req in requires: - new = False - if req in banned_requires: - return False +def clean_python_req(req, add_python=True): + """Strip version information from req.""" + if req.find("#") == 0: + return "" + ret = req.rstrip("\n\r").strip() + i = ret.find(";") + if i > 0: + ret = ret[:i] + i = ret.find("<") + if i > 0: + ret = ret[:i] + i = ret.find("\n") + if i > 0: + ret = ret[:i] + i = ret.find(">") + if i > 0: + ret = ret[:i] + i = ret.find("=") + if i > 0: + ret = ret[:i] + i = ret.find("#") + if i > 0: + ret = ret[:i] + i = ret.find("!") + if i > 0: + ret = ret[:i] - # Try dashes instead of underscores as some ecosystems are inconsistent in their naming - req2 = req.replace("_", "-") - if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires: - # Since this is done for python add a buildreq just in case (might not be correct though) - buildreqs.add(req2) - requires.add(req2) - return True - - # Try reversing the case of the first letter as some ecosystems are inconsistent in their naming - if len(req) > 1: - if req[0].isupper(): - req2 = req[0].lower() + req[1:] - else: - req2 = req[0].upper() + req[1:] - if req not in buildreqs and req2 in packages and req2 not in requires and req2 not in banned_requires: - # Since this is done for python add a buildreq just in case (might not be correct though) - buildreqs.add(req2) - requires.add(req2) - return True - - if req not in buildreqs and req not in packages and not override: - if req: - print("requirement '{}' not found in buildreqs or os_packages, skipping".format(req)) - return False - if new: - # print("Adding requirement:", req) - requires.add(req) - return new - - -def add_pkgconfig_buildreq(preq, conf32, cache=False): - """Format preq as pkgconfig req and add to buildreqs.""" - if conf32: - req = "pkgconfig(32" + preq + ")" - add_buildreq(req, cache) - req = "pkgconfig(" + preq + ")" - return add_buildreq(req, cache) - - -def configure_ac_line(line, conf32): - """Parse configure_ac line and add appropriate buildreqs.""" - # print("----\n", line, "\n----") - # ignore comments - if line.startswith('#'): - return - - pat_reqs = [(r"AC_CHECK_FUNC\([tgetent]", ["ncurses-devel"]), - ("PROG_INTLTOOL", ["intltool"]), - ("GETTEXT_PACKAGE", ["gettext", "perl(XML::Parser)"]), - ("AM_GLIB_GNU_GETTEXT", ["gettext", "perl(XML::Parser)"]), - ("GTK_DOC_CHECK", ["gtk-doc", "gtk-doc-dev", "libxslt-bin", "docbook-xml"]), - ("AC_PROG_SED", ["sed"]), - ("AC_PROG_GREP", ["grep"])] - - for pat, reqs in pat_reqs: - if pat in line: - for req in reqs: - add_buildreq(req) - - line = line.strip() - - # XFCE uses an equivalent to PKG_CHECK_MODULES, handle them both the same - for style in [r"PKG_CHECK_MODULES\((.*?)\)", r"XDT_CHECK_PACKAGE\((.*?)\)"]: - match = re.search(style, line) - L = [] - if match: - L = match.group(1).split(",") - if len(L) > 1: - rqlist = L[1].strip() - for req in parse_modules_list(rqlist): - add_pkgconfig_buildreq(req, conf32) - - # PKG_CHECK_EXISTS(MODULES, action-if-found, action-if-not-found) - match = re.search(r"PKG_CHECK_EXISTS\((.*?)\)", line) - if match: - L = match.group(1).split(",") - rqlist = L[0].strip() - for req in parse_modules_list(rqlist): - add_pkgconfig_buildreq(req, conf32) + ret = ret.strip() + # is ret actually a valid (non-empty) string? + if ret and add_python: + ret = ret.strip() + # use the dictionary to translate funky names to our current pgk names + ret = util.translate(ret) + return ret def is_number(num_str): @@ -190,6 +104,39 @@ def is_version(num_str): return False +def parse_modules_list(modules_string, is_cmake=False): + """Parse the modules_string for the list of modules, stripping out the version requirements.""" + if is_cmake: + modules = [m for m in re.split(r'\s*([><]?=|\${?[^}]*}?)\s*', modules_string)] + modules = filter(None, modules) + else: + modules = [m.strip('[]') for m in modules_string.split()] + res = [] + next_is_ver = False + for mod in modules: + if next_is_ver: + next_is_ver = False + continue + + if any(s in mod for s in ['<', '>', '=']): + next_is_ver = True + continue + + if is_number(mod): + continue + + if is_version(mod): + continue + + if mod.startswith('$'): + continue + + if len(mod) >= 2: + res.append(mod) + + return res + + def parse_go_mod(path): """Parse go.mod file for build requirements. @@ -234,83 +181,6 @@ def parse_go_mod(path): return reqs -def parse_modules_list(modules_string, is_cmake=False): - """Parse the modules_string for the list of modules, stripping out the version requirements.""" - if is_cmake: - modules = [m for m in re.split(r'\s*([><]?=|\${?[^}]*}?)\s*', modules_string)] - modules = filter(None, modules) - else: - modules = [m.strip('[]') for m in modules_string.split()] - res = [] - next_is_ver = False - for mod in modules: - if next_is_ver: - next_is_ver = False - continue - - if any(s in mod for s in ['<', '>', '=']): - next_is_ver = True - continue - - if is_number(mod): - continue - - if is_version(mod): - continue - - if mod.startswith('$'): - continue - - if len(mod) >= 2: - res.append(mod) - - return res - - -def parse_configure_ac(filename, conf32): - """Parse the configure.ac file for build requirements.""" - buf = "" - depth = 0 - # print("Configure parse: ", filename) - buildpattern.set_build_pattern("configure_ac", 1) - f = util.open_auto(filename, "r") - while 1: - c = f.read(1) - if not c: - break - if c == "(": - depth += 1 - if c == ")" and depth > 0: - depth -= 1 - if c != "\n": - buf += c - if c == "\n" and depth == 0: - configure_ac_line(buf, conf32) - buf = "" - - configure_ac_line(buf, conf32) - f.close() - - -def parse_cargo_toml(filename, packages): - """Update build requirements using Cargo.toml. - - Set the build requirements for building rust programs using cargo. - """ - global cargo_bin - buildpattern.set_build_pattern("cargo", 1) - add_buildreq("rustc") - with util.open_auto(filename, "r") as ctoml: - cargo = toml.loads(ctoml.read()) - if cargo.get("bin") or os.path.exists(os.path.join(os.path.dirname(filename), "src/main.rs")): - cargo_bin = True - if not cargo.get("dependencies"): - return - for cdep in cargo["dependencies"]: - if add_buildreq(cdep): - add_requires(cdep, packages) - - def _get_desc_field(field, desc): """Get a field value from an R package DESCRIPTION file. @@ -400,541 +270,642 @@ def _get_r_provides(): return set(provides) -def parse_r_description(filename, packages): - """Update build/runtime requirements according to the R package description.""" - deps = [] - with util.open_auto(filename, "r") as desc: - content = desc.read() - deps = _get_desc_field("Depends", content) - deps.extend(_get_desc_field("Imports", content)) - deps.extend(_get_desc_field("LinkingTo", content)) - r_provides = _get_r_provides() - for dep in deps: - if dep == 'R': - continue - if dep in r_provides: - continue - pkg = 'R-' + dep - if pkg in packages: - add_buildreq(pkg) - add_requires(pkg, packages) - else: - print("CRAN package '{}' not found in os_packages, skipping".format(pkg)) +class Requirements(object): + """Handle package build and runtime requiremnts.""" + def __init__(self, url): + """Initialize Default requirements settings.""" + self.banned_requires = set(["futures", + "configparser"]) + self.buildreqs = set() + self.buildreqs_cache = set() + self.requires = set() + self.extra_cmake = set() + self.extra_cmake_openmpi = set() + self.verbose = False + self.cargo_bin = False + self.pypi_provides = None + self.pypi_requires = set() + self.banned_buildreqs = set(["llvm-devel", + "gcj", + "pkgconfig(dnl)", + "pkgconfig(hal)", + "tslib-0.0", + "pkgconfig(parallels-sdk)", + "oslo-python", + "libxml2No-python", + "futures", + "configparser"]) + self.autoreconf_reqs = ["gettext-bin", + "automake-dev", + "automake", + "m4", + "libtool", + "libtool-dev", + "pkg-config-dev"] + if "gnome.org" in url: + self.add_buildreq("buildreq-gnome") + if "kde.org" in url or "https://github.com/KDE" in url: + self.add_buildreq("buildreq-kde") -def set_build_req(): - """Add build requirements based on the buildpattern pattern.""" - if buildpattern.default_pattern == "maven": - maven_reqs = ["apache-maven", - "openjdk-dev", - "mvn-aether-core", - "mvn-aopalliance", - "mvn-cdi-api", - "mvn-commons-cli", - "mvn-commons-codec", - "mvn-commons-io", - "mvn-commons-lang", - "mvn-commons-lang3", - "mvn-commons-logging", - "mvn-guice", - "mvn-guava", - "mvn-httpcomponents-client", - "mvn-httpcomponents-core", - "mvn-jsoup", - "mvn-jsr305", - "mvn-wagon", - "mvn-sisu", - "mvn-plexus-cipher", - "mvn-plexus-classworlds", - "mvn-plexus-containers", - "mvn-plexus-interpolation", - "mvn-sonatype-plexus-sec-dispatcher", - "mvn-plexus-utils", - "mvn-slf4j"] - for req in maven_reqs: - add_buildreq(req) + def add_buildreq(self, req, cache=False): + """Add req to the global buildreqs set if req is not banned.""" + new = True + req.strip() + if req in self.banned_buildreqs: + return False + if req in self.buildreqs: + new = False + if self.verbose and new: + print(" Adding buildreq:", req) - if buildpattern.default_pattern == "ruby": - add_buildreq("ruby") - add_buildreq("rubygem-rdoc") - if buildpattern.default_pattern == "cargo": - add_buildreq("rustc") + self.buildreqs.add(req) + if cache and new: + self.buildreqs_cache.add(req) + return new + def add_requires(self, req, packages, override=False): + """Add req to the global requires set if it is present in buildreqs and packages and is not banned.""" + new = True + req = req.strip() + if req in self.requires: + new = False + if req in self.banned_requires: + return False -def rakefile(filename, gems): - """Scan Rakefile for build requirements.""" - with util.open_auto(filename, "r") as f: - lines = f.readlines() + # Try dashes instead of underscores as some ecosystems are inconsistent in their naming + req2 = req.replace("_", "-") + if req not in self.buildreqs and req2 in packages and req2 not in self.requires and req2 not in self.banned_requires: + # Since this is done for python add a buildreq just in case (might not be correct though) + self.buildreqs.add(req2) + self.requires.add(req2) + return True - pat = re.compile(r"^require '(.*)'$") - for line in lines: - match = pat.search(line) - if match: - s = match.group(1) - if s != "rubygems" and s in gems: - print("Rakefile-dep: " + gems[s]) - add_buildreq(gems[s]) + # Try reversing the case of the first letter as some ecosystems are inconsistent in their naming + if len(req) > 1: + if req[0].isupper(): + req2 = req[0].lower() + req[1:] else: - print("Rakefile-new: rubygem-" + s) + req2 = req[0].upper() + req[1:] + if req not in self.buildreqs and req2 in packages and req2 not in self.requires and req2 not in self.banned_requires: + # Since this is done for python add a buildreq just in case (might not be correct though) + self.buildreqs.add(req2) + self.requires.add(req2) + return True + if req not in self.buildreqs and req not in packages and not override: + if req: + print("requirement '{}' not found in buildreqs or os_packages, skipping".format(req)) + return False + if new: + # print("Adding requirement:", req) + self.requires.add(req) + return new -def parse_cmake(filename, cmake_modules, conf32): - """Scan a .cmake or CMakeLists.txt file for what's it's actually looking for.""" - findpackage = re.compile(r"^[^#]*find_package\((\w+)\b.*\)", re.I) - pkgconfig = re.compile(r"^[^#]*pkg_check_modules\s*\(\w+ (.*)\)", re.I) - pkg_search_modifiers = {'REQUIRED', 'QUIET', 'NO_CMAKE_PATH', - 'NO_CMAKE_ENVIRONMENT_PATH', 'IMPORTED_TARGET'} - extractword = re.compile(r'(?:"([^"]+)"|(\S+))(.*)') + def add_pkgconfig_buildreq(self, preq, conf32, cache=False): + """Format preq as pkgconfig req and add to buildreqs.""" + if conf32: + req = "pkgconfig(32" + preq + ")" + self.add_buildreq(req, cache) + req = "pkgconfig(" + preq + ")" + return self.add_buildreq(req, cache) - with util.open_auto(filename, "r") as f: - lines = f.readlines() - for line in lines: - match = findpackage.search(line) + def configure_ac_line(self, line, conf32): + """Parse configure_ac line and add appropriate buildreqs.""" + # print("----\n", line, "\n----") + # ignore comments + if line.startswith('#'): + return + + pat_reqs = [(r"AC_CHECK_FUNC\([tgetent]", ["ncurses-devel"]), + ("PROG_INTLTOOL", ["intltool"]), + ("GETTEXT_PACKAGE", ["gettext", "perl(XML::Parser)"]), + ("AM_GLIB_GNU_GETTEXT", ["gettext", "perl(XML::Parser)"]), + ("GTK_DOC_CHECK", ["gtk-doc", "gtk-doc-dev", "libxslt-bin", "docbook-xml"]), + ("AC_PROG_SED", ["sed"]), + ("AC_PROG_GREP", ["grep"])] + + for pat, reqs in pat_reqs: + if pat in line: + for req in reqs: + self.add_buildreq(req) + + line = line.strip() + + # XFCE uses an equivalent to PKG_CHECK_MODULES, handle them both the same + for style in [r"PKG_CHECK_MODULES\((.*?)\)", r"XDT_CHECK_PACKAGE\((.*?)\)"]: + match = re.search(style, line) + L = [] + if match: + L = match.group(1).split(",") + if len(L) > 1: + rqlist = L[1].strip() + for req in parse_modules_list(rqlist): + self.add_pkgconfig_buildreq(req, conf32) + + # PKG_CHECK_EXISTS(MODULES, action-if-found, action-if-not-found) + match = re.search(r"PKG_CHECK_EXISTS\((.*?)\)", line) if match: - module = match.group(1) - try: - pkg = cmake_modules[module] - add_buildreq(pkg) - except Exception: - pass + L = match.group(1).split(",") + rqlist = L[0].strip() + for req in parse_modules_list(rqlist): + self.add_pkgconfig_buildreq(req, conf32) - match = pkgconfig.search(line) - if match: - rest = match.group(1) - while rest: - wordmatch = extractword.search(rest) - if not wordmatch: - break - rest = wordmatch.group(3) - if wordmatch.group(2) in pkg_search_modifiers: - continue - # Only one of the two groups can match at a time - module = wordmatch.group(1) - if not module: - module = wordmatch.group(2) - # We have a match, so strip out any version info - for m in parse_modules_list(module, is_cmake=True): - add_pkgconfig_buildreq(m, conf32) + def parse_configure_ac(self, filename, conf32): + """Parse the configure.ac file for build requirements.""" + buf = "" + depth = 0 + # print("Configure parse: ", filename) + buildpattern.set_build_pattern("configure_ac", 1) + f = util.open_auto(filename, "r") + while 1: + c = f.read(1) + if not c: + break + if c == "(": + depth += 1 + if c == ")" and depth > 0: + depth -= 1 + if c != "\n": + buf += c + if c == "\n" and depth == 0: + self.configure_ac_line(buf, conf32) + buf = "" + self.configure_ac_line(buf, conf32) + f.close() + def parse_cargo_toml(self, filename, packages): + """Update build requirements using Cargo.toml. -def qmake_profile(filename, qt_modules): - """Scan .pro file for build requirements.""" - with util.open_auto(filename, "r") as f: - lines = f.readlines() + Set the build requirements for building rust programs using cargo. + """ + buildpattern.set_build_pattern("cargo", 1) + self.add_buildreq("rustc") + with util.open_auto(filename, "r") as ctoml: + cargo = toml.loads(ctoml.read()) + if cargo.get("bin") or os.path.exists(os.path.join(os.path.dirname(filename), "src/main.rs")): + self.cargo_bin = True + if not cargo.get("dependencies"): + return + for cdep in cargo["dependencies"]: + if self.add_buildreq(cdep): + self.add_requires(cdep, packages) - pat = re.compile(r"(QT|QT_PRIVATE|QT_FOR_CONFIG).*=\s*(.*)\s*") - for line in lines: - match = pat.search(line) - if not match: - continue - s = match.group(2) - for module in s.split(): - module = re.sub('-private$', '', module) - try: - pc = qt_modules[module] - add_buildreq('pkgconfig({})'.format(pc)) - except Exception: - pass - - -def clean_python_req(req, add_python=True): - """Strip version information from req.""" - if req.find("#") == 0: - return "" - ret = req.rstrip("\n\r").strip() - i = ret.find(";") - if i > 0: - ret = ret[:i] - i = ret.find("<") - if i > 0: - ret = ret[:i] - i = ret.find("\n") - if i > 0: - ret = ret[:i] - i = ret.find(">") - if i > 0: - ret = ret[:i] - i = ret.find("=") - if i > 0: - ret = ret[:i] - i = ret.find("#") - if i > 0: - ret = ret[:i] - i = ret.find("!") - if i > 0: - ret = ret[:i] - - ret = ret.strip() - # is ret actually a valid (non-empty) string? - if ret and add_python: - ret = ret.strip() - # use the dictionary to translate funky names to our current pgk names - ret = util.translate(ret) - return ret - - -def grab_python_requirements(descfile, packages): - """Add python requirements from requirements.txt file.""" - if "/demo/" in descfile: - return - if "/doc/" in descfile: - return - if "/docs/" in descfile: - return - if "/example/" in descfile: - return - if "/test/" in descfile: - return - if "/tests/" in descfile: - return - - with util.open_auto(descfile, "r") as f: - lines = f.readlines() - - for line in lines: - # don't add the test section - if clean_python_req(line) == '[test]': - break - if clean_python_req(line) == '[testing]': - break - if clean_python_req(line) == '[dev]': - break - if clean_python_req(line) == '[doc]': - break - if clean_python_req(line) == '[docs]': - break - if 'pytest' in line: - continue - if clean_python_req(line) == 'mock': - continue - add_requires(clean_python_req(line), packages) - - -def get_python_build_version_from_classifier(filename): - """Detect if setup should use distutils3 only. - - Uses "Programming Language :: Python :: [2,3] :: Only" classifiers in the - setup.py file. Defaults to distutils3 if no such classifiers are found. - """ - with util.open_auto(filename) as setup_file: - data = setup_file.read() - - if "Programming Language :: Python :: 3 :: Only" in data: - return "distutils3" - - return "distutils3" - - -def add_setup_py_requires(filename, packages): - """Detect build requirements listed in setup.py in the install_requires and setup_requires lists. - - Handles the following patterns: - install_requires='one' - install_requires=['one', 'two', 'three'] - install_requires=['one', - 'two', - 'three'] - setup_requires=[ - 'one>=2.1', # >=2.1 is removed - 'two', - 'three' - ] - setuptools.setup( - setup_requires=['one', 'two'], - ...) - setuptools.setup(setup_requires=['one', 'two'], ...) - - Does not evaluate variables for security purposes - """ - multiline = False - with util.open_auto(filename) as f: - lines = f.readlines() - - for line in lines: - if "install_requires" in line or "setup_requires" in line: - req = "install_requires" in line - # find the value for *_requires - line = line.split("=", 1) - if len(line) == 2: - line = line[1].strip() - else: - # skip because this could be a conditionally extended list - # we only want to automatically detect the core packages + def parse_r_description(self, filename, packages): + """Update build/runtime requirements according to the R package description.""" + deps = [] + with util.open_auto(filename, "r") as desc: + content = desc.read() + deps = _get_desc_field("Depends", content) + deps.extend(_get_desc_field("Imports", content)) + deps.extend(_get_desc_field("LinkingTo", content)) + r_provides = _get_r_provides() + for dep in deps: + if dep == 'R': continue - - # easy, one-line case - if line.startswith("[") and "]" in line: - # remove the leading [ and split off everthing after the ] - line = line[1:].split("]")[0] - for item in line.split(','): - item = item.strip() - try: - # eval the string and add requirements - dep = clean_python_req(ast.literal_eval(item), False) - add_buildreq(dep) - if req: - add_requires(dep, packages) - - except Exception: - # do not fail, the line contained a variable and - # had to be skipped - pass - + if dep in r_provides: continue - - # more complicated, multi-line list. - # this sets the py_dep_string with the current line, which - # is the beginning of a multi-line list. - elif line.startswith("["): - multiline = True - line = line.lstrip("[") - - # if the line doesn't start with '[' it is the case where - # there is (should be) a single dependency as a string + pkg = 'R-' + dep + if pkg in packages: + self.add_buildreq(pkg) + self.add_requires(pkg, packages) else: - line = line.strip() + print("CRAN package '{}' not found in os_packages, skipping".format(pkg)) + + def set_build_req(self): + """Add build requirements based on the buildpattern pattern.""" + if buildpattern.default_pattern == "maven": + maven_reqs = ["apache-maven", + "openjdk-dev", + "mvn-aether-core", + "mvn-aopalliance", + "mvn-cdi-api", + "mvn-commons-cli", + "mvn-commons-codec", + "mvn-commons-io", + "mvn-commons-lang", + "mvn-commons-lang3", + "mvn-commons-logging", + "mvn-guice", + "mvn-guava", + "mvn-httpcomponents-client", + "mvn-httpcomponents-core", + "mvn-jsoup", + "mvn-jsr305", + "mvn-wagon", + "mvn-sisu", + "mvn-plexus-cipher", + "mvn-plexus-classworlds", + "mvn-plexus-containers", + "mvn-plexus-interpolation", + "mvn-sonatype-plexus-sec-dispatcher", + "mvn-plexus-utils", + "mvn-slf4j"] + for req in maven_reqs: + self.add_buildreq(req) + + if buildpattern.default_pattern == "ruby": + self.add_buildreq("ruby") + self.add_buildreq("rubygem-rdoc") + if buildpattern.default_pattern == "cargo": + self.add_buildreq("rustc") + + def rakefile(self, filename, gems): + """Scan Rakefile for build requirements.""" + with util.open_auto(filename, "r") as f: + lines = f.readlines() + + pat = re.compile(r"^require '(.*)'$") + for line in lines: + match = pat.search(line) + if match: + s = match.group(1) + if s != "rubygems" and s in gems: + print("Rakefile-dep: " + gems[s]) + self.add_buildreq(gems[s]) + else: + print("Rakefile-new: rubygem-" + s) + + def parse_cmake(self, filename, cmake_modules, conf32): + """Scan a .cmake or CMakeLists.txt file for what's it's actually looking for.""" + findpackage = re.compile(r"^[^#]*find_package\((\w+)\b.*\)", re.I) + pkgconfig = re.compile(r"^[^#]*pkg_check_modules\s*\(\w+ (.*)\)", re.I) + pkg_search_modifiers = {'REQUIRED', 'QUIET', 'NO_CMAKE_PATH', + 'NO_CMAKE_ENVIRONMENT_PATH', 'IMPORTED_TARGET'} + extractword = re.compile(r'(?:"([^"]+)"|(\S+))(.*)') + + with util.open_auto(filename, "r") as f: + lines = f.readlines() + for line in lines: + match = findpackage.search(line) + if match: + module = match.group(1) try: - dep = clean_python_req(ast.literal_eval(line), False) - add_buildreq(dep) - if req: - add_requires(dep, packages) - + pkg = cmake_modules[module] + self.add_buildreq(pkg) except Exception: - # Do not fail, just keep looking pass + match = pkgconfig.search(line) + if match: + rest = match.group(1) + while rest: + wordmatch = extractword.search(rest) + if not wordmatch: + break + rest = wordmatch.group(3) + if wordmatch.group(2) in pkg_search_modifiers: + continue + # Only one of the two groups can match at a time + module = wordmatch.group(1) + if not module: + module = wordmatch.group(2) + # We have a match, so strip out any version info + for m in parse_modules_list(module, is_cmake=True): + self.add_pkgconfig_buildreq(m, conf32) + + def qmake_profile(self, filename, qt_modules): + """Scan .pro file for build requirements.""" + with util.open_auto(filename, "r") as f: + lines = f.readlines() + + pat = re.compile(r"(QT|QT_PRIVATE|QT_FOR_CONFIG).*=\s*(.*)\s*") + for line in lines: + match = pat.search(line) + if not match: + continue + s = match.group(2) + for module in s.split(): + module = re.sub('-private$', '', module) + try: + pc = qt_modules[module] + self.add_buildreq('pkgconfig({})'.format(pc)) + except Exception: + pass + + def grab_python_requirements(self, descfile, packages): + """Add python requirements from requirements.txt file.""" + if "/demo/" in descfile: + return + if "/doc/" in descfile: + return + if "/docs/" in descfile: + return + if "/example/" in descfile: + return + if "/test/" in descfile: + return + if "/tests/" in descfile: + return + + with util.open_auto(descfile, "r") as f: + lines = f.readlines() + + for line in lines: + # don't add the test section + if clean_python_req(line) == '[test]': + break + if clean_python_req(line) == '[testing]': + break + if clean_python_req(line) == '[dev]': + break + if clean_python_req(line) == '[doc]': + break + if clean_python_req(line) == '[docs]': + break + if 'pytest' in line: + continue + if clean_python_req(line) == 'mock': + continue + self.add_requires(clean_python_req(line), packages) + + def add_setup_py_requires(self, filename, packages): + """Detect build requirements listed in setup.py in the install_requires and setup_requires lists. + + Handles the following patterns: + install_requires='one' + install_requires=['one', 'two', 'three'] + install_requires=['one', + 'two', + 'three'] + setup_requires=[ + 'one>=2.1', # >=2.1 is removed + 'two', + 'three' + ] + setuptools.setup( + setup_requires=['one', 'two'], + ...) + setuptools.setup(setup_requires=['one', 'two'], ...) + + Does not evaluate variables for security purposes + """ + multiline = False + with util.open_auto(filename) as f: + lines = f.readlines() + + for line in lines: + if "install_requires" in line or "setup_requires" in line: + req = "install_requires" in line + # find the value for *_requires + line = line.split("=", 1) + if len(line) == 2: + line = line[1].strip() + else: + # skip because this could be a conditionally extended list + # we only want to automatically detect the core packages + continue + + # easy, one-line case + if line.startswith("[") and "]" in line: + # remove the leading [ and split off everthing after the ] + line = line[1:].split("]")[0] + for item in line.split(','): + item = item.strip() + try: + # eval the string and add requirements + dep = clean_python_req(ast.literal_eval(item), False) + self.add_buildreq(dep) + if req: + self.add_requires(dep, packages) + + except Exception: + # do not fail, the line contained a variable and + # had to be skipped + pass + + continue + + # more complicated, multi-line list. + # this sets the py_dep_string with the current line, which + # is the beginning of a multi-line list. + elif line.startswith("["): + multiline = True + line = line.lstrip("[") + + # if the line doesn't start with '[' it is the case where + # there is (should be) a single dependency as a string + else: + line = line.strip() + try: + dep = clean_python_req(ast.literal_eval(line), False) + self.add_buildreq(dep) + if req: + self.add_requires(dep, packages) + + except Exception: + # Do not fail, just keep looking + pass + + continue + + # if multiline was set above when a multi-line list was + # detected, for each line until the end bracket is found attempt to + # add the line as a buildreq + if multiline: + # if end bracket found, reset the flag + if "]" in line: + multiline = False + line = line.split("]")[0] + + try: + dep = ast.literal_eval(line.split('#')[0].strip(' ,\n')) + dep = clean_python_req(dep) + self.add_buildreq(dep) + if req: + self.add_requires(dep, packages) + + except Exception: + # do not fail, the line contained a variable and had to + # be skipped + pass + + def parse_catkin_deps(self, cmakelists_file, conf32): + """Determine requirements for catkin packages.""" + f = util.open_auto(cmakelists_file, "r") + lines = f.readlines() + pat = re.compile(r"^find_package.*\(.*(catkin)(?: REQUIRED *)?(?:COMPONENTS (?P.*))?\)$") + catkin = False + for line in lines: + match = pat.search(line) + if not match: continue - # if multiline was set above when a multi-line list was - # detected, for each line until the end bracket is found attempt to - # add the line as a buildreq - if multiline: - # if end bracket found, reset the flag - if "]" in line: - multiline = False - line = line.split("]")[0] + # include catkin's required components + comp = match.group("comp") + if comp: + for curr in comp.split(" "): + self.add_pkgconfig_buildreq(curr, conf32) - try: - dep = ast.literal_eval(line.split('#')[0].strip(' ,\n')) - dep = clean_python_req(dep) - add_buildreq(dep) - if req: - add_requires(dep, packages) + catkin = True - except Exception: - # do not fail, the line contained a variable and had to - # be skipped - pass + # catkin find_package() function will always rely on CMAKE_PREFIX_PATH + # make sure we keep it consistent with CMAKE_INSTALL_PREFIX otherwise + # it'll never be able to find its modules + if catkin: + for curr in ["catkin", "catkin_pkg", "empy", "googletest"]: + self.add_buildreq(curr) + self.extra_cmake.add("-DCMAKE_PREFIX_PATH=/usr") + self.extra_cmake.add("-DCATKIN_BUILD_BINARY_PACKAGE=ON") + self.extra_cmake.add("-DSETUPTOOLS_DEB_LAYOUT=OFF") -def parse_catkin_deps(cmakelists_file, conf32): - """Determine requirements for catkin packages.""" - f = util.open_auto(cmakelists_file, "r") - lines = f.readlines() - pat = re.compile(r"^find_package.*\(.*(catkin)(?: REQUIRED *)?(?:COMPONENTS (?P.*))?\)$") - catkin = False + def scan_for_configure(self, dirn, tname, dlpath, config): + """Scan the package directory for build files to determine build pattern.""" + if buildpattern.default_pattern == "distutils36": + self.add_buildreq("buildreq-distutils36") + elif buildpattern.default_pattern == "distutils3": + self.add_buildreq("buildreq-distutils3") + elif buildpattern.default_pattern == "golang": + self.add_buildreq("buildreq-golang") + elif buildpattern.default_pattern == "cmake": + self.add_buildreq("buildreq-cmake") + elif buildpattern.default_pattern == "configure": + self.add_buildreq("buildreq-configure") + elif buildpattern.default_pattern == "qmake": + self.add_buildreq("buildreq-qmake") + elif buildpattern.default_pattern == "cpan": + self.add_buildreq("buildreq-cpan") + elif buildpattern.default_pattern == "scons": + self.add_buildreq("buildreq-scons") + elif buildpattern.default_pattern == "R": + self.add_buildreq("buildreq-R") + self.parse_r_description(os.path.join(dirn, "DESCRIPTION"), config.os_packages) + elif buildpattern.default_pattern == "phpize": + self.add_buildreq("buildreq-php") + elif buildpattern.default_pattern == "nginx": + self.add_buildreq("buildreq-nginx") - for line in lines: - match = pat.search(line) + count = 0 + for dirpath, _, files in os.walk(dirn): + default_score = 2 if dirpath == dirn else 1 - if not match: - continue - - # include catkin's required components - comp = match.group("comp") - if comp: - for curr in comp.split(" "): - add_pkgconfig_buildreq(curr, conf32) - - catkin = True - - # catkin find_package() function will always rely on CMAKE_PREFIX_PATH - # make sure we keep it consistent with CMAKE_INSTALL_PREFIX otherwise - # it'll never be able to find its modules - if catkin: - for curr in ["catkin", "catkin_pkg", "empy", "googletest"]: - add_buildreq(curr) - - extra_cmake.add("-DCMAKE_PREFIX_PATH=/usr") - extra_cmake.add("-DCATKIN_BUILD_BINARY_PACKAGE=ON") - extra_cmake.add("-DSETUPTOOLS_DEB_LAYOUT=OFF") - - -def is_qmake_pro(f): - """Test if file extension is pro and not hidden.""" - return f.endswith(".pro") and not f.startswith(".") - - -def scan_for_configure(dirn, tname, dlpath, config): - """Scan the package directory for build files to determine build pattern.""" - global pypi_provides - global pypi_requires - if buildpattern.default_pattern == "distutils36": - add_buildreq("buildreq-distutils36") - elif buildpattern.default_pattern == "distutils3": - add_buildreq("buildreq-distutils3") - elif buildpattern.default_pattern == "golang": - add_buildreq("buildreq-golang") - elif buildpattern.default_pattern == "cmake": - add_buildreq("buildreq-cmake") - elif buildpattern.default_pattern == "configure": - add_buildreq("buildreq-configure") - elif buildpattern.default_pattern == "qmake": - add_buildreq("buildreq-qmake") - elif buildpattern.default_pattern == "cpan": - add_buildreq("buildreq-cpan") - elif buildpattern.default_pattern == "scons": - add_buildreq("buildreq-scons") - elif buildpattern.default_pattern == "R": - add_buildreq("buildreq-R") - parse_r_description(os.path.join(dirn, "DESCRIPTION"), config.os_packages) - elif buildpattern.default_pattern == "phpize": - add_buildreq("buildreq-php") - elif buildpattern.default_pattern == "nginx": - add_buildreq("buildreq-nginx") - - count = 0 - for dirpath, _, files in os.walk(dirn): - default_score = 2 if dirpath == dirn else 1 - - if any(f.endswith(".go") for f in files): - add_buildreq("buildreq-golang") - buildpattern.set_build_pattern("golang", default_score) - - if "go.mod" in files: - if "Makefile" not in files: - # Go packages usually have make build systems so far - # so only use go directly if we can't find a Makefile + if any(f.endswith(".go") for f in files): + self.add_buildreq("buildreq-golang") buildpattern.set_build_pattern("golang", default_score) - add_buildreq("buildreq-golang") - if buildpattern.default_pattern == "golang-mod" or buildpattern.default_pattern == "godep": - config.set_gopath = False - mod_path = os.path.join(dirpath, "go.mod") - reqs = parse_go_mod(mod_path) - for req in reqs: - # req[0] is a SCM url segment in the form, repo/XXX/dependency-name - # req[1] is the version of the dependency - pkg = "go-" + req[0].replace("/", "-") - add_buildreq(pkg) - if buildpattern.default_pattern == "godep": - add_requires(pkg, config.os_packages) - if "CMakeLists.txt" in files and "configure.ac" not in files: - add_buildreq("buildreq-cmake") - buildpattern.set_build_pattern("cmake", default_score) + if "go.mod" in files: + if "Makefile" not in files: + # Go packages usually have make build systems so far + # so only use go directly if we can't find a Makefile + buildpattern.set_build_pattern("golang", default_score) + self.add_buildreq("buildreq-golang") + if buildpattern.default_pattern == "golang-mod" or buildpattern.default_pattern == "godep": + config.set_gopath = False + mod_path = os.path.join(dirpath, "go.mod") + reqs = parse_go_mod(mod_path) + for req in reqs: + # req[0] is a SCM url segment in the form, repo/XXX/dependency-name + # req[1] is the version of the dependency + pkg = "go-" + req[0].replace("/", "-") + self.add_buildreq(pkg) + if buildpattern.default_pattern == "godep": + self.add_requires(pkg, config.os_packages) - srcdir = os.path.abspath(os.path.join(dirn, "clr-build", config.cmake_srcdir or "..")) - if os.path.samefile(dirpath, srcdir): - parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt"), config.config_opts.get('32bit')) - - if "configure" in files and os.access(dirpath + '/configure', os.X_OK): - buildpattern.set_build_pattern("configure", default_score) - elif any(is_qmake_pro(f) for f in files): - add_buildreq("buildreq-qmake") - buildpattern.set_build_pattern("qmake", default_score) - - if "requires.txt" in files: - grab_python_requirements(dirpath + '/requires.txt', config.os_packages) - - if "setup.py" in files: - add_buildreq("buildreq-distutils3") - add_setup_py_requires(dirpath + '/setup.py', config.os_packages) - python_pattern = get_python_build_version_from_classifier(dirpath + '/setup.py') - buildpattern.set_build_pattern(python_pattern, default_score) - - if "Makefile.PL" in files or "Build.PL" in files: - buildpattern.set_build_pattern("cpan", default_score) - add_buildreq("buildreq-cpan") - - if "SConstruct" in files: - add_buildreq("buildreq-scons") - buildpattern.set_build_pattern("scons", default_score) - - if "requirements.txt" in files: - grab_python_requirements(dirpath + '/requirements.txt', config.os_packages) - - if "meson.build" in files: - add_buildreq("buildreq-meson") - buildpattern.set_build_pattern("meson", default_score) - - if "build.xml" in files: - add_buildreq("apache-ant") - buildpattern.set_build_pattern("ant", default_score) - - for name in files: - if name.lower() == "cargo.toml" and dirpath == dirn: - parse_cargo_toml(os.path.join(dirpath, name), config.os_packages) - if name.lower().startswith("configure."): - parse_configure_ac(os.path.join(dirpath, name), config.config_opts.get('32bit')) - if name.lower().startswith("rakefile") and buildpattern.default_pattern == "ruby": - rakefile(os.path.join(dirpath, name), config.gems) - if name.endswith(".pro") and buildpattern.default_pattern == "qmake": - qmake_profile(os.path.join(dirpath, name), config.qt_modules) - if name.lower() == "makefile": - buildpattern.set_build_pattern("make", default_score) - if name.lower() == "autogen.sh": - buildpattern.set_build_pattern("autogen", default_score) - if name.lower() == "cmakelists.txt": + if "CMakeLists.txt" in files and "configure.ac" not in files: + self.add_buildreq("buildreq-cmake") buildpattern.set_build_pattern("cmake", default_score) - if (name.lower() == "cmakelists.txt" or name.endswith(".cmake")) \ - and buildpattern.default_pattern == "cmake": - parse_cmake(os.path.join(dirpath, name), config.cmake_modules, config.config_opts.get('32bit')) - can_reconf = os.path.exists(os.path.join(dirn, "configure.ac")) - if not can_reconf: - can_reconf = os.path.exists(os.path.join(dirn, "configure.in")) + srcdir = os.path.abspath(os.path.join(dirn, "clr-build", config.cmake_srcdir or "..")) + if os.path.samefile(dirpath, srcdir): + self.parse_catkin_deps(os.path.join(srcdir, "CMakeLists.txt"), config.config_opts.get('32bit')) - if can_reconf and config.autoreconf: - print("Patches touch configure.*, adding autoreconf stage") - for breq in autoreconf_reqs: - add_buildreq(breq) - else: - config.autoreconf = False + if "configure" in files and os.access(dirpath + '/configure', os.X_OK): + buildpattern.set_build_pattern("configure", default_score) + elif any(is_qmake_pro(f) for f in files): + self.add_buildreq("buildreq-qmake") + buildpattern.set_build_pattern("qmake", default_score) - if buildpattern.default_pattern == "distutils3": - # First look for a local override - pypi_json = "" - pypi_file = os.path.join(dlpath, "pypi.json") - if os.path.isfile(pypi_file): - with open(pypi_file, "r") as pfile: - pypi_json = pfile.read() + if "requires.txt" in files: + self.grab_python_requirements(dirpath + '/requires.txt', config.os_packages) + + if "setup.py" in files: + self.add_buildreq("buildreq-distutils3") + self.add_setup_py_requires(dirpath + '/setup.py', config.os_packages) + python_pattern = get_python_build_version_from_classifier(dirpath + '/setup.py') + buildpattern.set_build_pattern(python_pattern, default_score) + + if "Makefile.PL" in files or "Build.PL" in files: + buildpattern.set_build_pattern("cpan", default_score) + self.add_buildreq("buildreq-cpan") + + if "SConstruct" in files: + self.add_buildreq("buildreq-scons") + buildpattern.set_build_pattern("scons", default_score) + + if "requirements.txt" in files: + self.grab_python_requirements(dirpath + '/requirements.txt', config.os_packages) + + if "meson.build" in files: + self.add_buildreq("buildreq-meson") + buildpattern.set_build_pattern("meson", default_score) + + if "build.xml" in files: + self.add_buildreq("apache-ant") + buildpattern.set_build_pattern("ant", default_score) + + for name in files: + if name.lower() == "cargo.toml" and dirpath == dirn: + self.parse_cargo_toml(os.path.join(dirpath, name), config.os_packages) + if name.lower().startswith("configure."): + self.parse_configure_ac(os.path.join(dirpath, name), config.config_opts.get('32bit')) + if name.lower().startswith("rakefile") and buildpattern.default_pattern == "ruby": + self.rakefile(os.path.join(dirpath, name), config.gems) + if name.endswith(".pro") and buildpattern.default_pattern == "qmake": + self.qmake_profile(os.path.join(dirpath, name), config.qt_modules) + if name.lower() == "makefile": + buildpattern.set_build_pattern("make", default_score) + if name.lower() == "autogen.sh": + buildpattern.set_build_pattern("autogen", default_score) + if name.lower() == "cmakelists.txt": + buildpattern.set_build_pattern("cmake", default_score) + if (name.lower() == "cmakelists.txt" or name.endswith(".cmake")) \ + and buildpattern.default_pattern == "cmake": + self.parse_cmake(os.path.join(dirpath, name), config.cmake_modules, config.config_opts.get('32bit')) + + can_reconf = os.path.exists(os.path.join(dirn, "configure.ac")) + if not can_reconf: + can_reconf = os.path.exists(os.path.join(dirn, "configure.in")) + if can_reconf and config.autoreconf: + print("Patches touch configure.*, adding autoreconf stage") + for breq in self.autoreconf_reqs: + self.add_buildreq(breq) else: - # Try and grab the pypi details for the package - if config.alias: - tname = config.alias - pypi_name = pypidata.get_pypi_name(tname) - pypi_json = pypidata.get_pypi_metadata(pypi_name) - if pypi_json: - try: - package_pypi = json.loads(pypi_json) - except json.JSONDecodeError: - package_pypi = {} - if package_pypi.get("name"): - pypi_provides = package_pypi["name"] - if package_pypi.get("requires"): - pypi_requires = set(package_pypi["requires"]) - if package_pypi.get("license"): - # The license field is freeform, might be worth looking at though - print(f"Pypi says the license is: {package_pypi['license']}") - if package_pypi.get("summary"): - specdescription.assign_summary(package_pypi["summary"], 4) + config.autoreconf = False - print("Buildreqs : ", end="") - for lic in sorted(buildreqs): - if count > 4: - count = 0 - print("\nBuildreqs : ", end="") - count = count + 1 - print(lic + " ", end="") - print("") + if buildpattern.default_pattern == "distutils3": + # First look for a local override + pypi_json = "" + pypi_file = os.path.join(dlpath, "pypi.json") + if os.path.isfile(pypi_file): + with open(pypi_file, "r") as pfile: + pypi_json = pfile.read() + else: + # Try and grab the pypi details for the package + if config.alias: + tname = config.alias + pypi_name = pypidata.get_pypi_name(tname) + pypi_json = pypidata.get_pypi_metadata(pypi_name) + if pypi_json: + try: + package_pypi = json.loads(pypi_json) + except json.JSONDecodeError: + package_pypi = {} + if package_pypi.get("name"): + self.pypi_provides = package_pypi["name"] + if package_pypi.get("requires"): + self.pypi_requires = set(package_pypi["requires"]) + if package_pypi.get("license"): + # The license field is freeform, might be worth looking at though + print(f"Pypi says the license is: {package_pypi['license']}") + if package_pypi.get("summary"): + specdescription.assign_summary(package_pypi["summary"], 4) - -def load_specfile(specfile): - """Load specfile object with necessary buildreq data.""" - specfile.buildreqs = buildreqs - specfile.requires = requires - specfile.cargo_bin = cargo_bin - specfile.extra_cmake += " " + " ".join(extra_cmake) - specfile.pypi_provides = pypi_provides - specfile.pypi_requires = sorted(pypi_requires) - specfile.extra_cmake_openmpi += " " + " ".join(extra_cmake_openmpi) + print("Buildreqs : ", end="") + for lic in sorted(self.buildreqs): + if count > 4: + count = 0 + print("\nBuildreqs : ", end="") + count = count + 1 + print(lic + " ", end="") + print("") diff --git a/autospec/check.py b/autospec/check.py index a1d355f..3da2c16 100644 --- a/autospec/check.py +++ b/autospec/check.py @@ -23,7 +23,6 @@ import os import re import buildpattern -import buildreq import count import tarball import util @@ -55,7 +54,7 @@ def check_regression(pkg_dir, skip_tests): util.write_out(os.path.join(pkg_dir, "testresults"), res_str) -def scan_for_tests(src_dir, config): +def scan_for_tests(src_dir, config, requirements): """Scan source directory for test files and set tests_config accordingly.""" global tests_config @@ -166,11 +165,11 @@ def scan_for_tests(src_dir, config): break if "tox.ini" in files: - buildreq.add_buildreq("tox") - buildreq.add_buildreq("pytest") - buildreq.add_buildreq("virtualenv") - buildreq.add_buildreq("pluggy") - buildreq.add_buildreq("py-python") + requirements.add_buildreq("tox") + requirements.add_buildreq("pytest") + requirements.add_buildreq("virtualenv") + requirements.add_buildreq("pluggy") + requirements.add_buildreq("py-python") def load_specfile(specfile): diff --git a/autospec/config.py b/autospec/config.py index 9db9370..6639b58 100644 --- a/autospec/config.py +++ b/autospec/config.py @@ -28,7 +28,6 @@ import textwrap from collections import OrderedDict import buildpattern -import buildreq import check import license import tarball @@ -394,11 +393,11 @@ class Config(object): os.remove("skip_test_suite") write_config(config_f, path) - def create_buildreq_cache(self, path, version): + def create_buildreq_cache(self, path, version, buildreqs_cache): """Make the buildreq_cache file.""" content = self.read_conf_file(os.path.join(path, "buildreq_cache")) # don't create an empty cache file - if len(buildreq.buildreqs_cache) < 1: + if len(buildreqs_cache) < 1: try: # file was possibly added to git so we should clean it up os.unlink(content) @@ -406,9 +405,9 @@ class Config(object): pass return if not content: - pkgs = sorted(buildreq.buildreqs_cache) + pkgs = sorted(buildreqs_cache) else: - pkgs = sorted(set(content[1:]).union(buildreq.buildreqs_cache)) + pkgs = sorted(set(content[1:]).union(buildreqs_cache)) with open(os.path.join(path, 'buildreq_cache'), "w") as cachefile: cachefile.write("\n".join([version] + pkgs)) self.config_files.add('buildreq_cache') @@ -604,7 +603,7 @@ class Config(object): write_out(filename, wrapper.fill(description) + "\n") - def parse_config_files(self, path, bump, filemanager, version): + def parse_config_files(self, path, bump, filemanager, version, requirements): """Parse the various configuration files that may exist in the package directory.""" packages_file = None @@ -701,35 +700,35 @@ class Config(object): content = self.read_conf_file(os.path.join(path, "buildreq_ban")) for banned in content: print("Banning build requirement: %s." % banned) - buildreq.banned_buildreqs.add(banned) - buildreq.buildreqs.discard(banned) - buildreq.buildreqs_cache.discard(banned) + requirements.banned_buildreqs.add(banned) + requirements.buildreqs.discard(banned) + requirements.buildreqs_cache.discard(banned) content = self.read_conf_file(os.path.join(path, "pkgconfig_ban")) for banned in content: banned = "pkgconfig(%s)" % banned print("Banning build requirement: %s." % banned) - buildreq.banned_buildreqs.add(banned) - buildreq.buildreqs.discard(banned) - buildreq.buildreqs_cache.discard(banned) + requirements.banned_buildreqs.add(banned) + requirements.buildreqs.discard(banned) + requirements.buildreqs_cache.discard(banned) content = self.read_conf_file(os.path.join(path, "requires_ban")) for banned in content: print("Banning runtime requirement: %s." % banned) - buildreq.banned_requires.add(banned) - buildreq.requires.discard(banned) + requirements.banned_requires.add(banned) + requirements.requires.discard(banned) content = self.read_conf_file(os.path.join(path, "buildreq_add")) for extra in content: print("Adding additional build requirement: %s." % extra) - buildreq.add_buildreq(extra) + requirements.add_buildreq(extra) cache_file = os.path.join(path, "buildreq_cache") content = self.read_conf_file(cache_file) if content and content[0] == version: for extra in content[1:]: print("Adding additional build (cache) requirement: %s." % extra) - buildreq.add_buildreq(extra) + requirements.add_buildreq(extra) else: try: os.unlink(cache_file) @@ -742,12 +741,12 @@ class Config(object): for extra in content: extra = "pkgconfig(%s)" % extra print("Adding additional build requirement: %s." % extra) - buildreq.add_buildreq(extra) + requirements.add_buildreq(extra) content = self.read_conf_file(os.path.join(path, "requires_add")) for extra in content: print("Adding additional runtime requirement: %s." % extra) - buildreq.add_requires(extra, self.os_packages, override=True) + requirements.add_requires(extra, self.os_packages, override=True) content = self.read_conf_file(os.path.join(path, "excludes")) for exclude in content: @@ -898,20 +897,20 @@ class Config(object): if self.config_opts['use_clang']: self.config_opts['funroll-loops'] = False - buildreq.add_buildreq("llvm") + requirements.add_buildreq("llvm") if self.config_opts['32bit']: - buildreq.add_buildreq("glibc-libc32") - buildreq.add_buildreq("glibc-dev32") - buildreq.add_buildreq("gcc-dev32") - buildreq.add_buildreq("gcc-libgcc32") - buildreq.add_buildreq("gcc-libstdc++32") + requirements.add_buildreq("glibc-libc32") + requirements.add_buildreq("glibc-dev32") + requirements.add_buildreq("gcc-dev32") + requirements.add_buildreq("gcc-libgcc32") + requirements.add_buildreq("gcc-libstdc++32") if self.config_opts['openmpi']: - buildreq.add_buildreq("openmpi-dev") - buildreq.add_buildreq("modules") + requirements.add_buildreq("openmpi-dev") + requirements.add_buildreq("modules") # MPI testsuites generally require "openssh" - buildreq.add_buildreq("openssh") + requirements.add_buildreq("openssh") self.prep_prepend = self.read_script_file(os.path.join(path, "prep_prepend")) if os.path.isfile(os.path.join(path, "prep_append")): diff --git a/autospec/specfiles.py b/autospec/specfiles.py index b5f333a..6d7e609 100644 --- a/autospec/specfiles.py +++ b/autospec/specfiles.py @@ -25,7 +25,6 @@ import time import types from collections import OrderedDict -import buildreq import tarball from util import _file_write from util import open_auto @@ -34,13 +33,14 @@ from util import open_auto class Specfile(object): """Holds data and methods needed to write the spec file.""" - def __init__(self, url, version, name, release, config): + def __init__(self, url, version, name, release, config, requirements): """Add default information for specfile template.""" self.url = url self.version = version self.name = name self.release = release self.config = config + self.requirements = requirements self.specfile = None self.sources = {"unit": [], "gcov": [], "tmpfile": [], "archive": [], "destination": [], "godep": []} self.source_index = {} @@ -49,10 +49,6 @@ class Specfile(object): self.licenses = [] self.license_files = [] self.packages = OrderedDict() - self.requires = set() - self.buildreqs = [] - self.pypi_provides = None - self.pypi_requires = [] self.default_desc = "" self.locales = [] self.default_pattern = "" @@ -70,7 +66,8 @@ class Specfile(object): self.custom_extras = {} self.keyid = "" self.email = "" - self.cargo_bin = False + self.extra_cmake = config.extra_cmake + " " + " ".join(requirements.extra_cmake) + self.extra_cmake_openmpi = config.extra_cmake_openmpi + " " + " ".join(requirements.extra_cmake_openmpi) def write_spec(self, path): """Write spec file.""" @@ -186,16 +183,16 @@ class Specfile(object): "staticdev32"]: continue # honor requires_ban for manual overrides - if "{}-{}".format(self.name, pkg) in buildreq.banned_requires: + if "{}-{}".format(self.name, pkg) in self.requirements.banned_requires: continue self._write("Requires: {}-{} = %{{version}}-%{{release}}\n".format(self.name, pkg)) - for pkg in sorted(self.requires): + for pkg in sorted(self.requirements.requires): self._write("Requires: {}\n".format(pkg)) def write_buildreq(self): """Write build requirements.""" - for req in sorted(self.buildreqs): + for req in sorted(self.requirements.buildreqs): self._write("BuildRequires : {}\n".format(req)) def write_strip_command(self): @@ -297,9 +294,9 @@ class Specfile(object): if pkg == "python3": self._write("Requires: python3-core\n") - if self.pypi_provides: - self._write(f"Provides: pypi({self.pypi_provides})\n") - for req in self.pypi_requires: + if self.requirements.pypi_provides: + self._write(f"Provides: pypi({self.requirements.pypi_provides})\n") + for req in self.requirements.pypi_requires: self._write(f"Requires: pypi({req})\n") if pkg == "perl": @@ -411,7 +408,7 @@ class Specfile(object): '-DCMAKE_INSTALL_LIBDIR=$MPI_LIB -DCMAKE_INSTALL_INCLUDEDIR=$MPI_INCLUDE -DLIB_INSTALL_DIR=$MPI_LIB \\\n' \ '-DBUILD_SHARED_LIBS:BOOL=ON -DLIB_SUFFIX=64 \\\n' \ '-DCMAKE_AR=/usr/bin/gcc-ar -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_RANLIB=/usr/bin/gcc-ranlib \\\n' - self._write_strip("{} {} {}".format(cmake_string, self.config.cmake_srcdir, self.config.extra_cmake_openmpi)) + self._write_strip("{} {} {}".format(cmake_string, self.config.cmake_srcdir, self.extra_cmake_openmpi)) def write_prep(self, ruby_pattern=False): """Write prep section to spec file.""" @@ -1491,7 +1488,7 @@ class Specfile(object): def write_cmake_pattern(self): """Write cmake pattern to spec file.""" - if self.config.extra_make == "" or self.config.extra_cmake == " ": + if self.config.extra_make == "" or self.extra_cmake == " ": self.config.extra_make = "VERBOSE=1" self.write_prep() self.write_lang_c(export_epoch=True) @@ -1502,7 +1499,7 @@ class Specfile(object): self._write_strip("mkdir -p clr-build") self._write_strip("pushd clr-build") self.write_variables() - self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.config.extra_cmake)) + self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake)) self.write_profile_payload("cmake") @@ -1519,7 +1516,7 @@ class Specfile(object): self.need_avx2_flags = saved_avx2flags self._write_strip('export CFLAGS="$CFLAGS -march=haswell -m64"') self._write_strip('export CXXFLAGS="$CXXFLAGS -march=haswell -m64"') - self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.config.extra_cmake)) + self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake)) self.write_make_line() self._write_strip("popd") @@ -1533,7 +1530,7 @@ class Specfile(object): self.need_avx512_flags = saved_avx512flags self._write_strip('export CFLAGS="$CFLAGS -march=skylake-avx512 -m64 "') self._write_strip('export CXXFLAGS="$CXXFLAGS -march=skylake-avx512 -m64 "') - self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.config.extra_cmake)) + self._write_strip("%cmake {} {}".format(self.config.cmake_srcdir, self.extra_cmake)) self.write_make_line() self._write_strip("popd") @@ -1546,7 +1543,7 @@ class Specfile(object): self._write_strip("%cmake -DLIB_INSTALL_DIR:PATH=/usr/lib32 " "-DCMAKE_INSTALL_LIBDIR=/usr/lib32 " "-DLIB_SUFFIX=32 " - "{} {} ".format(self.config.cmake_srcdir, self.config.extra_cmake)) + "{} {} ".format(self.config.cmake_srcdir, self.extra_cmake)) self.write_make_line() self._write_strip("unset PKG_CONFIG_PATH") self._write_strip("popd") @@ -1636,7 +1633,7 @@ class Specfile(object): self._write_strip("\n") self._write_strip("%install") self.write_install_prepend() - if self.cargo_bin: + if self.requirements.cargo_bin: self._write_strip("cargo install --frozen --root /") self._write_strip("cargo clean") self._write_strip("install -d -p %{buildroot}" + src_dir) diff --git a/autospec/tarball.py b/autospec/tarball.py index b426314..e8c8506 100644 --- a/autospec/tarball.py +++ b/autospec/tarball.py @@ -26,7 +26,6 @@ from collections import OrderedDict import build import buildpattern -import buildreq import download from util import call, do_regex, get_sha1sum, print_fatal, write_out @@ -359,12 +358,6 @@ def name_and_version(name_arg, version_arg, filemanager, config): if not giturl: giturl = "https://github.com/" + match.group(1).strip() + "/" + repo + ".git" - if "gnome.org" in url: - buildreq.add_buildreq("buildreq-gnome") - - if "kde.org" in url or "https://github.com/KDE" in url: - buildreq.add_buildreq("buildreq-kde") - # SQLite tarballs use 7 digit versions, e.g 3290000 = 3.29.0, 3081002 = 3.8.10.2 if "sqlite.org" in url: major = version[0] diff --git a/tests/test_build.py b/tests/test_build.py index d466afd..6a59083 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -3,6 +3,7 @@ import tempfile import os from unittest.mock import patch, mock_open, MagicMock import build +import buildreq import config import files @@ -18,7 +19,6 @@ class TestBuildpattern(unittest.TestCase): build.must_restart = 0 build.base_path = None build.download_path = None - build.buildreq.buildreqs = set() def test_setup_workingdir(self): """ @@ -33,34 +33,40 @@ class TestBuildpattern(unittest.TestCase): """ Test simple_pattern_pkgconfig with match """ + reqs = buildreq.Requirements("") build.simple_pattern_pkgconfig('line to test for testpkg.xyz', r'testpkg.xyz', 'testpkg', - False) - self.assertIn('pkgconfig(testpkg)', build.buildreq.buildreqs) + False, + reqs) + self.assertIn('pkgconfig(testpkg)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_simple_pattern_pkgconfig_32bit(self): """ Test simple_pattern_pkgconfig with match and 32bit option set """ + reqs = buildreq.Requirements("") build.simple_pattern_pkgconfig('line to test for testpkg.zyx', r'testpkg.zyx', 'testpkgz', - True) - self.assertIn('pkgconfig(32testpkgz)', build.buildreq.buildreqs) - self.assertIn('pkgconfig(testpkgz)', build.buildreq.buildreqs) + True, + reqs) + self.assertIn('pkgconfig(32testpkgz)', reqs.buildreqs) + self.assertIn('pkgconfig(testpkgz)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_simple_pattern_pkgconfig_no_match(self): """ Test simple_pattern_pkgconfig with no match, nothing should be modified """ + reqs = buildreq.Requirements("") build.simple_pattern_pkgconfig('line to test for somepkg.xyz', r'testpkg.xyz', 'testpkg', - False) - self.assertEqual(build.buildreq.buildreqs, set()) + False, + reqs) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_simple_pattern(self): @@ -69,20 +75,24 @@ class TestBuildpattern(unittest.TestCase): simple_pattern and simple_pattern_pkgconfig is the string that is added to buildreq.buildreqs. """ + reqs = buildreq.Requirements("") build.simple_pattern('line to test for testpkg.xyz', r'testpkg.xyz', - 'testpkg') - self.assertIn('testpkg', build.buildreq.buildreqs) + 'testpkg', + reqs) + self.assertIn('testpkg', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_simple_pattern_no_match(self): """ Test simple_pattern with no match, nothing should be modified """ + reqs = buildreq.Requirements("") build.simple_pattern('line to test for somepkg.xyz', r'testpkg.xyz', - 'testpkg') - self.assertEqual(build.buildreq.buildreqs, set()) + 'testpkg', + reqs) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_failed_pattern_no_match(self): @@ -90,8 +100,9 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with no match """ conf = config.Config() - build.failed_pattern('line to test for failure: somepkg', conf, r'(test)', 0) - self.assertEqual(build.buildreq.buildreqs, set()) + reqs = buildreq.Requirements("") + build.failed_pattern('line to test for failure: somepkg', conf, reqs, r'(test)', 0) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_failed_pattern_no_buildtool(self): @@ -100,8 +111,9 @@ class TestBuildpattern(unittest.TestCase): match in failed_commands. """ conf = config.Config() - build.failed_pattern('line to test for failure: testpkg', conf, r'(test)', 0) - self.assertEqual(build.buildreq.buildreqs, set()) + reqs = buildreq.Requirements("") + build.failed_pattern('line to test for failure: testpkg', conf, reqs, r'(test)', 0) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_failed_pattern_no_buildtool_match(self): @@ -109,9 +121,10 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool unset and match in failed_commands """ conf = config.Config() + reqs = buildreq.Requirements("") conf.setup_patterns() - build.failed_pattern('line to test for failure: lex', conf, r'(lex)', 0) - self.assertIn('flex', build.buildreq.buildreqs) + build.failed_pattern('line to test for failure: lex', conf, reqs, r'(lex)', 0) + self.assertIn('flex', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_pkgconfig(self): @@ -119,12 +132,14 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to pkgconfig """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.xyz', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='pkgconfig') - self.assertIn('pkgconfig(testpkg)', build.buildreq.buildreqs) + self.assertIn('pkgconfig(testpkg)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_R(self): @@ -133,13 +148,15 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.r', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='R') - self.assertIn('R-testpkg', build.buildreq.buildreqs) - self.assertIn('R-testpkg', build.buildreq.requires) + self.assertIn('R-testpkg', reqs.buildreqs) + self.assertIn('R-testpkg', reqs.requires) self.assertEqual(build.must_restart, 1) def test_failed_pattern_perl(self): @@ -147,12 +164,14 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to perl """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.pl', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='perl') - self.assertIn('perl(testpkg)', build.buildreq.buildreqs) + self.assertIn('perl(testpkg)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_pypi(self): @@ -160,12 +179,14 @@ class TestBuildpattern(unittest.TestCase): Test failed_pattern with buildtool set to pypi """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.py', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='pypi') - self.assertIn('testpkg-python', build.buildreq.buildreqs) + self.assertIn('testpkg-python', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_ruby(self): @@ -174,12 +195,14 @@ class TestBuildpattern(unittest.TestCase): config.gems, it should just prepend 'rubygem-' to the package name. """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg.rb', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='ruby') - self.assertIn('rubygem-testpkg', build.buildreq.buildreqs) + self.assertIn('rubygem-testpkg', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_ruby_gem_match(self): @@ -190,12 +213,14 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: test/unit', conf, + reqs, r'(test/unit)', 0, # verbose=0 buildtool='ruby') - self.assertIn('rubygem-test-unit', build.buildreq.buildreqs) + self.assertIn('rubygem-test-unit', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_ruby_table(self): @@ -205,12 +230,14 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: test/unit', conf, + reqs, r'(test/unit)', 0, # verbose=0 buildtool='ruby table') - self.assertIn('rubygem-test-unit', build.buildreq.buildreqs) + self.assertIn('rubygem-test-unit', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_ruby_table_no_match(self): @@ -219,12 +246,14 @@ class TestBuildpattern(unittest.TestCase): config.gems. This should not modify anything. """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='ruby table') - self.assertEqual(build.buildreq.buildreqs, set()) + self.assertEqual(reqs.buildreqs, set()) self.assertEqual(build.must_restart, 0) def test_failed_pattern_maven(self): @@ -233,12 +262,14 @@ class TestBuildpattern(unittest.TestCase): config.maven_jars, it should just prepend 'mvn-' to the package name. """ conf = config.Config() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: testpkg', conf, + reqs, r'(testpkg)', 0, # verbose=0 buildtool='maven') - self.assertIn('mvn-testpkg', build.buildreq.buildreqs) + self.assertIn('mvn-testpkg', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_failed_pattern_maven_match(self): @@ -249,12 +280,14 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") build.failed_pattern('line to test for failure: aether', conf, + reqs, r'(aether)', 0, # verbose=0 buildtool='maven') - self.assertIn('mvn-aether-core', build.buildreq.buildreqs) + self.assertIn('mvn-aether-core', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_parse_buildroot_log_fail(self): @@ -337,6 +370,7 @@ class TestBuildpattern(unittest.TestCase): conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") conf.config_opts['32bit'] = True call_backup = build.util.call build.util.call = mock_util_call @@ -347,12 +381,12 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) build.util.call = call_backup - self.assertIn('pkgconfig(Qt)', build.buildreq.buildreqs) - self.assertIn('pkgconfig(32Qt)', build.buildreq.buildreqs) + self.assertIn('pkgconfig(Qt)', reqs.buildreqs) + self.assertIn('pkgconfig(32Qt)', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_parse_build_results_simple_pats(self): @@ -365,6 +399,7 @@ class TestBuildpattern(unittest.TestCase): conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") call_backup = build.util.call build.util.call = mock_util_call fm = files.FileManager(conf) @@ -374,11 +409,11 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) build.util.call = call_backup - self.assertIn('httpd-dev', build.buildreq.buildreqs) + self.assertIn('httpd-dev', reqs.buildreqs) self.assertEqual(build.must_restart, 1) def test_parse_build_results_failed_pats(self): @@ -388,6 +423,7 @@ class TestBuildpattern(unittest.TestCase): """ conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") call_backup = build.util.call open_auto_backup = build.util.open_auto build.util.call = MagicMock(return_value=None) @@ -398,11 +434,11 @@ class TestBuildpattern(unittest.TestCase): for error in builderrors: if not error.startswith('#'): input, output = error.strip('\n').split('|') - build.buildreq.buildreqs = set() + reqs.buildreqs = set() build.util.open_auto = mock_open(read_data=input) - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) - self.assertIn(output, build.buildreq.buildreqs) + self.assertIn(output, reqs.buildreqs) self.assertGreater(build.must_restart, 0) # Restoring functions @@ -418,6 +454,7 @@ class TestBuildpattern(unittest.TestCase): conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") call_backup = build.util.call build.util.call = mock_util_call fm = files.FileManager(conf) @@ -433,7 +470,7 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) build.util.call = call_backup @@ -453,6 +490,7 @@ class TestBuildpattern(unittest.TestCase): conf = config.Config() conf.setup_patterns() + reqs = buildreq.Requirements("") call_backup = build.util.call build.util.call = mock_util_call fm = files.FileManager(conf) @@ -470,7 +508,7 @@ class TestBuildpattern(unittest.TestCase): m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - build.parse_build_results('testname', 0, fm, conf) + build.parse_build_results('testname', 0, fm, conf, reqs) build.util.call = call_backup diff --git a/tests/test_buildreq.py b/tests/test_buildreq.py index 09d23e3..c4458fe 100644 --- a/tests/test_buildreq.py +++ b/tests/test_buildreq.py @@ -10,116 +10,105 @@ import config class TestBuildreq(unittest.TestCase): - @classmethod - def setUpClass(self): - """ - Class setup method to configure necessary modules - """ - buildreq.banned_buildreqs.add('bannedreq') - def setUp(self): """ Test setup method to reset the buildreq module """ - buildreq.banned_requires = set() - buildreq.buildreqs = set() - buildreq.requires = set() - buildreq.verbose = False - buildreq.cargo_bin = False - buildreq.buildpattern.pattern_strength = 0 + self.reqs = buildreq.Requirements("") + self.reqs.banned_buildreqs.add('bannedreq') def test_add_buildreq(self): """ Test add_buildreq with unbanned new req. Follow up by asserting that trying to add the same req a second time results in a False return. """ - self.assertTrue(buildreq.add_buildreq('testreq')) - self.assertIn('testreq', buildreq.buildreqs) - self.assertFalse(buildreq.add_buildreq('testreq')) + self.assertTrue(self.reqs.add_buildreq('testreq')) + self.assertIn('testreq', self.reqs.buildreqs) + self.assertFalse(self.reqs.add_buildreq('testreq')) def test_add_buildreq_banned(self): """ Test add_buildreq with banned new req """ - self.assertFalse(buildreq.add_buildreq('bannedreq')) - self.assertNotIn('bannedreq', buildreq.buildreqs) + self.assertFalse(self.reqs.add_buildreq('bannedreq')) + self.assertNotIn('bannedreq', self.reqs.buildreqs) def test_add_requires(self): """ Test add_requires with unbanned new req already present in buildreqs but not yet present in requires """ - buildreq.add_buildreq('testreq') - self.assertTrue(buildreq.add_requires('testreq', ['testreq'])) - self.assertIn('testreq', buildreq.requires) + self.reqs.add_buildreq('testreq') + self.assertTrue(self.reqs.add_requires('testreq', ['testreq'])) + self.assertIn('testreq', self.reqs.requires) def test_add_requires_not_in_buildreqs(self): """ Test add_requires with unbanned new req not present in buildreqs. """ - self.assertFalse(buildreq.add_requires('testreq', [])) - self.assertNotIn('testreq', buildreq.requires) + self.assertFalse(self.reqs.add_requires('testreq', [])) + self.assertNotIn('testreq', self.reqs.requires) def test_add_pkgconfig_buildreq(self): """ Test add_pkgconfig_buildreq with config_opts['32bit'] set to False """ - self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', False)) - self.assertIn('pkgconfig(testreq)', buildreq.buildreqs) + self.assertTrue(self.reqs.add_pkgconfig_buildreq('testreq', False)) + self.assertIn('pkgconfig(testreq)', self.reqs.buildreqs) def test_add_pkgconfig_buildreq_32bit(self): """ Test add_pkgconfig_buildreq with config_opts['32bit'] set to True """ - self.assertTrue(buildreq.add_pkgconfig_buildreq('testreq', True)) - self.assertIn('pkgconfig(testreq)', buildreq.buildreqs) - self.assertIn('pkgconfig(32testreq)', buildreq.buildreqs) + self.assertTrue(self.reqs.add_pkgconfig_buildreq('testreq', True)) + self.assertIn('pkgconfig(testreq)', self.reqs.buildreqs) + self.assertIn('pkgconfig(32testreq)', self.reqs.buildreqs) def test_configure_ac_line(self): """ Test configure_ac_line with standard pattern """ - buildreq.configure_ac_line('AC_CHECK_FUNC\([tgetent])', False) - self.assertIn('ncurses-devel', buildreq.buildreqs) + self.reqs.configure_ac_line('AC_CHECK_FUNC\([tgetent])', False) + self.assertIn('ncurses-devel', self.reqs.buildreqs) def test_configure_ac_line_comment(self): """ Test configure_ac_line with commented line """ - buildreq.configure_ac_line('# AC_CHECK_FUNC\([tgetent])', False) - self.assertEqual(buildreq.buildreqs, set()) + self.reqs.configure_ac_line('# AC_CHECK_FUNC\([tgetent])', False) + self.assertEqual(self.reqs.buildreqs, set()) def test_configure_ac_line_pkg_check_modules(self): """ Test the somewhat complicated logic of configure_ac_line check for the PKG_CHECK_MODULES\((.*?)\) line. """ - buildreq.configure_ac_line( + self.reqs.configure_ac_line( 'PKG_CHECK_MODULES(prefix, ' '[module > 2 module2 < 2], ' 'action-if-found, action-if-not-found)', False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(module)', 'pkgconfig(module2)'])) def test_configure_ac_line_xdt_check_package(self): """ Test configure_ac_line for the XFCE version of PKG_CHECK_MODULES """ - buildreq.configure_ac_line( + self.reqs.configure_ac_line( 'XDT_CHECK_PACKAGE(prefix, ' '[module = 2 module2 > 9], ' 'action-if-found, action-if-not-found)', False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(module)', 'pkgconfig(module2)'])) def test_configure_ac_line_pkg_check_exists(self): """ Test configure_ac_line for the PKG_CHECK_EXISTS macro """ - buildreq.configure_ac_line('PKG_CHECK_EXISTS([module1 > 1 module2], ' + self.reqs.configure_ac_line('PKG_CHECK_EXISTS([module1 > 1 module2], ' 'action-if-found, ' 'action-if-not-found)', False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(module1)', 'pkgconfig(module2)'])) def test_parse_configure_ac(self): @@ -127,6 +116,7 @@ class TestBuildreq(unittest.TestCase): Test parse_configure_ac with changing () depths and package requirements """ + buildreq.buildpattern.pattern_strength = 0 open_name = 'buildreq.util.open_auto' content = 'AC_CHECK_FUNC([tgetent])\n' \ 'XDT_CHECK_PACKAGE(prefix, ' \ @@ -139,10 +129,10 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_configure_ac(os.path.join(tmpd, 'fname'), False) + self.reqs.parse_configure_ac(os.path.join(tmpd, 'fname'), False) self.assertEqual(buildreq.buildpattern.default_pattern, 'configure_ac') - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['gettext', 'perl(XML::Parser)', 'pkgconfig(module2)', @@ -200,15 +190,16 @@ class TestBuildreq(unittest.TestCase): open_name = 'buildreq.util.open_auto' content = 'does not matter, let us mock' m_open = mock_open(read_data=content) + buildreq.buildpattern.pattern_strength = 0 with patch(open_name, m_open, create=True): - buildreq.parse_cargo_toml('filename', ['dep1', 'dep2', 'dep3']) + self.reqs.parse_cargo_toml('filename', ['dep1', 'dep2', 'dep3']) buildreq.os.path.exists = exists_backup buildreq.toml.loads = loads_backup - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['rustc', 'dep1', 'dep2', 'dep3'])) - self.assertTrue(buildreq.cargo_bin) + self.assertTrue(self.reqs.cargo_bin) self.assertEqual(buildreq.buildpattern.default_pattern, 'cargo') def test_set_build_req_maven(self): @@ -218,8 +209,8 @@ class TestBuildreq(unittest.TestCase): case the overall package list changes in the future. """ buildreq.buildpattern.default_pattern = 'maven' - buildreq.set_build_req() - self.assertIn('apache-maven', buildreq.buildreqs) + self.reqs.set_build_req() + self.assertIn('apache-maven', self.reqs.buildreqs) def test_set_build_req_ruby(self): """ @@ -228,8 +219,8 @@ class TestBuildreq(unittest.TestCase): case the overall package list changes in the future. """ buildreq.buildpattern.default_pattern = 'ruby' - buildreq.set_build_req() - self.assertIn('ruby', buildreq.buildreqs) + self.reqs.set_build_req() + self.assertIn('ruby', self.reqs.buildreqs) def test_set_build_req_cargo(self): """ @@ -238,8 +229,8 @@ class TestBuildreq(unittest.TestCase): case the overall package list changes in the future. """ buildreq.buildpattern.default_pattern = 'cargo' - buildreq.set_build_req() - self.assertIn('rustc', buildreq.buildreqs) + self.reqs.set_build_req() + self.assertIn('rustc', self.reqs.buildreqs) def test_rakefile(self): """ @@ -251,9 +242,9 @@ class TestBuildreq(unittest.TestCase): content = "line1\nrequire 'bundler/gem_tasks'\nline3\nrequire 'nope'" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.rakefile('filename', conf.gems) + self.reqs.rakefile('filename', conf.gems) - self.assertEqual(buildreq.buildreqs, set(['rubygem-rubygems-tasks'])) + self.assertEqual(self.reqs.buildreqs, set(['rubygem-rubygems-tasks'])) def test_clean_python_req(self): """ @@ -282,32 +273,32 @@ class TestBuildreq(unittest.TestCase): Test grab_python_requirements with a reasonable requirements file """ # buildreqs must include the requires also - buildreq.buildreqs = set(['req1', 'req2', 'req7']) + self.reqs.buildreqs = set(['req1', 'req2', 'req7']) open_name = 'buildreq.util.open_auto' content = 'req1 <= 1.2.3\n' \ 'req2 >= 1.55\n' \ 'req7 == 3.3.3\n' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3']) + self.reqs.grab_python_requirements('filename', ['req1', 'req2', 'req3']) - self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2', 'req7'])) def test_grab_python_requirements_strange_file(self): """ Test grab_python_requirements with a poorly written file """ # buildreqs must include the requires also - buildreq.buildreqs = set(['req1', 'req2', 'req7']) + self.reqs.buildreqs = set(['req1', 'req2', 'req7']) open_name = 'buildreq.util.open_auto' content = ' req1 <= 1.2.3\n ' \ 'req2 >= 1.55 \n' \ ' req7 == 3.3.3\n ' m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.grab_python_requirements('filename', ['req1', 'req2', 'req3']) + self.reqs.grab_python_requirements('filename', ['req1', 'req2', 'req3']) - self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req7'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2', 'req7'])) def test_add_setup_py_requires(self): """ @@ -319,10 +310,10 @@ class TestBuildreq(unittest.TestCase): "setup_requires=['req2']" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) - self.assertEqual(buildreq.requires, set(['req1'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2'])) + self.assertEqual(self.reqs.requires, set(['req1'])) def test_add_setup_py_requires_multiline(self): """ @@ -334,10 +325,10 @@ class TestBuildreq(unittest.TestCase): "'req3']\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3'])) - self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2', 'req3'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2', 'req3'])) def test_add_setup_py_requires_multiline_formatted(self): """ @@ -352,10 +343,10 @@ class TestBuildreq(unittest.TestCase): "]\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2', 'req3']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2', 'req3'])) - self.assertEqual(buildreq.requires, set(['req1', 'req2', 'req3'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2', 'req3'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2', 'req3'])) def test_add_setup_py_requires_multiline_variable(self): """ @@ -370,10 +361,10 @@ class TestBuildreq(unittest.TestCase): "]\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) - self.assertEqual(buildreq.requires, set(['req1', 'req2'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2'])) def test_add_setup_py_requires_variable(self): """ @@ -383,10 +374,10 @@ class TestBuildreq(unittest.TestCase): content = "install_requires=[reqname, 'req1', 'req2']\n" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', ['req1', 'req2']) + self.reqs.add_setup_py_requires('filename', ['req1', 'req2']) - self.assertEqual(buildreq.buildreqs, set(['req1', 'req2'])) - self.assertEqual(buildreq.requires, set(['req1', 'req2'])) + self.assertEqual(self.reqs.buildreqs, set(['req1', 'req2'])) + self.assertEqual(self.reqs.requires, set(['req1', 'req2'])) def test_add_setup_py_requires_single_variable(self): """ @@ -396,10 +387,10 @@ class TestBuildreq(unittest.TestCase): content = "install_requires=reqname" m_open = mock_open(read_data=content) with patch(open_name, m_open, create=True): - buildreq.add_setup_py_requires('filename', []) + self.reqs.add_setup_py_requires('filename', []) - self.assertEqual(buildreq.buildreqs, set()) - self.assertEqual(buildreq.requires, set()) + self.assertEqual(self.reqs.buildreqs, set()) + self.assertEqual(self.reqs.requires, set()) def test_setup_py3_version_classifier(self): """ @@ -432,9 +423,9 @@ class TestBuildreq(unittest.TestCase): open(os.path.join(tmpd, 'SConstruct'), 'w').close() open(os.path.join(tmpd, 'meson.build'), 'w').close() - buildreq.scan_for_configure(tmpd, "", "", conf) + self.reqs.scan_for_configure(tmpd, "", "", conf) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['buildreq-golang', 'buildreq-cmake', 'buildreq-scons', 'buildreq-distutils3', 'buildreq-meson'])) def test_scan_for_configure_pypi(self): @@ -443,6 +434,7 @@ class TestBuildreq(unittest.TestCase): pattern to test pypi metadata handling. """ conf = config.Config() + buildreq.buildpattern.pattern_strength = 0 orig_summary = buildreq.specdescription.default_summary orig_sscore = buildreq.specdescription.default_summary_score orig_pypi_name = buildreq.pypidata.get_pypi_name @@ -459,7 +451,7 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: os.mkdir(os.path.join(tmpd, 'subdir')) open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close() - buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) + self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) ssummary = buildreq.specdescription.default_summary buildreq.specdescription.default_summary = orig_summary @@ -467,8 +459,8 @@ class TestBuildreq(unittest.TestCase): buildreq.pypidata.get_pypi_name = orig_pypi_name buildreq.pypidata.get_pypi_metadata = orig_pypi_meta - self.assertEqual(buildreq.pypi_provides, name) - self.assertEqual(buildreq.pypi_requires, set(requires)) + self.assertEqual(self.reqs.pypi_provides, name) + self.assertEqual(self.reqs.pypi_requires, set(requires)) self.assertEqual(ssummary, summary) def test_scan_for_configure_pypi_override(self): @@ -477,6 +469,7 @@ class TestBuildreq(unittest.TestCase): pattern to test pypi metadata file override handling. """ conf = config.Config() + buildreq.buildpattern.pattern_strength = 0 open_name = 'buildreq.open' orig_summary = buildreq.specdescription.default_summary orig_sscore = buildreq.specdescription.default_summary_score @@ -492,14 +485,14 @@ class TestBuildreq(unittest.TestCase): open(os.path.join(tmpd, 'subdir', 'setup.py'), 'w').close() open(os.path.join(tmpd, 'pypi.json'), 'w').close() with patch(open_name, m_open, create=True): - buildreq.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) + self.reqs.scan_for_configure(os.path.join(tmpd, 'subdir'), "", tmpd, conf) ssummary = buildreq.specdescription.default_summary buildreq.specdescription.default_summary = orig_summary buildreq.specdescription.default_summary_score = orig_sscore - self.assertEqual(buildreq.pypi_provides, name) - self.assertEqual(buildreq.pypi_requires, set(requires)) + self.assertEqual(self.reqs.pypi_provides, name) + self.assertEqual(self.reqs.pypi_requires, set(requires)) self.assertEqual(ssummary, summary) def test_parse_cmake_pkg_check_modules(self): @@ -513,9 +506,9 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)'])) def test_parse_cmake_pkg_check_modules_whitespace(self): @@ -529,9 +522,9 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(gio-unix-2.0)', 'pkgconfig(glib-2.0)'])) def test_parse_cmake_pkg_check_modules_in_a_comment(self): @@ -548,9 +541,9 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set([])) def test_parse_cmake_pkg_check_modules_variables(self): @@ -564,9 +557,9 @@ class TestBuildreq(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), conf.cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['pkgconfig(libavcodec)', 'pkgconfig(libavutil)'])) def test_parse_cmake_find_package(self): @@ -587,9 +580,9 @@ find_package(different_name) with tempfile.TemporaryDirectory() as tmpd: with open(os.path.join(tmpd, 'fname'), 'w') as f: f.write(content) - buildreq.parse_cmake(os.path.join(tmpd, 'fname'), cmake_modules, False) + self.reqs.parse_cmake(os.path.join(tmpd, 'fname'), cmake_modules, False) - self.assertEqual(buildreq.buildreqs, + self.assertEqual(self.reqs.buildreqs, set(['valid', 'another_name'])) def test_r_desc_field_begin(self): @@ -695,8 +688,8 @@ find_package(different_name) content = 'Depends: pkg1' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertTrue('R-pkg1' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertTrue('R-pkg1' in self.reqs.buildreqs) def test_parse_r_desc_imports(self): """Test parsing of an R description Imports field.""" @@ -705,8 +698,8 @@ find_package(different_name) content = 'Imports: pkg2' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertTrue('R-pkg2' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertTrue('R-pkg2' in self.reqs.buildreqs) def test_parse_r_desc_linkingto(self): """Test parsing of an R description LinkingTo field.""" @@ -715,8 +708,8 @@ find_package(different_name) content = 'LinkingTo: pkg3' m_open = mock_open(read_data=content) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertTrue('R-pkg3' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertTrue('R-pkg3' in self.reqs.buildreqs) def test_parse_r_desc_multiple(self): """Test parsing of an R description file that captures multiple fields.""" @@ -736,13 +729,13 @@ find_package(different_name) ] m_open = mock_open(read_data='\n'.join(content)) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertFalse('R-foo' in buildreq.buildreqs) - self.assertFalse('R-bar' in buildreq.buildreqs) - self.assertTrue('R-pkg1' in buildreq.buildreqs) - self.assertTrue('R-pkg2' in buildreq.buildreqs) - self.assertTrue('R-pkg3' in buildreq.buildreqs) - self.assertTrue('R-pkg4' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertFalse('R-foo' in self.reqs.buildreqs) + self.assertFalse('R-bar' in self.reqs.buildreqs) + self.assertTrue('R-pkg1' in self.reqs.buildreqs) + self.assertTrue('R-pkg2' in self.reqs.buildreqs) + self.assertTrue('R-pkg3' in self.reqs.buildreqs) + self.assertTrue('R-pkg4' in self.reqs.buildreqs) def test_parse_r_desc_not_in_os(self): """Test parsing of an R description file with some non-OS packages.""" @@ -756,10 +749,10 @@ find_package(different_name) ] m_open = mock_open(read_data='\n'.join(content)) with patch(open_name, m_open): - buildreq.parse_r_description('filename', pkgs) - self.assertTrue('R-pkg1' in buildreq.buildreqs) - self.assertFalse('R-pkg2' in buildreq.buildreqs) - self.assertFalse('R-pkg3' in buildreq.buildreqs) + self.reqs.parse_r_description('filename', pkgs) + self.assertTrue('R-pkg1' in self.reqs.buildreqs) + self.assertFalse('R-pkg2' in self.reqs.buildreqs) + self.assertFalse('R-pkg3' in self.reqs.buildreqs) if __name__ == '__main__': diff --git a/tests/test_check.py b/tests/test_check.py index f8925f6..ba153a0 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -3,6 +3,7 @@ import tempfile import unittest from unittest.mock import mock_open, patch +import buildreq import check import config @@ -28,7 +29,6 @@ class TestTest(unittest.TestCase): def setUp(self): check.tests_config = '' check.tarball.name = '' - check.buildreq.buildreqs = set() check.buildpattern.default_pattern = "make" def test_check_regression(self): @@ -88,6 +88,7 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with makecheck suite """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.in']) @@ -95,7 +96,7 @@ class TestTest(unittest.TestCase): m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "configure" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -106,13 +107,14 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with makecheck suite via Makefile.am """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.am']) m_open = mock_open() with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "configure_ac" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -123,11 +125,12 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with perlcheck suite """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.PL']) check.buildpattern.default_pattern = "cpan" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" self.assertEqual(check.tests_config, 'make TEST_VERBOSE=1 test') @@ -136,6 +139,7 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with perlcheck suite via Makefile.in """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['Makefile.in']) @@ -143,7 +147,7 @@ class TestTest(unittest.TestCase): m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "cpan" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -153,6 +157,7 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with setup.py suite """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['setup.py']) @@ -160,7 +165,7 @@ class TestTest(unittest.TestCase): m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "distutils3" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -172,6 +177,7 @@ class TestTest(unittest.TestCase): """ Test scan_for_tests with cmake suite """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['CMakeLists.txt']) @@ -179,7 +185,7 @@ class TestTest(unittest.TestCase): m_open = mock_open(read_data=content) with patch(self.open_name, m_open, create=True): check.buildpattern.default_pattern = "cmake" - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup check.buildpattern.default_pattern = "make" @@ -191,12 +197,13 @@ class TestTest(unittest.TestCase): Test scan_for_tests with tox.ini in the files list, should add several build requirements """ + reqs = buildreq.Requirements("") conf = config.Config() listdir_backup = os.listdir check.os.listdir = mock_generator(['tox.ini']) - check.scan_for_tests('pkgdir', conf) + check.scan_for_tests('pkgdir', conf, reqs) check.os.listdir = listdir_backup - self.assertEqual(check.buildreq.buildreqs, + self.assertEqual(reqs.buildreqs, set(['tox', 'pytest', 'virtualenv', diff --git a/tests/test_infile_update_spec.py b/tests/test_infile_update_spec.py index b10d2a2..a1a60a2 100644 --- a/tests/test_infile_update_spec.py +++ b/tests/test_infile_update_spec.py @@ -1,5 +1,6 @@ import unittest +import buildreq import config import infile_update_spec import specfiles @@ -9,7 +10,7 @@ class TestUpdateSpecfile(unittest.TestCase): def setUp(self): # url, version, name, release url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz" - self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1', config.Config()) + self.specfile = specfiles.Specfile(url, '1.1.1', 'test_pkg', '1', config.Config(), buildreq.Requirements(url)) self.bb_dict = { "DEPENDS": "ncurses gettext-native", diff --git a/tests/test_specfile.py b/tests/test_specfile.py index 9868bc7..2d51ca9 100644 --- a/tests/test_specfile.py +++ b/tests/test_specfile.py @@ -1,5 +1,6 @@ import unittest import unittest.mock +import buildreq import config import specfiles @@ -15,7 +16,8 @@ class TestSpecfileWrite(unittest.TestCase): conf = config.Config() conf.config_opts['dev_requires_extras'] = False url = "http://www.testpkg.com/testpkg/pkg-1.0.tar.gz" - self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2', conf) + reqs = buildreq.Requirements(url) + self.specfile = specfiles.Specfile(url, '1.0', 'pkg', '2', conf, reqs) def mock_write(string): self.WRITES.append(string) @@ -100,8 +102,8 @@ class TestSpecfileWrite(unittest.TestCase): self.specfile.packages["autostart"] = ["autostart"] self.specfile.packages["bin"] = [] self.specfile.packages["lib"] = ["package.so"] - self.specfile.requires.add("pkg1") - self.specfile.requires.add("pkg2") + self.specfile.requirements.requires.add("pkg1") + self.specfile.requirements.requires.add("pkg2") self.specfile.config.config_opts['no_autostart'] = True self.specfile.write_main_subpackage_requires() expect = ["Requires: pkg-bin = %{version}-%{release}\n", @@ -122,8 +124,8 @@ class TestSpecfileWrite(unittest.TestCase): self.specfile.packages["ignore"] = [] self.specfile.packages["dev"] = [] self.specfile.packages["active-units"] = [] - self.specfile.requires.add("pkg1") - self.specfile.requires.add("pkg2") + self.specfile.requirements.requires.add("pkg1") + self.specfile.requirements.requires.add("pkg2") self.specfile.write_main_subpackage_requires() expect = ["Requires: pkg-autostart = %{version}-%{release}\n", "Requires: pkg-bin = %{version}-%{release}\n", @@ -143,7 +145,7 @@ class TestSpecfileWrite(unittest.TestCase): """ test write_buildreq with unsorted list of build requirements. """ - self.specfile.buildreqs = ["python", "ruby", "go"] + self.specfile.requirements.buildreqs = ["python", "ruby", "go"] self.specfile.write_buildreq() expect = ["BuildRequires : go\n", "BuildRequires : python\n", @@ -195,8 +197,8 @@ class TestSpecfileWrite(unittest.TestCase): self.specfile.packages["python"] = ["pyfile1", "pyfile2"] self.specfile.packages["dev"] = ["dev1", "dev2"] self.specfile.packages["pack"] = ["packf1"] - self.specfile.requires = ["pep8", "pylint", "pycurl"] - self.specfile.buildreqs = ["pep8", "pycurl"] + self.specfile.requirements.requires = ["pep8", "pylint", "pycurl"] + self.specfile.requirements.buildreqs = ["pep8", "pycurl"] self.specfile.write_files_header() expect = ["\n%package data\n", "Summary: data components for the pkg package.\n",