mirror of
https://github.com/clearlinux/common.git
synced 2026-08-26 16:28:11 +00:00
Add go-builder.py and autospecnewgo target
This change enables recurive dependency resolution and packaging when creating new go packages. To do so the 'autospecnewgo' target is created and the autospecnew target is slightly modified to align make target exit code expectations for failure handling.
This commit is contained in:
William Douglas
committed by
Patrick McCarty
Patrick McCarty
parent
8a2da3412d
commit
460ce05217
@@ -233,6 +233,54 @@ autospecnew: preautospecnew-checks localreponotice
|
||||
printf "** NOTICE: A patch with changes is available in the file $(TOPLVL)/packages/$(NAME)/for-review.txt\n"; \
|
||||
printf "** Please look through this file and if you are unsure, please submit for code review with git send-email\n"; \
|
||||
printf "**\n\n"; \
|
||||
else \
|
||||
echo "Autospec of $(NAME) failed."; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
else \
|
||||
echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
#help autospecnewgo: Creates a new autospec package with (dependencies)
|
||||
#help for a given URL=$(URL) with NAME=$(NAME). Several files used by
|
||||
#help autospec will be created in the help process.
|
||||
#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
|
||||
#help to autospec.
|
||||
#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
|
||||
autospecnewgo: preautospecnew-checks localreponotice
|
||||
@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
|
||||
echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
|
||||
exit 1; \
|
||||
fi
|
||||
-$(MAKE) clone_$(NAME)
|
||||
@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
|
||||
echo "no remote repository found, creating new package repository and running autospec"; \
|
||||
mkdir -p $(TOPLVL)/packages/$(NAME); \
|
||||
( \
|
||||
cd $(TOPLVL)/packages/$(NAME); \
|
||||
git init; \
|
||||
git remote add origin $(PKG_BASE_URL)/$(NAME); \
|
||||
$(call gitoliteurl,packages/$(NAME)); \
|
||||
$(call subjectprefix,$(NAME)); \
|
||||
); \
|
||||
printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
|
||||
python3 $(TOPLVL)/projects/common/gowrap.py \
|
||||
--target packages/$(NAME) \
|
||||
--integrity \
|
||||
--config "$(AUTOSPEC_CONF)" \
|
||||
--name $(NAME) \
|
||||
--archives $(ARCHIVES) \
|
||||
--mock-config $(MOCK_CONFIG_VAL) \
|
||||
--mock-opts "$(MOCK_OPTS)" \
|
||||
$(URL); \
|
||||
if [ $$? -eq 0 ]; then \
|
||||
$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
|
||||
$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
|
||||
python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
|
||||
python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show); \
|
||||
else \
|
||||
exit 1; \
|
||||
fi; \
|
||||
else \
|
||||
echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
|
||||
|
||||
Executable
+285
@@ -0,0 +1,285 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# gowrap.py - part of autospec
|
||||
# Copyright (C) 2019 Intel Corporation
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
def parse_args():
|
||||
"""Parse commandline arguments."""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("url", help="Location of package archive")
|
||||
parser.add_argument("-t", "--target", dest="target", action="store",
|
||||
default=None,
|
||||
help="Target location to create or reuse")
|
||||
parser.add_argument("-c", "--config", dest="config", action="store",
|
||||
default="/usr/share/defaults/autospec/autospec.conf",
|
||||
help="Set configuration file to use")
|
||||
parser.add_argument("-n", "--name", action="store", dest="name", default="",
|
||||
help="Override the package name")
|
||||
parser.add_argument("-i", "--integrity", action="store_true",
|
||||
default=False,
|
||||
help="Search for package signature from source URL and "
|
||||
"attempt to verify package")
|
||||
parser.add_argument("-m", "--mock-config", action="store", default="clear",
|
||||
help="Value to pass with Mock's -r option. Defaults to "
|
||||
"\"clear\", meaning that Mock will use "
|
||||
"/etc/mock/clear.cfg.")
|
||||
parser.add_argument("-o", "--mock-opts", action="store", default="",
|
||||
help="Arbitrary options to pass down to mock when "
|
||||
"building a package.")
|
||||
parser.add_argument('-a', "--archives", action="store",
|
||||
dest="archives", default=[], nargs='*',
|
||||
help="tarball URLs for additional source archives and"
|
||||
" a location for the sources to be extacted to (e.g."
|
||||
" http://example.com/downloads/dependency.tar.gz"
|
||||
" /directory/relative/to/extract/root )")
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def missing_dependencies(name):
|
||||
"""Check if the build failed due to missing dependencies."""
|
||||
root_log = os.path.join("packages", name, "results", "root.log")
|
||||
if not os.path.exists(root_log):
|
||||
return False
|
||||
with open(root_log, "r") as lfile:
|
||||
for line in lfile.readlines():
|
||||
if "No matching package to install" in line:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def already_built(name, version):
|
||||
"""Check if the package and version are already built."""
|
||||
if not os.path.exists(os.path.join("packages", name, "rpms")):
|
||||
return False
|
||||
vpath = os.path.join("packages", name, "versions")
|
||||
if not os.path.exists(vpath):
|
||||
return False
|
||||
with open(vpath, "r") as vfile:
|
||||
if version not in [x.strip() for x in vfile.readlines()]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def build(path, args, name, url, version):
|
||||
"""Try and build a package."""
|
||||
if version:
|
||||
ver_arg = ["-v", version]
|
||||
else:
|
||||
ver_arg = []
|
||||
if already_built(name, version):
|
||||
return True
|
||||
print(f"Trying to autospec {name} - {version}")
|
||||
|
||||
proc = subprocess.run(["python3",
|
||||
f"{path}/../autospec/autospec/autospec.py", url,
|
||||
"-c", args.config, "-t", f"packages/{name}",
|
||||
"-n", name, "-m", args.mock_config,
|
||||
"-o", args.mock_opts, "-i"] + ver_arg, capture_output=True)
|
||||
if proc.returncode == 0:
|
||||
subprocess.run(["make", "link-new-rpms", f"PKG_REPO_DIR=packages/{name}"],
|
||||
capture_output=True)
|
||||
return proc.returncode == 0
|
||||
|
||||
|
||||
def parse_go_mod(path):
|
||||
"""Parse go.mod file for build requirements.
|
||||
|
||||
File content looks as follows:
|
||||
|
||||
module example.com/foo/bar
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v0.3.1
|
||||
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
"github.com/spf13/cobra" v0.0.3
|
||||
github.com/spf13/pflag v1.0.3 // indirect
|
||||
)
|
||||
|
||||
Need to handle all require lines including //indirect.
|
||||
Skip requires that use .git for now. May need to be handled
|
||||
differently.
|
||||
"""
|
||||
reqs = []
|
||||
with open(path, "r") as gfile:
|
||||
dep_start = False
|
||||
for line in gfile.readlines():
|
||||
# Ideally the mod file is generated and the format is
|
||||
# always correct but add a few defenses just in case
|
||||
line = line.strip()
|
||||
if line.startswith("//"):
|
||||
# Skip comments
|
||||
continue
|
||||
if dep_start:
|
||||
# End of the require section
|
||||
if line.startswith(")"):
|
||||
break
|
||||
req = line.split()[:2]
|
||||
req[0] = req[0].replace('"', '')
|
||||
if req[0].endswith(".git"):
|
||||
continue
|
||||
reqs.append(req)
|
||||
continue
|
||||
if line.startswith("require ("):
|
||||
dep_start = True
|
||||
return reqs
|
||||
|
||||
|
||||
def get_dependencies(name):
|
||||
"""Return path to the go.mod file if it exists."""
|
||||
command = None
|
||||
path = None
|
||||
pdir = os.path.join("packages", name)
|
||||
reqs = []
|
||||
for fname in os.listdir(pdir):
|
||||
# Find the archive
|
||||
if fname.endswith("zip"):
|
||||
command = ["unzip", fname]
|
||||
elif ".tar." in fname:
|
||||
command = ["tar", "xf", fname]
|
||||
else:
|
||||
command = None
|
||||
|
||||
if command:
|
||||
# Decompress archive
|
||||
tdir = tempfile.mkdtemp()
|
||||
tfile = os.path.join(tdir, fname)
|
||||
shutil.copyfile(os.path.join(pdir, fname), tfile)
|
||||
proc = subprocess.run(command, cwd=tdir, capture_output=True)
|
||||
if not proc.returncode:
|
||||
# inspect contents for a "go.sum" file
|
||||
for root, _, files in os.walk(tdir):
|
||||
if "go.mod" in files:
|
||||
path = f"{name}.gomod"
|
||||
shutil.copyfile(os.path.join(root, "go.mod"), path)
|
||||
reqs += parse_go_mod(path)
|
||||
os.unlink(path)
|
||||
shutil.rmtree(tdir)
|
||||
|
||||
return reqs
|
||||
|
||||
|
||||
def encode_requirements(requirement):
|
||||
"""Encode the project's module and version."""
|
||||
# Default dependency base proxy url
|
||||
base_url = "https://proxy.golang.org/"
|
||||
# Encoding based on
|
||||
# https://tip.golang.org/cmd/go/#hdr-Module_proxy_protocol
|
||||
encoded_module = ""
|
||||
for char in requirement[0]:
|
||||
if char.isupper():
|
||||
encoded_module += "!" + char.lower()
|
||||
else:
|
||||
encoded_module += char
|
||||
encoded_version = ""
|
||||
for char in requirement[1]:
|
||||
if char.isupper():
|
||||
encoded_version += "!" + char.lower()
|
||||
else:
|
||||
encoded_version += char
|
||||
return (encoded_module, encoded_version)
|
||||
|
||||
|
||||
def initialize_package(name, url, version, path, args):
|
||||
"""Setup package and try to build it."""
|
||||
if already_built(name, version):
|
||||
return True
|
||||
print(f"Trying to autospecnew {name} - {version}")
|
||||
|
||||
proc = subprocess.run(["make", "autospecnew", f"URL={url}", f"NAME={name}", f"SETVERSION={version}"], capture_output=True)
|
||||
if f"{name} already exists at" in proc.stdout.decode("utf-8"):
|
||||
return build(path, args, name, url, version)
|
||||
return proc.returncode == 0
|
||||
|
||||
|
||||
def build_recursive(path, args, name, url, version=None, success=None):
|
||||
"""Try and recursively build packages."""
|
||||
if not success:
|
||||
# First package being built, common utilities already run
|
||||
# so just use autospec directly
|
||||
success = {}
|
||||
ret = build(path, args, name, url, version)
|
||||
else:
|
||||
ret = initialize_package(name, url, version, path, args)
|
||||
|
||||
if ret:
|
||||
# Completed so we are done!
|
||||
return True
|
||||
|
||||
if not missing_dependencies(name):
|
||||
# Failed but not due to missing dependencies, bail
|
||||
print(f"Unknown error building {name} - {version}")
|
||||
return False
|
||||
|
||||
reqs = get_dependencies(name)
|
||||
if not reqs:
|
||||
# Didn't try and add anything so don't need to build again
|
||||
print(f"Build failed with no missing requirements: {name} - {version}")
|
||||
return False
|
||||
|
||||
print(f"First attempt to build failed due to missing dependencies {name} - {version}\n")
|
||||
for req in reqs:
|
||||
ereq = encode_requirements(req)
|
||||
rname = "go-" + req[0].replace("/", "-")
|
||||
rurl = f"https://proxy.golang.org/{ereq[0]}/@v/list"
|
||||
if (rname, req[1]) in success:
|
||||
# Rebuild detected, figure out if it is okay or not
|
||||
if success[(rname, req[1])]:
|
||||
# Already succeeded building req short circuit success
|
||||
continue
|
||||
else:
|
||||
# Encountered a build loop, bail out as this likely
|
||||
# requires manual fixing
|
||||
print(f"Detected build loop when building {rname} - {req[1]}")
|
||||
return True
|
||||
success[(rname, req[1])] = False
|
||||
if not build_recursive(path, args, rname, rurl, req[1], success):
|
||||
print(f"Failed to build dependency {rname} - {req[1]}")
|
||||
return False
|
||||
|
||||
success[(rname, req[1])] = True
|
||||
subprocess.run(["make", "repodel"], cwd=f"packages/{rname}", capture_output=True)
|
||||
subprocess.run(["make", "repostage"], cwd=f"packages/{rname}", capture_output=True)
|
||||
|
||||
# Create the repo with the new packages added
|
||||
subprocess.run(["make", "localrepocreate"], capture_output=True)
|
||||
# Retry previously failed build assuming dependencies got added
|
||||
print(f"Rebuilding after resolving dependencies {name} - {version}")
|
||||
ret = build(path, args, name, url, version)
|
||||
print("")
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def main():
|
||||
"""Start program execution."""
|
||||
args = parse_args()
|
||||
path = os.path.dirname(os.path.realpath(__file__))
|
||||
return build_recursive(path, args, args.name, args.url)
|
||||
|
||||
if __name__ == '__main__':
|
||||
if not main():
|
||||
sys.exit(-1)
|
||||
print("\n\nBuild completed\n\n")
|
||||
Reference in New Issue
Block a user