10 Commits

Author SHA1 Message Date
HNO3Miracle 2e17027941 Add module root detect, and add k8s.io support 2026-06-06 22:15:17 +08:00
HNO3Miracle 9df9dfbeb5 [Fix] fix hash version date. 2026-06-06 15:40:18 +08:00
HeliC829 33eb67c31e docs: document pkg.go.dev API migration
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-22 22:33:22 +08:00
HeliC829 1f596caf57 refactor: migrate metadata discovery to pkgsite
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-22 22:30:54 +08:00
HeliC829 e28166a773 feat(pkgsite): add pkg.go.dev v1beta client
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-22 22:26:20 +08:00
HeliC829 dbbf920578 feat(pack): fall back to git archive when source download fails
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-22 22:23:02 +08:00
HeliC829 2758d5f404 refactor(pack): split runtime and test dependencies
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-22 22:20:38 +08:00
HeliC829 437d50ba0f feat(hosters): add go.yaml.in short host mapping
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-22 22:18:13 +08:00
HeliC829 f41dfb5711 fix(spec): pass package path to library subpackage description
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-22 22:15:35 +08:00
HeliC829 1aa3ad45d8 chore: ignore local build artifacts
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-22 22:11:09 +08:00
12 changed files with 1906 additions and 375 deletions
+6
View File
@@ -1,4 +1,5 @@
# Binaries for programs and plugins
go2spec
*.exe
*.exe~
*.dll
@@ -24,6 +25,11 @@ go.work.sum
# env file
.env
# Local go2spec validation/output artifacts
comparison-runs/
go-github-sandrolain-httpcache/
go-github-sandrolain-httpcache_*.orig.tar.gz
# Editor/IDE
.idea/
.vscode/
+21 -63
View File
@@ -3,7 +3,6 @@ package main
import (
"fmt"
"log"
"os"
"os/exec"
"regexp"
"slices"
@@ -14,9 +13,6 @@ import (
)
var (
// describeRegexp parses the count and revision part of the “git describe --long” output.
describeRegexp = regexp.MustCompile(`-\d+-g([0-9a-f]+)\s*$`)
// semverRegexp checks if a string is a valid Go semver,
// from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
// with leading "v" added.
@@ -24,10 +20,22 @@ var (
// uversionPrereleaseRegexp checks for upstream pre-release
// so that '-' can be replaced with '~' in pkgVersionFromGit.
// To be kept in sync with the regexp portion of uversionmanglePattern in spec.go
uversionPrereleaseRegexp = regexp.MustCompile(`(\d)[_\.\-\+]?(RC|rc|pre|dev|beta|alpha)[.]?(\d*)$`)
packagingDateNow = time.Now
)
func packagingDateString() string {
return packagingDateNow().Format("20060102")
}
func shortCommitHash(hash string) string {
if len(hash) > 7 {
return hash[:7]
}
return hash
}
// pkgVersionFromGit determines the actual version to be packaged
// from the git repository status and user preference.
// Besides returning the upstream version, the "upstream" struct
@@ -42,8 +50,10 @@ func pkgVersionFromGit(gitdir string, u *upstream, preferredRev string, forcePre
var cmd *exec.Cmd // the temporary shell commands we execute
// If the user specifies a valid tag as the preferred revision, that tag should be used without additional heuristics.
if u.rr != nil {
if out, err := u.rr.VCS.Tags(gitdir); err == nil && slices.Contains(out, preferredRev) {
if preferredRev != "" {
cmd = exec.Command("git", "tag", "--list", preferredRev)
cmd.Dir = gitdir
if out, err := cmd.Output(); err == nil && slices.Contains(strings.Fields(string(out)), preferredRev) {
latestTag = preferredRev
}
}
@@ -110,50 +120,6 @@ func pkgVersionFromGit(gitdir string, u *upstream, preferredRev string, forcePre
// Packaging @master (prerelease)
mainVer := ""
//if u.hasRelease {
// mainVer = u.version + "+"
//}
// Find committer date, UNIX timestamp
cmd = exec.Command("git", "log", "--pretty=format:%ct", "-n1", "--no-show-signature")
cmd.Dir = gitdir
lastCommitUnixBytes, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("git log: %w", err)
}
lastCommitUnix, err := strconv.ParseInt(strings.TrimSpace(string(lastCommitUnixBytes)), 0, 64)
if err != nil {
return "", fmt.Errorf("parse last commit date: %w", err)
}
dateStr := time.Unix(lastCommitUnix, 0).UTC().Format("20060102")
// Fetch commit hash
cmd = exec.Command("git", "describe", "--long", "--tags")
cmd.Dir = gitdir
lastCommitHash := ""
describeBytes, err := cmd.Output()
if err != nil {
// In case there are no tags at all, we just use the sha of the current commit
cmd = exec.Command("git", "rev-parse", "--short", "HEAD")
cmd.Dir = gitdir
cmd.Stderr = os.Stderr
revparseBytes, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("git rev-parse: %w", err)
}
lastCommitHash = strings.TrimSpace(string(revparseBytes))
u.commitIsh = lastCommitHash
} else {
submatches := describeRegexp.FindSubmatch(describeBytes)
if submatches == nil {
return "", fmt.Errorf("git describe output %q does not match expected format", string(describeBytes))
}
lastCommitHash = string(submatches[1])
u.commitIsh = strings.TrimSpace(string(describeBytes))
}
// Fetch full commit hash
cmd = exec.Command("git", "rev-parse", "HEAD")
cmd.Dir = gitdir
@@ -162,18 +128,10 @@ func pkgVersionFromGit(gitdir string, u *upstream, preferredRev string, forcePre
return "", fmt.Errorf("git rev-parse HEAD: %w", err)
}
fullCommitHash := strings.TrimSpace(string(fullCommitHashBytes))
lastCommitHash := shortCommitHash(fullCommitHash)
u.commitIsh = lastCommitHash
u.version = fmt.Sprintf("%sgit%s.%s",
mainVer,
time.Unix(lastCommitUnix, 0).UTC().Format("20060102"),
lastCommitHash)
if u.hasRelease {
// have tag: 1.2.3.20250101+git9f107c8
u.version = fmt.Sprintf("%s.%s+git%s", u.version, dateStr, lastCommitHash)
} else {
// without tag: 0.git20250101.96ee002 96ee0021ea0fb9174681b8004d8deba3c499d7f5
u.version = fmt.Sprintf("0+git%s.%s\n%%define commit_id %s", dateStr, lastCommitHash, fullCommitHash)
}
// Snapshot versions are based on the packaging date, not the commit date.
u.version = fmt.Sprintf("0+git%s.%s\n%%define commit_id %s", packagingDateString(), lastCommitHash, fullCommitHash)
return u.version, nil
}
+15 -21
View File
@@ -13,8 +13,7 @@ import (
//go:embed description.json
var descriptionJSONBytes []byte
// reformatForControl reformats the wrapped description
// to conform to Debians control format.
// reformatForControl reformats wrapped text for the RPM spec's %description.
func reformatForControl(raw string) string {
output := ""
next_prefix := ""
@@ -47,8 +46,8 @@ func reformatForControl(raw string) string {
return output
}
// markdownToLongDescription converts Markdown to plain text
// and reformat it for expanded description in debian/control.
// markdownToLongDescription converts Markdown to plain text for the RPM spec's
// %description section.
func markdownToLongDescription(markdown string) (string, error) {
r, _ := glamour.NewTermRenderer(
glamour.WithStylesFromJSONBytes(descriptionJSONBytes),
@@ -63,23 +62,17 @@ func markdownToLongDescription(markdown string) (string, error) {
return reformatForControl(out), nil
}
// getDescriptionForGopkg reads from README.md (or equivalent) from GitHub,
// intended for extended description in debian/control.
// getLongDescriptionForGopkg reads README.md (or equivalent) from pkg.go.dev,
// intended for the RPM spec's %description section.
func getLongDescriptionForGopkg(gopkg string) (string, error) {
owner, repo, err := findGitHubRepo(gopkg)
info, err := getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
return "", fmt.Errorf("find github repo: %w", err)
return "", fmt.Errorf("get pkgsite metadata: %w", err)
}
rr, _, err := gitHub.Repositories.GetReadme(context.TODO(), owner, repo, nil)
if err != nil {
return "", fmt.Errorf("get readme: %w", err)
}
content, err := rr.GetContent()
if err != nil {
return "", fmt.Errorf("get content: %w", err)
if info.Module.Readme == nil || strings.TrimSpace(info.Module.Readme.Contents) == "" {
return "", fmt.Errorf("pkgsite module %q has no README", info.Module.Path)
}
content := info.Module.Readme.Contents
// Supported filename suffixes are from
// https://github.com/github/markup/blob/master/README.md
@@ -88,10 +81,11 @@ func getLongDescriptionForGopkg(gopkg string) (string, error) {
// fairly involved, but itd be the most correct solution to the problem at
// hand. Our current code just knows markdown, which is good enough since
// most (Go?) projects in fact use markdown for their README files.
if !strings.HasSuffix(rr.GetName(), "md") &&
!strings.HasSuffix(rr.GetName(), "markdown") &&
!strings.HasSuffix(rr.GetName(), "mdown") &&
!strings.HasSuffix(rr.GetName(), "mkdn") {
readmeName := strings.ToLower(info.Module.Readme.Filepath)
if !strings.HasSuffix(readmeName, "md") &&
!strings.HasSuffix(readmeName, "markdown") &&
!strings.HasSuffix(readmeName, "mdown") &&
!strings.HasSuffix(readmeName, "mkdn") {
return reformatForControl(content), nil
}
-3
View File
@@ -4,11 +4,9 @@ go 1.25.3
require (
github.com/charmbracelet/glamour v0.10.0
github.com/google/go-github/v60 v60.0.0
github.com/mattn/go-isatty v0.0.20
github.com/sandrolain/httpcache v1.4.0
golang.org/x/net v0.47.0
golang.org/x/tools/go/vcs v0.1.0-deprecated
)
require (
@@ -22,7 +20,6 @@ require (
github.com/charmbracelet/x/exp/slice v0.0.0-20250327172914-2fdc97757edf // indirect
github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/dlclark/regexp2 v1.11.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
-10
View File
@@ -28,13 +28,6 @@ github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQ
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github/v60 v60.0.0 h1:oLG98PsLauFvvu4D/YPxq374jhSxFYdzQGNCyONLfn8=
github.com/google/go-github/v60 v60.0.0/go.mod h1:ByhX2dP9XT9o/ll2yXAu2VD8l5eNVg8hD4Cr0S/LmQk=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
@@ -76,6 +69,3 @@ golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4=
golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+4 -37
View File
@@ -3,26 +3,11 @@ package main
import (
"net/http"
"os"
"time"
"github.com/google/go-github/v60/github"
"github.com/sandrolain/httpcache"
)
var (
gitHub *github.Client
)
// TokenTransport implements http.RoundTripper for Bearer token authentication
type TokenTransport struct {
Token string
Transport http.RoundTripper
}
func (t *TokenTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req.Header.Add("Authorization", "Bearer "+t.Token)
return t.Transport.RoundTrip(req)
}
func printHelp() {
helpText := `go2spec - A tool to package Go modules into RPM spec files.
@@ -41,28 +26,10 @@ If there are no commands provided, the tool will default to executing the 'pack'
}
func main() {
token := os.Getenv("GITHUB_TOKEN")
var client *http.Client
if token != "" {
// Use token authentication for better rate limits
client = &http.Client{
Transport: &TokenTransport{
Token: token,
Transport: httpcache.NewMemoryCacheTransport(),
},
}
} else {
// Fallback to basic auth if token is not provided
transport := github.BasicAuthTransport{
Username: os.Getenv("GITHUB_USERNAME"),
Password: os.Getenv("GITHUB_PASSWORD"),
OTP: os.Getenv("GITHUB_OTP"),
Transport: httpcache.NewMemoryCacheTransport(),
}
client = transport.Client()
pkgsiteHTTPClient = &http.Client{
Timeout: 30 * time.Second,
Transport: httpcache.NewMemoryCacheTransport(),
}
gitHub = github.NewClient(client)
args := os.Args[1:]
+145 -98
View File
@@ -3,128 +3,175 @@ package main
import (
"context"
"fmt"
"net/http"
"html"
"path"
"regexp"
"sort"
"strings"
"golang.org/x/net/html"
)
// To update, use:
// curl -s https://api.github.com/licenses | jq '.[].key'
var githubLicenseToSPDXLicense = map[string]string{
//"agpl-3.0"
"apache-2.0": "Apache-2.0",
"artistic-2.0": "Artistic-2.0",
"bsd-2-clause": "BSD-2-Clause",
"bsd-3-clause": "BSD-3-Clause",
"cc0-1.0": "CC0-1.0",
//"epl-1.0" (eclipse public license)
"gpl-2.0": "GPL-2.0-only",
"gpl-3.0": "GPL-3.0-only",
"isc": "ISC",
"lgpl-2.1": "LGPL-2.1-only",
"lgpl-3.0": "LGPL-3.0-only",
"mit": "MIT",
"mpl-2.0": "MPL-2.0",
//"unlicense"
func getRepoURLForGopkg(gopkg string) (string, error) {
info, err := getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
return "", err
}
repoURL := strings.TrimSpace(info.Module.RepoURL)
if repoURL == "" {
return "", fmt.Errorf("pkgsite module %q has no repository URL", info.Module.Path)
}
return gitCloneURLFromRepoURL(repoURL), nil
}
var githubRegexp = regexp.MustCompile(`github\.com/([^/]+/[^/]+)`)
var (
htmlTagRegexp = regexp.MustCompile(`<[^>]*>`)
markdownImageRegex = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`)
markdownLinkRegex = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`)
packagePrefixRegex = regexp.MustCompile(`^Package\s+\S+\s+`)
)
func findGitHubOwnerRepo(gopkg string) (string, error) {
if strings.HasPrefix(gopkg, "github.com/") {
return strings.TrimPrefix(gopkg, "github.com/"), nil
// cleanSummaryCandidate turns a godoc synopsis or README line into an
// RPM-style Summary by stripping markup, keeping the first sentence, dropping
// the leading "Package foo" convention, and capitalizing the result.
func cleanSummaryCandidate(summary string) string {
summary = html.UnescapeString(strings.TrimSpace(summary))
summary = markdownImageRegex.ReplaceAllString(summary, "")
summary = markdownLinkRegex.ReplaceAllString(summary, "$1")
summary = htmlTagRegexp.ReplaceAllString(summary, " ")
summary = strings.ReplaceAll(summary, "`", "")
summary = strings.Join(strings.Fields(summary), " ")
summary = strings.Trim(summary, " \t\n\r#*-_")
if end := strings.Index(summary, ". "); end >= 0 {
summary = summary[:end]
}
resp, err := http.Get("https://" + gopkg + "?go-get=1")
if err != nil {
return "", fmt.Errorf("HTTP get: %w", err)
summary = packagePrefixRegex.ReplaceAllString(summary, "")
if summary != "" && summary[0] >= 'a' && summary[0] <= 'z' {
summary = string(summary[0]-('a'-'A')) + summary[1:]
}
defer resp.Body.Close()
z := html.NewTokenizer(resp.Body)
for {
tt := z.Next()
if tt == html.ErrorToken {
return "", fmt.Errorf("%q is not on GitHub", gopkg)
}
token := z.Token()
if token.Data != "meta" {
if summary == "" ||
strings.HasPrefix(summary, "[!") ||
strings.HasPrefix(summary, "![") ||
strings.HasPrefix(summary, "<!--") ||
strings.ContainsAny(summary, "<>") {
return ""
}
return strings.TrimSuffix(summary, ".")
}
// summaryFromReadme returns the first prose line of a README, skipping code
// fences, headings, HTML comments, badges, admonitions, and images.
func summaryFromReadme(markdown string) string {
inFence := false
lines := strings.Split(markdown, "\n")
for i, line := range lines {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "```") {
inFence = !inFence
continue
}
var meta struct {
name, content string
}
for _, attr := range token.Attr {
if attr.Key == "name" {
meta.name = attr.Val
}
if attr.Key == "content" {
meta.content = attr.Val
if i+1 < len(lines) {
next := strings.TrimSpace(lines[i+1])
if next != "" && strings.Trim(next, "=-") == "" {
continue
}
}
match := func(name string, length int) string {
if f := strings.Fields(meta.content); meta.name == name && len(f) == length {
if f[0] != gopkg {
return ""
}
if repoMatch := githubRegexp.FindStringSubmatch(f[2]); repoMatch != nil {
return strings.TrimSuffix(repoMatch[1], ".git")
}
}
return ""
if inFence ||
line == "" ||
strings.HasPrefix(line, "#") ||
strings.HasPrefix(line, "[!") ||
strings.HasPrefix(line, "![") ||
strings.HasPrefix(line, "<!--") ||
strings.HasPrefix(line, "<p align=") ||
strings.HasPrefix(line, "---") ||
strings.Trim(line, "=-") == "" {
continue
}
if repo := match("go-import", 3); repo != "" {
return repo, nil
}
if repo := match("go-source", 4); repo != "" {
return repo, nil
if summary := cleanSummaryCandidate(line); summary != "" {
return summary
}
}
return ""
}
func findGitHubRepo(gopkg string) (owner string, repo string, _ error) {
ownerrepo, err := findGitHubOwnerRepo(gopkg)
if err != nil {
return "", "", fmt.Errorf("find GitHub owner repo: %w", err)
func unusableSummary(summary, gopkg string, info *pkgsiteInfo) bool {
if summary == "" {
return true
}
parts := strings.Split(ownerrepo, "/")
if got, want := len(parts), 2; got != want {
return "", "", fmt.Errorf("invalid GitHub repo: %q does not follow owner/repo", repo)
}
return parts[0], parts[1], nil
base := path.Base(gopkg)
moduleBase := path.Base(info.Module.Path)
return strings.EqualFold(summary, base) ||
strings.EqualFold(summary, moduleBase) ||
strings.EqualFold(summary, info.Package.Name)
}
func getLicenseForGopkg(gopkg string) (string, error) {
owner, repo, err := findGitHubRepo(gopkg)
if err != nil {
return "", fmt.Errorf("find GitHub repo: %w", err)
}
rl, _, err := gitHub.Repositories.License(context.TODO(), owner, repo)
if err != nil {
return "", fmt.Errorf("get license for Go package: %w", err)
}
if license, ok := githubLicenseToSPDXLicense[rl.GetLicense().GetKey()]; ok {
return license, nil
}
return "TODO", nil
}
// getDescriptionForGopkg gets the package description from GitHub,
// intended for the synopsis or the short description in debian/control.
func getDescriptionForGopkg(gopkg string) (string, error) {
owner, repo, err := findGitHubRepo(gopkg)
if err != nil {
return "", fmt.Errorf("find GitHub repo: %w", err)
}
rr, _, err := gitHub.Repositories.Get(context.TODO(), owner, repo)
info, err := getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
return "", err
}
return strings.TrimSpace(rr.GetDescription()), nil
licenses := info.Module.Licenses
if len(licenses) == 0 {
licenses = info.Package.Licenses
}
return pkgsiteLicenseExpression(licenses), nil
}
func providesFromPkgsitePackages(gopkg string, info *pkgsiteInfo, packages []pkgsitePackageSummary) []string {
seen := map[string]bool{gopkg: true}
provides := []string{gopkg}
if info == nil || gopkg != info.Module.Path {
return provides
}
for _, p := range packages {
path := strings.TrimSpace(p.Path)
if path == "" || seen[path] || !p.IsRedistributable {
continue
}
seen[path] = true
provides = append(provides, path)
}
sort.Strings(provides)
return provides
}
func getProvidesForGopkg(gopkg string) ([]string, error) {
info, err := getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
return []string{gopkg}, err
}
if gopkg != info.Module.Path {
return []string{gopkg}, nil
}
packages, err := getPkgsiteModulePackages(context.TODO(), info.Module.Path, info.Module.Version)
if err != nil {
return []string{gopkg}, err
}
return providesFromPkgsitePackages(gopkg, info, packages), nil
}
// getDescriptionForGopkg gets the package synopsis from pkg.go.dev,
// intended for the summary in the RPM spec.
func getDescriptionForGopkg(gopkg string) (string, error) {
info, err := getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
return "", err
}
description := cleanSummaryCandidate(info.Package.Synopsis)
if unusableSummary(description, gopkg, info) {
description = ""
}
if description == "" && info.Module.Readme != nil {
description = summaryFromReadme(info.Module.Readme.Contents)
if unusableSummary(description, gopkg, info) {
description = ""
}
}
if description == "" {
return "", fmt.Errorf("no usable synopsis for %q", gopkg)
}
return description, nil
}
+386 -107
View File
@@ -1,9 +1,12 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"golang.org/x/net/publicsuffix"
"io"
"log"
"net/http"
@@ -11,10 +14,9 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
"golang.org/x/net/publicsuffix"
"golang.org/x/tools/go/vcs"
)
type packageType int
@@ -29,19 +31,23 @@ const (
// upstream describes the upstream repo we are about to package.
type upstream struct {
rr *vcs.RepoRoot
tarPath string // path to the downloaded or generated orig tarball tempfile
compression string // compression method, either "gz" or "xz"
version string // upstream version number, e.g. 0.0~git20180204.1d24609
tag string // Latest upstream tag, if any
commitIsh string // commit-ish corresponding to upstream version to be packaged
remote string // git remote, set to short hostname if upstream git history is included
firstMain string // import path of the first main package within repo, if any
vendorDirs []string // all vendor sub directories, relative to the repo directory
repoDeps []string // the repository paths of all dependencies (e.g. github.com/zyedidia/glob)
hasGodeps bool // whether the Godeps/_workspace directory exists
hasRelease bool // whether any release tags exist, for debian/watch
isRelease bool // whether what we end up packaging is a tagged release
repoURL string
modulePath string
tarPath string // path to the downloaded or generated orig tarball tempfile
compression string // compression method, either "gz" or "xz"
version string // upstream version number, e.g. 0.0~git20180204.1d24609
tag string // Latest upstream tag, if any
commitIsh string // commit-ish corresponding to upstream version to be packaged
remote string // git remote, set to short hostname if upstream git history is included
firstMain string // import path of the first main package within repo, if any
packageName string // package name of the requested package from pkg.go.dev
vendorDirs []string // all vendor sub directories, relative to the repo directory
repoDeps []string // all non-stdlib imports needed for build or tests
repoRunDeps []string // non-stdlib imports needed by normal builds, excluding test-only imports
repoTestDeps []string // non-stdlib imports needed only by tests
hasGodeps bool // whether the Godeps/_workspace directory exists
hasRelease bool // whether any release tags exist, for debian/watch
isRelease bool // whether what we end up packaging is a tagged release
}
var errUnsupportedHoster = errors.New("unsupported hoster")
@@ -118,29 +124,104 @@ func downloadFile(filename, url string) error {
return nil
}
// get downloads the specified Go package into the provided GOPATH,
// get downloads the source module into the provided GOPATH,
// checking out the specified revision if non-empty.
func (u *upstream) get(gopath, repo, rev string) error {
func (u *upstream) get(gopath, sourceRepo, requestedPath, rev string) error {
done := make(chan struct{})
defer close(done)
go progressSize("go get", filepath.Join(gopath, "src"), done)
go progressSize("git clone", filepath.Join(gopath, "src"), done)
rr, err := vcs.RepoRootForImportPath(repo, false)
info, err := getPkgsiteInfo(context.TODO(), requestedPath)
if err != nil {
return fmt.Errorf("get repo root: %w", err)
return fmt.Errorf("get pkgsite metadata: %w", err)
}
u.rr = rr
dir := filepath.Join(gopath, "src", rr.Root)
u.packageName = info.Package.Name
u.modulePath = info.Module.Path
u.repoURL = gitCloneURLFromRepoURL(info.Module.RepoURL)
if u.repoURL == "" {
return fmt.Errorf("pkgsite module %q has no repository URL", info.Module.Path)
}
dir := filepath.Join(gopath, "src", sourceRepo)
if err := os.MkdirAll(filepath.Dir(dir), 0755); err != nil {
return fmt.Errorf("mkdir clone parent: %w", err)
}
cmd := exec.Command("git", "clone", u.repoURL, dir)
cmd.Env = passthroughEnv()
cmd.Stderr = os.Stderr
log.Println("get: Running", cmd)
if err := cmd.Run(); err != nil {
return fmt.Errorf("git clone: %w", err)
}
if rev != "" {
// Run "git clone {repo} {dir}" and "git checkout {tag}"
return rr.VCS.CreateAtRev(dir, rr.Repo, rev)
cmd = exec.Command("git", "-c", "advice.detachedHead=false", "checkout", rev)
cmd.Dir = dir
cmd.Env = passthroughEnv()
cmd.Stderr = os.Stderr
log.Println("get: Running", cmd, "in", cmd.Dir)
if err := cmd.Run(); err != nil {
return fmt.Errorf("git checkout %q: %w", rev, err)
}
}
// Run "git clone {repo} {dir}" (or the equivalent command for hg, svn, bzr)
return rr.VCS.Create(dir, rr.Repo)
return nil
}
func (u *upstream) tarballUrl() (string, error) {
repo := strings.TrimSuffix(u.rr.Repo, ".git")
func guessedPackageType(u *upstream) packageType {
if u.packageName == "main" || (u.packageName == "" && u.firstMain != "") {
return typeProgram
}
return typeLibrary
}
// sourceImportPathForPackage returns the module root used for source checkout
// and tarball generation, while callers keep the requested import path for
// naming and go_import_path.
func sourceImportPathForPackage(requestedPath string, info *pkgsiteInfo) string {
if info != nil && info.Package.ModulePath != "" {
return info.Package.ModulePath
}
if info != nil && info.Module.Path != "" {
return info.Module.Path
}
return requestedPath
}
func goImportPathFromArgument(arg string) (string, bool, error) {
arg = strings.TrimSpace(arg)
if strings.HasPrefix(arg, "go(") || strings.HasSuffix(arg, ")") {
m := regexp.MustCompile(`^go\(([^()]+)\)$`).FindStringSubmatch(arg)
if m == nil {
return "", false, fmt.Errorf("invalid Go RPM capability %q", arg)
}
return strings.TrimSpace(m[1]), true, nil
}
return arg, false, nil
}
// gitCloneURLFromRepoURL converts repository URLs returned by pkg.go.dev into
// something git clone understands. pkg.go.dev reports golang.org/x/* modules
// with a cs.opensource.google browser URL, so rewrite those to go.googlesource.com.
func gitCloneURLFromRepoURL(repoURL string) string {
repoURL = strings.TrimSuffix(strings.TrimSpace(repoURL), ".git")
u, err := url.Parse(repoURL)
if err != nil || u.Host != "cs.opensource.google" {
return repoURL
}
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
if len(parts) == 3 && parts[0] == "go" && parts[1] == "x" {
return "https://go.googlesource.com/" + parts[2]
}
return repoURL
}
func (u *upstream) tarballURLForRef(ref, compression string) (string, error) {
repo := strings.TrimSuffix(u.repoURL, ".git")
if repo == "" {
return "", fmt.Errorf("repository URL is empty")
}
repoU, err := url.Parse(repo)
if err != nil {
return "", fmt.Errorf("parse URL: %w", err)
@@ -149,26 +230,138 @@ func (u *upstream) tarballUrl() (string, error) {
switch repoU.Host {
case "github.com":
return fmt.Sprintf("%s/archive/%s.tar.%s",
repo, u.tag, u.compression), nil
repo, ref, compression), nil
case "gitlab.com", "salsa.debian.org":
parts := strings.Split(repoU.Path, "/")
if len(parts) < 3 {
return "", fmt.Errorf("incomplete repo URL: %s", u.rr.Repo)
return "", fmt.Errorf("incomplete repo URL: %s", u.repoURL)
}
project := parts[2]
project := strings.TrimSuffix(parts[len(parts)-1], ".git")
return fmt.Sprintf("%s/-/archive/%s/%s-%s.tar.%s",
repo, u.tag, project, u.tag, u.compression), nil
repo, ref, project, ref, compression), nil
case "git.sr.ht":
return fmt.Sprintf("%s/archive/%s.tar.%s",
repo, u.tag, u.compression), nil
repo, ref, compression), nil
case "codeberg.org":
return fmt.Sprintf("%s/archive/%s.tar.%s",
repo, u.tag, u.compression), nil
repo, ref, compression), nil
default:
return "", errUnsupportedHoster
}
}
func (u *upstream) tarballUrl() (string, error) {
return u.tarballURLForRef(u.tag, u.compression)
}
func moduleProxyEscapedPath(modulePath string) string {
parts := strings.Split(strings.Trim(modulePath, "/"), "/")
for i, part := range parts {
parts[i] = moduleProxyEscapeString(part)
}
return strings.Join(parts, "/")
}
// moduleProxyEscapeString applies the case-escape required by the Go module
// proxy protocol: uppercase letters become "!" followed by lowercase.
// See https://go.dev/ref/mod#goproxy-protocol.
func moduleProxyEscapeString(s string) string {
var b strings.Builder
for _, r := range s {
if r >= 'A' && r <= 'Z' {
b.WriteByte('!')
b.WriteRune(r + ('a' - 'A'))
continue
}
b.WriteRune(r)
}
return strings.ReplaceAll(url.PathEscape(b.String()), "%21", "!")
}
// moduleProxyVersionForSpec leaves RPM macros unchanged and otherwise applies
// module proxy escaping so versions like v1.0.0-RC1 become v1.0.0-!r!c1.
func moduleProxyVersionForSpec(version string) string {
if strings.Contains(version, "%{") {
return version
}
return moduleProxyEscapeString(version)
}
func repoRootImportPath(repoURL string) string {
u, err := url.Parse(strings.TrimSuffix(repoURL, ".git"))
if err != nil {
return ""
}
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
switch u.Host {
case "github.com", "gitlab.com", "codeberg.org", "salsa.debian.org":
if len(parts) >= 2 {
return u.Host + "/" + strings.Join(parts[:2], "/")
}
}
return ""
}
// moduleUsesRepoSubdir reports whether modulePath lives in a repository
// subdirectory. A trailing /vN semantic import version suffix is treated as
// part of the repo-root module path, not as a subdirectory.
func moduleUsesRepoSubdir(modulePath, repoURL string) bool {
root := repoRootImportPath(repoURL)
if root == "" || !strings.HasPrefix(modulePath, root+"/") {
return false
}
rest := strings.TrimPrefix(modulePath, root+"/")
return !regexp.MustCompile(`^v\d+$`).MatchString(rest)
}
// sourceRefForSpec picks the git ref for Source0, preferring RPM macros when
// the upstream version maps cleanly to %{commit_id} or %{version}.
func (u *upstream) sourceRefForSpec() (string, error) {
ref := u.tag
if strings.Contains(u.version, "commit_id") {
ref = "%{commit_id}"
} else if u.isRelease && u.tag == "v"+u.version {
ref = "v%{version}"
} else if u.isRelease && u.tag == u.version {
ref = "%{version}"
} else if ref == "" {
ref = u.commitIsh
}
if ref == "" {
return "", fmt.Errorf("source reference is empty")
}
return ref, nil
}
// sourceURLForSpec returns the Source0 URL for the spec file. It prefers
// hoster tarballs for repo-root modules, and uses proxy.golang.org zips for
// repo subdirectories or unsupported hosters. It returns an error for
// commit-pinned subdirectory modules because the proxy requires a canonical
// pseudo-version.
func (u *upstream) sourceURLForSpec(modulePath string) (string, error) {
ref, err := u.sourceRefForSpec()
if err != nil {
return "", err
}
if moduleUsesRepoSubdir(modulePath, u.repoURL) {
if strings.Contains(ref, "commit_id") {
return "", fmt.Errorf("module proxy Source0 for commit-pinned subdirectory module %q requires a canonical pseudo-version", modulePath)
}
return fmt.Sprintf("https://proxy.golang.org/%s/@v/%s.zip#/%%{_name}-%%{version}.zip",
moduleProxyEscapedPath(modulePath), moduleProxyVersionForSpec(ref)), nil
}
tarURL, err := u.tarballURLForRef(ref, "gz")
if err == nil {
return fmt.Sprintf("%s#/%%{_name}-%%{version}.tar.gz", tarURL), nil
}
if err != errUnsupportedHoster || modulePath == "" || strings.Contains(ref, "commit_id") {
return "", err
}
return fmt.Sprintf("https://proxy.golang.org/%s/@v/%s.zip#/%%{_name}-%%{version}.zip",
moduleProxyEscapedPath(modulePath), moduleProxyVersionForSpec(ref)), nil
}
func (u *upstream) tarballFromHoster() error {
tarURL, err := u.tarballUrl()
if err != nil {
@@ -204,7 +397,7 @@ func (u *upstream) tar(gopath, repo string) error {
} else if err == errUnsupportedHoster {
log.Printf("INFO: Hoster does not provide release tarball\n")
} else {
return fmt.Errorf("tarball from hoster: %w", err)
log.Printf("WARNING: Could not download release tarball from hoster, falling back to local git archive: %v\n", err)
}
}
}
@@ -263,10 +456,74 @@ func (u *upstream) findMains(gopath, repo string) error {
return nil
}
type goListPackage struct {
ImportPath string
Imports []string
TestImports []string
XTestImports []string
Error *struct {
Err string
}
}
func shouldIgnoreGoDependency(p, repo string) bool {
if p == "" {
return true
}
// Strip packages that are included in the repository we are packaging.
if strings.HasPrefix(p, repo+"/") || p == repo {
return true
}
if p == "C" {
// TODO: maybe parse the comments to figure out C deps from pkg-config files?
return true
}
return false
}
func addGoDependencies(deps map[string]bool, repo string, imports []string) {
for _, p := range imports {
if shouldIgnoreGoDependency(p, repo) {
continue
}
deps[p] = true
}
}
func sortedDependencySet(deps map[string]bool) []string {
result := make([]string, 0, len(deps))
for dep := range deps {
result = append(result, dep)
}
sort.Strings(result)
return result
}
// setRepoDependencies stores three views of imports: repoDeps is the union for
// BuildRequires, repoRunDeps is non-test imports for library Requires, and
// repoTestDeps is imports used only by tests.
func (u *upstream) setRepoDependencies(runtimeDeps, testDeps map[string]bool) {
buildDeps := make(map[string]bool, len(runtimeDeps)+len(testDeps))
for dep := range runtimeDeps {
buildDeps[dep] = true
}
testOnlyDeps := make(map[string]bool)
for dep := range testDeps {
buildDeps[dep] = true
if !runtimeDeps[dep] {
testOnlyDeps[dep] = true
}
}
u.repoDeps = sortedDependencySet(buildDeps)
u.repoRunDeps = sortedDependencySet(runtimeDeps)
u.repoTestDeps = sortedDependencySet(testOnlyDeps)
}
func (u *upstream) findDependencies(gopath, repo string) error {
log.Printf("Determining dependencies\n")
cmd := exec.Command("go", "list", "-e", "-f", "{{join .Imports \"\\n\"}}\n{{join .TestImports \"\\n\"}}\n{{join .XTestImports \"\\n\"}}", repo+"/...")
cmd := exec.Command("go", "list", "-e", "-json", repo+"/...")
cmd.Dir = filepath.Join(gopath, "src", repo)
cmd.Env = passthroughEnv()
cmd.Stderr = os.Stderr
@@ -276,33 +533,37 @@ func (u *upstream) findDependencies(gopath, repo string) error {
log.Println("WARNING: In findDependencies:", fmt.Errorf("%q: %w", cmd.Args, err))
// See https://bugs.debian.org/992610
log.Printf("Retrying without appending \"/...\" to repo")
cmd = exec.Command("go", "list", "-e", "-f", "{{join .Imports \"\\n\"}}\n{{join .TestImports \"\\n\"}}\n{{join .XTestImports \"\\n\"}}", repo)
cmd = exec.Command("go", "list", "-e", "-json", repo)
cmd.Dir = filepath.Join(gopath, "src", repo)
cmd.Env = passthroughEnv()
cmd.Stderr = os.Stderr
out, err = cmd.Output()
if err != nil {
log.Println("WARNING: In findDependencies:", fmt.Errorf("%q: %w", cmd.Args, err))
return fmt.Errorf("go list dependencies: %q: %w", cmd.Args, err)
}
}
godependencies := make(map[string]bool)
for _, p := range strings.Split(strings.TrimSpace(string(out)), "\n") {
if p == "" {
continue // skip separators between import types
runtimeDeps := make(map[string]bool)
testDeps := make(map[string]bool)
dec := json.NewDecoder(strings.NewReader(string(out)))
for {
var pkg goListPackage
err := dec.Decode(&pkg)
if errors.Is(err, io.EOF) {
break
}
// Strip packages that are included in the repository we are packaging.
if strings.HasPrefix(p, repo+"/") || p == repo {
continue
if err != nil {
return fmt.Errorf("decode go list dependency output: %w", err)
}
if p == "C" {
// TODO: maybe parse the comments to figure out C deps from pkg-config files?
} else {
godependencies[p] = true
if pkg.Error != nil && pkg.Error.Err != "" {
log.Printf("WARNING: go list reported package load error for %q: %s\n", pkg.ImportPath, pkg.Error.Err)
}
addGoDependencies(runtimeDeps, repo, pkg.Imports)
addGoDependencies(testDeps, repo, pkg.TestImports)
addGoDependencies(testDeps, repo, pkg.XTestImports)
}
if len(godependencies) == 0 {
if len(runtimeDeps) == 0 && len(testDeps) == 0 {
return nil
}
@@ -317,51 +578,34 @@ func (u *upstream) findDependencies(gopath, repo string) error {
}
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
delete(godependencies, line)
delete(runtimeDeps, line)
delete(testDeps, line)
}
// Resolve all packages to the root of their repository.
//roots := make(map[string]bool)
//for dep := range godependencies {
// rr, err := vcs.RepoRootForImportPath(dep, false)
// if err != nil {
// log.Printf("Could not determine repo path for import path %q: %v\n", dep, err)
// continue
// }
// roots[rr.Root] = true
//}
//u.repoDeps = make([]string, 0, len(godependencies))
//for root := range roots {
// u.repoDeps = append(u.repoDeps, root)
//}
// Alternatively, just list all import paths as dependencies.
u.repoDeps = make([]string, 0, len(godependencies))
for dep := range godependencies {
u.repoDeps = append(u.repoDeps, dep)
}
u.setRepoDependencies(runtimeDeps, testDeps)
return nil
}
// makeUpstreamSourceTarball downloads the specified Go package from the Internet,
// makeUpstreamSourceTarball downloads the source module from the Internet,
// checks out the specified revision, determines the version number, removes
// vendored dependencies, and creates a tarball of the upstream source code.
// It also discovers main packages and dependencies, and may check out a release
// tag different from the requested revision when pkgVersionFromGit selects one.
// It returns an upstream struct describing the downloaded package.
func makeUpstreamSourceTarball(repo, revision string, forcePrerelease bool) (*upstream, error) {
func makeUpstreamSourceTarball(requestedPath, sourceRepo, revision string, forcePrerelease bool) (*upstream, error) {
gopath, err := os.MkdirTemp("", "pack-tmp")
if err != nil {
return nil, fmt.Errorf("create tmp dir: %w", err)
}
defer os.RemoveAll(gopath)
repoDir := filepath.Join(gopath, "src", repo)
repoDir := filepath.Join(gopath, "src", sourceRepo)
var u upstream
log.Printf("Downloading %q\n", repo+"/...")
if err := u.get(gopath, repo, revision); err != nil {
return nil, fmt.Errorf("go get: %w", err)
log.Printf("Downloading %q\n", sourceRepo+"/...")
if err := u.get(gopath, sourceRepo, requestedPath, revision); err != nil {
return nil, fmt.Errorf("get source: %w", err)
}
// Verify early this repository uses git (we call pkgVersionFromGit later):
@@ -389,22 +633,43 @@ func makeUpstreamSourceTarball(repo, revision string, forcePrerelease bool) (*up
log.Printf("Determining upstream version number\n")
u.version, err = pkgVersionFromGit(repoDir, &u, revision, forcePrerelease)
preferredRevision := revision
if preferredRevision == "" {
// Reuses the cache populated by u.get above when available.
if info, err := getPkgsiteInfo(context.TODO(), requestedPath); err == nil {
preferredRevision = info.Package.Version
log.Printf("Using pkg.go.dev latest version %q as the preferred release", preferredRevision)
}
}
u.version, err = pkgVersionFromGit(repoDir, &u, preferredRevision, forcePrerelease)
if err != nil {
return nil, fmt.Errorf("get package version from Git: %w", err)
}
log.Printf("Package version is %q\n", u.version)
if err := u.findMains(gopath, repo); err != nil {
if u.isRelease && u.tag != "" {
// pkgVersionFromGit may select a release tag different from the revision
// passed by the user; dependency and tarball discovery must use that tree.
cmd := exec.Command("git", "-c", "advice.detachedHead=false", "checkout", u.tag)
cmd.Dir = repoDir
cmd.Env = passthroughEnv()
cmd.Stderr = os.Stderr
log.Println("makeUpstreamSourceTarball: Running", cmd, "in", cmd.Dir)
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("git checkout release tag %q: %w", u.tag, err)
}
}
if err := u.findMains(gopath, sourceRepo); err != nil {
return nil, fmt.Errorf("find mains: %w", err)
}
if err := u.findDependencies(gopath, repo); err != nil {
if err := u.findDependencies(gopath, sourceRepo); err != nil {
return nil, fmt.Errorf("find dependencies: %w", err)
}
if err := u.tar(gopath, repo); err != nil {
if err := u.tar(gopath, sourceRepo); err != nil {
return nil, fmt.Errorf("tar: %w", err)
}
@@ -474,6 +739,7 @@ func shortHostName(gopkg string, allowUnknownHoster bool) (host string, err erro
"gitlab.com": "gitlab",
"go.bug.st": "bugst",
"go.cypherpunks.ru": "cypherpunks",
"go.yaml.in": "yaml",
"go.mongodb.org": "mongodb",
"go.opentelemetry.io": "opentelemetry",
"go.step.sm": "step",
@@ -627,19 +893,35 @@ func mainPack(args []string, usage func()) {
}
gitRevision = strings.TrimSpace(gitRevision)
gopkg := flagSet.Arg(0)
gopkg, fromGoCapability, err := goImportPathFromArgument(flagSet.Arg(0))
if err != nil {
log.Fatalf("Verifying arguments: %v", err)
}
// Remove URL scheme if present (https://, http://, git://, etc.)
gopkg = strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(gopkg, "https://"), "http://"), "git://")
// Verify that the provided argument is a valid Go package import path
rr, err := vcs.RepoRootForImportPath(gopkg, false)
// Verify the provided argument using the official pkg.go.dev API. Keep the
// requested path for naming and go_import_path; use the module path only for
// source checkout and tarball generation.
info, err := getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
log.Fatalf("Verifying arguments: %v — did you specify a Go package import path?", err)
}
if gopkg != rr.Root {
log.Printf("Continuing with repository root %q instead of specified import path %q", rr.Root, gopkg)
gopkg = rr.Root
if fromGoCapability {
modulePath := sourceImportPathForPackage(gopkg, info)
if modulePath != "" && modulePath != gopkg {
log.Printf("Using module root %q to package requested Go capability go(%s)", modulePath, gopkg)
gopkg = modulePath
info, err = getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
log.Fatalf("Verifying module root %q: %v", gopkg, err)
}
}
}
sourceGopkg := sourceImportPathForPackage(gopkg, info)
if sourceGopkg != gopkg {
log.Printf("Using module root %q as source checkout for specified import path %q", sourceGopkg, gopkg)
}
// Set default source and binary package names.
@@ -685,25 +967,33 @@ func mainPack(args []string, usage func()) {
// NOTE: directory existence is checked after determining final openRuyiSrc
// Create a tarball of the upstream source
u, err := makeUpstreamSourceTarball(gopkg, gitRevision, forcePrerelease)
u, err := makeUpstreamSourceTarball(gopkg, sourceGopkg, gitRevision, forcePrerelease)
if err != nil {
log.Fatalf("Could not create a tarball of the upstream source: %v\n", err)
}
if pkgType == typeGuess {
if u.firstMain != "" {
log.Printf("Assuming you are packaging a program (because %q defines a main package), use -type to override\n", u.firstMain)
switch guessedPackageType(u) {
case typeProgram:
if u.packageName == "main" {
log.Printf("Assuming you are packaging a program (because pkg.go.dev reports package %q as main), use -type to override\n", gopkg)
} else {
log.Printf("Assuming you are packaging a program (because %q defines a main package), use -type to override\n", u.firstMain)
}
pkgType = typeProgram
openRuyiSrc = nameFromGopkg(gopkg, pkgType, customProgPkgName, allowUnknownHoster)
} else {
default:
if u.firstMain != "" {
log.Printf("Found main package %q, but pkg.go.dev reports root package %q; assuming library, use -type to override\n", u.firstMain, u.packageName)
}
pkgType = typeLibrary
}
}
// Now that we know the final package name, check output directory
info, err := os.Stat(openRuyiSrc)
dirInfo, err := os.Stat(openRuyiSrc)
if err == nil {
if !info.IsDir() {
if !dirInfo.IsDir() {
log.Fatalf("%q exists but is not a directory\n", openRuyiSrc)
}
entries, err := os.ReadDir(openRuyiSrc)
@@ -734,19 +1024,8 @@ func mainPack(args []string, usage func()) {
log.Fatalf("Could not create repository: %v\n", err)
}
seen := make(map[string]bool)
pkgdependencies := make([]string, 0, len(u.repoDeps))
for _, dep := range u.repoDeps {
pkgname := nameFromGopkg(dep, typeLibrary, "", allowUnknownHoster)
if !seen[pkgname] {
seen[pkgname] = true
pkgdependencies = append(pkgdependencies, pkgname)
}
}
// optional: sort.Strings(debdependencies)
if err := writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, u.version,
pkgType, pkgdependencies, u); err != nil {
pkgType, u); err != nil {
log.Fatalf("Could not create spec file: %v\n", err)
}
+282
View File
@@ -0,0 +1,282 @@
# pkg.go.dev API migration notes
## Goal
This migration replaces the old metadata discovery path based on GitHub API calls, `?go-get=1` HTML parsing, and `golang.org/x/tools/go/vcs` with the official pkg.go.dev API (`https://pkg.go.dev/v1beta`) plus Go module proxy fallbacks.
The main reason is that RPM spec generation should follow Go module semantics, not just Git repository layout. pkg.go.dev already resolves module paths, versions, README content, SPDX license metadata, repository URLs, and ambiguous import paths in the same ecosystem used by `go get` and `go mod`.
## Code changes
- Added `pkgsite.go` as the single pkg.go.dev API client.
- Added `pkgsite_test.go` covering API error retry, ambiguity handling, summary cleanup, license expression generation, module proxy escaping, Source0 generation, and type detection.
- Removed GitHub API client setup and credential handling from `main.go`.
- Removed `github.com/google/go-github/v60` and deprecated `golang.org/x/tools/go/vcs` dependencies from `go.mod`.
- Replaced GitHub README/license/repository lookups in `metadata.go` and `description.go` with pkgsite module/package metadata.
- Reworked `pack.go` to use pkgsite module path, package name, version, and repository URL.
- Reworked `spec.go` so `URL` and `Source0` come from pkgsite/repository/module-proxy information rather than hard-coded GitHub owner/repo parsing.
## Change classification
### API and data-source changes
These changes replace old discovery sources with Go ecosystem sources, without being output features by themselves:
- Added the pkg.go.dev v1beta client with retry handling, response-size cap, in-process metadata cache, and the HTTP cache transport wired in `main.go`.
- Removed the GitHub API client, credential/token handling, GitHub README/license lookups, `?go-get=1` HTML scraping, and `golang.org/x/tools/go/vcs` repo-root guessing.
- Replaced old metadata sources with pkgsite module/package metadata for module path, package name, repo URL, README, synopsis, license, version, imports, and ambiguity candidates.
- Added Go module proxy zip URL support and module-proxy path escaping; `.mod` parsing is still future work.
- Replaced `vcs.VCS.Tags` with local `git tag --list` over the cloned repository, and replaced VCS abstraction checkout with direct `git clone` / `git checkout`.
- Replaced the flattened `go list -f` dependency scan with structured `go list -e -json`.
- Removed `go-github`, `x/tools/go/vcs`, and obsolete transitive dependencies from `go.mod` / `go.sum`.
### Feature enhancements independent of the API migration
These are output, policy, or robustness improvements that could exist with any metadata source:
- Preserve the user-requested import path for package naming and `%define go_import_path`, while using the resolved module path for source checkout and tarball generation.
- Preserve `/vN` semantic-import-version suffixes in generated RPM names.
- Add RPM-safe summary cleanup helpers that strip markup, badges, README headings, and Go doc `Package ...` prefixes.
- Split dependencies into normal imports, test-only imports, and their union; `BuildRequires` uses the union, while library runtime `Requires` uses normal imports only. This still depends on a heuristic import-path-to-RPM conversion, so resolving imports to owning modules remains future work.
- Derive runtime/build dependency sets inside `writeSpec` instead of passing a flattened dependency list.
- Fix the missing `gopkg` argument in `writeRPMLibrarySubpackage`.
- Fall back from failed hoster tarball downloads to local `git archive` instead of failing immediately.
- Add `go.yaml.in -> yaml` to the short host-name mapping.
- Add migration notes and regression tests for retry/error handling, ambiguity, summary cleanup, license expressions, proxy escaping, Source0 generation, type detection, dependency splitting, and spec asset paths.
### Feature enhancements enabled by the new API/data sources
These user-visible improvements depend on pkgsite, module-proxy, or structured Go metadata:
- Generate real `License:` values from pkgsite SPDX license types, including multi-license SPDX expressions.
- Generate real `URL:` values for non-GitHub modules and normalize `cs.opensource.google/go/x/*` to `go.googlesource.com/*`.
- Generate `Summary:` from pkgsite package synopsis, with pkgsite README fallback and cleanup.
- Generate long `%description` from pkgsite `module.readme.contents`.
- Prefer pkgsite's latest module version, then validate it against local Git tags and check out the selected release tag before dependency/tarball discovery.
- Resolve module paths and ambiguous import paths using pkgsite candidates instead of repository-root guessing.
- Use module-proxy `Source0` zips for monorepo submodules and unsupported hosters while keeping hoster tarballs as the primary source for normal repo-root modules.
- Refuse invalid module-proxy `Source0` for commit-pinned subdirectory modules by leaving `Source0: TODO` unless a canonical pseudo-version is available.
- Use pkgsite README and license file paths to emit exact `%doc` / `%license` entries such as `Readme` or `License`, and omit `%doc` when no README exists.
- Classify program/library type from pkgsite `Package.Name == "main"`; local `main` discovery remains only as a fallback.
Future enhancements that depend on the same data-source migration include per-package `Provides` from `/v1beta/packages/{module}`, cleaner `BuildRequires` from module proxy `.mod` files, cgo detection from imports, multi-module repository fan-out, and import-path-to-owning-module resolution.
## Fields improved by official API data
| Field | Old behavior | New behavior |
|---|---|---|
| Module path | `vcs.RepoRootForImportPath` / repo root guessing | `package.modulePath` and ambiguity candidates from pkgsite |
| Version | `git describe` over repository tags | pkgsite latest module version is preferred, then validated against Git tags |
| Summary | GitHub repo description or README fallback | pkgsite package synopsis, cleaned for RPM Summary |
| License | GitHub license API or `TODO` | pkgsite SPDX license types |
| Repository URL | GitHub-only parsing / `TODO` for non-GitHub | pkgsite `module.repoUrl`; `cs.opensource.google/go/x/*` is normalized to `go.googlesource.com/*` |
| Source0 | Hoster table only | Hoster table plus `proxy.golang.org` zip fallback |
| Type guess | Any `package main` in repo meant program | Requested package `name == "main"` means program; command subpackages no longer misclassify libraries. If pkgsite reports no package name, local `main` detection remains a fallback |
## License handling
The old `githubLicenseToSPDXLicense` path converted GitHub license API keys such as `mit` or `apache-2.0` into local SPDX strings. That local conversion is intentionally removed because the new metadata source already returns SPDX license identifiers.
Current behavior:
- `getLicenseForGopkg` reads `module.licenses` from pkgsite and falls back to `package.licenses` only if the module has no license data.
- `pkgsiteLicenseExpression` uses `licenses[].types` exactly as returned by pkgsite. It does not translate GitHub keys or infer license names from filenames.
- If one license file has multiple pkgsite-detected types, they are joined with `OR`, for example `(MIT OR OFL-1.1)`.
- If multiple license files apply, the per-file groups are joined with `AND`.
- Top-level license files are preferred over nested license files, because nested files often belong to vendored or generated content.
- If pkgsite returns no license types, the generated `License:` field stays `TODO` instead of guessing.
- `%license` file entries use pkgsite `licenses[].filePath`, sanitized for spec output, so casing such as `License` is preserved.
This means `License:` correctness now depends on pkg.go.dev's official license detection data, not on GitHub's license API or a local GitHub-key-to-SPDX mapping. The generator still owns only the RPM expression policy: top-level preference, per-file `OR`, cross-file `AND`, duplicate removal, stable sorting, and `TODO` when the API does not provide SPDX types.
## Important behavior changes
- `/vN` module paths are preserved in generated package names, for example `github.com/alecthomas/chroma/v2` becomes `go-github-alecthomas-chroma-v2`. This is intentional because pkgsite reports the module path, not only the Git repository root.
- Summary differences versus the original tool are expected. The new summaries come from pkgsite package synopsis or cleaned README text.
- `golang.org/x/*` URLs now point at `go.googlesource.com/*` and Source0 uses `proxy.golang.org`.
- Monorepo submodules such as `github.com/charmbracelet/x/ansi` use module-proxy Source0 so the source is scoped to that module instead of the whole monorepo tarball.
- Root libraries with command subpackages, such as `go.yaml.in/yaml/v4`, are treated as libraries by default. Use `-type` to override when a package should be generated as a program or combined package.
- Requested subpackage paths are preserved for package naming and `%define go_import_path`; the module root is used only for source checkout and tarball generation.
- Dependency discovery now keeps normal imports separate from `TestImports` and `XTestImports`. `BuildRequires` uses the union, while library runtime `Requires` uses normal imports only.
- Commit-pinned subdirectory modules no longer emit invalid module-proxy `Source0` URLs; a canonical pseudo-version is required before the proxy can serve those sources, otherwise `Source0` is left as `TODO` for manual handling.
## Dependency data
The old implementation did not run a full compile to discover dependencies. It ran:
```bash
go list -e -f '{{join .Imports "\n"}}
{{join .TestImports "\n"}}
{{join .XTestImports "\n"}}' <repo>/...
```
Then it removed imports that are part of the packaged repository and imports from the Go standard library. The remaining import paths were all written as RPM `BuildRequires`. This means regular compile imports, internal test imports, and external test-package imports were already included, but they were merged into one set and emitted only as `BuildRequires`; no `%check` or test-only bucket existed.
The pkg.go.dev API has package-level dependency data, but it is not a complete module-level RPM dependency source:
- `/v1beta/package/{path}?imports=true` returns the direct imports for one package. These are package import paths, not module paths or RPM package names.
- `/v1beta/packages/{module}` enumerates packages in a module, but does not return the dependency graph by itself.
- `/v1beta/module/{module}` exposes module metadata such as version and repository URL. Although the OpenAPI schema contains a `goModContents` field, the field is currently empty or partial in `v1beta` responses; do not use it. Fetch `https://proxy.golang.org/<module>/@v/<version>.mod` directly instead.
For cleaner `BuildRequires`, the preferred future approach is to fetch the official module proxy `.mod` file for the selected version and parse it with `golang.org/x/mod/modfile`. Direct `require` entries can represent normal module dependencies, while `// indirect` entries can be handled separately or ignored depending on RPM policy. Package-level pkgsite `imports=true` can still be useful for cgo detection, identifying direct package imports, and resolving import paths back to modules when needed.
Test-only dependencies are now tracked separately by reading structured `go list -json` output. The generator stores normal imports, test-only imports, and their union separately. Today the union is emitted as `BuildRequires`, and only normal imports are emitted as library runtime `Requires`. Because import paths are still collapsed to RPM provides by a heuristic, a test-only import that shares the same top-level provide as a runtime import can still appear in runtime `Requires`; resolving imports to owning module paths is a follow-up. A future RPM-macro-specific enhancement can emit the test-only set as `%check`-scoped dependencies or another separate generated list.
If a dependency is needed only by tests but does not appear in the generated spec, the likely causes are:
- `go list -e -json` may return partial package data when a package cannot be loaded. The current code logs package load errors; these diagnostics still need policy decisions for when generation should fail.
- Only the default build context is analyzed. Test files behind non-default build tags, GOOS, GOARCH, cgo settings, or downstream RPM-specific tags are not visible unless those settings are passed to `go list`.
- `passthroughEnv` currently preserves only a small environment allowlist. It does not pass variables such as `GOFLAGS`, `GOOS`, `GOARCH`, `CGO_ENABLED`, `GOPROXY`, `GONOSUMDB`, or `GOPRIVATE`, so local settings that would make additional test files or private/proxy dependencies visible can be lost.
- Imports equal to the packaged repo path or prefixed by `<repo>/` are dropped as internal packages. This is correct for normal subpackages, but can be wrong for monorepos or nested modules where a same-prefix import is actually a separate packaged module.
- `convertDependenciesToRPM` collapses import paths to a top-level path such as `github.com/user/repo`, with only a simple `/vN` exception. Nested modules, vanity paths, and monorepo submodules can therefore be converted to the wrong RPM virtual provide, making a dependency look missing.
- Test cgo imports are represented as `C`, and the current code explicitly ignores `C`, so C compiler, pkg-config, and C library requirements used only by tests are not derived.
- Generated test files are not considered because `go list` does not run `go generate`.
The safer design is to keep dependency provenance all the way through generation: normal imports, in-package test imports, external test-package imports, cgo usage, and module `require` data should be stored as separate sets. The spec writer can then emit ordinary build dependencies and test-only dependencies independently, instead of filtering a flattened import list after the fact.
## Examples verified
### `go.yaml.in/yaml/v4`
pkgsite reports the root package name as `yaml`, while `cmd/go-yaml` is a separate `main` subpackage. This package is not part of the `go.mod` comparison set; it was manually run as a regression check for library/program detection. The generated spec is now a library package:
```spec
Name: go-yaml-yaml-v4
Version: 4.0.0~rc4
Summary: Implements YAML 1.1/1.2 encoding and decoding for Go programs
License: Apache-2.0
BuildArch: noarch
BuildSystem: golangmodules
Provides: go(go.yaml.in/yaml/v4) = %{version}
```
### `golang.org/x/net`
Old output used `TODO` metadata. New output uses official metadata and module proxy Source0:
```spec
License: BSD-3-Clause
URL: https://go.googlesource.com/net
Source0: https://proxy.golang.org/golang.org/x/net/@v/v%{version}.zip#/%{_name}-%{version}.zip
```
### `github.com/charmbracelet/x/ansi`
This is a submodule inside the `github.com/charmbracelet/x` monorepo. Source0 now points at the module proxy zip:
```spec
Name: go-github-charmbracelet-x-ansi
Source0: https://proxy.golang.org/github.com/charmbracelet/x/ansi/@v/v%{version}.zip#/%{_name}-%{version}.zip
```
### Dependency split validation
Recent manual validation confirmed that test/build-only imports are excluded from library runtime `Requires`:
- `github.com/charmbracelet/lipgloss`: `go(github.com/aymanbagabas/go-udiff)` appears in `BuildRequires` only.
- `github.com/aliyun/alibabacloud-oss-go-sdk-v2`: `go(github.com/stretchr/testify)` appears in `BuildRequires` only.
- `google.golang.org/grpc`: `go(google.golang.org/protobuf/testing)` appears in `BuildRequires` only.
## Final comparison artifacts
The latest full comparison run is:
```text
comparison-runs/go-mod-all-final-20260522184105/
```
Important files:
- `summary.txt` - readable per-package comparison.
- `fields-full.tsv` - full field-level data for original/current output.
- `diff-summary-full.tsv` - compact list of differing fields.
This run used all 27 `require` entries from `go.mod`.
Headline result:
- Current implementation: 27/27 packages generated successfully.
- Current generated specs: no `Summary`, `License`, `URL`, or `Source0` field remains `TODO`.
- Original implementation: 2 packages failed in the comparison baseline.
Notable improvements:
- `golang.org/x/*` packages now have real URL, License, and Source0 fields.
- `github.com/charmbracelet/glamour` and `github.com/charmbracelet/x/exp/slice` generated successfully in the current implementation.
- Multi-license modules such as `github.com/alecthomas/chroma/v2` now get SPDX expressions like `(MIT OR OFL-1.1)`.
Older `comparison-runs/` directories are intermediate states from the migration and review process. Use `go-mod-all-final-20260522184105` as the current reference.
Additional ad-hoc validation after the comparison run covered `github.com/charmbracelet/bubbletea`, `github.com/charmbracelet/lipgloss`, `github.com/go-openapi/testify`, `github.com/hashicorp/go-secure-stdlib`, `github.com/aliyun/alibabacloud-oss-go-sdk-v2`, `github.com/aymerick/douceur`, `google.golang.org/grpc`, `github.com/apache/arrow`, `github.com/aws/smithy-go`, and `github.com/charmbracelet/x`. These runs verified the dependency split for normal module cases and exposed the legacy GOPATH / package-collection cases tracked below.
## Verification commands
```bash
go test ./...
go run . pack go.yaml.in/yaml/v4
go run . pack golang.org/x/net
go run . pack github.com/charmbracelet/x/ansi
```
## Caveats
- pkg.go.dev API is currently `v1beta`; schema and behavior may change.
- pkg.go.dev only covers public modules. Private modules still need a different path.
- Newly tagged versions may have pkgsite indexing delay.
- `pkgsiteInfoCache` is in-process only and not version-keyed yet, so requesting two versions of the same module in one process can reuse the first cached result.
- There is also an in-memory HTTP cache transport for pkgsite requests, so cache behavior has two process-local layers.
- pkgsite response bodies are capped by `pkgsiteMaxResponseBytes`.
- pkgsite requests currently retry up to 3 times with fixed linear sleeps on transport errors, HTTP 429, or HTTP 5xx responses.
- Module-proxy zip Source0 archives unpack as module-version paths such as `module@vX.Y.Z`; generated specs may still need `%setup`/`%autosetup` adjustments depending on downstream RPM macros.
- Subdirectory modules still need an end-to-end checkout/layout review. The spec `Source0` may use module-proxy zip correctly, but dependency discovery currently clones the repository and may not analyze the actual submodule directory layout.
- For requested subpath programs, `%define _name` follows the requested path while hoster tarballs usually unpack using the repository or module root name. This can require `%setup`/`%autosetup -n` handling and needs a dedicated fix before declaring subpath program output fully buildable.
## Possible split-out work: multi-module and legacy repositories
This section is not part of the current implementation plan. It records cases that may need a separate design pass after the pkgsite migration and dependency-splitting behavior are validated.
`github.com/hashicorp/go-secure-stdlib` is a package-collection repository, not a normal single Go module. pkgsite reports the repository root with `hasGoMod: false`, while subpaths such as `github.com/hashicorp/go-secure-stdlib/base62` are separate modules with their own `go.mod`, versions, tags, dependencies, and potentially licenses. `/v1beta/packages/github.com/hashicorp/go-secure-stdlib` is module-scoped, so it returns only packages in the root pseudo-module, not sibling modules.
`hasGoMod: false` alone is not enough to classify a repository as a package collection. `github.com/aymerick/douceur` is a legacy GOPATH-style repository: pkgsite reports `hasGoMod: false`, but `/v1beta/packages` returns several importable packages under the same root rather than separate sibling modules. These repositories need GOPATH-mode dependency discovery, not fan-out into submodules.
Today, `hasGoMod: false` repositories can generate silently incomplete dependencies because `go list -json` runs in module-mode defaults and may match no packages. This was observed with `github.com/aymerick/douceur` and `github.com/apache/arrow`; GOPATH-mode discovery or a refusal policy is needed in the split-out work.
The generator should distinguish three shapes:
1. **Single module with many packages**, such as `golang.org/x/text`: one module path, one version, one dependency closure. Use `/v1beta/packages/{module}` to emit `Provides: go(<pkgpath>) = %{version}` for each redistributable package in the module, and keep `go list -json ./...` for dependency discovery.
2. **Legacy GOPATH repository**, such as `github.com/aymerick/douceur`: no `go.mod`, but one repository-level package set. Treat it like a single package set, but run dependency discovery in GOPATH mode and still emit per-package `Provides`.
3. **Multi-module repository / package collection**, such as `github.com/hashicorp/go-secure-stdlib` or `github.com/charmbracelet/x`: multiple subdirectories are independent modules. Do not fold them into one source RPM, because versions, Source0 paths, dependencies, and licenses can diverge.
Default behavior for confirmed package-collection roots should be refusal with a clear diagnostic, not generation of a mostly empty root spec. The diagnostic should explain that sibling modules were detected and suggest packaging a concrete submodule path, for example `github.com/hashicorp/go-secure-stdlib/base62`. An opt-in `--fanout` or `--submodules=a,b,c` mode can later generate one independent spec per discovered submodule.
Currently, packaging a bare package-collection root such as `github.com/charmbracelet/x` fails at pkgsite lookup time with "not found"; the clearer diagnostic described here is not implemented.
Submodule discovery should avoid GitHub-specific APIs. Preferred sources are `git ls-remote --tags <repoUrl>` grouped by tags like `<subdir>/vX.Y.Z`, followed by pkgsite `/v1beta/module/{repo}/{subdir}` probes to keep only paths with `hasGoMod: true`. Each discovered submodule must run the normal single-module pipeline independently: resolve its pkgsite version, Source0, license, package list, normal/test dependencies, and spec name.
This feature depends on two dependency fixes:
- Resolve import paths to owning module paths before converting them to RPM virtual provides. For example, `github.com/hashicorp/go-secure-stdlib/base62` must map to the submodule RPM name, not the repository-root name.
- Compare internal imports against the module path being packaged, not just the repository prefix. Sibling modules share a repository prefix but are external dependencies from Go module and RPM perspectives.
## Follow-up work
- Use the documented `/v1beta/packages/{module}` endpoint to enumerate all packages in a module and improve `library+program` / `program+library` generation.
- Validate and fix subdirectory module checkout/layout so dependency discovery analyzes the module subdirectory while `Source0` remains scoped to the module.
- Validate and fix requested subpath program specs so `_name`, `Source0`, and `%setup` agree on the extracted source directory.
- Use Go module proxy `.mod` files and `golang.org/x/mod/modfile` to derive direct module dependencies for cleaner `BuildRequires`.
- Emit the tracked test-only dependency set through RPM-macro-specific `%check` dependency mechanisms if the target macro set supports it.
- Use pkgsite imports or source scanning to detect cgo and improve `ExclusiveArch` / C toolchain BuildRequires.
- Improve Source0 and `%setup` semantics for module-proxy zip sources.
- Improve retry behavior by honoring `Retry-After`, adding jitter, and using exponential backoff.
- Add fixture tests for real pkgsite API responses to catch future `v1beta` schema drift.
### Dependency follow-up plan
When implementing dependency handling, keep these work items together so the known gaps above are not reintroduced:
1. Decide which `go list -json` package load errors should be fatal instead of warnings, so partial results cannot silently hide dependencies.
2. Make the analyzed build context explicit: preserve or configure `GOFLAGS`, build tags, `GOOS`, `GOARCH`, `CGO_ENABLED`, proxy/private-module settings, and any required code generation before dependency discovery.
3. Resolve import paths to module paths before converting them to RPM virtual provides. Multi-module same-prefix filtering is tracked separately in the split-out repository-shape work above.
4. Track cgo separately, including test-only `import "C"`, so C compiler, pkg-config, and C library requirements can be emitted when tests need them.
5. Emit the already-separated test-only dependency set through target-specific `%check` dependency macros when available.
6. Add regression fixtures for external test packages, build-tagged tests, cgo tests, generated test files, and vanity paths. Nested-module and monorepo same-prefix fixtures belong with the split-out repository-shape work above.
+349
View File
@@ -0,0 +1,349 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"sync"
"time"
)
const pkgsiteAPIBase = "https://pkg.go.dev/v1beta"
const pkgsiteMaxResponseBytes = 20 << 20
var (
pkgsiteHTTPClient = http.DefaultClient
pkgsiteMu sync.Mutex
pkgsiteInfoCache = make(map[string]*pkgsiteInfo)
)
type pkgsiteInfo struct {
Package pkgsitePackage
Module pkgsiteModule
}
type pkgsitePackage struct {
ModulePath string `json:"modulePath"`
Version string `json:"version"`
Path string `json:"path"`
Name string `json:"name"`
Synopsis string `json:"synopsis"`
Imports []string `json:"imports"`
Licenses []pkgsiteLicense `json:"licenses"`
IsRedistributable bool `json:"isRedistributable"`
IsStandardLibrary bool `json:"isStandardLibrary"`
AmbiguousCandidates []pkgsiteCandidate
}
type pkgsiteModule struct {
Path string `json:"path"`
Version string `json:"version"`
RepoURL string `json:"repoUrl"`
Readme *pkgsiteReadme `json:"readme"`
Licenses []pkgsiteLicense `json:"licenses"`
IsRedistributable bool `json:"isRedistributable"`
IsStandardLibrary bool `json:"isStandardLibrary"`
HasGoMod bool `json:"hasGoMod"`
}
type pkgsiteReadme struct {
Contents string `json:"contents"`
Filepath string `json:"filepath"`
}
type pkgsiteLicense struct {
Types []string `json:"types"`
FilePath string `json:"filePath"`
Contents string `json:"contents"`
}
type pkgsiteCandidate struct {
ModulePath string `json:"modulePath"`
PackagePath string `json:"packagePath"`
}
type pkgsitePackagesResponse struct {
ModulePath string `json:"modulePath"`
Version string `json:"version"`
IsStandardLibrary bool `json:"isStandardLibrary"`
Packages pkgsitePackageItems `json:"packages"`
}
type pkgsitePackageItems struct {
Items []pkgsitePackageSummary `json:"items"`
Total int `json:"total"`
NextPageToken string `json:"nextPageToken"`
}
type pkgsitePackageSummary struct {
Path string `json:"path"`
Name string `json:"name"`
Synopsis string `json:"synopsis"`
IsRedistributable bool `json:"isRedistributable"`
}
type pkgsiteAPIError struct {
Code int `json:"code"`
Message string `json:"message"`
Fixes []string `json:"fixes"`
Candidates []pkgsiteCandidate `json:"candidates"`
Status string `json:"-"`
}
func (e *pkgsiteAPIError) Error() string {
if e.Message != "" {
return e.Message
}
if e.Status != "" {
return e.Status
}
if e.Code != 0 {
return fmt.Sprintf("pkgsite API error: HTTP %d", e.Code)
}
return "pkgsite API error"
}
func pkgsiteEscapedPath(importPath string) string {
parts := strings.Split(strings.Trim(importPath, "/"), "/")
for i, part := range parts {
parts[i] = url.PathEscape(part)
}
return strings.Join(parts, "/")
}
func pkgsiteURL(endpoint, importPath string, values url.Values) string {
u := pkgsiteAPIBase + "/" + endpoint + "/" + pkgsiteEscapedPath(importPath)
if len(values) > 0 {
u += "?" + values.Encode()
}
return u
}
// pkgsiteGetJSON performs a pkg.go.dev v1beta GET with up to 3 attempts and
// linear backoff for transport errors, HTTP 429, and 5xx responses.
func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url.Values, v any) error {
client := pkgsiteHTTPClient
if client == nil {
client = http.DefaultClient
}
var lastErr error
for attempt := 1; attempt <= 3; attempt++ {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pkgsiteURL(endpoint, importPath, values), nil)
if err != nil {
return fmt.Errorf("create pkgsite request: %w", err)
}
req.Header.Set("Accept", "application/json")
resp, err := client.Do(req)
if err != nil {
lastErr = fmt.Errorf("pkgsite request: %w", err)
if attempt < 3 {
time.Sleep(time.Duration(attempt) * time.Second)
continue
}
return lastErr
}
body, err := io.ReadAll(io.LimitReader(resp.Body, pkgsiteMaxResponseBytes+1))
resp.Body.Close()
if err != nil {
return fmt.Errorf("read pkgsite response: %w", err)
}
if len(body) > pkgsiteMaxResponseBytes {
return fmt.Errorf("pkgsite response exceeds %d bytes", pkgsiteMaxResponseBytes)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
var apiErr pkgsiteAPIError
if err := json.Unmarshal(body, &apiErr); err != nil || apiErr.Error() == "pkgsite API error" {
apiErr.Message = strings.TrimSpace(string(body))
}
apiErr.Code = resp.StatusCode
apiErr.Status = resp.Status
lastErr = &apiErr
if attempt < 3 && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500) {
time.Sleep(time.Duration(attempt) * time.Second)
continue
}
return lastErr
}
if err := json.Unmarshal(body, v); err != nil {
return fmt.Errorf("decode pkgsite response: %w", err)
}
return nil
}
return lastErr
}
func getPkgsitePackage(ctx context.Context, gopkg, modulePath string) (pkgsitePackage, error) {
values := url.Values{}
values.Set("imports", "true")
values.Set("licenses", "true")
if modulePath != "" {
values.Set("module", modulePath)
}
var p pkgsitePackage
err := pkgsiteGetJSON(ctx, "package", gopkg, values, &p)
if err == nil {
return p, nil
}
var apiErr *pkgsiteAPIError
if modulePath != "" || !isPkgsiteAPIError(err, &apiErr) || len(apiErr.Candidates) == 0 {
return pkgsitePackage{}, err
}
// Ambiguous import paths return candidate module/package pairs. Retry once
// with the longest module path, which is the most specific match.
best := apiErr.Candidates[0]
for _, candidate := range apiErr.Candidates[1:] {
if len(candidate.ModulePath) > len(best.ModulePath) {
best = candidate
}
}
if best.ModulePath == "" {
return pkgsitePackage{}, err
}
return getPkgsitePackage(ctx, gopkg, best.ModulePath)
}
func isPkgsiteAPIError(err error, target **pkgsiteAPIError) bool {
return errors.As(err, target)
}
func getPkgsiteModule(ctx context.Context, modulePath, version string) (pkgsiteModule, error) {
values := url.Values{}
values.Set("licenses", "true")
values.Set("readme", "true")
if version != "" {
values.Set("version", version)
}
var m pkgsiteModule
if err := pkgsiteGetJSON(ctx, "module", modulePath, values, &m); err != nil {
return pkgsiteModule{}, err
}
return m, nil
}
func getPkgsiteModulePackages(ctx context.Context, modulePath, version string) ([]pkgsitePackageSummary, error) {
values := url.Values{}
values.Set("limit", "1000")
if version != "" {
values.Set("version", version)
}
var packages []pkgsitePackageSummary
seenPaths := make(map[string]bool)
var response pkgsitePackagesResponse
if err := pkgsiteGetJSON(ctx, "packages", modulePath, values, &response); err != nil {
return nil, err
}
for _, p := range response.Packages.Items {
p.Path = strings.TrimSpace(p.Path)
if p.Path == "" || seenPaths[p.Path] || !p.IsRedistributable {
continue
}
seenPaths[p.Path] = true
packages = append(packages, p)
}
sort.Slice(packages, func(i, j int) bool {
return packages[i].Path < packages[j].Path
})
return packages, nil
}
func getPkgsiteInfo(ctx context.Context, gopkg string) (*pkgsiteInfo, error) {
pkgsiteMu.Lock()
if info := pkgsiteInfoCache[gopkg]; info != nil {
pkgsiteMu.Unlock()
return info, nil
}
pkgsiteMu.Unlock()
p, err := getPkgsitePackage(ctx, gopkg, "")
if err != nil {
return nil, fmt.Errorf("get pkgsite package: %w", err)
}
if p.ModulePath == "" {
return nil, fmt.Errorf("pkgsite package %q has empty module path", gopkg)
}
m, err := getPkgsiteModule(ctx, p.ModulePath, p.Version)
if err != nil {
return nil, fmt.Errorf("get pkgsite module: %w", err)
}
info := &pkgsiteInfo{Package: p, Module: m}
pkgsiteMu.Lock()
pkgsiteInfoCache[gopkg] = info
if p.Path == p.ModulePath {
// Only cache module-path lookups when the package data also describes
// the module root. Subpackage metadata would give callers the wrong
// package name for later module-root lookups.
pkgsiteInfoCache[p.ModulePath] = info
}
pkgsiteMu.Unlock()
return info, nil
}
func pkgsiteLicenseExpression(licenses []pkgsiteLicense) string {
topLevel := make([]pkgsiteLicense, 0, len(licenses))
for _, license := range licenses {
if !strings.Contains(strings.Trim(license.FilePath, "/"), "/") {
topLevel = append(topLevel, license)
}
}
if len(topLevel) > 0 {
// Prefer root license files; subdirectory licenses often describe vendored
// or generated code that should not affect the spec License field.
licenses = topLevel
}
seenGroups := make(map[string]bool)
for _, license := range licenses {
seenTypes := make(map[string]bool)
for _, typ := range license.Types {
typ = strings.TrimSpace(typ)
if typ != "" {
seenTypes[typ] = true
}
}
if len(seenTypes) == 0 {
continue
}
types := make([]string, 0, len(seenTypes))
for typ := range seenTypes {
types = append(types, typ)
}
sort.Strings(types)
group := types[0]
if len(types) > 1 {
group = "(" + strings.Join(types, " OR ") + ")"
}
seenGroups[group] = true
}
if len(seenGroups) == 0 {
return "TODO"
}
groups := make([]string, 0, len(seenGroups))
for group := range seenGroups {
groups = append(groups, group)
}
sort.Strings(groups)
return strings.Join(groups, " AND ")
}
+574
View File
@@ -0,0 +1,574 @@
package main
import (
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
func TestPkgsiteLicenseExpression(t *testing.T) {
tests := []struct {
name string
licenses []pkgsiteLicense
want string
}{
{
name: "prefers top-level licenses",
licenses: []pkgsiteLicense{
{FilePath: "LICENSE", Types: []string{"MIT"}},
{FilePath: "internal/LICENSE", Types: []string{"Apache-2.0"}},
},
want: "MIT",
},
{
name: "joins separate license files with AND",
licenses: []pkgsiteLicense{
{FilePath: "LICENSE", Types: []string{"MIT"}},
{FilePath: "COPYING", Types: []string{"BSD-3-Clause"}},
},
want: "BSD-3-Clause AND MIT",
},
{
name: "joins multiple matches in one license file with OR",
licenses: []pkgsiteLicense{
{FilePath: "LICENSE", Types: []string{"MIT", "Apache-2.0"}},
},
want: "(Apache-2.0 OR MIT)",
},
{
name: "uses TODO when pkgsite has no SPDX type",
licenses: []pkgsiteLicense{{FilePath: "LICENSE"}},
want: "TODO",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := pkgsiteLicenseExpression(tt.licenses); got != tt.want {
t.Fatalf("pkgsiteLicenseExpression() = %q, want %q", got, tt.want)
}
})
}
}
func TestPkgsiteLicenseFilePaths(t *testing.T) {
licenses := []pkgsiteLicense{
{FilePath: "License", Types: []string{"MIT"}},
{FilePath: "internal/LICENSE", Types: []string{"Apache-2.0"}},
{FilePath: "License", Types: []string{"MIT"}},
}
got := strings.Join(pkgsiteLicenseFilePaths(licenses), ",")
if want := "License"; got != want {
t.Fatalf("pkgsiteLicenseFilePaths() = %q, want %q", got, want)
}
}
func TestCleanSpecAssetPath(t *testing.T) {
tests := []struct {
in string
want string
}{
{in: "Readme", want: "Readme"},
{in: "/docs/readme.md", want: "docs/readme.md"},
{in: "../README.md", want: ""},
{in: "docs/../README.md", want: ""},
}
for _, tt := range tests {
if got := cleanSpecAssetPath(tt.in); got != tt.want {
t.Fatalf("cleanSpecAssetPath(%q) = %q, want %q", tt.in, got, tt.want)
}
}
}
func TestSummaryFromReadme(t *testing.T) {
readme := `
# Project
[![Build Status](https://example.invalid/badge.svg)](https://example.invalid)
<p align="center">
This is the first useful line.
`
if got, want := summaryFromReadme(readme), "This is the first useful line"; got != want {
t.Fatalf("summaryFromReadme() = %q, want %q", got, want)
}
}
func TestUnusableSummary(t *testing.T) {
info := &pkgsiteInfo{
Package: pkgsitePackage{Name: "runewidth"},
Module: pkgsiteModule{Path: "github.com/mattn/go-runewidth"},
}
if !unusableSummary("go-runewidth", "github.com/mattn/go-runewidth", info) {
t.Fatalf("expected basename summary to be unusable")
}
if unusableSummary("Determines terminal display width", "github.com/mattn/go-runewidth", info) {
t.Fatalf("expected descriptive summary to be usable")
}
}
func TestCleanSummaryCandidate(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
{name: "strips html", in: `<p align="center">Package termenv styles terminals.</p>`, want: "Styles terminals"},
{name: "strips godoc package prefix", in: "Package isatty implements interface to isatty.", want: "Implements interface to isatty"},
{name: "strips markdown image", in: `![testing](https://example.invalid) Package css parses CSS.`, want: "Parses CSS"},
{name: "strips markdown link and code", in: "A [`Writer`](https://example.invalid) for ANSI output.", want: "A Writer for ANSI output"},
{name: "uses first sentence", in: "Package termenv styles terminals. It supports colors.", want: "Styles terminals"},
{name: "rejects html-only", in: `<p align="center">`, want: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := cleanSummaryCandidate(tt.in); got != tt.want {
t.Fatalf("cleanSummaryCandidate() = %q, want %q", got, tt.want)
}
})
}
}
func TestTarballURLForRef(t *testing.T) {
tests := []struct {
name string
repoURL string
ref string
want string
}{
{
name: "github",
repoURL: "https://github.com/google/go-cmp",
ref: "v0.7.0",
want: "https://github.com/google/go-cmp/archive/v0.7.0.tar.gz",
},
{
name: "gitlab subgroup",
repoURL: "https://gitlab.com/group/subgroup/project",
ref: "v1.2.3",
want: "https://gitlab.com/group/subgroup/project/-/archive/v1.2.3/project-v1.2.3.tar.gz",
},
{
name: "sourcehut",
repoURL: "https://git.sr.ht/~user/project",
ref: "v1.0.0",
want: "https://git.sr.ht/~user/project/archive/v1.0.0.tar.gz",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
u := upstream{repoURL: tt.repoURL}
got, err := u.tarballURLForRef(tt.ref, "gz")
if err != nil {
t.Fatalf("tarballURLForRef() returned error: %v", err)
}
if got != tt.want {
t.Fatalf("tarballURLForRef() = %q, want %q", got, tt.want)
}
})
}
}
func TestGitCloneURLFromRepoURL(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
{
name: "keeps github clone URL",
in: "https://github.com/google/go-cmp.git",
want: "https://github.com/google/go-cmp",
},
{
name: "converts Go source browser URL",
in: "https://cs.opensource.google/go/x/net",
want: "https://go.googlesource.com/net",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := gitCloneURLFromRepoURL(tt.in); got != tt.want {
t.Fatalf("gitCloneURLFromRepoURL(%q) = %q, want %q", tt.in, got, tt.want)
}
})
}
}
func TestGuessedPackageTypeUsesRootPackageName(t *testing.T) {
tests := []struct {
name string
u upstream
want packageType
}{
{
name: "library root with command subpackage stays library",
u: upstream{packageName: "yaml", firstMain: "go.yaml.in/yaml/v4/cmd/go-yaml"},
want: typeLibrary,
},
{
name: "library root without command subpackage stays library",
u: upstream{packageName: "yaml"},
want: typeLibrary,
},
{
name: "root main is program",
u: upstream{packageName: "main", firstMain: "example.com/tool"},
want: typeProgram,
},
{
name: "fallback to old main detection when pkgsite name missing",
u: upstream{firstMain: "example.com/tool"},
want: typeProgram,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := guessedPackageType(&tt.u); got != tt.want {
t.Fatalf("guessedPackageType() = %v, want %v", got, tt.want)
}
})
}
}
func TestSourceImportPathForPackageUsesModuleRoot(t *testing.T) {
info := &pkgsiteInfo{Package: pkgsitePackage{ModulePath: "github.com/example/project"}}
if got, want := sourceImportPathForPackage("github.com/example/project/cmd/tool", info), "github.com/example/project"; got != want {
t.Fatalf("sourceImportPathForPackage() = %q, want %q", got, want)
}
info = &pkgsiteInfo{Module: pkgsiteModule{Path: "github.com/example/module"}}
if got, want := sourceImportPathForPackage("github.com/example/module/pkg", info), "github.com/example/module"; got != want {
t.Fatalf("sourceImportPathForPackage(module fallback) = %q, want %q", got, want)
}
if got, want := sourceImportPathForPackage("example.com/no-module", nil), "example.com/no-module"; got != want {
t.Fatalf("sourceImportPathForPackage(nil) = %q, want %q", got, want)
}
}
func TestGoImportPathFromArgument(t *testing.T) {
got, isCapability, err := goImportPathFromArgument("go(k8s.io/api/apps/v1)")
if err != nil {
t.Fatalf("goImportPathFromArgument() returned error: %v", err)
}
if got != "k8s.io/api/apps/v1" || !isCapability {
t.Fatalf("goImportPathFromArgument() = %q, %v; want k8s.io/api/apps/v1, true", got, isCapability)
}
got, isCapability, err = goImportPathFromArgument("k8s.io/api/apps/v1")
if err != nil {
t.Fatalf("goImportPathFromArgument() returned error: %v", err)
}
if got != "k8s.io/api/apps/v1" || isCapability {
t.Fatalf("goImportPathFromArgument() = %q, %v; want k8s.io/api/apps/v1, false", got, isCapability)
}
if _, _, err := goImportPathFromArgument("go(k8s.io/api/apps/v1"); err == nil {
t.Fatalf("goImportPathFromArgument() succeeded for malformed capability")
}
}
func TestSetRepoDependenciesSeparatesRuntimeAndTestOnly(t *testing.T) {
u := upstream{}
u.setRepoDependencies(
map[string]bool{
"github.com/runtime/dep": true,
"github.com/shared/dep": true,
},
map[string]bool{
"github.com/shared/dep": true,
"github.com/testonly/dep": true,
"github.com/testonly/dep2": true,
},
)
if got, want := strings.Join(u.repoRunDeps, ","), "github.com/runtime/dep,github.com/shared/dep"; got != want {
t.Fatalf("repoRunDeps = %q, want %q", got, want)
}
if got, want := strings.Join(u.repoTestDeps, ","), "github.com/testonly/dep,github.com/testonly/dep2"; got != want {
t.Fatalf("repoTestDeps = %q, want %q", got, want)
}
if got, want := strings.Join(u.repoDeps, ","), "github.com/runtime/dep,github.com/shared/dep,github.com/testonly/dep,github.com/testonly/dep2"; got != want {
t.Fatalf("repoDeps = %q, want %q", got, want)
}
}
func TestModuleProxyEscapedPath(t *testing.T) {
if got, want := moduleProxyEscapedPath("github.com/Azure/azure-sdk-for-go"), "github.com/!azure/azure-sdk-for-go"; got != want {
t.Fatalf("moduleProxyEscapedPath() = %q, want %q", got, want)
}
}
func TestModuleProxyVersionForSpec(t *testing.T) {
if got, want := moduleProxyVersionForSpec("v1.0.0-RC1"), "v1.0.0-!r!c1"; got != want {
t.Fatalf("moduleProxyVersionForSpec() = %q, want %q", got, want)
}
if got, want := moduleProxyVersionForSpec("v%{version}"), "v%{version}"; got != want {
t.Fatalf("moduleProxyVersionForSpec() = %q, want %q", got, want)
}
}
func TestModuleUsesRepoSubdir(t *testing.T) {
tests := []struct {
name string
modulePath string
repoURL string
want bool
}{
{
name: "real submodule",
modulePath: "github.com/charmbracelet/x/ansi",
repoURL: "https://github.com/charmbracelet/x",
want: true,
},
{
name: "semantic import version suffix is not a repo subdir",
modulePath: "github.com/aymanbagabas/go-osc52/v2",
repoURL: "https://github.com/aymanbagabas/go-osc52",
want: false,
},
{
name: "vanity module does not match repo host path",
modulePath: "go.yaml.in/yaml/v4",
repoURL: "https://github.com/yaml/go-yaml",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := moduleUsesRepoSubdir(tt.modulePath, tt.repoURL); got != tt.want {
t.Fatalf("moduleUsesRepoSubdir() = %v, want %v", got, tt.want)
}
})
}
}
func TestGetPkgsitePackageRetriesAmbiguousPathWithLongestModule(t *testing.T) {
oldClient := pkgsiteHTTPClient
defer func() { pkgsiteHTTPClient = oldClient }()
var requested []string
pkgsiteHTTPClient = &http.Client{
Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
requested = append(requested, req.URL.RawQuery)
if req.URL.Query().Get("module") == "" {
return &http.Response{
StatusCode: http.StatusBadRequest,
Status: "400 Bad Request",
Body: io.NopCloser(strings.NewReader(`{
"message":"ambiguous package path",
"candidates":[
{"modulePath":"example.com/a","packagePath":"example.com/a/b/c"},
{"modulePath":"example.com/a/b","packagePath":"example.com/a/b/c"}
]
}`)),
Header: make(http.Header),
}, nil
}
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Body: io.NopCloser(strings.NewReader(`{
"modulePath":"example.com/a/b",
"version":"v1.2.3",
"path":"example.com/a/b/c",
"name":"c"
}`)),
Header: make(http.Header),
}, nil
}),
}
p, err := getPkgsitePackage(t.Context(), "example.com/a/b/c", "")
if err != nil {
t.Fatalf("getPkgsitePackage() returned error: %v", err)
}
if p.ModulePath != "example.com/a/b" {
t.Fatalf("ModulePath = %q, want %q", p.ModulePath, "example.com/a/b")
}
if len(requested) != 2 || requested[1] != "imports=true&licenses=true&module=example.com%2Fa%2Fb" {
t.Fatalf("requests = %#v, want retry with longest module path", requested)
}
}
func TestProvidesFromPkgsitePackagesForModuleRoot(t *testing.T) {
info := &pkgsiteInfo{Module: pkgsiteModule{Path: "k8s.io/api"}}
got := strings.Join(providesFromPkgsitePackages("k8s.io/api", info, []pkgsitePackageSummary{
{Path: "k8s.io/api/apps/v1", IsRedistributable: true},
{Path: "k8s.io/api/apps/v1beta1", IsRedistributable: true},
{Path: "k8s.io/api/internal", IsRedistributable: false},
}), ",")
want := "k8s.io/api,k8s.io/api/apps/v1,k8s.io/api/apps/v1beta1"
if got != want {
t.Fatalf("providesFromPkgsitePackages() = %q, want %q", got, want)
}
}
func TestProvidesFromPkgsitePackagesForSubpackage(t *testing.T) {
info := &pkgsiteInfo{Module: pkgsiteModule{Path: "k8s.io/api"}}
got := strings.Join(providesFromPkgsitePackages("k8s.io/api/apps/v1", info, []pkgsitePackageSummary{
{Path: "k8s.io/api/apps/v1beta1", IsRedistributable: true},
}), ",")
want := "k8s.io/api/apps/v1"
if got != want {
t.Fatalf("providesFromPkgsitePackages() = %q, want %q", got, want)
}
}
func TestConvertDependenciesToRPMKeepsKubernetesImportPaths(t *testing.T) {
got := strings.Join(convertDependenciesToRPM([]string{
"k8s.io/apimachinery/pkg/apis/meta/v1",
"k8s.io/apimachinery/pkg/runtime",
"sigs.k8s.io/randfill",
}), ",")
want := "go(k8s.io/apimachinery/pkg/apis/meta/v1),go(k8s.io/apimachinery/pkg/runtime),go(sigs.k8s.io/randfill)"
if got != want {
t.Fatalf("convertDependenciesToRPM() = %q, want %q", got, want)
}
}
func TestSourceURLForSpecUsesCommitIDMacro(t *testing.T) {
u := upstream{
repoURL: "https://github.com/example/project",
version: "0+git20260522.abcdef\n%define commit_id 0123456789abcdef",
}
got, err := u.sourceURLForSpec("github.com/example/project")
if err != nil {
t.Fatalf("sourceURLForSpec() returned error: %v", err)
}
want := "https://github.com/example/project/archive/%{commit_id}.tar.gz#/%{_name}-%{version}.tar.gz"
if got != want {
t.Fatalf("sourceURLForSpec() = %q, want %q", got, want)
}
}
func TestSourceURLForSpecKeepsVersionMacroForMatchingReleaseTag(t *testing.T) {
u := upstream{
repoURL: "https://github.com/example/project",
version: "1.2.3",
tag: "v1.2.3",
isRelease: true,
}
got, err := u.sourceURLForSpec("github.com/example/project")
if err != nil {
t.Fatalf("sourceURLForSpec() returned error: %v", err)
}
want := "https://github.com/example/project/archive/v%{version}.tar.gz#/%{_name}-%{version}.tar.gz"
if got != want {
t.Fatalf("sourceURLForSpec() = %q, want %q", got, want)
}
}
func TestSourceURLForSpecFallsBackToModuleProxy(t *testing.T) {
u := upstream{
repoURL: "https://go.googlesource.com/net",
version: "0.55.0",
tag: "v0.55.0",
isRelease: true,
}
got, err := u.sourceURLForSpec("golang.org/x/net")
if err != nil {
t.Fatalf("sourceURLForSpec() returned error: %v", err)
}
want := "https://proxy.golang.org/golang.org/x/net/@v/v%{version}.zip#/%{_name}-%{version}.zip"
if got != want {
t.Fatalf("sourceURLForSpec() = %q, want %q", got, want)
}
}
func TestSourceURLForSpecUsesModuleProxyForRepoSubmodule(t *testing.T) {
u := upstream{
repoURL: "https://github.com/charmbracelet/x",
version: "0.1.0",
tag: "v0.1.0",
isRelease: true,
}
got, err := u.sourceURLForSpec("github.com/charmbracelet/x/ansi")
if err != nil {
t.Fatalf("sourceURLForSpec() returned error: %v", err)
}
want := "https://proxy.golang.org/github.com/charmbracelet/x/ansi/@v/v%{version}.zip#/%{_name}-%{version}.zip"
if got != want {
t.Fatalf("sourceURLForSpec() = %q, want %q", got, want)
}
}
func TestSourceURLForSpecRejectsCommitIDForRepoSubmodule(t *testing.T) {
u := upstream{
repoURL: "https://github.com/charmbracelet/x",
version: "0+git20260522.abcdef\n%define commit_id abcdef1234567890",
}
_, err := u.sourceURLForSpec("github.com/charmbracelet/x/ansi")
if err == nil {
t.Fatalf("sourceURLForSpec() succeeded for commit-pinned submodule, want error")
}
if !strings.Contains(err.Error(), "canonical pseudo-version") {
t.Fatalf("sourceURLForSpec() error = %v, want canonical pseudo-version message", err)
}
}
func TestPkgVersionFromGitUsesPackagingDateAndSevenCharHash(t *testing.T) {
oldNow := packagingDateNow
packagingDateNow = func() time.Time {
return time.Date(2025, 8, 8, 12, 0, 0, 0, time.UTC)
}
defer func() { packagingDateNow = oldNow }()
dir := t.TempDir()
runGit(t, dir, nil, "init")
if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module example.com/project\n"), 0644); err != nil {
t.Fatalf("write go.mod: %v", err)
}
runGit(t, dir, nil, "add", "go.mod")
runGit(t, dir, []string{
"GIT_AUTHOR_NAME=Test",
"GIT_AUTHOR_EMAIL=test@example.invalid",
"GIT_COMMITTER_NAME=Test",
"GIT_COMMITTER_EMAIL=test@example.invalid",
"GIT_AUTHOR_DATE=1999-01-02T03:04:05Z",
"GIT_COMMITTER_DATE=1999-01-02T03:04:05Z",
}, "commit", "-m", "initial")
fullHash := strings.TrimSpace(runGit(t, dir, nil, "rev-parse", "HEAD"))
want := "0+git20250808." + fullHash[:7] + "\n%define commit_id " + fullHash
u := upstream{}
got, err := pkgVersionFromGit(dir, &u, "", false)
if err != nil {
t.Fatalf("pkgVersionFromGit() returned error: %v", err)
}
if got != want {
t.Fatalf("pkgVersionFromGit() = %q, want %q", got, want)
}
if strings.Contains(got, "19990102") {
t.Fatalf("pkgVersionFromGit() used commit date: %q", got)
}
if u.commitIsh != fullHash[:7] {
t.Fatalf("commitIsh = %q, want %q", u.commitIsh, fullHash[:7])
}
}
func runGit(t *testing.T, dir string, env []string, args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = dir
cmd.Env = append(os.Environ(), env...)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, out)
}
return string(out)
}
+124 -36
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"log"
"os"
@@ -10,8 +11,67 @@ import (
"strings"
)
type specAssetFiles struct {
licenseFiles []string
readmeFile string
}
func cleanSpecAssetPath(path string) string {
path = filepath.ToSlash(strings.TrimSpace(path))
path = strings.TrimLeft(path, "/")
if path == "" || strings.HasPrefix(path, "../") || strings.Contains(path, "/../") || path == ".." {
return ""
}
return path
}
func pkgsiteLicenseFilePaths(licenses []pkgsiteLicense) []string {
topLevel := make([]pkgsiteLicense, 0, len(licenses))
for _, license := range licenses {
if !strings.Contains(strings.Trim(license.FilePath, "/"), "/") {
topLevel = append(topLevel, license)
}
}
if len(topLevel) > 0 {
licenses = topLevel
}
seen := make(map[string]bool)
paths := make([]string, 0, len(licenses))
for _, license := range licenses {
path := cleanSpecAssetPath(license.FilePath)
if path == "" || seen[path] {
continue
}
seen[path] = true
paths = append(paths, path)
}
sort.Strings(paths)
return paths
}
func getSpecAssetFilesForGopkg(gopkg string) (specAssetFiles, error) {
info, err := getPkgsiteInfo(context.TODO(), gopkg)
if err != nil {
return specAssetFiles{}, err
}
licenses := info.Module.Licenses
if len(licenses) == 0 {
licenses = info.Package.Licenses
}
assets := specAssetFiles{
licenseFiles: pkgsiteLicenseFilePaths(licenses),
}
if info.Module.Readme != nil {
assets.readmeFile = cleanSpecAssetPath(info.Module.Readme.Filepath)
}
return assets, nil
}
func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version string,
pkgType packageType, dependencies []string, u *upstream) error {
pkgType packageType, u *upstream) error {
f, err := os.Create(filepath.Join(dir, "", openRuyiSrc+".spec"))
if err != nil {
@@ -35,14 +95,23 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
log.Printf("Could not determine license for %q: %v\n", gopkg, err)
license = "TODO"
}
assetFiles, err := getSpecAssetFilesForGopkg(gopkg)
if err != nil {
log.Printf("Could not determine license/doc file paths for %q: %v\n", gopkg, err)
}
provides, err := getProvidesForGopkg(gopkg)
if err != nil {
log.Printf("Could not determine package provides for %q: %v\n", gopkg, err)
provides = []string{gopkg}
}
upstreamName := filepath.Base(gopkg)
if regexp.MustCompile(`^v\d+$`).MatchString(upstreamName) {
upstreamName = filepath.Base(filepath.Dir(gopkg))
}
owner, repo, err := findGitHubRepo(gopkg)
repoURL, err := getRepoURLForGopkg(gopkg)
if err != nil {
owner = "TODO"
repo = "TODO"
log.Printf("Could not determine repository URL for %q: %v\n", gopkg, err)
repoURL = "TODO"
}
// Write the spec file content
@@ -72,8 +141,8 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
// Header
fmt.Fprintf(f, "Name: %s\n", openRuyiSrc)
// Some times typeLibrary is treat as typeProgram,
// So we add an additional Name line, and keep one of those mannually
// Program packages may need manual naming cleanup; keep the library-form
// name visible as a second Name line until that workflow is redesigned.
switch pkgType {
case typeProgram:
fmt.Fprintf(f, "Name: %s\n", openRuyiLib)
@@ -83,15 +152,18 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
fmt.Fprintf(f, "Release: %%autorelease\n")
fmt.Fprintf(f, "Summary: %s\n", description)
fmt.Fprintf(f, "License: %s\n", license)
fmt.Fprintf(f, "URL: https://github.com/%s/%s\n", owner, repo)
fmt.Fprintf(f, "URL: %s\n", repoURL)
fmt.Fprintf(f, "#!RemoteAsset\n")
// If the computed version text contains a commit_id definition (see pkgVersionFromGit),
// use the commit_id tarball instead of v%{version}.tar.gz
if strings.Contains(u.version, "commit_id") {
fmt.Fprintf(f, "Source0: https://github.com/%s/%s/archive/%%{commit_id}.tar.gz#/%%{_name}-%%{version}.tar.gz\n", owner, repo)
} else {
fmt.Fprintf(f, "Source0: https://github.com/%s/%s/archive/v%%{version}.tar.gz#/%%{_name}-%%{version}.tar.gz\n", owner, repo)
sourceModulePath := u.modulePath
if sourceModulePath == "" {
sourceModulePath = gopkg
}
sourceURL, err := u.sourceURLForSpec(sourceModulePath)
if err != nil {
log.Printf("Could not determine remote source URL for %q: %v\n", gopkg, err)
sourceURL = "TODO"
}
fmt.Fprintf(f, "Source0: %s\n", sourceURL)
switch pkgType {
case typeLibrary:
@@ -105,26 +177,28 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
fmt.Fprintf(f, "BuildRequires: go\n")
fmt.Fprintf(f, "BuildRequires: go-rpm-macros\n")
// And other BuildRequires from dependencies
rpmDeps := convertDependenciesToRPM(u.repoDeps)
sort.Strings(rpmDeps)
for _, dep := range rpmDeps {
rpmBuildDeps := convertDependenciesToRPM(u.repoDeps)
sort.Strings(rpmBuildDeps)
for _, dep := range rpmBuildDeps {
fmt.Fprintf(f, "BuildRequires: %s\n", dep)
}
rpmRuntimeDeps := convertDependenciesToRPM(u.repoRunDeps)
sort.Strings(rpmRuntimeDeps)
// For different package types, write different sections
switch pkgType {
case typeLibrary:
writeRPMLibraryPackage(f, gopkg, openRuyiLib, longdescription, rpmDeps)
writeRPMLibraryPackage(f, provides, openRuyiLib, longdescription, rpmRuntimeDeps)
case typeProgram:
log.Printf("Nothing to do for program package.\n")
// TODO: what can this be used for? ExclusiveArch %%{go_arches}?
// writeRPMProgramPackage(f, gopkg, openRuyiProgram, longdescription)
case typeLibraryProgram:
writeRPMLibraryPackage(f, gopkg, openRuyiLib, longdescription, rpmDeps)
writeRPMLibraryPackage(f, provides, openRuyiLib, longdescription, rpmRuntimeDeps)
writeRPMProgramSubpackage(f, gopkg, openRuyiProgram, openRuyiSrc, description)
case typeProgramLibrary:
//writeRPMProgramPackage(f, gopkg, openRuyiProgram, longdescription)
writeRPMLibrarySubpackage(f, gopkg, openRuyiLib, openRuyiSrc, longdescription, rpmDeps)
writeRPMLibrarySubpackage(f, provides, gopkg, openRuyiLib, openRuyiSrc, longdescription, rpmRuntimeDeps)
default:
log.Fatalf("Invalid pkgType %d in writeRPMSpec(), aborting", pkgType)
}
@@ -135,7 +209,7 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
fmt.Fprintf(f, "\n")
// %files
writeRPMFilesSection(f, openRuyiSrc, openRuyiLib, openRuyiProgram, pkgType)
writeRPMFilesSection(f, openRuyiSrc, openRuyiLib, openRuyiProgram, pkgType, assetFiles)
// %changelog
fmt.Fprintf(f, "%%changelog\n")
@@ -145,9 +219,11 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
}
// For library package
func writeRPMLibraryPackage(f *os.File, gopkg, openRuyiLib, longdesc string, deps []string) {
func writeRPMLibraryPackage(f *os.File, provides []string, openRuyiLib, longdesc string, deps []string) {
fmt.Fprintf(f, "\n")
fmt.Fprintf(f, "Provides: go(%s) = %%{version}\n", gopkg)
for _, provide := range provides {
fmt.Fprintf(f, "Provides: go(%s) = %%{version}\n", provide)
}
fmt.Fprintf(f, "\n")
// 库包的运行时依赖
if len(deps) > 0 {
@@ -159,11 +235,13 @@ func writeRPMLibraryPackage(f *os.File, gopkg, openRuyiLib, longdesc string, dep
}
// For library subpackage
func writeRPMLibrarySubpackage(f *os.File, gopkg, openRuyiLib, openRuyiSrc, longdesc string, deps []string) {
func writeRPMLibrarySubpackage(f *os.File, provides []string, gopkg, openRuyiLib, openRuyiSrc, longdesc string, deps []string) {
fmt.Fprintf(f, "\n")
fmt.Fprintf(f, "%%package -n %s\n", openRuyiLib)
fmt.Fprintf(f, "Summary: Development files of %s\n", filepath.Base(gopkg))
fmt.Fprintf(f, "Provides: go(%s) = %%{version}\n", gopkg)
for _, provide := range provides {
fmt.Fprintf(f, "Provides: go(%s) = %%{version}\n", provide)
}
fmt.Fprintf(f, "BuildArch: noarch\n")
if len(deps) > 0 {
for _, dep := range deps {
@@ -175,7 +253,7 @@ func writeRPMLibrarySubpackage(f *os.File, gopkg, openRuyiLib, openRuyiSrc, long
fmt.Fprintf(f, "%%description -n %s\n", openRuyiLib)
fmt.Fprintf(f, "%s\n", longdesc)
fmt.Fprintf(f, "\n")
fmt.Fprintf(f, "This package provides the Go source files of %s for development.\n")
fmt.Fprintf(f, "This package provides the Go source files of %s for development.\n", gopkg)
}
// For program subpackage
@@ -190,45 +268,50 @@ func writeRPMProgramSubpackage(f *os.File, gopkg, openRuyiProgram, openRuyiSrc,
fmt.Fprintf(f, "This package contains the %s executable.\n", filepath.Base(gopkg))
}
func writeRPMFilesSection(f *os.File, openRuyiSrc, openRuyiLib, openRuyiProgram string, pkgType packageType) {
func writeLicenseAndDocFiles(f *os.File, assetFiles specAssetFiles, includeDoc bool) {
for _, licenseFile := range assetFiles.licenseFiles {
fmt.Fprintf(f, "%%license %s\n", licenseFile)
}
if includeDoc && assetFiles.readmeFile != "" {
fmt.Fprintf(f, "%%doc %s\n", assetFiles.readmeFile)
}
}
func writeRPMFilesSection(f *os.File, openRuyiSrc, openRuyiLib, openRuyiProgram string, pkgType packageType, assetFiles specAssetFiles) {
switch pkgType {
case typeLibrary:
fmt.Fprintf(f, "%%files\n")
fmt.Fprintf(f, "%%license LICENSE*\n")
fmt.Fprintf(f, "%%doc README*\n")
writeLicenseAndDocFiles(f, assetFiles, true)
fmt.Fprintf(f, "%%{go_sys_gopath}/%%{go_import_path}\n")
fmt.Fprintf(f, "\n")
case typeProgram:
fmt.Fprintf(f, "%%files\n")
fmt.Fprintf(f, "%%license LICENSE*\n")
fmt.Fprintf(f, "%%doc README*\n")
writeLicenseAndDocFiles(f, assetFiles, true)
fmt.Fprintf(f, "%%{_bindir}/%%{_name}\n")
fmt.Fprintf(f, "\n")
case typeLibraryProgram:
// 库包文件(主包)
fmt.Fprintf(f, "%%files\n")
fmt.Fprintf(f, "%%license LICENSE*\n")
fmt.Fprintf(f, "%%doc README*\n")
writeLicenseAndDocFiles(f, assetFiles, true)
fmt.Fprintf(f, "%%{go_sys_gopath}/%%{go_import_path}\n")
fmt.Fprintf(f, "\n")
// 程序子包文件
fmt.Fprintf(f, "%%files -n %s\n", openRuyiProgram)
fmt.Fprintf(f, "%%license LICENSE*\n")
writeLicenseAndDocFiles(f, assetFiles, false)
fmt.Fprintf(f, "%%{_bindir}/%%{_name}\n")
fmt.Fprintf(f, "\n")
case typeProgramLibrary:
// 程序主包文件
fmt.Fprintf(f, "%%files\n")
fmt.Fprintf(f, "%%license LICENSE*\n")
fmt.Fprintf(f, "%%doc README*\n")
writeLicenseAndDocFiles(f, assetFiles, true)
fmt.Fprintf(f, "%%{_bindir}/%%{_name}\n")
fmt.Fprintf(f, "\n")
// 库子包文件
fmt.Fprintf(f, "%%files -n %s\n", openRuyiLib)
fmt.Fprintf(f, "%%license LICENSE*\n")
writeLicenseAndDocFiles(f, assetFiles, false)
fmt.Fprintf(f, "%%{go_sys_gopath}/%%{go_import_path}\n")
fmt.Fprintf(f, "\n")
}
@@ -255,6 +338,11 @@ func convertDependenciesToRPM(goPkgs []string) []string {
topLevelPkgs := make(map[string]bool)
for _, goPkg := range goPkgs {
if strings.HasPrefix(goPkg, "k8s.io/") || strings.HasPrefix(goPkg, "sigs.k8s.io/") {
topLevelPkgs[goPkg] = true
continue
}
// 提取顶级包:github.com/user/repo/subpkg -> github.com/user/repo
parts := strings.Split(goPkg, "/")
var topLevel string