merge: include ordered files output

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

.gitignore

@@ -25,6 +25,11 @@ go.work.sum
 # env file
 .env
 
+# Local go2spec validation/output artifacts
+comparison-runs/
+go-github-sandrolain-httpcache/
+go-github-sandrolain-httpcache_*.orig.tar.gz
+
 # Editor/IDE
 .idea/
 .vscode/

metadata.go

@@ -6,6 +6,7 @@ import (
 	"html"
 	"path"
 	"regexp"
+	"sort"
 	"strings"
 )
 
@@ -116,6 +117,41 @@ func getLicenseForGopkg(gopkg string) (string, error) {
 	return pkgsiteLicenseExpression(licenses), nil
 }
 
+func providesFromPkgsitePackages(gopkg string, info *pkgsiteInfo, packages []pkgsitePackageSummary) []string {
+	seen := map[string]bool{gopkg: true}
+	provides := []string{gopkg}
+	if info == nil || gopkg != info.Module.Path {
+		return provides
+	}
+
+	for _, p := range packages {
+		path := strings.TrimSpace(p.Path)
+		if path == "" || seen[path] || !p.IsRedistributable {
+			continue
+		}
+		seen[path] = true
+		provides = append(provides, path)
+	}
+	sort.Strings(provides)
+	return provides
+}
+
+func getProvidesForGopkg(gopkg string) ([]string, error) {
+	info, err := getPkgsiteInfo(context.TODO(), gopkg)
+	if err != nil {
+		return []string{gopkg}, err
+	}
+	if gopkg != info.Module.Path {
+		return []string{gopkg}, nil
+	}
+
+	packages, err := getPkgsiteModulePackages(context.TODO(), info.Module.Path, info.Module.Version)
+	if err != nil {
+		return []string{gopkg}, err
+	}
+	return providesFromPkgsitePackages(gopkg, info, packages), nil
+}
+
 // getDescriptionForGopkg gets the package synopsis from pkg.go.dev,
 // intended for the summary in the RPM spec.
 func getDescriptionForGopkg(gopkg string) (string, error) {

pack.go

@@ -126,7 +126,7 @@ func downloadFile(filename, url string) error {
 
 // get downloads the source module into the provided GOPATH,
 // checking out the specified revision if non-empty.
-func (u *upstream) get(gopath, sourceRepo, requestedPath, rev string) error {
+func (u *upstream) get(gopath, sourceRepo, requestedPath, rev string, forcePrerelease bool) error {
 	done := make(chan struct{})
 	defer close(done)
 	go progressSize("git clone", filepath.Join(gopath, "src"), done)
@@ -147,16 +147,29 @@ func (u *upstream) get(gopath, sourceRepo, requestedPath, rev string) error {
 		return fmt.Errorf("mkdir clone parent: %w", err)
 	}
 
-	cmd := exec.Command("git", "clone", u.repoURL, dir)
-	cmd.Env = passthroughEnv()
-	cmd.Stderr = os.Stderr
-	log.Println("get: Running", cmd)
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("git clone: %w", err)
+	cloneRef := shallowCloneRef(info, rev, forcePrerelease)
+	shallowCloneSucceeded := false
+	if cloneRef != "" {
+		if err := runGitClone(u.repoURL, dir, cloneRef, true); err == nil {
+			shallowCloneSucceeded = true
+			log.Printf("Shallow cloned %q at %q", u.repoURL, cloneRef)
+		} else {
+			log.Printf("WARNING: shallow git clone at %q failed, falling back to full clone: %v", cloneRef, err)
+			if removeErr := os.RemoveAll(dir); removeErr != nil {
+				return fmt.Errorf("remove failed shallow clone dir: %w", removeErr)
+			}
+			if err := runGitClone(u.repoURL, dir, "", false); err != nil {
+				return fmt.Errorf("git clone: %w", err)
+			}
+		}
+	} else {
+		if err := runGitClone(u.repoURL, dir, "", false); err != nil {
+			return fmt.Errorf("git clone: %w", err)
+		}
 	}
 
-	if rev != "" {
-		cmd = exec.Command("git", "-c", "advice.detachedHead=false", "checkout", rev)
+	if rev != "" && !(shallowCloneSucceeded && rev == cloneRef) {
+		cmd := exec.Command("git", "-c", "advice.detachedHead=false", "checkout", rev)
 		cmd.Dir = dir
 		cmd.Env = passthroughEnv()
 		cmd.Stderr = os.Stderr
@@ -168,6 +181,35 @@ func (u *upstream) get(gopath, sourceRepo, requestedPath, rev string) error {
 	return nil
 }
 
+func shallowCloneRef(info *pkgsiteInfo, rev string, forcePrerelease bool) string {
+	if forcePrerelease || rev != "" {
+		return ""
+	}
+	if info != nil {
+		return strings.TrimSpace(info.Package.Version)
+	}
+	return ""
+}
+
+func gitCloneArgs(repoURL, dir, ref string, shallow bool) []string {
+	args := []string{"-c", "advice.detachedHead=false", "clone"}
+	if shallow {
+		args = append(args, "--depth=1", "--single-branch")
+		if ref != "" {
+			args = append(args, "--branch", ref)
+		}
+	}
+	return append(args, repoURL, dir)
+}
+
+func runGitClone(repoURL, dir, ref string, shallow bool) error {
+	cmd := exec.Command("git", gitCloneArgs(repoURL, dir, ref, shallow)...)
+	cmd.Env = passthroughEnv()
+	cmd.Stderr = os.Stderr
+	log.Println("get: Running", cmd)
+	return cmd.Run()
+}
+
 func guessedPackageType(u *upstream) packageType {
 	if u.packageName == "main" || (u.packageName == "" && u.firstMain != "") {
 		return typeProgram
@@ -188,6 +230,18 @@ func sourceImportPathForPackage(requestedPath string, info *pkgsiteInfo) string
 	return requestedPath
 }
 
+func goImportPathFromArgument(arg string) (string, bool, error) {
+	arg = strings.TrimSpace(arg)
+	if strings.HasPrefix(arg, "go(") || strings.HasSuffix(arg, ")") {
+		m := regexp.MustCompile(`^go\(([^()]+)\)$`).FindStringSubmatch(arg)
+		if m == nil {
+			return "", false, fmt.Errorf("invalid Go RPM capability %q", arg)
+		}
+		return strings.TrimSpace(m[1]), true, nil
+	}
+	return arg, false, nil
+}
+
 // gitCloneURLFromRepoURL converts repository URLs returned by pkg.go.dev into
 // something git clone understands. pkg.go.dev reports golang.org/x/* modules
 // with a cs.opensource.google browser URL, so rewrite those to go.googlesource.com.
@@ -592,7 +646,7 @@ func makeUpstreamSourceTarball(requestedPath, sourceRepo, revision string, force
 	var u upstream
 
 	log.Printf("Downloading %q\n", sourceRepo+"/...")
-	if err := u.get(gopath, sourceRepo, requestedPath, revision); err != nil {
+	if err := u.get(gopath, sourceRepo, requestedPath, revision, forcePrerelease); err != nil {
 		return nil, fmt.Errorf("get source: %w", err)
 	}
 
@@ -881,7 +935,10 @@ func mainPack(args []string, usage func()) {
 	}
 
 	gitRevision = strings.TrimSpace(gitRevision)
-	gopkg := flagSet.Arg(0)
+	gopkg, fromGoCapability, err := goImportPathFromArgument(flagSet.Arg(0))
+	if err != nil {
+		log.Fatalf("Verifying arguments: %v", err)
+	}
 
 	// Remove URL scheme if present (https://, http://, git://, etc.)
 	gopkg = strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(gopkg, "https://"), "http://"), "git://")
@@ -893,6 +950,17 @@ func mainPack(args []string, usage func()) {
 	if err != nil {
 		log.Fatalf("Verifying arguments: %v — did you specify a Go package import path?", err)
 	}
+	if fromGoCapability {
+		modulePath := sourceImportPathForPackage(gopkg, info)
+		if modulePath != "" && modulePath != gopkg {
+			log.Printf("Using module root %q to package requested Go capability go(%s)", modulePath, gopkg)
+			gopkg = modulePath
+			info, err = getPkgsiteInfo(context.TODO(), gopkg)
+			if err != nil {
+				log.Fatalf("Verifying module root %q: %v", gopkg, err)
+			}
+		}
+	}
 	sourceGopkg := sourceImportPathForPackage(gopkg, info)
 	if sourceGopkg != gopkg {
 		log.Printf("Using module root %q as source checkout for specified import path %q", sourceGopkg, gopkg)

pkgsite.go

@@ -68,6 +68,26 @@ type pkgsiteCandidate struct {
 	PackagePath string `json:"packagePath"`
 }
 
+type pkgsitePackagesResponse struct {
+	ModulePath        string              `json:"modulePath"`
+	Version           string              `json:"version"`
+	IsStandardLibrary bool                `json:"isStandardLibrary"`
+	Packages          pkgsitePackageItems `json:"packages"`
+}
+
+type pkgsitePackageItems struct {
+	Items         []pkgsitePackageSummary `json:"items"`
+	Total         int                     `json:"total"`
+	NextPageToken string                  `json:"nextPageToken"`
+}
+
+type pkgsitePackageSummary struct {
+	Path              string `json:"path"`
+	Name              string `json:"name"`
+	Synopsis          string `json:"synopsis"`
+	IsRedistributable bool   `json:"isRedistributable"`
+}
+
 type pkgsiteAPIError struct {
 	Code       int                `json:"code"`
 	Message    string             `json:"message"`
@@ -215,6 +235,35 @@ func getPkgsiteModule(ctx context.Context, modulePath, version string) (pkgsiteM
 	return m, nil
 }
 
+func getPkgsiteModulePackages(ctx context.Context, modulePath, version string) ([]pkgsitePackageSummary, error) {
+	values := url.Values{}
+	values.Set("limit", "1000")
+	if version != "" {
+		values.Set("version", version)
+	}
+
+	var packages []pkgsitePackageSummary
+	seenPaths := make(map[string]bool)
+	var response pkgsitePackagesResponse
+	if err := pkgsiteGetJSON(ctx, "packages", modulePath, values, &response); err != nil {
+		return nil, err
+	}
+
+	for _, p := range response.Packages.Items {
+		p.Path = strings.TrimSpace(p.Path)
+		if p.Path == "" || seenPaths[p.Path] || !p.IsRedistributable {
+			continue
+		}
+		seenPaths[p.Path] = true
+		packages = append(packages, p)
+	}
+
+	sort.Slice(packages, func(i, j int) bool {
+		return packages[i].Path < packages[j].Path
+	})
+	return packages, nil
+}
+
 func getPkgsiteInfo(ctx context.Context, gopkg string) (*pkgsiteInfo, error) {
 	pkgsiteMu.Lock()
 	if info := pkgsiteInfoCache[gopkg]; info != nil {

pkgsite_test.go

@@ -91,6 +91,30 @@ func TestCleanSpecAssetPath(t *testing.T) {
 	}
 }
 
+func TestWriteLicenseAndDocFilesOrdersDocBeforeSortedLicenses(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "files")
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatalf("create temp file: %v", err)
+	}
+	writeLicenseAndDocFiles(f, specAssetFiles{
+		licenseFiles: []string{"zLICENSE", "COPYING", "LICENSE"},
+		readmeFile:   "README.md",
+	}, true)
+	if err := f.Close(); err != nil {
+		t.Fatalf("close temp file: %v", err)
+	}
+
+	content, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read temp file: %v", err)
+	}
+	want := "%doc README.md\n%license COPYING\n%license LICENSE\n%license zLICENSE\n"
+	if got := string(content); got != want {
+		t.Fatalf("writeLicenseAndDocFiles() = %q, want %q", got, want)
+	}
+}
+
 func TestSummaryFromReadme(t *testing.T) {
 	readme := `
 # Project
@@ -259,6 +283,83 @@ func TestSourceImportPathForPackageUsesModuleRoot(t *testing.T) {
 	}
 }
 
+func TestGoImportPathFromArgument(t *testing.T) {
+	got, isCapability, err := goImportPathFromArgument("go(k8s.io/api/apps/v1)")
+	if err != nil {
+		t.Fatalf("goImportPathFromArgument() returned error: %v", err)
+	}
+	if got != "k8s.io/api/apps/v1" || !isCapability {
+		t.Fatalf("goImportPathFromArgument() = %q, %v; want k8s.io/api/apps/v1, true", got, isCapability)
+	}
+
+	got, isCapability, err = goImportPathFromArgument("k8s.io/api/apps/v1")
+	if err != nil {
+		t.Fatalf("goImportPathFromArgument() returned error: %v", err)
+	}
+	if got != "k8s.io/api/apps/v1" || isCapability {
+		t.Fatalf("goImportPathFromArgument() = %q, %v; want k8s.io/api/apps/v1, false", got, isCapability)
+	}
+
+	if _, _, err := goImportPathFromArgument("go(k8s.io/api/apps/v1"); err == nil {
+		t.Fatalf("goImportPathFromArgument() succeeded for malformed capability")
+	}
+}
+
+func TestShallowCloneRef(t *testing.T) {
+	info := &pkgsiteInfo{Package: pkgsitePackage{Version: "v1.2.3"}}
+	tests := []struct {
+		name            string
+		info            *pkgsiteInfo
+		rev             string
+		forcePrerelease bool
+		want            string
+	}{
+		{
+			name: "skips shallow clone for explicit revision",
+			info: info,
+			rev:  "v1.2.4",
+			want: "",
+		},
+		{
+			name: "uses pkgsite latest version",
+			info: info,
+			want: "v1.2.3",
+		},
+		{
+			name:            "skips shallow clone for prerelease packaging",
+			info:            info,
+			forcePrerelease: true,
+			want:            "",
+		},
+		{
+			name: "handles missing pkgsite info",
+			want: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := shallowCloneRef(tt.info, tt.rev, tt.forcePrerelease); got != tt.want {
+				t.Fatalf("shallowCloneRef() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGitCloneArgs(t *testing.T) {
+	got := strings.Join(gitCloneArgs("https://example.com/repo", "/tmp/repo", "v1.2.3", true), " ")
+	want := "-c advice.detachedHead=false clone --depth=1 --single-branch --branch v1.2.3 https://example.com/repo /tmp/repo"
+	if got != want {
+		t.Fatalf("gitCloneArgs(shallow) = %q, want %q", got, want)
+	}
+
+	got = strings.Join(gitCloneArgs("https://example.com/repo", "/tmp/repo", "", false), " ")
+	want = "-c advice.detachedHead=false clone https://example.com/repo /tmp/repo"
+	if got != want {
+		t.Fatalf("gitCloneArgs(full) = %q, want %q", got, want)
+	}
+}
+
 func TestSetRepoDependenciesSeparatesRuntimeAndTestOnly(t *testing.T) {
 	u := upstream{}
 	u.setRepoDependencies(
@@ -382,6 +483,42 @@ func TestGetPkgsitePackageRetriesAmbiguousPathWithLongestModule(t *testing.T) {
 	}
 }
 
+func TestProvidesFromPkgsitePackagesForModuleRoot(t *testing.T) {
+	info := &pkgsiteInfo{Module: pkgsiteModule{Path: "k8s.io/api"}}
+	got := strings.Join(providesFromPkgsitePackages("k8s.io/api", info, []pkgsitePackageSummary{
+		{Path: "k8s.io/api/apps/v1", IsRedistributable: true},
+		{Path: "k8s.io/api/apps/v1beta1", IsRedistributable: true},
+		{Path: "k8s.io/api/internal", IsRedistributable: false},
+	}), ",")
+	want := "k8s.io/api,k8s.io/api/apps/v1,k8s.io/api/apps/v1beta1"
+	if got != want {
+		t.Fatalf("providesFromPkgsitePackages() = %q, want %q", got, want)
+	}
+}
+
+func TestProvidesFromPkgsitePackagesForSubpackage(t *testing.T) {
+	info := &pkgsiteInfo{Module: pkgsiteModule{Path: "k8s.io/api"}}
+	got := strings.Join(providesFromPkgsitePackages("k8s.io/api/apps/v1", info, []pkgsitePackageSummary{
+		{Path: "k8s.io/api/apps/v1beta1", IsRedistributable: true},
+	}), ",")
+	want := "k8s.io/api/apps/v1"
+	if got != want {
+		t.Fatalf("providesFromPkgsitePackages() = %q, want %q", got, want)
+	}
+}
+
+func TestConvertDependenciesToRPMKeepsKubernetesImportPaths(t *testing.T) {
+	got := strings.Join(convertDependenciesToRPM([]string{
+		"k8s.io/apimachinery/pkg/apis/meta/v1",
+		"k8s.io/apimachinery/pkg/runtime",
+		"sigs.k8s.io/randfill",
+	}), ",")
+	want := "go(k8s.io/apimachinery/pkg/apis/meta/v1),go(k8s.io/apimachinery/pkg/runtime),go(sigs.k8s.io/randfill)"
+	if got != want {
+		t.Fatalf("convertDependenciesToRPM() = %q, want %q", got, want)
+	}
+}
+
 func TestSourceURLForSpecUsesCommitIDMacro(t *testing.T) {
 	u := upstream{
 		repoURL: "https://github.com/example/project",

spec.go

@@ -99,6 +99,11 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
 	if err != nil {
 		log.Printf("Could not determine license/doc file paths for %q: %v\n", gopkg, err)
 	}
+	provides, err := getProvidesForGopkg(gopkg)
+	if err != nil {
+		log.Printf("Could not determine package provides for %q: %v\n", gopkg, err)
+		provides = []string{gopkg}
+	}
 	upstreamName := filepath.Base(gopkg)
 	if regexp.MustCompile(`^v\d+$`).MatchString(upstreamName) {
 		upstreamName = filepath.Base(filepath.Dir(gopkg))
@@ -183,17 +188,17 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
 	// For different package types, write different sections
 	switch pkgType {
 	case typeLibrary:
-		writeRPMLibraryPackage(f, gopkg, openRuyiLib, longdescription, rpmRuntimeDeps)
+		writeRPMLibraryPackage(f, provides, openRuyiLib, longdescription, rpmRuntimeDeps)
 	case typeProgram:
 		log.Printf("Nothing to do for program package.\n")
 	// TODO: what can this be used for? ExclusiveArch %%{go_arches}?
 	//	writeRPMProgramPackage(f, gopkg, openRuyiProgram, longdescription)
 	case typeLibraryProgram:
-		writeRPMLibraryPackage(f, gopkg, openRuyiLib, longdescription, rpmRuntimeDeps)
+		writeRPMLibraryPackage(f, provides, openRuyiLib, longdescription, rpmRuntimeDeps)
 		writeRPMProgramSubpackage(f, gopkg, openRuyiProgram, openRuyiSrc, description)
 	case typeProgramLibrary:
 		//writeRPMProgramPackage(f, gopkg, openRuyiProgram, longdescription)
-		writeRPMLibrarySubpackage(f, gopkg, openRuyiLib, openRuyiSrc, longdescription, rpmRuntimeDeps)
+		writeRPMLibrarySubpackage(f, provides, gopkg, openRuyiLib, openRuyiSrc, longdescription, rpmRuntimeDeps)
 	default:
 		log.Fatalf("Invalid pkgType %d in writeRPMSpec(), aborting", pkgType)
 	}
@@ -214,9 +219,11 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
 }
 
 // For library package
-func writeRPMLibraryPackage(f *os.File, gopkg, openRuyiLib, longdesc string, deps []string) {
+func writeRPMLibraryPackage(f *os.File, provides []string, openRuyiLib, longdesc string, deps []string) {
 	fmt.Fprintf(f, "\n")
-	fmt.Fprintf(f, "Provides:       go(%s) = %%{version}\n", gopkg)
+	for _, provide := range provides {
+		fmt.Fprintf(f, "Provides:       go(%s) = %%{version}\n", provide)
+	}
 	fmt.Fprintf(f, "\n")
 	// 库包的运行时依赖
 	if len(deps) > 0 {
@@ -228,11 +235,13 @@ func writeRPMLibraryPackage(f *os.File, gopkg, openRuyiLib, longdesc string, dep
 }
 
 // For library subpackage
-func writeRPMLibrarySubpackage(f *os.File, gopkg, openRuyiLib, openRuyiSrc, longdesc string, deps []string) {
+func writeRPMLibrarySubpackage(f *os.File, provides []string, gopkg, openRuyiLib, openRuyiSrc, longdesc string, deps []string) {
 	fmt.Fprintf(f, "\n")
 	fmt.Fprintf(f, "%%package -n %s\n", openRuyiLib)
 	fmt.Fprintf(f, "Summary:        Development files of %s\n", filepath.Base(gopkg))
-	fmt.Fprintf(f, "Provides:       go(%s) = %%{version}\n", gopkg)
+	for _, provide := range provides {
+		fmt.Fprintf(f, "Provides:       go(%s) = %%{version}\n", provide)
+	}
 	fmt.Fprintf(f, "BuildArch:      noarch\n")
 	if len(deps) > 0 {
 		for _, dep := range deps {
@@ -260,12 +269,14 @@ func writeRPMProgramSubpackage(f *os.File, gopkg, openRuyiProgram, openRuyiSrc,
 }
 
 func writeLicenseAndDocFiles(f *os.File, assetFiles specAssetFiles, includeDoc bool) {
-	for _, licenseFile := range assetFiles.licenseFiles {
-		fmt.Fprintf(f, "%%license %s\n", licenseFile)
-	}
 	if includeDoc && assetFiles.readmeFile != "" {
 		fmt.Fprintf(f, "%%doc %s\n", assetFiles.readmeFile)
 	}
+	licenseFiles := append([]string(nil), assetFiles.licenseFiles...)
+	sort.Strings(licenseFiles)
+	for _, licenseFile := range licenseFiles {
+		fmt.Fprintf(f, "%%license %s\n", licenseFile)
+	}
 }
 
 func writeRPMFilesSection(f *os.File, openRuyiSrc, openRuyiLib, openRuyiProgram string, pkgType packageType, assetFiles specAssetFiles) {
@@ -329,6 +340,11 @@ func convertDependenciesToRPM(goPkgs []string) []string {
 	topLevelPkgs := make(map[string]bool)
 
 	for _, goPkg := range goPkgs {
+		if strings.HasPrefix(goPkg, "k8s.io/") || strings.HasPrefix(goPkg, "sigs.k8s.io/") {
+			topLevelPkgs[goPkg] = true
+			continue
+		}
+
 		// 提取顶级包：github.com/user/repo/subpkg -> github.com/user/repo
 		parts := strings.Split(goPkg, "/")
 		var topLevel string