misaka00251/go2spec
1
0
Fork 3
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

Add update function #8

Merged
misaka00251 merged 3 commits from HeliC829/go2spec:add-update-function into master 2026-07-09 04:12:32 +00:00
Conversation 1 Commits 3 Files Changed 10
+1887 -10
10 changed files with 1887 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1
View File
@@ -6,6 +6,7 @@ require (
github.com/charmbracelet/glamour v0.10.0
github.com/mattn/go-isatty v0.0.20
github.com/sandrolain/httpcache v1.4.0
golang.org/x/mod v0.29.0
golang.org/x/net v0.47.0
)
go.sum
+2
View File
@@ -60,6 +60,8 @@ github.com/yuin/goldmark-emoji v1.0.5 h1:EMVWyCGPlXJfUXBXpuMu+ii3TIaxbVBnEX9uaDC
github.com/yuin/goldmark-emoji v1.0.5/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U=
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
main.go
+3
View File
@@ -16,6 +16,7 @@ Usage:
Available Commands:
pack Package the Go modules into an RPM spec file
update Check existing Go module spec files for updates
help Show this help message
Use "go2spec [command] --help" for more information about a command.
@@ -46,6 +47,8 @@ func main() {
println("Executing 'pack' command...")
// Actual packing logic would go here
mainPack(args[1:], nil)
case "update":
os.Exit(mainUpdate(args[1:]))
default:
// Default to 'pack' command if no command is provided
println("No command provided. Defaulting to 'pack' command...")
pack.go
+146 -1
View File
@@ -2,6 +2,8 @@ package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
@@ -48,6 +50,7 @@ type upstream struct {
hasGodeps bool // whether the Godeps/_workspace directory exists
hasRelease bool // whether any release tags exist, for debian/watch
isRelease bool // whether what we end up packaging is a tagged release
sha256 string // SHA256 checksum of the source tarball
}
var errUnsupportedHoster = errors.New("unsupported hoster")
@@ -124,6 +127,24 @@ func downloadFile(filename, url string) error {
return nil
}
// fetchText fetches the content of a URL and returns it as a string.
// Returns an error if the HTTP status is not 200.
func fetchText(url string) (string, error) {
resp, err := http.Get(url)
if err != nil {
return "", fmt.Errorf("http get: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return "", fmt.Errorf("response: %s", resp.Status)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return "", fmt.Errorf("read body: %w", err)
}
return string(body), nil
}
// get downloads the source module into the provided GOPATH,
// checking out the specified revision if non-empty.
func (u *upstream) get(gopath, sourceRepo, requestedPath, rev string) error {
@@ -364,7 +385,131 @@ func (u *upstream) tarballFromHoster() error {
close(done)
return err
if err != nil {
return err
}
u.sha256 = u.fetchChecksumFromReleasePage()
if u.sha256 == "" {
sum, err := fileSHA256(u.tarPath)
if err != nil {
log.Printf("INFO: Could not compute source tarball sha256: %v", err)
} else {
u.sha256 = sum
}
}
return nil
}
// fetchChecksumFromReleasePage fetches the GitHub/GitLab release page and
// attempts to extract a SHA-256 checksum for the source tarball from the
// page content. Some projects include the checksum in the release body
// markdown, e.g. "sha256: <hex>" or "SHA256: <hex>".
func (u *upstream) fetchChecksumFromReleasePage() string {
releaseURL := u.releasePageURL()
if releaseURL == "" {
return ""
}
log.Printf("Fetching release page %s for checksum", releaseURL)
content, err := fetchText(releaseURL)
if err != nil {
log.Printf("INFO: Could not fetch release page: %v", err)
return ""
}
return extractSHA256FromHTML(content, u.tag)
}
// releasePageURL constructs the release page URL for the current tag.
// For GitHub: https://github.com/{owner}/{repo}/releases/tag/{tag}
// For GitLab: https://gitlab.com/{owner}/{repo}/-/releases/{tag}
func (u *upstream) releasePageURL() string {
repo := strings.TrimSuffix(u.repoURL, ".git")
if repo == "" || u.tag == "" {
return ""
}
repoU, err := url.Parse(repo)
if err != nil {
return ""
}
switch repoU.Host {
case "github.com":
return fmt.Sprintf("%s/releases/tag/%s", repo, u.tag)
case "gitlab.com", "salsa.debian.org":
return fmt.Sprintf("%s/-/releases/%s", repo, u.tag)
default:
return ""
}
}
// extractSHA256FromHTML searches HTML content for a SHA-256 checksum
// associated with the tarball for the given tag. It looks for patterns like:
//
// sha256: <64-hex-chars>
// SHA256: <64-hex-chars>
// sha256sum: <64-hex-chars>
//
// It also tries to match checksums near tarball-related filenames.
func extractSHA256FromHTML(html, tag string) string {
lower := strings.ToLower(html)
patterns := []*regexp.Regexp{
regexp.MustCompile(`sha-?256(?:sum)?[:\s]+([a-f0-9]{64})`),
regexp.MustCompile(`sha-?256(?:sum)?[:\s]*([a-f0-9]{64})`),
}
for _, re := range patterns {
if m := re.FindStringSubmatch(lower); len(m) > 1 && isValidSHA256(m[1]) {
return m[1]
}
}
// Fallback: look for a 64-char hex string near "tar" or "source" keywords
tarRe := regexp.MustCompile(`(?:tar|source)[^a-f0-9]{0,80}([a-f0-9]{64})`)
if m := tarRe.FindStringSubmatch(lower); len(m) > 1 {
if isValidSHA256(m[1]) {
return m[1]
}
}
revTarRe := regexp.MustCompile(`([a-f0-9]{64})[^a-f0-9]{0,80}(?:tar|source)`)
if m := revTarRe.FindStringSubmatch(lower); len(m) > 1 {
if isValidSHA256(m[1]) {
return m[1]
}
}
return ""
}
// isValidSHA256 reports whether s is a valid lowercase hexadecimal string
// of exactly 64 characters (the length of a SHA-256 digest).
func isValidSHA256(s string) bool {
if len(s) != 64 {
return false
}
for _, c := range s {
if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) {
return false
}
}
return true
}
func fileSHA256(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
return "", fmt.Errorf("open: %w", err)
}
defer f.Close()
sum := sha256.New()
if _, err := io.Copy(sum, f); err != nil {
return "", fmt.Errorf("hash: %w", err)
}
return hex.EncodeToString(sum.Sum(nil)), nil
}
func (u *upstream) tar(gopath, repo string) error {
pkgsite.go
+85 -8
View File
@@ -14,13 +14,14 @@ import (
"time"
)
const pkgsiteAPIBase = "https://pkg.go.dev/v1beta"
const pkgsiteMaxResponseBytes = 20 << 20
var (
pkgsiteAPIBase = "https://pkg.go.dev/v1beta"
pkgsiteHTTPClient = http.DefaultClient
pkgsiteMu sync.Mutex
pkgsiteInfoCache = make(map[string]*pkgsiteInfo)
pkgsiteRetryLog func(string)
)
type pkgsiteInfo struct {
@@ -105,8 +106,6 @@ func pkgsiteURL(endpoint, importPath string, values url.Values) string {
return u
}
// pkgsiteGetJSON performs a pkg.go.dev v1beta GET with up to 3 attempts and
// linear backoff for transport errors, HTTP 429, and 5xx responses.
func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url.Values, v any) error {
client := pkgsiteHTTPClient
if client == nil {
@@ -114,7 +113,7 @@ func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url
}
var lastErr error
for attempt := 1; attempt <= 3; attempt++ {
for attempt := 1; attempt <= 5; attempt++ {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pkgsiteURL(endpoint, importPath, values), nil)
if err != nil {
return fmt.Errorf("create pkgsite request: %w", err)
@@ -124,8 +123,13 @@ func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url
resp, err := client.Do(req)
if err != nil {
lastErr = fmt.Errorf("pkgsite request: %w", err)
if attempt < 3 {
time.Sleep(time.Duration(attempt) * time.Second)
if attempt < 5 {
backoff := time.Duration(attempt*2) * time.Second
reportPkgsiteRetry(" RETRY: pkgsite %s %s request failed (%v), retry %d/5 in %s",
endpoint, importPath, err, attempt+1, backoff)
if err := sleepWithContext(ctx, backoff); err != nil {
return err
}
continue
}
return lastErr
@@ -148,8 +152,12 @@ func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url
apiErr.Code = resp.StatusCode
apiErr.Status = resp.Status
lastErr = &apiErr
if attempt < 3 && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500) {
time.Sleep(time.Duration(attempt) * time.Second)
if attempt < 5 && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500) {
backoff := pkgsiteRetryBackoff(resp, attempt)
reportPkgsiteStatusRetry(endpoint, importPath, resp.StatusCode, &apiErr, attempt+1, backoff)
if err := sleepWithContext(ctx, backoff); err != nil {
return err
}
continue
}
return lastErr
@@ -163,6 +171,75 @@ func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url
return lastErr
}
func sleepWithContext(ctx context.Context, d time.Duration) error {
if d <= 0 {
select {
case <-ctx.Done():
return ctx.Err()
default:
return nil
}
}
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func pkgsiteRetryBackoff(resp *http.Response, attempt int) time.Duration {
if resp != nil && resp.StatusCode == http.StatusTooManyRequests {
if retryAfter, ok := parseRetryAfter(resp.Header.Get("Retry-After")); ok {
return retryAfter
}
return time.Duration(attempt*10) * time.Second
}
return time.Duration(attempt*5) * time.Second
}
func parseRetryAfter(v string) (time.Duration, bool) {
v = strings.TrimSpace(v)
if v == "" {
return 0, false
}
if seconds, err := time.ParseDuration(v + "s"); err == nil && seconds > 0 {
return seconds, true
}
for _, layout := range []string{time.RFC1123, time.RFC1123Z, time.RFC850, time.ANSIC} {
t, err := time.Parse(layout, v)
if err == nil {
delay := time.Until(t)
if delay > 0 {
return delay, true
}
return time.Second, true
}
}
return 0, false
}
func reportPkgsiteRetry(format string, args ...any) {
if pkgsiteRetryLog != nil {
pkgsiteRetryLog(fmt.Sprintf(format, args...))
}
}
func reportPkgsiteStatusRetry(endpoint, importPath string, statusCode int, apiErr *pkgsiteAPIError, nextAttempt int, backoff time.Duration) {
switch statusCode {
case http.StatusTooManyRequests:
reportPkgsiteRetry(" RATE: %s limited on %s, retry %d/5 in %s",
importPath, endpoint, nextAttempt, backoff)
default:
reportPkgsiteRetry(" RETRY: %s %s HTTP %d, retry %d/5 in %s",
importPath, endpoint, statusCode, nextAttempt, backoff)
}
}
func getPkgsitePackage(ctx context.Context, gopkg, modulePath string) (pkgsitePackage, error) {
values := url.Values{}
values.Set("imports", "true")
pkgsite_test.go
+72
View File
@@ -1,12 +1,16 @@
package main
import (
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
@@ -62,6 +66,22 @@ func TestPkgsiteLicenseExpression(t *testing.T) {
}
}
func TestFileSHA256(t *testing.T) {
path := filepath.Join(t.TempDir(), "archive.tar.gz")
if err := os.WriteFile(path, []byte("source archive"), 0o644); err != nil {
t.Fatalf("write archive: %v", err)
}
got, err := fileSHA256(path)
if err != nil {
t.Fatalf("fileSHA256: %v", err)
}
want := "6ad189ace456a83fade855d5a647cd8ad9e7966da4404b1187218dca3d9eddaa"
if got != want {
t.Fatalf("fileSHA256() = %q, want %q", got, want)
}
}
func TestPkgsiteLicenseFilePaths(t *testing.T) {
licenses := []pkgsiteLicense{
{FilePath: "License", Types: []string{"MIT"}},
@@ -538,3 +558,55 @@ func runGit(t *testing.T, dir string, env []string, args ...string) string {
}
return string(out)
}
func TestPkgsiteGetJSONLogsRateLimitRetry(t *testing.T) {
var mu sync.Mutex
var logs []string
attempts := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.Header().Set("Retry-After", "1")
w.WriteHeader(http.StatusTooManyRequests)
_, _ = w.Write([]byte(`{"message":"slow down"}`))
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"path":"example.com/mod","version":"v1.2.3"}`))
}))
defer srv.Close()
oldBase := pkgsiteAPIBase
oldClient := pkgsiteHTTPClient
oldLog := pkgsiteRetryLog
t.Cleanup(func() {
pkgsiteAPIBase = oldBase
pkgsiteHTTPClient = oldClient
pkgsiteRetryLog = oldLog
})
pkgsiteAPIBase = srv.URL
pkgsiteHTTPClient = srv.Client()
pkgsiteRetryLog = func(msg string) {
mu.Lock()
defer mu.Unlock()
logs = append(logs, msg)
}
var mod pkgsiteModule
if err := pkgsiteGetJSON(context.Background(), "module", "example.com/mod", url.Values{}, &mod); err != nil {
t.Fatalf("pkgsiteGetJSON: %v", err)
}
if attempts != 2 {
t.Fatalf("expected 2 attempts, got %d", attempts)
}
mu.Lock()
defer mu.Unlock()
if len(logs) == 0 {
t.Fatal("expected rate limit log message")
}
if !strings.Contains(logs[0], "example.com/mod limited on module") || !strings.Contains(logs[0], "retry 2/5 in 1s") {
t.Fatalf("unexpected log message: %q", logs[0])
}
}
spec.go
+5 -1
View File
@@ -148,7 +148,11 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st
fmt.Fprintf(f, "Summary: %s\n", description)
fmt.Fprintf(f, "License: %s\n", license)
fmt.Fprintf(f, "URL: %s\n", repoURL)
fmt.Fprintf(f, "#!RemoteAsset\n")
if u.sha256 != "" {
fmt.Fprintf(f, "#!RemoteAsset: sha256:%s\n", u.sha256)
} else {
fmt.Fprintf(f, "#!RemoteAsset\n")
}
sourceModulePath := u.modulePath
if sourceModulePath == "" {
sourceModulePath = gopkg
update.go
+625
View File
@@ -0,0 +1,625 @@
package main
import (
"bufio"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"time"
"golang.org/x/mod/semver"
)
var downloadRetryLog func(string)
var errRemoteAssetSkip = errors.New("skip remote asset sha256")
type downloadCandidate struct {
source string
url string
}
func repoRefForVersion(version string) string {
v := strings.TrimSpace(version)
if !strings.HasPrefix(v, "v") {
v = "v" + v
}
if canonical := semver.Canonical(v); canonical != "" {
return canonical
}
return v
}
func specVersionForVersion(version string) string {
return strings.TrimPrefix(repoRefForVersion(version), "v")
}
func verifyDownload(ctx context.Context, spec *specInfo) error {
if spec.GoImportPath == "" {
return fmt.Errorf("no go_import_path")
}
if spec.LatestVersion == "" {
return fmt.Errorf("no latest version")
}
info, err := getPkgsiteInfo(ctx, spec.GoImportPath)
if err != nil {
return fmt.Errorf("get pkgsite info: %w", err)
}
candidates, archiveErr, err := downloadCandidatesForSpec(spec, info)
if err != nil {
return err
}
if archiveErr != nil {
reportDownloadRetry(" FALLBACK: %s repo archive unavailable (%v), trying module proxy",
spec.GoImportPath, archiveErr)
}
return verifyDownloadCandidates(ctx, spec.GoImportPath, candidates)
}
func downloadCandidatesForSpec(spec *specInfo, info *pkgsiteInfo) ([]downloadCandidate, error, error) {
repoURL := gitCloneURLFromRepoURL(info.Module.RepoURL)
if repoURL == "" {
return nil, nil, fmt.Errorf("no repository URL")
}
version := spec.LatestVersion
if !strings.HasPrefix(version, "v") {
version = "v" + version
}
repoRef := repoRefForVersion(version)
sourceModulePath := sourceImportPathForPackage(spec.GoImportPath, info)
proxyURL := fmt.Sprintf("https://proxy.golang.org/%s/@v/%s.zip",
moduleProxyEscapedPath(sourceModulePath),
moduleProxyVersionForSpec(version))
if moduleUsesRepoSubdir(sourceModulePath, repoURL) {
return []downloadCandidate{
{source: "proxy", url: proxyURL},
}, nil, nil
}
tarURL, err := (&upstream{repoURL: repoURL}).tarballURLForRef(repoRef, "gz")
if err != nil {
return []downloadCandidate{
{source: "proxy", url: proxyURL},
}, err, nil
}
return []downloadCandidate{
{source: "repo", url: tarURL},
{source: "proxy", url: proxyURL},
}, nil, nil
}
func remoteAssetSHA256(ctx context.Context, spec *specInfo, newVersion string) (string, error) {
sourceURL, err := resolvedSource0URL(spec.FilePath, sourceVersionForSpec(newVersion))
if err != nil {
return "", err
}
releaseURL, tag := releasePageURLForSource(sourceURL)
if releaseURL == "" {
reportDownloadRetry(" HASH: %s no release checksum page, downloading Source0", spec.GoImportPath)
return downloadAssetSHA256(ctx, sourceURL)
}
content, err := fetchTextContext(ctx, releaseURL)
if err == nil {
if sum := extractSHA256FromHTML(content, tag); sum != "" {
return sum, nil
}
}
reportDownloadRetry(" HASH: %s release checksum unavailable, downloading Source0", spec.GoImportPath)
return downloadAssetSHA256(ctx, sourceURL)
}
func sourceVersionForSpec(version string) string {
return specVersionForVersion(version)
}
func verifyDownloadCandidates(ctx context.Context, importPath string, candidates []downloadCandidate) error {
var errs []string
for i, candidate := range candidates {
err := verifyHTTPRequest(ctx, candidate.url)
if err == nil {
if i > 0 {
reportDownloadRetry(" OK: %s verified via %s after fallback", importPath, candidate.source)
}
return nil
}
errs = append(errs, fmt.Sprintf("%s: %v", candidate.source, err))
if i+1 < len(candidates) {
reportDownloadRetry(" FALLBACK: %s %s probe failed (%v), trying %s",
importPath, candidate.source, err, candidates[i+1].source)
}
}
return errors.New(strings.Join(errs, "; "))
}
func verifyHTTPRequest(ctx context.Context, downloadURL string) error {
var lastErr error
for attempt := 1; attempt <= 3; attempt++ {
if err := verifyHTTPHead(ctx, downloadURL); err == nil {
return nil
} else {
lastErr = err
}
if err := verifyHTTPRangeGet(ctx, downloadURL); err == nil {
return nil
} else {
lastErr = err
}
if attempt < 3 && isRetryableDownloadError(lastErr) {
backoff := time.Duration(attempt*2) * time.Second
reportDownloadRetry(" RETRY: download probe %s failed (%v), retry %d/3 in %s",
downloadURL, lastErr, attempt+1, backoff)
if err := sleepWithContext(ctx, backoff); err != nil {
return err
}
continue
}
break
}
return lastErr
}
func verifyHTTPHead(ctx context.Context, downloadURL string) error {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, downloadURL, nil)
if err != nil {
return fmt.Errorf("create HEAD request: %w", err)
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("HEAD request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
return fmt.Errorf("HTTP %d from %s", resp.StatusCode, downloadURL)
}
if resp.ContentLength == 0 {
return fmt.Errorf("HEAD response has no content length from %s", downloadURL)
}
return nil
}
func verifyHTTPRangeGet(ctx context.Context, downloadURL string) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
if err != nil {
return fmt.Errorf("create GET request: %w", err)
}
req.Header.Set("Range", "bytes=0-0")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("GET request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
return fmt.Errorf("HTTP %d from %s", resp.StatusCode, downloadURL)
}
buf := make([]byte, 1)
n, err := resp.Body.Read(buf)
if err != nil && err != io.EOF {
return fmt.Errorf("read download probe: %w", err)
}
if n == 0 {
return fmt.Errorf("empty download from %s", downloadURL)
}
return nil
}
func isRetryableDownloadError(err error) bool {
if err == nil {
return false
}
var netErr net.Error
if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) {
return true
}
msg := err.Error()
return strings.Contains(msg, "context deadline exceeded") ||
strings.Contains(msg, "Client.Timeout exceeded") ||
strings.Contains(msg, "connection reset by peer") ||
strings.Contains(msg, "EOF") ||
strings.Contains(msg, "HTTP 408") ||
strings.Contains(msg, "HTTP 425") ||
strings.Contains(msg, "HTTP 429") ||
strings.Contains(msg, "HTTP 500") ||
strings.Contains(msg, "HTTP 502") ||
strings.Contains(msg, "HTTP 503") ||
strings.Contains(msg, "HTTP 504")
}
func reportDownloadRetry(format string, args ...any) {
if downloadRetryLog != nil {
downloadRetryLog(fmt.Sprintf(format, args...))
}
}
func readSpecLines(path string) ([]string, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
var lines []string
scanner := bufio.NewScanner(f)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
func writeSpecLines(path string, lines []string) error {
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
w := bufio.NewWriter(f)
for i, line := range lines {
if _, err := w.WriteString(line); err != nil {
return err
}
if i < len(lines)-1 {
if _, err := w.WriteString("\n"); err != nil {
return err
}
}
}
if len(lines) > 0 {
if _, err := w.WriteString("\n"); err != nil {
return err
}
}
return w.Flush()
}
func updateSpecVersion(spec *specInfo, newVersion, remoteSHA256 string, updateRemoteAsset bool) error {
lines, err := readSpecLines(spec.FilePath)
if err != nil {
return err
}
versionLineRe := regexp.MustCompile(`^(\s*Version:\s*)(\S+)(.*)$`)
autoChangelogRe := regexp.MustCompile(`^(\s*)%\{\?autochangelog\}(\s*)$`)
remoteAssetHashRe := regexp.MustCompile(`^(\s*)#!RemoteAsset(?::\s*sha256:[0-9a-fA-F]+)?(\s*)$`)
source0LineRe := regexp.MustCompile(`^(\s*Source0:\s*)(\S+)(.*)$`)
updated := false
source0Index := -1
source0RemoteAssetIndex := -1
for i, line := range lines {
matches := versionLineRe.FindStringSubmatch(line)
if len(matches) == 4 {
lines[i] = matches[1] + newVersion + matches[3]
updated = true
}
matches = autoChangelogRe.FindStringSubmatch(line)
if len(matches) == 3 {
lines[i] = matches[1] + "%autochangelog" + matches[2]
}
if source0Index < 0 && source0LineRe.MatchString(line) {
source0Index = i
if i > 0 && remoteAssetHashRe.MatchString(lines[i-1]) {
source0RemoteAssetIndex = i - 1
}
}
}
if !updated {
return fmt.Errorf("Version line not found in %s", spec.FilePath)
}
if !updateRemoteAsset {
return writeSpecLines(spec.FilePath, lines)
}
if source0RemoteAssetIndex >= 0 {
matches := remoteAssetHashRe.FindStringSubmatch(lines[source0RemoteAssetIndex])
if remoteSHA256 != "" {
lines[source0RemoteAssetIndex] = matches[1] + "#!RemoteAsset: sha256:" + remoteSHA256 + matches[2]
} else {
lines[source0RemoteAssetIndex] = matches[1] + "#!RemoteAsset" + matches[2]
}
} else if source0Index >= 0 {
insert := "#!RemoteAsset"
if remoteSHA256 != "" {
insert = "#!RemoteAsset: sha256:" + remoteSHA256
}
lines = append(lines[:source0Index], append([]string{insert}, lines[source0Index:]...)...)
}
return writeSpecLines(spec.FilePath, lines)
}
type downloadResult struct {
spec *specInfo
err error
}
func verifyAndUpdateSpec(ctx context.Context, spec *specInfo, dryRun bool) error {
err := verifyDownload(ctx, spec)
if err != nil {
return fmt.Errorf("download verification failed: %w", err)
}
if !dryRun {
newVer := specVersionForVersion(spec.LatestVersion)
remoteSHA256, err := remoteAssetSHA256(ctx, spec, newVer)
updateRemoteAsset := true
if errors.Is(err, errRemoteAssetSkip) {
updateRemoteAsset = false
err = nil
}
if err != nil {
return fmt.Errorf("compute remote asset sha256: %w", err)
}
if err := updateSpecVersion(spec, newVer, remoteSHA256, updateRemoteAsset); err != nil {
return fmt.Errorf("update spec file: %w", err)
}
}
return nil
}
func processUpdate(ctx context.Context, spec *specInfo, dryRun bool) error {
err := verifyDownload(ctx, spec)
if err != nil {
return err
}
if !dryRun {
newVer := specVersionForVersion(spec.LatestVersion)
remoteSHA256, err := remoteAssetSHA256(ctx, spec, newVer)
updateRemoteAsset := true
if errors.Is(err, errRemoteAssetSkip) {
updateRemoteAsset = false
err = nil
}
if err != nil {
return fmt.Errorf("compute remote asset sha256: %w", err)
}
if err := updateSpecVersion(spec, newVer, remoteSHA256, updateRemoteAsset); err != nil {
return fmt.Errorf("update spec file: %w", err)
}
}
return nil
}
func resolvedSource0URL(path, newVersion string) (string, error) {
lines, err := readSpecLines(path)
if err != nil {
return "", err
}
macros := make(map[string]string)
defineRe := regexp.MustCompile(`^%(?:define|global)\s+(\S+)\s+(.+)$`)
fieldRe := regexp.MustCompile(`^(Name|Version):\s+(\S+)`)
source0Re := regexp.MustCompile(`^Source0:\s+(\S+)`)
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if matches := defineRe.FindStringSubmatch(trimmed); len(matches) == 3 {
macros[matches[1]] = strings.TrimSpace(matches[2])
continue
}
if matches := fieldRe.FindStringSubmatch(trimmed); len(matches) == 3 {
switch matches[1] {
case "Name":
macros["name"] = matches[2]
case "Version":
macros["version"] = newVersion
}
continue
}
if matches := source0Re.FindStringSubmatch(trimmed); len(matches) == 2 {
if source0UsesPinnedCommit(matches[1]) {
return "", fmt.Errorf("%w: Source0 uses pinned commit_id", errRemoteAssetSkip)
}
macros["version"] = newVersion
resolved := expandRPMMacros(matches[1], macros, 16)
if strings.Contains(resolved, "%{") {
return "", fmt.Errorf("unresolved Source0 macros in %s: %s", path, resolved)
}
return strings.SplitN(resolved, "#", 2)[0], nil
}
}
return "", fmt.Errorf("Source0 line not found in %s", path)
}
func source0UsesPinnedCommit(source string) bool {
return strings.Contains(source, "%{commit_id}") || strings.Contains(source, "%{?commit_id}")
}
func expandRPMMacros(s string, macros map[string]string, limit int) string {
re := regexp.MustCompile(`%\{(\??)([^}]+)\}`)
expanded := s
for i := 0; i < limit; i++ {
changed := false
next := re.ReplaceAllStringFunc(expanded, func(match string) string {
parts := re.FindStringSubmatch(match)
if len(parts) != 3 {
return match
}
optional := parts[1] == "?"
name := parts[2]
value, ok := macros[name]
if !ok {
if optional {
changed = true
return ""
}
return match
}
changed = true
return value
})
expanded = next
if !changed {
break
}
}
return expanded
}
func releasePageURLForSource(sourceURL string) (releaseURL, tag string) {
u, err := url.Parse(sourceURL)
if err != nil {
return "", ""
}
tag = sourceTagFromURLPath(u.Path)
if tag == "" {
return "", ""
}
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
switch u.Host {
case "github.com":
if len(parts) < 2 {
return "", ""
}
repo := fmt.Sprintf("%s://%s/%s/%s", u.Scheme, u.Host, parts[0], parts[1])
return fmt.Sprintf("%s/releases/tag/%s", repo, tag), tag
case "gitlab.com", "salsa.debian.org":
dashArchive := -1
for i := 0; i+1 < len(parts); i++ {
if parts[i] == "-" && parts[i+1] == "archive" {
dashArchive = i
break
}
}
if dashArchive <= 0 {
return "", ""
}
repo := fmt.Sprintf("%s://%s/%s", u.Scheme, u.Host, strings.Join(parts[:dashArchive], "/"))
return fmt.Sprintf("%s/-/releases/%s", repo, tag), tag
default:
return "", ""
}
}
func sourceTagFromURLPath(path string) string {
path, err := url.PathUnescape(path)
if err != nil {
return ""
}
const refsPrefix = "refs/tags/"
parts := strings.Split(strings.Trim(path, "/"), "/")
for i, part := range parts {
if part != "archive" || i+1 >= len(parts) {
continue
}
rest := strings.Join(parts[i+1:], "/")
if strings.HasPrefix(rest, refsPrefix) {
return trimArchiveSuffix(strings.TrimPrefix(rest, refsPrefix))
}
if idx := strings.Index(rest, "/"); idx >= 0 {
rest = rest[:idx]
}
return trimArchiveSuffix(rest)
}
return ""
}
func trimArchiveSuffix(s string) string {
for _, suffix := range []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tgz", ".zip"} {
if strings.HasSuffix(s, suffix) {
return strings.TrimSuffix(s, suffix)
}
}
return s
}
func fetchTextContext(ctx context.Context, fetchURL string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL, nil)
if err != nil {
return "", fmt.Errorf("create GET request: %w", err)
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("GET request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("HTTP %d from %s", resp.StatusCode, fetchURL)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return "", fmt.Errorf("read response: %w", err)
}
return string(body), nil
}
func downloadAssetSHA256(ctx context.Context, downloadURL string) (string, error) {
var lastErr error
for attempt := 1; attempt <= 3; attempt++ {
sum, err := fetchSHA256(ctx, downloadURL)
if err == nil {
return sum, nil
}
lastErr = err
if attempt < 3 && isRetryableDownloadError(lastErr) {
backoff := time.Duration(attempt*2) * time.Second
reportDownloadRetry(" RETRY: asset hash %s failed (%v), retry %d/3 in %s",
downloadURL, lastErr, attempt+1, backoff)
if err := sleepWithContext(ctx, backoff); err != nil {
return "", err
}
continue
}
break
}
return "", lastErr
}
func fetchSHA256(ctx context.Context, downloadURL string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
if err != nil {
return "", fmt.Errorf("create GET request: %w", err)
}
client := &http.Client{Timeout: 120 * time.Second}
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("GET request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
return "", fmt.Errorf("HTTP %d from %s", resp.StatusCode, downloadURL)
}
sum := sha256.New()
if _, err := io.Copy(sum, resp.Body); err != nil {
return "", fmt.Errorf("read download for sha256: %w", err)
}
return hex.EncodeToString(sum.Sum(nil)), nil
}
update_cli.go
+492
View File
@@ -0,0 +1,492 @@
package main
import (
"bufio"
"context"
"errors"
"flag"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/mattn/go-isatty"
"github.com/sandrolain/httpcache"
"golang.org/x/mod/semver"
)
type specInfo struct {
FilePath string
GoImportPath string
Version string
Name string
Source0 string
LatestVersion string
HasUpdate bool
Error string
}
func parseSpecFile(path string) (*specInfo, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
info := &specInfo{FilePath: path}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(line, "#") {
continue
}
if strings.HasPrefix(line, "%define go_import_path") || strings.HasPrefix(line, "%global go_import_path") {
parts := strings.Fields(line)
if len(parts) >= 3 {
info.GoImportPath = parts[2]
}
}
if strings.HasPrefix(line, "%define _name") || strings.HasPrefix(line, "%global _name") {
parts := strings.Fields(line)
if len(parts) >= 3 {
info.Name = parts[2]
}
}
if strings.HasPrefix(line, "Version:") {
parts := strings.Fields(line)
if len(parts) >= 2 {
info.Version = parts[1]
}
}
if strings.HasPrefix(line, "Source0:") {
parts := strings.Fields(line)
if len(parts) >= 2 {
info.Source0 = parts[1]
}
}
}
return info, scanner.Err()
}
func normalizeVersion(v string) string {
v = strings.TrimSpace(v)
if !strings.HasPrefix(v, "v") {
v = "v" + v
}
if semver.IsValid(v) {
return v
}
return ""
}
func checkLatestVersion(ctx context.Context, spec *specInfo) {
if spec.GoImportPath == "" {
spec.Error = "no go_import_path"
return
}
info, err := getPkgsiteInfo(ctx, spec.GoImportPath)
if err != nil {
spec.Error = fmt.Sprintf("pkgsite error: %v", err)
return
}
latestPkgVer := info.Package.Version
latestModVer := info.Module.Version
latestVer := latestPkgVer
if latestModVer != "" && (latestVer == "" || semver.Compare(normalizeVersion(latestModVer), normalizeVersion(latestVer)) > 0) {
latestVer = latestModVer
}
spec.LatestVersion = latestVer
currentNorm := normalizeVersion(spec.Version)
latestNorm := normalizeVersion(latestVer)
if currentNorm == "" || latestNorm == "" {
spec.HasUpdate = false
return
}
spec.HasUpdate = semver.Compare(latestNorm, currentNorm) > 0
}
func scanSpecDir(specDir string) ([]*specInfo, error) {
entries, err := os.ReadDir(specDir)
if err != nil {
return nil, err
}
var specs []*specInfo
for _, entry := range entries {
if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "go-") {
continue
}
specFiles, err := filepath.Glob(filepath.Join(specDir, entry.Name(), "*.spec"))
if err != nil {
continue
}
for _, sf := range specFiles {
spec, err := parseSpecFile(sf)
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: failed to parse %s: %v\n", sf, err)
continue
}
if spec.GoImportPath != "" {
specs = append(specs, spec)
}
}
}
return specs, nil
}
func printBox(title string, lines []string) {
width := 56
contentWidth := width - 4
fmt.Println("+" + strings.Repeat("-", width) + "+")
if title != "" {
padding := width - len(title)
left := padding / 2
right := padding - left
fmt.Printf("|%s%s%s|\n", strings.Repeat(" ", left), title, strings.Repeat(" ", right))
fmt.Println("+" + strings.Repeat("-", width) + "+")
}
for _, line := range lines {
if len(line) > contentWidth {
line = line[:contentWidth-3] + "..."
}
fmt.Printf("| %-*s |\n", contentWidth, line)
}
fmt.Println("+" + strings.Repeat("-", width) + "+")
}
type outputMsg struct {
text string
}
type counters struct {
scanned int32
withUpdate int32
scanErrors int32
modified int32
processed int32
dlErrors int32
upErrors int32
total int
}
type outputLoop struct {
ch chan outputMsg
counts *counters
done chan struct{}
exited chan struct{}
lastProgress []string
interactiveTTY bool
progressShown int
}
func newOutputLoop(c *counters) *outputLoop {
return &outputLoop{
ch: make(chan outputMsg, 64),
counts: c,
done: make(chan struct{}),
exited: make(chan struct{}),
interactiveTTY: isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()),
}
}
func (o *outputLoop) start() {
go o.loop()
}
func (o *outputLoop) loop() {
defer close(o.exited)
ticker := time.NewTicker(200 * time.Millisecond)
defer ticker.Stop()
for {
select {
case msg := <-o.ch:
o.printMessage(msg.text)
case <-ticker.C:
o.maybePrintProgress()
case <-o.done:
for {
select {
case msg := <-o.ch:
o.printMessage(msg.text)
default:
o.clearProgress()
return
}
}
}
}
}
func (o *outputLoop) maybePrintProgress() {
progress := o.currentProgressLines()
if sameProgressLines(progress, o.lastProgress) {
return
}
o.lastProgress = append(o.lastProgress[:0], progress...)
if o.interactiveTTY {
o.renderProgress(progress)
return
}
for _, line := range progress {
fmt.Println(line)
}
}
func (o *outputLoop) currentProgressLines() []string {
c := o.counts
scanned := atomic.LoadInt32(&c.scanned)
withUpdate := atomic.LoadInt32(&c.withUpdate)
scanErrors := atomic.LoadInt32(&c.scanErrors)
processed := atomic.LoadInt32(&c.processed)
modified := atomic.LoadInt32(&c.modified)
dlErrors := atomic.LoadInt32(&c.dlErrors)
upErrors := atomic.LoadInt32(&c.upErrors)
return []string{
fmt.Sprintf(" [scan] %d/%d updates:%d errors:%d",
scanned, c.total, withUpdate, scanErrors),
fmt.Sprintf(" [verify] %d/%d modified:%d errors:%d",
processed, withUpdate, modified, dlErrors+upErrors),
}
}
func (o *outputLoop) print(text string) {
o.ch <- outputMsg{text: text}
}
func (o *outputLoop) stop() {
close(o.done)
<-o.exited
}
func (o *outputLoop) printMessage(text string) {
o.clearProgress()
fmt.Println(text)
if o.interactiveTTY {
progress := o.currentProgressLines()
o.lastProgress = append(o.lastProgress[:0], progress...)
o.renderProgress(progress)
}
}
func (o *outputLoop) clearProgress() {
if o.interactiveTTY && o.progressShown > 0 {
fmt.Print("\r\033[2K")
for i := 1; i < o.progressShown; i++ {
fmt.Print("\033[1A\r\033[2K")
}
o.progressShown = 0
}
}
func (o *outputLoop) renderProgress(lines []string) {
o.clearProgress()
for i, line := range lines {
if i > 0 {
fmt.Print("\n")
}
fmt.Print(line)
}
o.progressShown = len(lines)
}
func sameProgressLines(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func mainUpdate(args []string) int {
fs := flag.NewFlagSet("update", flag.ContinueOnError)
fs.SetOutput(os.Stderr)
concurrency := fs.Int("j", 3, "number of concurrent checks")
dryRun := fs.Bool("n", false, "dry run: detect updates only, do not modify files")
fs.Usage = func() {
fmt.Fprintf(fs.Output(), "Usage: %s update [options] <SPECS_DIR>\n\n", os.Args[0])
fmt.Fprintf(fs.Output(), "Check Go module spec files for updates.\n\n")
fmt.Fprintf(fs.Output(), "Arguments:\n")
fmt.Fprintf(fs.Output(), " SPECS_DIR directory containing go-* subdirectories\n\n")
fmt.Fprintf(fs.Output(), "Options:\n")
fs.PrintDefaults()
}
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
if *concurrency < 1 {
*concurrency = 1
}
if fs.NArg() != 1 {
fs.Usage()
return 1
}
specDir := fs.Arg(0)
specs, err := scanSpecDir(specDir)
if err != nil {
fmt.Fprintf(os.Stderr, "Error scanning spec dir: %v\n", err)
return 1
}
if len(specs) == 0 {
fmt.Fprintf(os.Stderr, "No spec files with go_import_path found in %s\n", specDir)
return 1
}
pkgsiteHTTPClient = &http.Client{
Timeout: 30 * time.Second,
Transport: httpcache.NewMemoryCacheTransport(),
}
ctx := context.Background()
total := len(specs)
c := &counters{total: total}
out := newOutputLoop(c)
out.start()
pkgsiteRetryLog = out.print
downloadRetryLog = out.print
fmt.Printf("Checking %d spec files for updates...\n", total)
fmt.Printf("Parameters: spec_dir=%s concurrency=%d dry_run=%t\n", specDir, *concurrency, *dryRun)
updateCh := make(chan *specInfo, *concurrency)
var wg2 sync.WaitGroup
if !*dryRun {
for i := 0; i < *concurrency; i++ {
wg2.Add(1)
go func() {
defer wg2.Done()
for sp := range updateCh {
err := verifyDownload(ctx, sp)
if err != nil {
sp.Error = err.Error()
out.print(fmt.Sprintf(" FAIL: %s - %v", sp.GoImportPath, err))
atomic.AddInt32(&c.dlErrors, 1)
} else {
newVer := specVersionForVersion(sp.LatestVersion)
remoteSHA256, err := remoteAssetSHA256(ctx, sp, newVer)
updateRemoteAsset := true
if errors.Is(err, errRemoteAssetSkip) {
out.print(fmt.Sprintf(" SKIP: %s remote asset sha256 - %v", sp.GoImportPath, err))
updateRemoteAsset = false
err = nil
}
if err != nil {
sp.Error = err.Error()
out.print(fmt.Sprintf(" FAIL: %s - %v", sp.GoImportPath, err))
atomic.AddInt32(&c.upErrors, 1)
} else if err := updateSpecVersion(sp, newVer, remoteSHA256, updateRemoteAsset); err != nil {
sp.Error = err.Error()
out.print(fmt.Sprintf(" FAIL: %s - %v", sp.GoImportPath, err))
atomic.AddInt32(&c.upErrors, 1)
} else {
atomic.AddInt32(&c.modified, 1)
out.print(fmt.Sprintf(" OK: %s updated to %s", sp.GoImportPath, newVer))
}
}
atomic.AddInt32(&c.processed, 1)
}
}()
}
}
var wg1 sync.WaitGroup
sem1 := make(chan struct{}, *concurrency)
for _, spec := range specs {
wg1.Add(1)
sem1 <- struct{}{}
go func(s *specInfo) {
defer wg1.Done()
defer func() { <-sem1 }()
checkLatestVersion(ctx, s)
atomic.AddInt32(&c.scanned, 1)
needSend := false
if s.Error != "" {
atomic.AddInt32(&c.scanErrors, 1)
out.print(fmt.Sprintf(" ERROR: %s - %s", s.GoImportPath, s.Error))
} else if s.HasUpdate {
atomic.AddInt32(&c.withUpdate, 1)
out.print(fmt.Sprintf(" UPDATE: %-50s %s -> %s", s.GoImportPath, s.Version, s.LatestVersion))
needSend = true
}
if needSend && !*dryRun {
updateCh <- s
}
}(spec)
}
wg1.Wait()
close(updateCh)
if !*dryRun {
wg2.Wait()
}
pkgsiteRetryLog = nil
downloadRetryLog = nil
out.stop()
var errSpecs []*specInfo
for _, s := range specs {
if s.Error != "" {
errSpecs = append(errSpecs, s)
}
}
summaryLines := []string{
fmt.Sprintf("%-12s %d/%d", "Scanned:", total, total),
fmt.Sprintf("%-12s %d", "Updates:", atomic.LoadInt32(&c.withUpdate)),
fmt.Sprintf("%-12s %d", "Modified:", atomic.LoadInt32(&c.modified)),
}
if atomic.LoadInt32(&c.scanErrors) > 0 || atomic.LoadInt32(&c.dlErrors) > 0 || atomic.LoadInt32(&c.upErrors) > 0 {
totalErrors := atomic.LoadInt32(&c.scanErrors) + atomic.LoadInt32(&c.dlErrors) + atomic.LoadInt32(&c.upErrors)
summaryLines = append(summaryLines, fmt.Sprintf("%-12s %d", "Errors:", totalErrors))
}
if *dryRun {
summaryLines = append(summaryLines, fmt.Sprintf("%-12s %s", "Mode:", "dry run"))
}
fmt.Println()
printBox("SUMMARY", summaryLines)
if atomic.LoadInt32(&c.withUpdate) == 0 {
fmt.Println("\nAll modules are up to date.")
} else if *dryRun {
fmt.Println("\nDry run complete. No files were modified.")
} else {
fmt.Println()
printBox("UPDATE RESULT", []string{
fmt.Sprintf("%-12s %d", "Attempted:", atomic.LoadInt32(&c.withUpdate)),
fmt.Sprintf("%-12s %d", "Modified:", atomic.LoadInt32(&c.modified)),
fmt.Sprintf("%-12s %d", "Dl errors:", atomic.LoadInt32(&c.dlErrors)),
fmt.Sprintf("%-12s %d", "Up errors:", atomic.LoadInt32(&c.upErrors)),
})
}
if len(errSpecs) > 0 {
fmt.Printf("\n=== %d modules with errors ===\n", len(errSpecs))
for _, s := range errSpecs {
fmt.Printf(" %-50s %s\n", s.GoImportPath, s.Error)
}
}
return 0
}
update_test.go
+456
View File
@@ -0,0 +1,456 @@
package main
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
)
func TestUpdateSpecVersionPreservesFormatting(t *testing.T) {
t.Parallel()
dir := t.TempDir()
specPath := filepath.Join(dir, "test.spec")
original := strings.Join([]string{
"Name: golang-test",
"Version:\t1.2.3 # keep-comment",
"Release: 1%{?dist}",
"%changelog",
"%{?autochangelog}",
}, "\n")
if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil {
t.Fatalf("write spec: %v", err)
}
spec := &specInfo{FilePath: specPath}
if err := updateSpecVersion(spec, "2.0.0", strings.Repeat("a", 64), true); err != nil {
t.Fatalf("updateSpecVersion: %v", err)
}
updated, err := os.ReadFile(specPath)
if err != nil {
t.Fatalf("read spec: %v", err)
}
got := string(updated)
if !strings.Contains(got, "Version:\t2.0.0 # keep-comment") {
t.Fatalf("version line not updated as expected:\n%s", got)
}
if !strings.Contains(got, "%autochangelog\n") {
t.Fatalf("expected %%{{?autochangelog}} to be normalized, got:\n%s", got)
}
if strings.Contains(got, "%{?autochangelog}") {
t.Fatalf("expected %%{{?autochangelog}} to be removed, got:\n%s", got)
}
if !strings.HasSuffix(got, "\n") {
t.Fatalf("expected rewritten spec to keep trailing newline, got %q", got)
}
}
func TestUpdateSpecVersionReturnsErrorWhenVersionLineMissing(t *testing.T) {
t.Parallel()
dir := t.TempDir()
specPath := filepath.Join(dir, "test.spec")
if err := os.WriteFile(specPath, []byte("Name: golang-test\n"), 0o644); err != nil {
t.Fatalf("write spec: %v", err)
}
spec := &specInfo{FilePath: specPath}
if err := updateSpecVersion(spec, "2.0.0", strings.Repeat("a", 64), true); err == nil {
t.Fatal("expected updateSpecVersion to fail when Version line is missing")
}
}
func TestParseSpecFileAcceptsGlobalMacros(t *testing.T) {
t.Parallel()
dir := t.TempDir()
specPath := filepath.Join(dir, "test.spec")
original := strings.Join([]string{
"%global go_import_path github.com/example/project/cmd/tool",
"%global _name golang-github-example-project",
"Version: 1.2.3",
"Source0: https://example.invalid/source.tar.gz",
}, "\n")
if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil {
t.Fatalf("write spec: %v", err)
}
spec, err := parseSpecFile(specPath)
if err != nil {
t.Fatalf("parseSpecFile: %v", err)
}
if spec.GoImportPath != "github.com/example/project/cmd/tool" {
t.Fatalf("GoImportPath = %q", spec.GoImportPath)
}
if spec.Name != "golang-github-example-project" {
t.Fatalf("Name = %q", spec.Name)
}
}
func TestDownloadCandidatesUseModulePathForPackageImport(t *testing.T) {
t.Parallel()
spec := &specInfo{
GoImportPath: "github.com/example/project/cmd/tool",
LatestVersion: "v1.2.3",
}
info := &pkgsiteInfo{
Package: pkgsitePackage{ModulePath: "github.com/example/project"},
Module: pkgsiteModule{
Path: "github.com/example/project",
RepoURL: "https://github.com/example/project",
},
}
candidates, archiveErr, err := downloadCandidatesForSpec(spec, info)
if err != nil {
t.Fatalf("downloadCandidatesForSpec: %v", err)
}
if archiveErr != nil {
t.Fatalf("archiveErr = %v", archiveErr)
}
if len(candidates) != 2 {
t.Fatalf("len(candidates) = %d, want 2", len(candidates))
}
if got, want := candidates[0].url, "https://github.com/example/project/archive/v1.2.3.tar.gz"; got != want {
t.Fatalf("repo candidate = %q, want %q", got, want)
}
if got, want := candidates[1].url, "https://proxy.golang.org/github.com/example/project/@v/v1.2.3.zip"; got != want {
t.Fatalf("proxy candidate = %q, want %q", got, want)
}
}
func TestVerifyHTTPRequestFallsBackToRangeGet(t *testing.T) {
t.Parallel()
var sawHead atomic.Bool
var sawGet atomic.Bool
var gotRange atomic.Value
gotRange.Store("")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodHead:
sawHead.Store(true)
w.WriteHeader(http.StatusMethodNotAllowed)
case http.MethodGet:
sawGet.Store(true)
gotRange.Store(r.Header.Get("Range"))
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write([]byte("x"))
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
}))
defer srv.Close()
if err := verifyHTTPRequest(context.Background(), srv.URL); err != nil {
t.Fatalf("verifyHTTPRequest: %v", err)
}
if !sawHead.Load() {
t.Fatal("expected HEAD request to be attempted first")
}
if !sawGet.Load() {
t.Fatal("expected GET fallback to be attempted")
}
if gotRange.Load().(string) != "bytes=0-0" {
t.Fatalf("unexpected Range header: %q", gotRange.Load().(string))
}
}
func TestRepoRefForVersionStripsBuildMetadata(t *testing.T) {
t.Parallel()
tests := []struct {
in string
want string
}{
{in: "v2.74.0+incompatible", want: "v2.74.0"},
{in: "2.74.0+incompatible", want: "v2.74.0"},
{in: "v1.2.3-rc.1", want: "v1.2.3-rc.1"},
{in: "release-2026-06-12", want: "vrelease-2026-06-12"},
}
for _, tc := range tests {
if got := repoRefForVersion(tc.in); got != tc.want {
t.Fatalf("repoRefForVersion(%q) = %q, want %q", tc.in, got, tc.want)
}
}
}
func TestSourceVersionForSpecStripsBuildMetadata(t *testing.T) {
t.Parallel()
if got := sourceVersionForSpec("v2.74.0+incompatible"); got != "2.74.0" {
t.Fatalf("sourceVersionForSpec() = %q, want %q", got, "2.74.0")
}
}
func TestSpecVersionForVersionStripsIncompatible(t *testing.T) {
t.Parallel()
if got := specVersionForVersion("v2.74.0+incompatible"); got != "2.74.0" {
t.Fatalf("specVersionForVersion() = %q, want %q", got, "2.74.0")
}
}
func TestUpdateSpecVersionOnlyUpdatesSource0RemoteAsset(t *testing.T) {
t.Parallel()
dir := t.TempDir()
specPath := filepath.Join(dir, "test.spec")
source1Hash := strings.Repeat("b", 64)
source0Hash := strings.Repeat("c", 64)
original := strings.Join([]string{
"Name: golang-test",
"Version: 1.0.0",
"#!RemoteAsset: sha256:" + source1Hash,
"Source1: https://example.invalid/extra.tar.gz",
"Source0: https://example.invalid/source.tar.gz",
}, "\n")
if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil {
t.Fatalf("write spec: %v", err)
}
spec := &specInfo{FilePath: specPath}
if err := updateSpecVersion(spec, "2.0.0", source0Hash, true); err != nil {
t.Fatalf("updateSpecVersion: %v", err)
}
updated, err := os.ReadFile(specPath)
if err != nil {
t.Fatalf("read spec: %v", err)
}
got := string(updated)
if !strings.Contains(got, "#!RemoteAsset: sha256:"+source1Hash+"\nSource1:") {
t.Fatalf("Source1 RemoteAsset was unexpectedly changed:\n%s", got)
}
if !strings.Contains(got, "#!RemoteAsset: sha256:"+source0Hash+"\nSource0:") {
t.Fatalf("Source0 RemoteAsset was not inserted correctly:\n%s", got)
}
}
func TestUpdateSpecVersionClearsRemoteAssetWhenHashEmpty(t *testing.T) {
t.Parallel()
dir := t.TempDir()
specPath := filepath.Join(dir, "test.spec")
sourceHash := strings.Repeat("d", 64)
original := strings.Join([]string{
"Name: golang-test",
"Version: 1.0.0",
"#!RemoteAsset: sha256:" + sourceHash,
"Source0: https://example.invalid/%{commit_id}.tar.gz",
}, "\n")
if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil {
t.Fatalf("write spec: %v", err)
}
spec := &specInfo{FilePath: specPath}
if err := updateSpecVersion(spec, "2.0.0", "", true); err != nil {
t.Fatalf("updateSpecVersion: %v", err)
}
updated, err := os.ReadFile(specPath)
if err != nil {
t.Fatalf("read spec: %v", err)
}
got := string(updated)
if !strings.Contains(got, "Version: 2.0.0") {
t.Fatalf("version was not updated:\n%s", got)
}
if strings.Contains(got, "sha256:"+sourceHash) {
t.Fatalf("RemoteAsset hash should be cleared:\n%s", got)
}
if !strings.Contains(got, "#!RemoteAsset\nSource0:") {
t.Fatalf("RemoteAsset marker should be kept without hash:\n%s", got)
}
}
func TestUpdateSpecVersionKeepsRemoteAssetWhenUpdateDisabled(t *testing.T) {
t.Parallel()
dir := t.TempDir()
specPath := filepath.Join(dir, "test.spec")
sourceHash := strings.Repeat("d", 64)
original := strings.Join([]string{
"Name: golang-test",
"Version: 1.0.0",
"#!RemoteAsset: sha256:" + sourceHash,
"Source0: https://example.invalid/%{commit_id}.tar.gz",
}, "\n")
if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil {
t.Fatalf("write spec: %v", err)
}
spec := &specInfo{FilePath: specPath}
if err := updateSpecVersion(spec, "2.0.0", "", false); err != nil {
t.Fatalf("updateSpecVersion: %v", err)
}
updated, err := os.ReadFile(specPath)
if err != nil {
t.Fatalf("read spec: %v", err)
}
got := string(updated)
if !strings.Contains(got, "Version: 2.0.0") {
t.Fatalf("version was not updated:\n%s", got)
}
if !strings.Contains(got, "#!RemoteAsset: sha256:"+sourceHash) {
t.Fatalf("RemoteAsset hash should be unchanged:\n%s", got)
}
}
func TestResolvedSource0URLSkipsPinnedCommit(t *testing.T) {
t.Parallel()
dir := t.TempDir()
specPath := filepath.Join(dir, "test.spec")
original := strings.Join([]string{
"%define commit_id abcdef123456",
"Name: golang-test",
"Version: 1.0.0",
"Source0: https://example.invalid/%{commit_id}.tar.gz",
}, "\n")
if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil {
t.Fatalf("write spec: %v", err)
}
_, err := resolvedSource0URL(specPath, "2.0.0")
if !errors.Is(err, errRemoteAssetSkip) {
t.Fatalf("resolvedSource0URL error = %v, want errRemoteAssetSkip", err)
}
}
func TestReleasePageURLForSource(t *testing.T) {
t.Parallel()
tests := []struct {
source string
wantURL string
wantTag string
}{
{
source: "https://github.com/apache/beam/archive/refs/tags/v2.74.0.tar.gz",
wantURL: "https://github.com/apache/beam/releases/tag/v2.74.0",
wantTag: "v2.74.0",
},
{
source: "https://github.com/charmbracelet/x/archive/refs/tags/term/v0.2.2.tar.gz",
wantURL: "https://github.com/charmbracelet/x/releases/tag/term/v0.2.2",
wantTag: "term/v0.2.2",
},
{
source: "https://gitlab.com/group/project/-/archive/v1.2.3/project-v1.2.3.tar.gz",
wantURL: "https://gitlab.com/group/project/-/releases/v1.2.3",
wantTag: "v1.2.3",
},
}
for _, tc := range tests {
gotURL, gotTag := releasePageURLForSource(tc.source)
if gotURL != tc.wantURL || gotTag != tc.wantTag {
t.Fatalf("releasePageURLForSource(%q) = (%q, %q), want (%q, %q)",
tc.source, gotURL, gotTag, tc.wantURL, tc.wantTag)
}
}
}
func TestExtractSHA256FromHTML(t *testing.T) {
t.Parallel()
want := strings.Repeat("a", 64)
html := "Release notes\nSHA256: " + want
if got := extractSHA256FromHTML(html, "v1.2.3"); got != want {
t.Fatalf("extractSHA256FromHTML() = %q, want %q", got, want)
}
}
func TestDownloadAssetSHA256(t *testing.T) {
t.Parallel()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("source archive"))
}))
defer srv.Close()
got, err := downloadAssetSHA256(context.Background(), srv.URL)
if err != nil {
t.Fatalf("downloadAssetSHA256: %v", err)
}
want := "6ad189ace456a83fade855d5a647cd8ad9e7966da4404b1187218dca3d9eddaa"
if got != want {
t.Fatalf("downloadAssetSHA256() = %q, want %q", got, want)
}
}
func TestVerifyDownloadCandidatesFallsBackToProxy(t *testing.T) {
t.Parallel()
var logsMu sync.Mutex
var logs []string
oldLog := downloadRetryLog
downloadRetryLog = func(msg string) {
logsMu.Lock()
defer logsMu.Unlock()
logs = append(logs, msg)
}
defer func() { downloadRetryLog = oldLog }()
repoSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusGatewayTimeout)
}))
defer repoSrv.Close()
proxySrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write([]byte("x"))
}))
defer proxySrv.Close()
err := verifyDownloadCandidates(context.Background(), "codeberg.org/go-fonts/liberation", []downloadCandidate{
{source: "repo", url: repoSrv.URL},
{source: "proxy", url: proxySrv.URL},
})
if err != nil {
t.Fatalf("verifyDownloadCandidates: %v", err)
}
logsMu.Lock()
defer logsMu.Unlock()
joined := strings.Join(logs, "\n")
if !strings.Contains(joined, "trying proxy") {
t.Fatalf("expected fallback log, got: %s", joined)
}
if !strings.Contains(joined, "verified via proxy after fallback") {
t.Fatalf("expected fallback success log, got: %s", joined)
}
}
func TestIsRetryableDownloadErrorRecognizesTimeout(t *testing.T) {
t.Parallel()
timeoutErr := &net.DNSError{IsTimeout: true}
if !isRetryableDownloadError(timeoutErr) {
t.Fatal("expected timeout net error to be retryable")
}
if !isRetryableDownloadError(errors.New("HTTP 504 from example")) {
t.Fatal("expected HTTP 504 to be retryable")
}
if isRetryableDownloadError(fmt.Errorf("HTTP 404 from example")) {
t.Fatal("did not expect HTTP 404 to be retryable")
}
}