diff --git a/go.mod b/go.mod index 5fbaabb..7ba5a75 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/charmbracelet/glamour v0.10.0 github.com/mattn/go-isatty v0.0.20 github.com/sandrolain/httpcache v1.4.0 + golang.org/x/mod v0.29.0 golang.org/x/net v0.47.0 ) diff --git a/go.sum b/go.sum index 67d4107..5d7f9f6 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/yuin/goldmark-emoji v1.0.5 h1:EMVWyCGPlXJfUXBXpuMu+ii3TIaxbVBnEX9uaDC github.com/yuin/goldmark-emoji v1.0.5/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/main.go b/main.go index 18b334e..42d6afd 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ Usage: Available Commands: pack Package the Go modules into an RPM spec file + update Check existing Go module spec files for updates help Show this help message Use "go2spec [command] --help" for more information about a command. @@ -46,6 +47,8 @@ func main() { println("Executing 'pack' command...") // Actual packing logic would go here mainPack(args[1:], nil) + case "update": + os.Exit(mainUpdate(args[1:])) default: // Default to 'pack' command if no command is provided println("No command provided. Defaulting to 'pack' command...") diff --git a/pack.go b/pack.go index 598594d..d1f9494 100644 --- a/pack.go +++ b/pack.go @@ -2,6 +2,8 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "flag" @@ -48,6 +50,7 @@ type upstream struct { hasGodeps bool // whether the Godeps/_workspace directory exists hasRelease bool // whether any release tags exist, for debian/watch isRelease bool // whether what we end up packaging is a tagged release + sha256 string // SHA256 checksum of the source tarball } var errUnsupportedHoster = errors.New("unsupported hoster") @@ -124,6 +127,24 @@ func downloadFile(filename, url string) error { return nil } +// fetchText fetches the content of a URL and returns it as a string. +// Returns an error if the HTTP status is not 200. +func fetchText(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("http get: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return "", fmt.Errorf("response: %s", resp.Status) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", fmt.Errorf("read body: %w", err) + } + return string(body), nil +} + // get downloads the source module into the provided GOPATH, // checking out the specified revision if non-empty. func (u *upstream) get(gopath, sourceRepo, requestedPath, rev string) error { @@ -364,7 +385,131 @@ func (u *upstream) tarballFromHoster() error { close(done) - return err + if err != nil { + return err + } + + u.sha256 = u.fetchChecksumFromReleasePage() + if u.sha256 == "" { + sum, err := fileSHA256(u.tarPath) + if err != nil { + log.Printf("INFO: Could not compute source tarball sha256: %v", err) + } else { + u.sha256 = sum + } + } + + return nil +} + +// fetchChecksumFromReleasePage fetches the GitHub/GitLab release page and +// attempts to extract a SHA-256 checksum for the source tarball from the +// page content. Some projects include the checksum in the release body +// markdown, e.g. "sha256: " or "SHA256: ". +func (u *upstream) fetchChecksumFromReleasePage() string { + releaseURL := u.releasePageURL() + if releaseURL == "" { + return "" + } + + log.Printf("Fetching release page %s for checksum", releaseURL) + content, err := fetchText(releaseURL) + if err != nil { + log.Printf("INFO: Could not fetch release page: %v", err) + return "" + } + + return extractSHA256FromHTML(content, u.tag) +} + +// releasePageURL constructs the release page URL for the current tag. +// For GitHub: https://github.com/{owner}/{repo}/releases/tag/{tag} +// For GitLab: https://gitlab.com/{owner}/{repo}/-/releases/{tag} +func (u *upstream) releasePageURL() string { + repo := strings.TrimSuffix(u.repoURL, ".git") + if repo == "" || u.tag == "" { + return "" + } + repoU, err := url.Parse(repo) + if err != nil { + return "" + } + + switch repoU.Host { + case "github.com": + return fmt.Sprintf("%s/releases/tag/%s", repo, u.tag) + case "gitlab.com", "salsa.debian.org": + return fmt.Sprintf("%s/-/releases/%s", repo, u.tag) + default: + return "" + } +} + +// extractSHA256FromHTML searches HTML content for a SHA-256 checksum +// associated with the tarball for the given tag. It looks for patterns like: +// +// sha256: <64-hex-chars> +// SHA256: <64-hex-chars> +// sha256sum: <64-hex-chars> +// +// It also tries to match checksums near tarball-related filenames. +func extractSHA256FromHTML(html, tag string) string { + lower := strings.ToLower(html) + + patterns := []*regexp.Regexp{ + regexp.MustCompile(`sha-?256(?:sum)?[:\s]+([a-f0-9]{64})`), + regexp.MustCompile(`sha-?256(?:sum)?[:\s]*([a-f0-9]{64})`), + } + + for _, re := range patterns { + if m := re.FindStringSubmatch(lower); len(m) > 1 && isValidSHA256(m[1]) { + return m[1] + } + } + + // Fallback: look for a 64-char hex string near "tar" or "source" keywords + tarRe := regexp.MustCompile(`(?:tar|source)[^a-f0-9]{0,80}([a-f0-9]{64})`) + if m := tarRe.FindStringSubmatch(lower); len(m) > 1 { + if isValidSHA256(m[1]) { + return m[1] + } + } + revTarRe := regexp.MustCompile(`([a-f0-9]{64})[^a-f0-9]{0,80}(?:tar|source)`) + if m := revTarRe.FindStringSubmatch(lower); len(m) > 1 { + if isValidSHA256(m[1]) { + return m[1] + } + } + + return "" +} + +// isValidSHA256 reports whether s is a valid lowercase hexadecimal string +// of exactly 64 characters (the length of a SHA-256 digest). +func isValidSHA256(s string) bool { + if len(s) != 64 { + return false + } + for _, c := range s { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + return false + } + } + return true +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open: %w", err) + } + defer f.Close() + + sum := sha256.New() + if _, err := io.Copy(sum, f); err != nil { + return "", fmt.Errorf("hash: %w", err) + } + return hex.EncodeToString(sum.Sum(nil)), nil } func (u *upstream) tar(gopath, repo string) error { diff --git a/pkgsite.go b/pkgsite.go index 1519950..9462759 100644 --- a/pkgsite.go +++ b/pkgsite.go @@ -14,13 +14,14 @@ import ( "time" ) -const pkgsiteAPIBase = "https://pkg.go.dev/v1beta" const pkgsiteMaxResponseBytes = 20 << 20 var ( + pkgsiteAPIBase = "https://pkg.go.dev/v1beta" pkgsiteHTTPClient = http.DefaultClient pkgsiteMu sync.Mutex pkgsiteInfoCache = make(map[string]*pkgsiteInfo) + pkgsiteRetryLog func(string) ) type pkgsiteInfo struct { @@ -105,8 +106,6 @@ func pkgsiteURL(endpoint, importPath string, values url.Values) string { return u } -// pkgsiteGetJSON performs a pkg.go.dev v1beta GET with up to 3 attempts and -// linear backoff for transport errors, HTTP 429, and 5xx responses. func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url.Values, v any) error { client := pkgsiteHTTPClient if client == nil { @@ -114,7 +113,7 @@ func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url } var lastErr error - for attempt := 1; attempt <= 3; attempt++ { + for attempt := 1; attempt <= 5; attempt++ { req, err := http.NewRequestWithContext(ctx, http.MethodGet, pkgsiteURL(endpoint, importPath, values), nil) if err != nil { return fmt.Errorf("create pkgsite request: %w", err) @@ -124,8 +123,13 @@ func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url resp, err := client.Do(req) if err != nil { lastErr = fmt.Errorf("pkgsite request: %w", err) - if attempt < 3 { - time.Sleep(time.Duration(attempt) * time.Second) + if attempt < 5 { + backoff := time.Duration(attempt*2) * time.Second + reportPkgsiteRetry(" RETRY: pkgsite %s %s request failed (%v), retry %d/5 in %s", + endpoint, importPath, err, attempt+1, backoff) + if err := sleepWithContext(ctx, backoff); err != nil { + return err + } continue } return lastErr @@ -148,8 +152,12 @@ func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url apiErr.Code = resp.StatusCode apiErr.Status = resp.Status lastErr = &apiErr - if attempt < 3 && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500) { - time.Sleep(time.Duration(attempt) * time.Second) + if attempt < 5 && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500) { + backoff := pkgsiteRetryBackoff(resp, attempt) + reportPkgsiteStatusRetry(endpoint, importPath, resp.StatusCode, &apiErr, attempt+1, backoff) + if err := sleepWithContext(ctx, backoff); err != nil { + return err + } continue } return lastErr @@ -163,6 +171,75 @@ func pkgsiteGetJSON(ctx context.Context, endpoint, importPath string, values url return lastErr } +func sleepWithContext(ctx context.Context, d time.Duration) error { + if d <= 0 { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } + } + + timer := time.NewTimer(d) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func pkgsiteRetryBackoff(resp *http.Response, attempt int) time.Duration { + if resp != nil && resp.StatusCode == http.StatusTooManyRequests { + if retryAfter, ok := parseRetryAfter(resp.Header.Get("Retry-After")); ok { + return retryAfter + } + return time.Duration(attempt*10) * time.Second + } + return time.Duration(attempt*5) * time.Second +} + +func parseRetryAfter(v string) (time.Duration, bool) { + v = strings.TrimSpace(v) + if v == "" { + return 0, false + } + if seconds, err := time.ParseDuration(v + "s"); err == nil && seconds > 0 { + return seconds, true + } + for _, layout := range []string{time.RFC1123, time.RFC1123Z, time.RFC850, time.ANSIC} { + t, err := time.Parse(layout, v) + if err == nil { + delay := time.Until(t) + if delay > 0 { + return delay, true + } + return time.Second, true + } + } + return 0, false +} + +func reportPkgsiteRetry(format string, args ...any) { + if pkgsiteRetryLog != nil { + pkgsiteRetryLog(fmt.Sprintf(format, args...)) + } +} + +func reportPkgsiteStatusRetry(endpoint, importPath string, statusCode int, apiErr *pkgsiteAPIError, nextAttempt int, backoff time.Duration) { + switch statusCode { + case http.StatusTooManyRequests: + reportPkgsiteRetry(" RATE: %s limited on %s, retry %d/5 in %s", + importPath, endpoint, nextAttempt, backoff) + default: + reportPkgsiteRetry(" RETRY: %s %s HTTP %d, retry %d/5 in %s", + importPath, endpoint, statusCode, nextAttempt, backoff) + } +} + func getPkgsitePackage(ctx context.Context, gopkg, modulePath string) (pkgsitePackage, error) { values := url.Values{} values.Set("imports", "true") diff --git a/pkgsite_test.go b/pkgsite_test.go index 158948a..0a75e57 100644 --- a/pkgsite_test.go +++ b/pkgsite_test.go @@ -1,12 +1,16 @@ package main import ( + "context" "io" "net/http" + "net/http/httptest" + "net/url" "os" "os/exec" "path/filepath" "strings" + "sync" "testing" "time" ) @@ -62,6 +66,22 @@ func TestPkgsiteLicenseExpression(t *testing.T) { } } +func TestFileSHA256(t *testing.T) { + path := filepath.Join(t.TempDir(), "archive.tar.gz") + if err := os.WriteFile(path, []byte("source archive"), 0o644); err != nil { + t.Fatalf("write archive: %v", err) + } + + got, err := fileSHA256(path) + if err != nil { + t.Fatalf("fileSHA256: %v", err) + } + want := "6ad189ace456a83fade855d5a647cd8ad9e7966da4404b1187218dca3d9eddaa" + if got != want { + t.Fatalf("fileSHA256() = %q, want %q", got, want) + } +} + func TestPkgsiteLicenseFilePaths(t *testing.T) { licenses := []pkgsiteLicense{ {FilePath: "License", Types: []string{"MIT"}}, @@ -538,3 +558,55 @@ func runGit(t *testing.T, dir string, env []string, args ...string) string { } return string(out) } + +func TestPkgsiteGetJSONLogsRateLimitRetry(t *testing.T) { + var mu sync.Mutex + var logs []string + attempts := 0 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + if attempts == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"message":"slow down"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"path":"example.com/mod","version":"v1.2.3"}`)) + })) + defer srv.Close() + + oldBase := pkgsiteAPIBase + oldClient := pkgsiteHTTPClient + oldLog := pkgsiteRetryLog + t.Cleanup(func() { + pkgsiteAPIBase = oldBase + pkgsiteHTTPClient = oldClient + pkgsiteRetryLog = oldLog + }) + + pkgsiteAPIBase = srv.URL + pkgsiteHTTPClient = srv.Client() + pkgsiteRetryLog = func(msg string) { + mu.Lock() + defer mu.Unlock() + logs = append(logs, msg) + } + + var mod pkgsiteModule + if err := pkgsiteGetJSON(context.Background(), "module", "example.com/mod", url.Values{}, &mod); err != nil { + t.Fatalf("pkgsiteGetJSON: %v", err) + } + if attempts != 2 { + t.Fatalf("expected 2 attempts, got %d", attempts) + } + mu.Lock() + defer mu.Unlock() + if len(logs) == 0 { + t.Fatal("expected rate limit log message") + } + if !strings.Contains(logs[0], "example.com/mod limited on module") || !strings.Contains(logs[0], "retry 2/5 in 1s") { + t.Fatalf("unexpected log message: %q", logs[0]) + } +} diff --git a/spec.go b/spec.go index 28de148..3561bf6 100644 --- a/spec.go +++ b/spec.go @@ -148,7 +148,11 @@ func writeSpec(dir, gopkg, openRuyiSrc, openRuyiLib, openRuyiProgram, version st fmt.Fprintf(f, "Summary: %s\n", description) fmt.Fprintf(f, "License: %s\n", license) fmt.Fprintf(f, "URL: %s\n", repoURL) - fmt.Fprintf(f, "#!RemoteAsset\n") + if u.sha256 != "" { + fmt.Fprintf(f, "#!RemoteAsset: sha256:%s\n", u.sha256) + } else { + fmt.Fprintf(f, "#!RemoteAsset\n") + } sourceModulePath := u.modulePath if sourceModulePath == "" { sourceModulePath = gopkg diff --git a/update.go b/update.go new file mode 100644 index 0000000..fb81954 --- /dev/null +++ b/update.go @@ -0,0 +1,625 @@ +package main + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "time" + + "golang.org/x/mod/semver" +) + +var downloadRetryLog func(string) + +var errRemoteAssetSkip = errors.New("skip remote asset sha256") + +type downloadCandidate struct { + source string + url string +} + +func repoRefForVersion(version string) string { + v := strings.TrimSpace(version) + if !strings.HasPrefix(v, "v") { + v = "v" + v + } + if canonical := semver.Canonical(v); canonical != "" { + return canonical + } + return v +} + +func specVersionForVersion(version string) string { + return strings.TrimPrefix(repoRefForVersion(version), "v") +} + +func verifyDownload(ctx context.Context, spec *specInfo) error { + if spec.GoImportPath == "" { + return fmt.Errorf("no go_import_path") + } + if spec.LatestVersion == "" { + return fmt.Errorf("no latest version") + } + + info, err := getPkgsiteInfo(ctx, spec.GoImportPath) + if err != nil { + return fmt.Errorf("get pkgsite info: %w", err) + } + + candidates, archiveErr, err := downloadCandidatesForSpec(spec, info) + if err != nil { + return err + } + if archiveErr != nil { + reportDownloadRetry(" FALLBACK: %s repo archive unavailable (%v), trying module proxy", + spec.GoImportPath, archiveErr) + } + return verifyDownloadCandidates(ctx, spec.GoImportPath, candidates) +} + +func downloadCandidatesForSpec(spec *specInfo, info *pkgsiteInfo) ([]downloadCandidate, error, error) { + repoURL := gitCloneURLFromRepoURL(info.Module.RepoURL) + if repoURL == "" { + return nil, nil, fmt.Errorf("no repository URL") + } + + version := spec.LatestVersion + if !strings.HasPrefix(version, "v") { + version = "v" + version + } + repoRef := repoRefForVersion(version) + sourceModulePath := sourceImportPathForPackage(spec.GoImportPath, info) + proxyURL := fmt.Sprintf("https://proxy.golang.org/%s/@v/%s.zip", + moduleProxyEscapedPath(sourceModulePath), + moduleProxyVersionForSpec(version)) + + if moduleUsesRepoSubdir(sourceModulePath, repoURL) { + return []downloadCandidate{ + {source: "proxy", url: proxyURL}, + }, nil, nil + } + + tarURL, err := (&upstream{repoURL: repoURL}).tarballURLForRef(repoRef, "gz") + if err != nil { + return []downloadCandidate{ + {source: "proxy", url: proxyURL}, + }, err, nil + } + + return []downloadCandidate{ + {source: "repo", url: tarURL}, + {source: "proxy", url: proxyURL}, + }, nil, nil +} + +func remoteAssetSHA256(ctx context.Context, spec *specInfo, newVersion string) (string, error) { + sourceURL, err := resolvedSource0URL(spec.FilePath, sourceVersionForSpec(newVersion)) + if err != nil { + return "", err + } + releaseURL, tag := releasePageURLForSource(sourceURL) + if releaseURL == "" { + reportDownloadRetry(" HASH: %s no release checksum page, downloading Source0", spec.GoImportPath) + return downloadAssetSHA256(ctx, sourceURL) + } + content, err := fetchTextContext(ctx, releaseURL) + if err == nil { + if sum := extractSHA256FromHTML(content, tag); sum != "" { + return sum, nil + } + } + reportDownloadRetry(" HASH: %s release checksum unavailable, downloading Source0", spec.GoImportPath) + return downloadAssetSHA256(ctx, sourceURL) +} + +func sourceVersionForSpec(version string) string { + return specVersionForVersion(version) +} + +func verifyDownloadCandidates(ctx context.Context, importPath string, candidates []downloadCandidate) error { + var errs []string + for i, candidate := range candidates { + err := verifyHTTPRequest(ctx, candidate.url) + if err == nil { + if i > 0 { + reportDownloadRetry(" OK: %s verified via %s after fallback", importPath, candidate.source) + } + return nil + } + errs = append(errs, fmt.Sprintf("%s: %v", candidate.source, err)) + if i+1 < len(candidates) { + reportDownloadRetry(" FALLBACK: %s %s probe failed (%v), trying %s", + importPath, candidate.source, err, candidates[i+1].source) + } + } + return errors.New(strings.Join(errs, "; ")) +} + +func verifyHTTPRequest(ctx context.Context, downloadURL string) error { + var lastErr error + for attempt := 1; attempt <= 3; attempt++ { + if err := verifyHTTPHead(ctx, downloadURL); err == nil { + return nil + } else { + lastErr = err + } + if err := verifyHTTPRangeGet(ctx, downloadURL); err == nil { + return nil + } else { + lastErr = err + } + if attempt < 3 && isRetryableDownloadError(lastErr) { + backoff := time.Duration(attempt*2) * time.Second + reportDownloadRetry(" RETRY: download probe %s failed (%v), retry %d/3 in %s", + downloadURL, lastErr, attempt+1, backoff) + if err := sleepWithContext(ctx, backoff); err != nil { + return err + } + continue + } + break + } + return lastErr +} + +func verifyHTTPHead(ctx context.Context, downloadURL string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, downloadURL, nil) + if err != nil { + return fmt.Errorf("create HEAD request: %w", err) + } + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("HEAD request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return fmt.Errorf("HTTP %d from %s", resp.StatusCode, downloadURL) + } + + if resp.ContentLength == 0 { + return fmt.Errorf("HEAD response has no content length from %s", downloadURL) + } + + return nil +} + +func verifyHTTPRangeGet(ctx context.Context, downloadURL string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return fmt.Errorf("create GET request: %w", err) + } + req.Header.Set("Range", "bytes=0-0") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("GET request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return fmt.Errorf("HTTP %d from %s", resp.StatusCode, downloadURL) + } + + buf := make([]byte, 1) + n, err := resp.Body.Read(buf) + if err != nil && err != io.EOF { + return fmt.Errorf("read download probe: %w", err) + } + if n == 0 { + return fmt.Errorf("empty download from %s", downloadURL) + } + + return nil +} + +func isRetryableDownloadError(err error) bool { + if err == nil { + return false + } + var netErr net.Error + if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) { + return true + } + msg := err.Error() + return strings.Contains(msg, "context deadline exceeded") || + strings.Contains(msg, "Client.Timeout exceeded") || + strings.Contains(msg, "connection reset by peer") || + strings.Contains(msg, "EOF") || + strings.Contains(msg, "HTTP 408") || + strings.Contains(msg, "HTTP 425") || + strings.Contains(msg, "HTTP 429") || + strings.Contains(msg, "HTTP 500") || + strings.Contains(msg, "HTTP 502") || + strings.Contains(msg, "HTTP 503") || + strings.Contains(msg, "HTTP 504") +} + +func reportDownloadRetry(format string, args ...any) { + if downloadRetryLog != nil { + downloadRetryLog(fmt.Sprintf(format, args...)) + } +} + +func readSpecLines(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + var lines []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines, scanner.Err() +} + +func writeSpecLines(path string, lines []string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + w := bufio.NewWriter(f) + for i, line := range lines { + if _, err := w.WriteString(line); err != nil { + return err + } + if i < len(lines)-1 { + if _, err := w.WriteString("\n"); err != nil { + return err + } + } + } + if len(lines) > 0 { + if _, err := w.WriteString("\n"); err != nil { + return err + } + } + return w.Flush() +} + +func updateSpecVersion(spec *specInfo, newVersion, remoteSHA256 string, updateRemoteAsset bool) error { + lines, err := readSpecLines(spec.FilePath) + if err != nil { + return err + } + + versionLineRe := regexp.MustCompile(`^(\s*Version:\s*)(\S+)(.*)$`) + autoChangelogRe := regexp.MustCompile(`^(\s*)%\{\?autochangelog\}(\s*)$`) + remoteAssetHashRe := regexp.MustCompile(`^(\s*)#!RemoteAsset(?::\s*sha256:[0-9a-fA-F]+)?(\s*)$`) + source0LineRe := regexp.MustCompile(`^(\s*Source0:\s*)(\S+)(.*)$`) + updated := false + source0Index := -1 + source0RemoteAssetIndex := -1 + for i, line := range lines { + matches := versionLineRe.FindStringSubmatch(line) + if len(matches) == 4 { + lines[i] = matches[1] + newVersion + matches[3] + updated = true + } + matches = autoChangelogRe.FindStringSubmatch(line) + if len(matches) == 3 { + lines[i] = matches[1] + "%autochangelog" + matches[2] + } + if source0Index < 0 && source0LineRe.MatchString(line) { + source0Index = i + if i > 0 && remoteAssetHashRe.MatchString(lines[i-1]) { + source0RemoteAssetIndex = i - 1 + } + } + } + if !updated { + return fmt.Errorf("Version line not found in %s", spec.FilePath) + } + if !updateRemoteAsset { + return writeSpecLines(spec.FilePath, lines) + } + if source0RemoteAssetIndex >= 0 { + matches := remoteAssetHashRe.FindStringSubmatch(lines[source0RemoteAssetIndex]) + if remoteSHA256 != "" { + lines[source0RemoteAssetIndex] = matches[1] + "#!RemoteAsset: sha256:" + remoteSHA256 + matches[2] + } else { + lines[source0RemoteAssetIndex] = matches[1] + "#!RemoteAsset" + matches[2] + } + } else if source0Index >= 0 { + insert := "#!RemoteAsset" + if remoteSHA256 != "" { + insert = "#!RemoteAsset: sha256:" + remoteSHA256 + } + lines = append(lines[:source0Index], append([]string{insert}, lines[source0Index:]...)...) + } + + return writeSpecLines(spec.FilePath, lines) +} + +type downloadResult struct { + spec *specInfo + err error +} + +func verifyAndUpdateSpec(ctx context.Context, spec *specInfo, dryRun bool) error { + err := verifyDownload(ctx, spec) + if err != nil { + return fmt.Errorf("download verification failed: %w", err) + } + + if !dryRun { + newVer := specVersionForVersion(spec.LatestVersion) + remoteSHA256, err := remoteAssetSHA256(ctx, spec, newVer) + updateRemoteAsset := true + if errors.Is(err, errRemoteAssetSkip) { + updateRemoteAsset = false + err = nil + } + if err != nil { + return fmt.Errorf("compute remote asset sha256: %w", err) + } + + if err := updateSpecVersion(spec, newVer, remoteSHA256, updateRemoteAsset); err != nil { + return fmt.Errorf("update spec file: %w", err) + } + } + + return nil +} + +func processUpdate(ctx context.Context, spec *specInfo, dryRun bool) error { + err := verifyDownload(ctx, spec) + if err != nil { + return err + } + + if !dryRun { + newVer := specVersionForVersion(spec.LatestVersion) + remoteSHA256, err := remoteAssetSHA256(ctx, spec, newVer) + updateRemoteAsset := true + if errors.Is(err, errRemoteAssetSkip) { + updateRemoteAsset = false + err = nil + } + if err != nil { + return fmt.Errorf("compute remote asset sha256: %w", err) + } + + if err := updateSpecVersion(spec, newVer, remoteSHA256, updateRemoteAsset); err != nil { + return fmt.Errorf("update spec file: %w", err) + } + } + + return nil +} + +func resolvedSource0URL(path, newVersion string) (string, error) { + lines, err := readSpecLines(path) + if err != nil { + return "", err + } + + macros := make(map[string]string) + defineRe := regexp.MustCompile(`^%(?:define|global)\s+(\S+)\s+(.+)$`) + fieldRe := regexp.MustCompile(`^(Name|Version):\s+(\S+)`) + source0Re := regexp.MustCompile(`^Source0:\s+(\S+)`) + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if matches := defineRe.FindStringSubmatch(trimmed); len(matches) == 3 { + macros[matches[1]] = strings.TrimSpace(matches[2]) + continue + } + if matches := fieldRe.FindStringSubmatch(trimmed); len(matches) == 3 { + switch matches[1] { + case "Name": + macros["name"] = matches[2] + case "Version": + macros["version"] = newVersion + } + continue + } + if matches := source0Re.FindStringSubmatch(trimmed); len(matches) == 2 { + if source0UsesPinnedCommit(matches[1]) { + return "", fmt.Errorf("%w: Source0 uses pinned commit_id", errRemoteAssetSkip) + } + macros["version"] = newVersion + resolved := expandRPMMacros(matches[1], macros, 16) + if strings.Contains(resolved, "%{") { + return "", fmt.Errorf("unresolved Source0 macros in %s: %s", path, resolved) + } + return strings.SplitN(resolved, "#", 2)[0], nil + } + } + + return "", fmt.Errorf("Source0 line not found in %s", path) +} + +func source0UsesPinnedCommit(source string) bool { + return strings.Contains(source, "%{commit_id}") || strings.Contains(source, "%{?commit_id}") +} + +func expandRPMMacros(s string, macros map[string]string, limit int) string { + re := regexp.MustCompile(`%\{(\??)([^}]+)\}`) + expanded := s + for i := 0; i < limit; i++ { + changed := false + next := re.ReplaceAllStringFunc(expanded, func(match string) string { + parts := re.FindStringSubmatch(match) + if len(parts) != 3 { + return match + } + optional := parts[1] == "?" + name := parts[2] + value, ok := macros[name] + if !ok { + if optional { + changed = true + return "" + } + return match + } + changed = true + return value + }) + expanded = next + if !changed { + break + } + } + return expanded +} + +func releasePageURLForSource(sourceURL string) (releaseURL, tag string) { + u, err := url.Parse(sourceURL) + if err != nil { + return "", "" + } + + tag = sourceTagFromURLPath(u.Path) + if tag == "" { + return "", "" + } + + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + switch u.Host { + case "github.com": + if len(parts) < 2 { + return "", "" + } + repo := fmt.Sprintf("%s://%s/%s/%s", u.Scheme, u.Host, parts[0], parts[1]) + return fmt.Sprintf("%s/releases/tag/%s", repo, tag), tag + case "gitlab.com", "salsa.debian.org": + dashArchive := -1 + for i := 0; i+1 < len(parts); i++ { + if parts[i] == "-" && parts[i+1] == "archive" { + dashArchive = i + break + } + } + if dashArchive <= 0 { + return "", "" + } + repo := fmt.Sprintf("%s://%s/%s", u.Scheme, u.Host, strings.Join(parts[:dashArchive], "/")) + return fmt.Sprintf("%s/-/releases/%s", repo, tag), tag + default: + return "", "" + } +} + +func sourceTagFromURLPath(path string) string { + path, err := url.PathUnescape(path) + if err != nil { + return "" + } + + const refsPrefix = "refs/tags/" + parts := strings.Split(strings.Trim(path, "/"), "/") + for i, part := range parts { + if part != "archive" || i+1 >= len(parts) { + continue + } + rest := strings.Join(parts[i+1:], "/") + if strings.HasPrefix(rest, refsPrefix) { + return trimArchiveSuffix(strings.TrimPrefix(rest, refsPrefix)) + } + if idx := strings.Index(rest, "/"); idx >= 0 { + rest = rest[:idx] + } + return trimArchiveSuffix(rest) + } + return "" +} + +func trimArchiveSuffix(s string) string { + for _, suffix := range []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tgz", ".zip"} { + if strings.HasSuffix(s, suffix) { + return strings.TrimSuffix(s, suffix) + } + } + return s +} + +func fetchTextContext(ctx context.Context, fetchURL string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL, nil) + if err != nil { + return "", fmt.Errorf("create GET request: %w", err) + } + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("GET request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP %d from %s", resp.StatusCode, fetchURL) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", fmt.Errorf("read response: %w", err) + } + return string(body), nil +} + +func downloadAssetSHA256(ctx context.Context, downloadURL string) (string, error) { + var lastErr error + for attempt := 1; attempt <= 3; attempt++ { + sum, err := fetchSHA256(ctx, downloadURL) + if err == nil { + return sum, nil + } + lastErr = err + if attempt < 3 && isRetryableDownloadError(lastErr) { + backoff := time.Duration(attempt*2) * time.Second + reportDownloadRetry(" RETRY: asset hash %s failed (%v), retry %d/3 in %s", + downloadURL, lastErr, attempt+1, backoff) + if err := sleepWithContext(ctx, backoff); err != nil { + return "", err + } + continue + } + break + } + return "", lastErr +} + +func fetchSHA256(ctx context.Context, downloadURL string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return "", fmt.Errorf("create GET request: %w", err) + } + + client := &http.Client{Timeout: 120 * time.Second} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("GET request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return "", fmt.Errorf("HTTP %d from %s", resp.StatusCode, downloadURL) + } + + sum := sha256.New() + if _, err := io.Copy(sum, resp.Body); err != nil { + return "", fmt.Errorf("read download for sha256: %w", err) + } + + return hex.EncodeToString(sum.Sum(nil)), nil +} diff --git a/update_cli.go b/update_cli.go new file mode 100644 index 0000000..13aed5c --- /dev/null +++ b/update_cli.go @@ -0,0 +1,492 @@ +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/mattn/go-isatty" + "github.com/sandrolain/httpcache" + "golang.org/x/mod/semver" +) + +type specInfo struct { + FilePath string + GoImportPath string + Version string + Name string + Source0 string + LatestVersion string + HasUpdate bool + Error string +} + +func parseSpecFile(path string) (*specInfo, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + info := &specInfo{FilePath: path} + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, "#") { + continue + } + if strings.HasPrefix(line, "%define go_import_path") || strings.HasPrefix(line, "%global go_import_path") { + parts := strings.Fields(line) + if len(parts) >= 3 { + info.GoImportPath = parts[2] + } + } + if strings.HasPrefix(line, "%define _name") || strings.HasPrefix(line, "%global _name") { + parts := strings.Fields(line) + if len(parts) >= 3 { + info.Name = parts[2] + } + } + if strings.HasPrefix(line, "Version:") { + parts := strings.Fields(line) + if len(parts) >= 2 { + info.Version = parts[1] + } + } + if strings.HasPrefix(line, "Source0:") { + parts := strings.Fields(line) + if len(parts) >= 2 { + info.Source0 = parts[1] + } + } + } + return info, scanner.Err() +} + +func normalizeVersion(v string) string { + v = strings.TrimSpace(v) + if !strings.HasPrefix(v, "v") { + v = "v" + v + } + if semver.IsValid(v) { + return v + } + return "" +} + +func checkLatestVersion(ctx context.Context, spec *specInfo) { + if spec.GoImportPath == "" { + spec.Error = "no go_import_path" + return + } + info, err := getPkgsiteInfo(ctx, spec.GoImportPath) + if err != nil { + spec.Error = fmt.Sprintf("pkgsite error: %v", err) + return + } + latestPkgVer := info.Package.Version + latestModVer := info.Module.Version + latestVer := latestPkgVer + if latestModVer != "" && (latestVer == "" || semver.Compare(normalizeVersion(latestModVer), normalizeVersion(latestVer)) > 0) { + latestVer = latestModVer + } + spec.LatestVersion = latestVer + currentNorm := normalizeVersion(spec.Version) + latestNorm := normalizeVersion(latestVer) + if currentNorm == "" || latestNorm == "" { + spec.HasUpdate = false + return + } + spec.HasUpdate = semver.Compare(latestNorm, currentNorm) > 0 +} + +func scanSpecDir(specDir string) ([]*specInfo, error) { + entries, err := os.ReadDir(specDir) + if err != nil { + return nil, err + } + var specs []*specInfo + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "go-") { + continue + } + specFiles, err := filepath.Glob(filepath.Join(specDir, entry.Name(), "*.spec")) + if err != nil { + continue + } + for _, sf := range specFiles { + spec, err := parseSpecFile(sf) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to parse %s: %v\n", sf, err) + continue + } + if spec.GoImportPath != "" { + specs = append(specs, spec) + } + } + } + return specs, nil +} + +func printBox(title string, lines []string) { + width := 56 + contentWidth := width - 4 + fmt.Println("+" + strings.Repeat("-", width) + "+") + if title != "" { + padding := width - len(title) + left := padding / 2 + right := padding - left + fmt.Printf("|%s%s%s|\n", strings.Repeat(" ", left), title, strings.Repeat(" ", right)) + fmt.Println("+" + strings.Repeat("-", width) + "+") + } + for _, line := range lines { + if len(line) > contentWidth { + line = line[:contentWidth-3] + "..." + } + fmt.Printf("| %-*s |\n", contentWidth, line) + } + fmt.Println("+" + strings.Repeat("-", width) + "+") +} + +type outputMsg struct { + text string +} + +type counters struct { + scanned int32 + withUpdate int32 + scanErrors int32 + modified int32 + processed int32 + dlErrors int32 + upErrors int32 + total int +} + +type outputLoop struct { + ch chan outputMsg + counts *counters + done chan struct{} + exited chan struct{} + lastProgress []string + interactiveTTY bool + progressShown int +} + +func newOutputLoop(c *counters) *outputLoop { + return &outputLoop{ + ch: make(chan outputMsg, 64), + counts: c, + done: make(chan struct{}), + exited: make(chan struct{}), + interactiveTTY: isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()), + } +} + +func (o *outputLoop) start() { + go o.loop() +} + +func (o *outputLoop) loop() { + defer close(o.exited) + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for { + select { + case msg := <-o.ch: + o.printMessage(msg.text) + case <-ticker.C: + o.maybePrintProgress() + case <-o.done: + for { + select { + case msg := <-o.ch: + o.printMessage(msg.text) + default: + o.clearProgress() + return + } + } + } + } +} + +func (o *outputLoop) maybePrintProgress() { + progress := o.currentProgressLines() + if sameProgressLines(progress, o.lastProgress) { + return + } + o.lastProgress = append(o.lastProgress[:0], progress...) + + if o.interactiveTTY { + o.renderProgress(progress) + return + } + + for _, line := range progress { + fmt.Println(line) + } +} + +func (o *outputLoop) currentProgressLines() []string { + c := o.counts + scanned := atomic.LoadInt32(&c.scanned) + withUpdate := atomic.LoadInt32(&c.withUpdate) + scanErrors := atomic.LoadInt32(&c.scanErrors) + processed := atomic.LoadInt32(&c.processed) + modified := atomic.LoadInt32(&c.modified) + dlErrors := atomic.LoadInt32(&c.dlErrors) + upErrors := atomic.LoadInt32(&c.upErrors) + + return []string{ + fmt.Sprintf(" [scan] %d/%d updates:%d errors:%d", + scanned, c.total, withUpdate, scanErrors), + fmt.Sprintf(" [verify] %d/%d modified:%d errors:%d", + processed, withUpdate, modified, dlErrors+upErrors), + } +} + +func (o *outputLoop) print(text string) { + o.ch <- outputMsg{text: text} +} + +func (o *outputLoop) stop() { + close(o.done) + <-o.exited +} + +func (o *outputLoop) printMessage(text string) { + o.clearProgress() + fmt.Println(text) + if o.interactiveTTY { + progress := o.currentProgressLines() + o.lastProgress = append(o.lastProgress[:0], progress...) + o.renderProgress(progress) + } +} + +func (o *outputLoop) clearProgress() { + if o.interactiveTTY && o.progressShown > 0 { + fmt.Print("\r\033[2K") + for i := 1; i < o.progressShown; i++ { + fmt.Print("\033[1A\r\033[2K") + } + o.progressShown = 0 + } +} + +func (o *outputLoop) renderProgress(lines []string) { + o.clearProgress() + for i, line := range lines { + if i > 0 { + fmt.Print("\n") + } + fmt.Print(line) + } + o.progressShown = len(lines) +} + +func sameProgressLines(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func mainUpdate(args []string) int { + fs := flag.NewFlagSet("update", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + concurrency := fs.Int("j", 3, "number of concurrent checks") + dryRun := fs.Bool("n", false, "dry run: detect updates only, do not modify files") + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "Usage: %s update [options] \n\n", os.Args[0]) + fmt.Fprintf(fs.Output(), "Check Go module spec files for updates.\n\n") + fmt.Fprintf(fs.Output(), "Arguments:\n") + fmt.Fprintf(fs.Output(), " SPECS_DIR directory containing go-* subdirectories\n\n") + fmt.Fprintf(fs.Output(), "Options:\n") + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if *concurrency < 1 { + *concurrency = 1 + } + if fs.NArg() != 1 { + fs.Usage() + return 1 + } + + specDir := fs.Arg(0) + specs, err := scanSpecDir(specDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error scanning spec dir: %v\n", err) + return 1 + } + if len(specs) == 0 { + fmt.Fprintf(os.Stderr, "No spec files with go_import_path found in %s\n", specDir) + return 1 + } + + pkgsiteHTTPClient = &http.Client{ + Timeout: 30 * time.Second, + Transport: httpcache.NewMemoryCacheTransport(), + } + + ctx := context.Background() + total := len(specs) + c := &counters{total: total} + + out := newOutputLoop(c) + out.start() + pkgsiteRetryLog = out.print + downloadRetryLog = out.print + + fmt.Printf("Checking %d spec files for updates...\n", total) + fmt.Printf("Parameters: spec_dir=%s concurrency=%d dry_run=%t\n", specDir, *concurrency, *dryRun) + + updateCh := make(chan *specInfo, *concurrency) + + var wg2 sync.WaitGroup + if !*dryRun { + for i := 0; i < *concurrency; i++ { + wg2.Add(1) + go func() { + defer wg2.Done() + for sp := range updateCh { + err := verifyDownload(ctx, sp) + if err != nil { + sp.Error = err.Error() + out.print(fmt.Sprintf(" FAIL: %s - %v", sp.GoImportPath, err)) + atomic.AddInt32(&c.dlErrors, 1) + } else { + newVer := specVersionForVersion(sp.LatestVersion) + remoteSHA256, err := remoteAssetSHA256(ctx, sp, newVer) + updateRemoteAsset := true + if errors.Is(err, errRemoteAssetSkip) { + out.print(fmt.Sprintf(" SKIP: %s remote asset sha256 - %v", sp.GoImportPath, err)) + updateRemoteAsset = false + err = nil + } + if err != nil { + sp.Error = err.Error() + out.print(fmt.Sprintf(" FAIL: %s - %v", sp.GoImportPath, err)) + atomic.AddInt32(&c.upErrors, 1) + } else if err := updateSpecVersion(sp, newVer, remoteSHA256, updateRemoteAsset); err != nil { + sp.Error = err.Error() + out.print(fmt.Sprintf(" FAIL: %s - %v", sp.GoImportPath, err)) + atomic.AddInt32(&c.upErrors, 1) + } else { + atomic.AddInt32(&c.modified, 1) + out.print(fmt.Sprintf(" OK: %s updated to %s", sp.GoImportPath, newVer)) + } + } + atomic.AddInt32(&c.processed, 1) + } + }() + } + } + + var wg1 sync.WaitGroup + sem1 := make(chan struct{}, *concurrency) + + for _, spec := range specs { + wg1.Add(1) + sem1 <- struct{}{} + go func(s *specInfo) { + defer wg1.Done() + defer func() { <-sem1 }() + + checkLatestVersion(ctx, s) + + atomic.AddInt32(&c.scanned, 1) + needSend := false + + if s.Error != "" { + atomic.AddInt32(&c.scanErrors, 1) + out.print(fmt.Sprintf(" ERROR: %s - %s", s.GoImportPath, s.Error)) + } else if s.HasUpdate { + atomic.AddInt32(&c.withUpdate, 1) + out.print(fmt.Sprintf(" UPDATE: %-50s %s -> %s", s.GoImportPath, s.Version, s.LatestVersion)) + needSend = true + } + + if needSend && !*dryRun { + updateCh <- s + } + }(spec) + } + + wg1.Wait() + close(updateCh) + + if !*dryRun { + wg2.Wait() + } + + pkgsiteRetryLog = nil + downloadRetryLog = nil + out.stop() + + var errSpecs []*specInfo + for _, s := range specs { + if s.Error != "" { + errSpecs = append(errSpecs, s) + } + } + + summaryLines := []string{ + fmt.Sprintf("%-12s %d/%d", "Scanned:", total, total), + fmt.Sprintf("%-12s %d", "Updates:", atomic.LoadInt32(&c.withUpdate)), + fmt.Sprintf("%-12s %d", "Modified:", atomic.LoadInt32(&c.modified)), + } + if atomic.LoadInt32(&c.scanErrors) > 0 || atomic.LoadInt32(&c.dlErrors) > 0 || atomic.LoadInt32(&c.upErrors) > 0 { + totalErrors := atomic.LoadInt32(&c.scanErrors) + atomic.LoadInt32(&c.dlErrors) + atomic.LoadInt32(&c.upErrors) + summaryLines = append(summaryLines, fmt.Sprintf("%-12s %d", "Errors:", totalErrors)) + } + if *dryRun { + summaryLines = append(summaryLines, fmt.Sprintf("%-12s %s", "Mode:", "dry run")) + } + fmt.Println() + printBox("SUMMARY", summaryLines) + + if atomic.LoadInt32(&c.withUpdate) == 0 { + fmt.Println("\nAll modules are up to date.") + } else if *dryRun { + fmt.Println("\nDry run complete. No files were modified.") + } else { + fmt.Println() + printBox("UPDATE RESULT", []string{ + fmt.Sprintf("%-12s %d", "Attempted:", atomic.LoadInt32(&c.withUpdate)), + fmt.Sprintf("%-12s %d", "Modified:", atomic.LoadInt32(&c.modified)), + fmt.Sprintf("%-12s %d", "Dl errors:", atomic.LoadInt32(&c.dlErrors)), + fmt.Sprintf("%-12s %d", "Up errors:", atomic.LoadInt32(&c.upErrors)), + }) + } + + if len(errSpecs) > 0 { + fmt.Printf("\n=== %d modules with errors ===\n", len(errSpecs)) + for _, s := range errSpecs { + fmt.Printf(" %-50s %s\n", s.GoImportPath, s.Error) + } + } + + return 0 +} diff --git a/update_test.go b/update_test.go new file mode 100644 index 0000000..9c9ab64 --- /dev/null +++ b/update_test.go @@ -0,0 +1,456 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" +) + +func TestUpdateSpecVersionPreservesFormatting(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + specPath := filepath.Join(dir, "test.spec") + original := strings.Join([]string{ + "Name: golang-test", + "Version:\t1.2.3 # keep-comment", + "Release: 1%{?dist}", + "%changelog", + "%{?autochangelog}", + }, "\n") + if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil { + t.Fatalf("write spec: %v", err) + } + + spec := &specInfo{FilePath: specPath} + if err := updateSpecVersion(spec, "2.0.0", strings.Repeat("a", 64), true); err != nil { + t.Fatalf("updateSpecVersion: %v", err) + } + + updated, err := os.ReadFile(specPath) + if err != nil { + t.Fatalf("read spec: %v", err) + } + + got := string(updated) + if !strings.Contains(got, "Version:\t2.0.0 # keep-comment") { + t.Fatalf("version line not updated as expected:\n%s", got) + } + if !strings.Contains(got, "%autochangelog\n") { + t.Fatalf("expected %%{{?autochangelog}} to be normalized, got:\n%s", got) + } + if strings.Contains(got, "%{?autochangelog}") { + t.Fatalf("expected %%{{?autochangelog}} to be removed, got:\n%s", got) + } + if !strings.HasSuffix(got, "\n") { + t.Fatalf("expected rewritten spec to keep trailing newline, got %q", got) + } +} + +func TestUpdateSpecVersionReturnsErrorWhenVersionLineMissing(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + specPath := filepath.Join(dir, "test.spec") + if err := os.WriteFile(specPath, []byte("Name: golang-test\n"), 0o644); err != nil { + t.Fatalf("write spec: %v", err) + } + + spec := &specInfo{FilePath: specPath} + if err := updateSpecVersion(spec, "2.0.0", strings.Repeat("a", 64), true); err == nil { + t.Fatal("expected updateSpecVersion to fail when Version line is missing") + } +} + +func TestParseSpecFileAcceptsGlobalMacros(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + specPath := filepath.Join(dir, "test.spec") + original := strings.Join([]string{ + "%global go_import_path github.com/example/project/cmd/tool", + "%global _name golang-github-example-project", + "Version: 1.2.3", + "Source0: https://example.invalid/source.tar.gz", + }, "\n") + if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil { + t.Fatalf("write spec: %v", err) + } + + spec, err := parseSpecFile(specPath) + if err != nil { + t.Fatalf("parseSpecFile: %v", err) + } + if spec.GoImportPath != "github.com/example/project/cmd/tool" { + t.Fatalf("GoImportPath = %q", spec.GoImportPath) + } + if spec.Name != "golang-github-example-project" { + t.Fatalf("Name = %q", spec.Name) + } +} + +func TestDownloadCandidatesUseModulePathForPackageImport(t *testing.T) { + t.Parallel() + + spec := &specInfo{ + GoImportPath: "github.com/example/project/cmd/tool", + LatestVersion: "v1.2.3", + } + info := &pkgsiteInfo{ + Package: pkgsitePackage{ModulePath: "github.com/example/project"}, + Module: pkgsiteModule{ + Path: "github.com/example/project", + RepoURL: "https://github.com/example/project", + }, + } + + candidates, archiveErr, err := downloadCandidatesForSpec(spec, info) + if err != nil { + t.Fatalf("downloadCandidatesForSpec: %v", err) + } + if archiveErr != nil { + t.Fatalf("archiveErr = %v", archiveErr) + } + if len(candidates) != 2 { + t.Fatalf("len(candidates) = %d, want 2", len(candidates)) + } + if got, want := candidates[0].url, "https://github.com/example/project/archive/v1.2.3.tar.gz"; got != want { + t.Fatalf("repo candidate = %q, want %q", got, want) + } + if got, want := candidates[1].url, "https://proxy.golang.org/github.com/example/project/@v/v1.2.3.zip"; got != want { + t.Fatalf("proxy candidate = %q, want %q", got, want) + } +} + +func TestVerifyHTTPRequestFallsBackToRangeGet(t *testing.T) { + t.Parallel() + + var sawHead atomic.Bool + var sawGet atomic.Bool + var gotRange atomic.Value + gotRange.Store("") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodHead: + sawHead.Store(true) + w.WriteHeader(http.StatusMethodNotAllowed) + case http.MethodGet: + sawGet.Store(true) + gotRange.Store(r.Header.Get("Range")) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("x")) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer srv.Close() + + if err := verifyHTTPRequest(context.Background(), srv.URL); err != nil { + t.Fatalf("verifyHTTPRequest: %v", err) + } + if !sawHead.Load() { + t.Fatal("expected HEAD request to be attempted first") + } + if !sawGet.Load() { + t.Fatal("expected GET fallback to be attempted") + } + if gotRange.Load().(string) != "bytes=0-0" { + t.Fatalf("unexpected Range header: %q", gotRange.Load().(string)) + } +} + +func TestRepoRefForVersionStripsBuildMetadata(t *testing.T) { + t.Parallel() + + tests := []struct { + in string + want string + }{ + {in: "v2.74.0+incompatible", want: "v2.74.0"}, + {in: "2.74.0+incompatible", want: "v2.74.0"}, + {in: "v1.2.3-rc.1", want: "v1.2.3-rc.1"}, + {in: "release-2026-06-12", want: "vrelease-2026-06-12"}, + } + + for _, tc := range tests { + if got := repoRefForVersion(tc.in); got != tc.want { + t.Fatalf("repoRefForVersion(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestSourceVersionForSpecStripsBuildMetadata(t *testing.T) { + t.Parallel() + + if got := sourceVersionForSpec("v2.74.0+incompatible"); got != "2.74.0" { + t.Fatalf("sourceVersionForSpec() = %q, want %q", got, "2.74.0") + } +} + +func TestSpecVersionForVersionStripsIncompatible(t *testing.T) { + t.Parallel() + + if got := specVersionForVersion("v2.74.0+incompatible"); got != "2.74.0" { + t.Fatalf("specVersionForVersion() = %q, want %q", got, "2.74.0") + } +} + +func TestUpdateSpecVersionOnlyUpdatesSource0RemoteAsset(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + specPath := filepath.Join(dir, "test.spec") + source1Hash := strings.Repeat("b", 64) + source0Hash := strings.Repeat("c", 64) + original := strings.Join([]string{ + "Name: golang-test", + "Version: 1.0.0", + "#!RemoteAsset: sha256:" + source1Hash, + "Source1: https://example.invalid/extra.tar.gz", + "Source0: https://example.invalid/source.tar.gz", + }, "\n") + if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil { + t.Fatalf("write spec: %v", err) + } + + spec := &specInfo{FilePath: specPath} + if err := updateSpecVersion(spec, "2.0.0", source0Hash, true); err != nil { + t.Fatalf("updateSpecVersion: %v", err) + } + + updated, err := os.ReadFile(specPath) + if err != nil { + t.Fatalf("read spec: %v", err) + } + got := string(updated) + + if !strings.Contains(got, "#!RemoteAsset: sha256:"+source1Hash+"\nSource1:") { + t.Fatalf("Source1 RemoteAsset was unexpectedly changed:\n%s", got) + } + if !strings.Contains(got, "#!RemoteAsset: sha256:"+source0Hash+"\nSource0:") { + t.Fatalf("Source0 RemoteAsset was not inserted correctly:\n%s", got) + } +} + +func TestUpdateSpecVersionClearsRemoteAssetWhenHashEmpty(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + specPath := filepath.Join(dir, "test.spec") + sourceHash := strings.Repeat("d", 64) + original := strings.Join([]string{ + "Name: golang-test", + "Version: 1.0.0", + "#!RemoteAsset: sha256:" + sourceHash, + "Source0: https://example.invalid/%{commit_id}.tar.gz", + }, "\n") + if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil { + t.Fatalf("write spec: %v", err) + } + + spec := &specInfo{FilePath: specPath} + if err := updateSpecVersion(spec, "2.0.0", "", true); err != nil { + t.Fatalf("updateSpecVersion: %v", err) + } + + updated, err := os.ReadFile(specPath) + if err != nil { + t.Fatalf("read spec: %v", err) + } + got := string(updated) + + if !strings.Contains(got, "Version: 2.0.0") { + t.Fatalf("version was not updated:\n%s", got) + } + if strings.Contains(got, "sha256:"+sourceHash) { + t.Fatalf("RemoteAsset hash should be cleared:\n%s", got) + } + if !strings.Contains(got, "#!RemoteAsset\nSource0:") { + t.Fatalf("RemoteAsset marker should be kept without hash:\n%s", got) + } +} + +func TestUpdateSpecVersionKeepsRemoteAssetWhenUpdateDisabled(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + specPath := filepath.Join(dir, "test.spec") + sourceHash := strings.Repeat("d", 64) + original := strings.Join([]string{ + "Name: golang-test", + "Version: 1.0.0", + "#!RemoteAsset: sha256:" + sourceHash, + "Source0: https://example.invalid/%{commit_id}.tar.gz", + }, "\n") + if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil { + t.Fatalf("write spec: %v", err) + } + + spec := &specInfo{FilePath: specPath} + if err := updateSpecVersion(spec, "2.0.0", "", false); err != nil { + t.Fatalf("updateSpecVersion: %v", err) + } + + updated, err := os.ReadFile(specPath) + if err != nil { + t.Fatalf("read spec: %v", err) + } + got := string(updated) + + if !strings.Contains(got, "Version: 2.0.0") { + t.Fatalf("version was not updated:\n%s", got) + } + if !strings.Contains(got, "#!RemoteAsset: sha256:"+sourceHash) { + t.Fatalf("RemoteAsset hash should be unchanged:\n%s", got) + } +} + +func TestResolvedSource0URLSkipsPinnedCommit(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + specPath := filepath.Join(dir, "test.spec") + original := strings.Join([]string{ + "%define commit_id abcdef123456", + "Name: golang-test", + "Version: 1.0.0", + "Source0: https://example.invalid/%{commit_id}.tar.gz", + }, "\n") + if err := os.WriteFile(specPath, []byte(original), 0o644); err != nil { + t.Fatalf("write spec: %v", err) + } + + _, err := resolvedSource0URL(specPath, "2.0.0") + if !errors.Is(err, errRemoteAssetSkip) { + t.Fatalf("resolvedSource0URL error = %v, want errRemoteAssetSkip", err) + } +} + +func TestReleasePageURLForSource(t *testing.T) { + t.Parallel() + + tests := []struct { + source string + wantURL string + wantTag string + }{ + { + source: "https://github.com/apache/beam/archive/refs/tags/v2.74.0.tar.gz", + wantURL: "https://github.com/apache/beam/releases/tag/v2.74.0", + wantTag: "v2.74.0", + }, + { + source: "https://github.com/charmbracelet/x/archive/refs/tags/term/v0.2.2.tar.gz", + wantURL: "https://github.com/charmbracelet/x/releases/tag/term/v0.2.2", + wantTag: "term/v0.2.2", + }, + { + source: "https://gitlab.com/group/project/-/archive/v1.2.3/project-v1.2.3.tar.gz", + wantURL: "https://gitlab.com/group/project/-/releases/v1.2.3", + wantTag: "v1.2.3", + }, + } + + for _, tc := range tests { + gotURL, gotTag := releasePageURLForSource(tc.source) + if gotURL != tc.wantURL || gotTag != tc.wantTag { + t.Fatalf("releasePageURLForSource(%q) = (%q, %q), want (%q, %q)", + tc.source, gotURL, gotTag, tc.wantURL, tc.wantTag) + } + } +} + +func TestExtractSHA256FromHTML(t *testing.T) { + t.Parallel() + + want := strings.Repeat("a", 64) + html := "Release notes\nSHA256: " + want + if got := extractSHA256FromHTML(html, "v1.2.3"); got != want { + t.Fatalf("extractSHA256FromHTML() = %q, want %q", got, want) + } +} + +func TestDownloadAssetSHA256(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("source archive")) + })) + defer srv.Close() + + got, err := downloadAssetSHA256(context.Background(), srv.URL) + if err != nil { + t.Fatalf("downloadAssetSHA256: %v", err) + } + want := "6ad189ace456a83fade855d5a647cd8ad9e7966da4404b1187218dca3d9eddaa" + if got != want { + t.Fatalf("downloadAssetSHA256() = %q, want %q", got, want) + } +} + +func TestVerifyDownloadCandidatesFallsBackToProxy(t *testing.T) { + t.Parallel() + + var logsMu sync.Mutex + var logs []string + oldLog := downloadRetryLog + downloadRetryLog = func(msg string) { + logsMu.Lock() + defer logsMu.Unlock() + logs = append(logs, msg) + } + defer func() { downloadRetryLog = oldLog }() + + repoSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusGatewayTimeout) + })) + defer repoSrv.Close() + + proxySrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("x")) + })) + defer proxySrv.Close() + + err := verifyDownloadCandidates(context.Background(), "codeberg.org/go-fonts/liberation", []downloadCandidate{ + {source: "repo", url: repoSrv.URL}, + {source: "proxy", url: proxySrv.URL}, + }) + if err != nil { + t.Fatalf("verifyDownloadCandidates: %v", err) + } + + logsMu.Lock() + defer logsMu.Unlock() + joined := strings.Join(logs, "\n") + if !strings.Contains(joined, "trying proxy") { + t.Fatalf("expected fallback log, got: %s", joined) + } + if !strings.Contains(joined, "verified via proxy after fallback") { + t.Fatalf("expected fallback success log, got: %s", joined) + } +} + +func TestIsRetryableDownloadErrorRecognizesTimeout(t *testing.T) { + t.Parallel() + + timeoutErr := &net.DNSError{IsTimeout: true} + if !isRetryableDownloadError(timeoutErr) { + t.Fatal("expected timeout net error to be retryable") + } + if !isRetryableDownloadError(errors.New("HTTP 504 from example")) { + t.Fatal("expected HTTP 504 to be retryable") + } + if isRetryableDownloadError(fmt.Errorf("HTTP 404 from example")) { + t.Fatal("did not expect HTTP 404 to be retryable") + } +}