From c25d97ac98f3463fc121c7474bbbec66d8315f00 Mon Sep 17 00:00:00 2001 From: "E. Lynette Rayle" Date: Wed, 11 Nov 2026 12:43:41 -0500 Subject: [PATCH 1/2] Automate immutable ScanCode license category updates Extend the existing daily license updater to refresh the ScanCode pin and checksum, regenerate npm data, validate both outputs, and maintain one reviewable update PR. Preserve provenance verification and data-only publication policy. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/fetch-licenses.yaml | 59 ++++-- cmd/license-categories/main.go | 66 +++++-- cmd/license-categories/update_scancode.go | 139 +++++++++++++ .../update_scancode_test.go | 187 ++++++++++++++++++ cmd/pr_body.go | 53 ++++- cmd/pr_body_test.go | 21 ++ .../license-categories/docs/operations.md | 41 +++- 7 files changed, 529 insertions(+), 37 deletions(-) create mode 100644 cmd/license-categories/update_scancode.go create mode 100644 cmd/license-categories/update_scancode_test.go diff --git a/.github/workflows/fetch-licenses.yaml b/.github/workflows/fetch-licenses.yaml index 9ce5700..59cdf09 100644 --- a/.github/workflows/fetch-licenses.yaml +++ b/.github/workflows/fetch-licenses.yaml @@ -4,12 +4,17 @@ on: workflow_dispatch: inputs: force_run: - description: 'Force run license extraction' + description: 'Force SPDX extraction and license category regeneration' + type: boolean required: false - default: 'false' + default: false schedule: - cron: '5 4 * * *' # Runs at 0405 UTC (0605 CET, 0005 ET, 2105 PT) +concurrency: + group: fetch-licenses + cancel-in-progress: false + jobs: fetch-licenses: runs-on: ubuntu-latest @@ -17,6 +22,7 @@ jobs: DT_STAMP: '' GENFILES_CHANGED: 'false' SRC_CHANGED: 'false' + FORCE_RUN: ${{ github.event.inputs.force_run || 'false' }} permissions: contents: write pull-requests: write @@ -30,6 +36,11 @@ jobs: - name: Checkout this repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Checkout official SPDX Repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -52,17 +63,16 @@ jobs: # --quiet: Exits with 1 if there are changes; otherwise, exits with 0 git diff --quiet -- cmd/licenses.json cmd/exceptions.json && src_changed=false if [ $src_changed != 'true' ]; then - if [ ${{ github.event.inputs.force_run }} == 'true' ]; then + if [ "$FORCE_RUN" == 'true' ]; then echo -e '***************\nNo changes in spdx json, but skipping abort due to force run\n***************' else - echo -e '***************\nABORTING: No changes to spdx json data\n***************' - exit 0 + echo 'No SPDX source changes; still checking ScanCode.' fi fi echo "SRC_CHANGED=$src_changed" >> $GITHUB_ENV - name: Run license extraction - if: ${{ env.SRC_CHANGED == 'true' || github.event.inputs.force_run == 'true' }} + if: ${{ env.SRC_CHANGED == 'true' || env.FORCE_RUN == 'true' }} run: | cd cmd echo "Current branch: $(git branch)" @@ -70,14 +80,17 @@ jobs: cd .. git log --oneline -n 5 + - name: Update ScanCode pin and generate npm license categories + run: go run ./cmd/license-categories -update-scancode -out packages/license-categories/src/generated.ts + - name: Check for changes in generated files id: genfiles-changes run: | genfiles_changed=true # --quiet: Exits with 1 if there are changes; otherwise, exits with 0 - git diff --quiet -- spdxexp/spdxlicenses/*.go && genfiles_changed=false + git diff --quiet -- spdxexp/spdxlicenses/*.go cmd/license-categories/main.go packages/license-categories/src/generated.ts && genfiles_changed=false if [ $genfiles_changed != 'true' ]; then - if [ ${{ github.event.inputs.force_run }} == 'true' ]; then + if [ "$FORCE_RUN" == 'true' ]; then echo -e '***************\nNo changes to generated files, but skipping abort due to force run\n***************' else # This happens when the generated files are already up to date @@ -88,18 +101,40 @@ jobs: fi echo "GENFILES_CHANGED=$genfiles_changed" >> $GITHUB_ENV + - name: Set up Node + if: ${{ env.GENFILES_CHANGED == 'true' || env.FORCE_RUN == 'true' }} + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 24 + cache: npm + cache-dependency-path: packages/license-categories/package-lock.json + + - name: Validate update before opening PR + if: ${{ env.GENFILES_CHANGED == 'true' || env.FORCE_RUN == 'true' }} + run: | + go test ./... + make verify-license-categories + npm ci --prefix packages/license-categories + npm test --prefix packages/license-categories + - name: Generate Pull Request description - if: ${{ env.GENFILES_CHANGED == 'true' || github.event.inputs.force_run == 'true' }} + if: ${{ env.GENFILES_CHANGED == 'true' || env.FORCE_RUN == 'true' }} run: go run ./cmd pr-body -o "${RUNNER_TEMP}/license-pr-body.md" - name: Create Pull Request - if: ${{ env.GENFILES_CHANGED == 'true' || github.event.inputs.force_run == 'true' }} + if: ${{ env.GENFILES_CHANGED == 'true' || env.FORCE_RUN == 'true' }} uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 with: token: ${{ secrets.GITHUB_TOKEN }} - commit-message: Add updated license files + commit-message: Update SPDX and ScanCode license data branch: auto-update-licenses base: main - title: "Update SPDX license files (${{ env.DT_STAMP }})" + title: "Update SPDX and ScanCode license data (${{ env.DT_STAMP }})" body-path: ${{ runner.temp }}/license-pr-body.md labels: 'auto-update,licenses' + add-paths: | + cmd/*.json + cmd/*.txt + cmd/license-categories/main.go + spdxexp/spdxlicenses/*.go + packages/license-categories/src/generated.ts diff --git a/cmd/license-categories/main.go b/cmd/license-categories/main.go index 988f459..12a5770 100644 --- a/cmd/license-categories/main.go +++ b/cmd/license-categories/main.go @@ -26,13 +26,13 @@ import ( ) const ( - scancodeCommit = "5ee8765870cd2ed1cb345645be94a799ca584c90" - defaultScancodeZipURL = "https://codeload.github.com/aboutcode-org/scancode-licensedb/zip/" + scancodeCommit - defaultScancodeZipSHA256 = "6b4b05f985bd7a520b30e7f8547778ae4cdd6760acc577ab11cebbde41f165bb" - defaultTimeout = 60 * time.Second - maxZipBytes = 100 * 1024 * 1024 // 100MB upper bound for the scancode zip download. - maxZipEntryBytes = 1 * 1024 * 1024 // 1MB upper bound per zip entry to guard against zip bombs. - outputFileMode = 0o644 // World-readable since the generated file is manually copied into other projects. + scancodeCommit = "5ee8765870cd2ed1cb345645be94a799ca584c90" + scancodeZipURL = "https://codeload.github.com/aboutcode-org/scancode-licensedb/zip/" + scancodeCommit + scancodeZipSHA256 = "6b4b05f985bd7a520b30e7f8547778ae4cdd6760acc577ab11cebbde41f165bb" + defaultTimeout = 60 * time.Second + maxZipBytes = 100 * 1024 * 1024 // 100MB upper bound for the scancode zip download. + maxZipEntryBytes = 1 * 1024 * 1024 // 1MB upper bound per zip entry to guard against zip bombs. + outputFileMode = 0o644 // World-readable since the generated file is manually copied into other projects. ) // excludedSpdxIDs contains SPDX IDs that should be ignored when generating license categories. @@ -79,10 +79,10 @@ type scancodeSource struct { zipSHA256 string } -var defaultScancodeSource = scancodeSource{ +var currentScancodeSource = scancodeSource{ commit: scancodeCommit, - zipURL: defaultScancodeZipURL, - zipSHA256: defaultScancodeZipSHA256, + zipURL: scancodeZipURL, + zipSHA256: scancodeZipSHA256, } type generationProvenance struct { @@ -105,6 +105,7 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error { outPath := fs.String("out", "-", "Output file path, or '-' for stdout") verifyPath := fs.String("verify", "", "Verify an existing file, ignoring its leading go-spdx, SPDX list version, and ScanCode provenance values") + updateScancode := fs.Bool("update-scancode", false, "Update the ScanCode pin to upstream main and regenerate (run from repository root)") err := fs.Parse(args) if err != nil { return err @@ -113,13 +114,28 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error { if *verifyPath != "" && *outPath != "-" { return errors.New("-verify cannot be combined with -out") } + if *verifyPath != "" && *updateScancode { + return errors.New("-verify cannot be combined with -update-scancode") + } - spdxIDs := spdxlicenses.GetLicenses() - records, err := loadScancodeBySpdx(context.Background(), client, defaultScancodeSource, maxZipBytes) + ctx := context.Background() + source := currentScancodeSource + var sourceFile []byte + var records map[string]licenseMetadata + if *updateScancode { + source, records, sourceFile, err = prepareScancodeUpdate(ctx, client, stderr) + if err != nil { + return fmt.Errorf("update scancode source: %w", err) + } + } + if records == nil { + records, err = loadScancodeBySpdx(ctx, client, source, maxZipBytes) + } if err != nil { return fmt.Errorf("load scancode data: %w", err) } + spdxIDs := spdxlicenses.GetLicenses() licenses, warnings, err := buildLicenseData(spdxIDs, records, excludedSpdxIDs, spdxCategoryOverrides) if err != nil { return err @@ -137,7 +153,7 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error { output, err := renderTypeScript(licenses, generationProvenance{ GoSPDXVersion: goSPDXVersion, SPDXLicenseListVersion: spdxlicenses.LicenseListVersion(), - ScancodeCommit: defaultScancodeSource.commit, + ScancodeCommit: source.commit, }) if err != nil { return err @@ -154,18 +170,30 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error { if *outPath == "-" { _, err = stdout.Write(output) - return err + } else { + err = os.WriteFile(*outPath, output, outputFileMode) } - - err = os.WriteFile(*outPath, output, outputFileMode) if err != nil { return fmt.Errorf("write output: %w", err) } + if sourceFile != nil { + if err := os.WriteFile(scancodePinPath, sourceFile, outputFileMode); err != nil { + return fmt.Errorf("write scancode pin: %w", err) + } + } return nil } func loadScancodeBySpdx(ctx context.Context, client *http.Client, source scancodeSource, maxZipBytes int64) (map[string]licenseMetadata, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, source.zipURL, http.NoBody) + content, err := downloadScancodeArchive(ctx, client, source.zipURL, maxZipBytes) + if err != nil { + return nil, err + } + return parseScancodeArchive(content, source) +} + +func downloadScancodeArchive(ctx context.Context, client *http.Client, url string, maxZipBytes int64) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) if err != nil { return nil, fmt.Errorf("create request: %w", err) } @@ -188,6 +216,10 @@ func loadScancodeBySpdx(ctx context.Context, client *http.Client, source scancod if int64(len(content)) > maxZipBytes { return nil, fmt.Errorf("zip response exceeds maximum size of %d bytes", maxZipBytes) } + return content, nil +} + +func parseScancodeArchive(content []byte, source scancodeSource) (map[string]licenseMetadata, error) { if err := verifySHA256(content, source.zipSHA256); err != nil { return nil, err } diff --git a/cmd/license-categories/update_scancode.go b/cmd/license-categories/update_scancode.go new file mode 100644 index 0000000..fde11f1 --- /dev/null +++ b/cmd/license-categories/update_scancode.go @@ -0,0 +1,139 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "io" + "net/http" + "os" + "regexp" + "strconv" +) + +const ( + scancodePinPath = "cmd/license-categories/main.go" + scancodeMainAPI = "https://api.github.com/repos/aboutcode-org/scancode-licensedb/commits/main" +) + +var fullCommitSHA = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func prepareScancodeUpdate(ctx context.Context, client *http.Client, stderr io.Writer) (scancodeSource, map[string]licenseMetadata, []byte, error) { + commit, err := latestScancodeCommit(ctx, client) + if err != nil { + return scancodeSource{}, nil, nil, err + } + if commit == currentScancodeSource.commit { + fmt.Fprintln(stderr, "ScanCode pin is already current.") //nolint:errcheck,revive // status output + return currentScancodeSource, nil, nil, nil + } + + source := scancodeSource{ + commit: commit, + zipURL: "https://codeload.github.com/aboutcode-org/scancode-licensedb/zip/" + commit, + } + content, err := downloadScancodeArchive(ctx, client, source.zipURL, maxZipBytes) + if err != nil { + return scancodeSource{}, nil, nil, err + } + digest := sha256.Sum256(content) + source.zipSHA256 = hex.EncodeToString(digest[:]) + records, err := parseScancodeArchive(content, source) + if err != nil { + return scancodeSource{}, nil, nil, err + } + pins, err := os.ReadFile(scancodePinPath) + if err != nil { + return scancodeSource{}, nil, nil, fmt.Errorf("read scancode pin: %w", err) + } + pins, err = replaceScancodePins(pins, source) + if err != nil { + return scancodeSource{}, nil, nil, err + } + fmt.Fprintf(stderr, "Updating ScanCode pin: %s -> %s\n", currentScancodeSource.commit, commit) //nolint:errcheck,revive // status output + return source, records, pins, nil +} + +func latestScancodeCommit(ctx context.Context, client *http.Client) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, scancodeMainAPI, http.NoBody) + if err != nil { + return "", fmt.Errorf("create ScanCode main request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github+json") + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("resolve ScanCode main: %w", err) + } + defer resp.Body.Close() //nolint:errcheck // best-effort close + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("resolve ScanCode main: unexpected status %s", resp.Status) + } + var response struct { + SHA string `json:"sha"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, maxZipEntryBytes)).Decode(&response); err != nil { + return "", fmt.Errorf("decode ScanCode main: %w", err) + } + if !fullCommitSHA.MatchString(response.SHA) { + return "", fmt.Errorf("ScanCode main returned invalid full commit SHA %q", response.SHA) + } + return response.SHA, nil +} + +func replaceScancodePins(contents []byte, source scancodeSource) ([]byte, error) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, scancodePinPath, contents, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("parse scancode pin: %w", err) + } + values := map[string]string{ + "scancodeCommit": source.commit, + "scancodeZipSHA256": source.zipSHA256, + } + type replacement struct { + start, end int + text string + } + var replacements []replacement + for _, decl := range file.Decls { + gen, ok := decl.(*ast.GenDecl) + if !ok || gen.Tok != token.CONST { + continue + } + for _, spec := range gen.Specs { + value, ok := spec.(*ast.ValueSpec) + if !ok || len(value.Names) != 1 || len(value.Values) != 1 { + continue + } + name := value.Names[0].Name + text, ok := values[name] + if !ok { + continue + } + literal, ok := value.Values[0].(*ast.BasicLit) + if !ok || literal.Kind != token.STRING { + return nil, fmt.Errorf("scancode pin %s must be a string literal", name) + } + start, end := fset.Position(literal.Pos()).Offset, fset.Position(literal.End()).Offset + replacements = append(replacements, replacement{start, end, strconv.Quote(text)}) + delete(values, name) + } + } + if len(values) != 0 { + return nil, fmt.Errorf("missing scancode pin constants: %v", values) + } + // Work backwards so changed literal lengths cannot shift later offsets. + for i := len(replacements) - 1; i >= 0; i-- { + edit := replacements[i] + updated := append([]byte{}, contents[:edit.start]...) + updated = append(updated, edit.text...) + contents = append(updated, contents[edit.end:]...) + } + return format.Source(contents) +} diff --git a/cmd/license-categories/update_scancode_test.go b/cmd/license-categories/update_scancode_test.go new file mode 100644 index 0000000..c3e47ad --- /dev/null +++ b/cmd/license-categories/update_scancode_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type scancodeTransport func(*http.Request) (*http.Response, error) + +func (transport scancodeTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return transport(req) +} + +func scancodeArchive(t *testing.T, data string) []byte { + t.Helper() + var content bytes.Buffer + writer := zip.NewWriter(&content) + entry, err := writer.Create("licensedb-revision/docs/mit.json") + require.NoError(t, err) + _, err = io.WriteString(entry, data) + require.NoError(t, err) + require.NoError(t, writer.Close()) + return content.Bytes() +} + +func TestLatestScancodeCommit(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name, body, wantError string + status int + }{ + {"valid", `{"sha":"` + strings.Repeat("a", 40) + `"}`, "", 200}, + {"rate limit", `{}`, "unexpected status", 403}, + {"server failure", `{}`, "unexpected status", 500}, + {"malformed", ``, "decode ScanCode", 200}, + {"missing", `{}`, "invalid full commit SHA", 200}, + {"short", `{"sha":"abc123"}`, "invalid full commit SHA", 200}, + {"unsafe", `{"sha":"refs/heads/main"}`, "invalid full commit SHA", 200}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + client := &http.Client{Transport: scancodeTransport(func(req *http.Request) (*http.Response, error) { + assert.Equal(t, scancodeMainAPI, req.URL.String()) + return &http.Response{StatusCode: tt.status, Status: fmt.Sprint(tt.status), Body: io.NopCloser(strings.NewReader(tt.body))}, nil + })} + commit, err := latestScancodeCommit(context.Background(), client) + if tt.wantError != "" { + require.ErrorContains(t, err, tt.wantError) + return + } + require.NoError(t, err) + assert.Equal(t, strings.Repeat("a", 40), commit) + }) + } +} + +func TestReplaceScancodePins(t *testing.T) { + t.Parallel() + input := []byte(`package main +// Keep this comment. +const ( + scancodeCommit = "old" + scancodeZipURL = "https://codeload.github.com/aboutcode-org/scancode-licensedb/zip/" + scancodeCommit + scancodeZipSHA256 = "old-checksum" +) +var spdxCategoryOverrides = map[string]string{"MIT": "Permissive"} +`) + source := scancodeSource{commit: strings.Repeat("a", 40), zipSHA256: strings.Repeat("b", 64)} + output, err := replaceScancodePins(input, source) + require.NoError(t, err) + assert.Contains(t, string(output), `"https://codeload.github.com/aboutcode-org/scancode-licensedb/zip/" + scancodeCommit`) + assert.Contains(t, string(output), strconv.Quote(source.commit)) + assert.Contains(t, string(output), strconv.Quote(source.zipSHA256)) + assert.Contains(t, string(output), "// Keep this comment.") + assert.Contains(t, string(output), `map[string]string{"MIT": "Permissive"}`) + _, err = replaceScancodePins([]byte("package main\n"), source) + require.ErrorContains(t, err, "missing scancode pin constants") + _, err = replaceScancodePins([]byte("package main\nconst scancodeCommit = other\n"), source) + require.ErrorContains(t, err, "must be a string literal") +} + +func TestRunUpdateScancode(t *testing.T) { + // Chdir and the synthetic current source are process-wide; do not run in parallel. + original, err := os.ReadFile("main.go") + require.NoError(t, err) + const newCommit = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + valid := scancodeArchive(t, `{"name":"MIT License","category":"Permissive","spdx_license_key":"MIT"}`) + for _, tt := range []struct { + name, commit, wantError string + archive []byte + matchingChecksum bool + }{ + {"new revision", newCommit, "", valid, false}, + {"unchanged revision", currentScancodeSource.commit, "", valid, true}, + {"unchanged checksum mismatch", currentScancodeSource.commit, "checksum mismatch", valid, false}, + {"bad zip", newCommit, "open zip", []byte("not a zip"), false}, + {"bad json", newCommit, "decode", scancodeArchive(t, "{"), false}, + {"empty records", newCommit, "no scancode SPDX records", scancodeArchive(t, `{}`), false}, + {"unrecognized category", newCommit, "has no description mapping", scancodeArchive(t, `{"name":"MIT","category":"New risk","spdx_license_key":"MIT"}`), false}, + } { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "cmd/license-categories"), 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(dir, scancodePinPath), original, 0o600)) + t.Chdir(dir) + if tt.matchingChecksum { + previous := currentScancodeSource + t.Cleanup(func() { currentScancodeSource = previous }) + currentScancodeSource.zipSHA256 = fmt.Sprintf("%x", sha256.Sum256(tt.archive)) + } + var requested []string + client := &http.Client{Transport: scancodeTransport(func(req *http.Request) (*http.Response, error) { + requested = append(requested, req.URL.String()) + body := tt.archive + if req.URL.String() == scancodeMainAPI { + body = []byte(`{"sha":"` + tt.commit + `"}`) + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body))}, nil + })} + var warnings bytes.Buffer + err := run([]string{"-update-scancode", "-out", "generated.ts"}, io.Discard, &warnings, client) + pins, readErr := os.ReadFile(scancodePinPath) + require.NoError(t, readErr) + if tt.wantError != "" { + require.ErrorContains(t, err, tt.wantError) + assert.Equal(t, original, pins) + _, err = os.Stat("generated.ts") + require.True(t, os.IsNotExist(err)) + return + } + require.NoError(t, err) + assert.Equal(t, []string{scancodeMainAPI, "https://codeload.github.com/aboutcode-org/scancode-licensedb/zip/" + tt.commit}, requested) + if tt.commit == currentScancodeSource.commit { + assert.Equal(t, original, pins) + assert.Contains(t, warnings.String(), "already current") + } else { + digest := sha256.Sum256(tt.archive) + assert.Contains(t, string(pins), fmt.Sprintf("%x", digest)) + assert.Contains(t, string(pins), newCommit) + } + output, err := os.ReadFile("generated.ts") + require.NoError(t, err) + assert.Contains(t, string(output), "// scancode-licensedb commit: "+tt.commit) + assert.Contains(t, string(output), "'MIT': {name: 'MIT License', category: 'Permissive'}") + assert.Contains(t, string(output), "'any-OSI': {name: 'any-OSI', category: 'Permissive'}") + assert.Contains(t, warnings.String(), "missing scancode metadata") + }) + } +} + +func TestPrepareScancodeUpdate_Unchanged(t *testing.T) { + t.Parallel() + requests := 0 + client := &http.Client{Transport: scancodeTransport(func(req *http.Request) (*http.Response, error) { + requests++ + assert.Equal(t, scancodeMainAPI, req.URL.String()) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"sha":"` + currentScancodeSource.commit + `"}`)), + }, nil + })} + source, records, pins, err := prepareScancodeUpdate(context.Background(), client, io.Discard) + require.NoError(t, err) + assert.Equal(t, currentScancodeSource, source) + assert.Nil(t, records) + assert.Nil(t, pins) + assert.Equal(t, 1, requests) +} + +func TestRunRejectsUpdatingDuringVerification(t *testing.T) { + t.Parallel() + err := run([]string{"-verify", "generated.ts", "-update-scancode"}, io.Discard, io.Discard, nil) + require.ErrorContains(t, err, "-verify cannot be combined with -update-scancode") +} diff --git a/cmd/pr_body.go b/cmd/pr_body.go index e95e74e..40a81d1 100644 --- a/cmd/pr_body.go +++ b/cmd/pr_body.go @@ -35,6 +35,20 @@ func generatePRBody(outputPath string) error { } body := buildPRBody(previousLicenses, currentLicenses, previousExceptions, currentExceptions) + previousCategories, err := readPreviousFile(repositoryRoot, "packages/license-categories/src/generated.ts") + if err != nil { + return err + } + // #nosec G304 -- path is built from the Git repository root and an internal constant. + currentCategories, err := os.ReadFile(filepath.Join(repositoryRoot, "packages/license-categories/src/generated.ts")) + if err != nil { + return fmt.Errorf("read generated license categories: %w", err) + } + summary, err := buildCategorySummary(previousCategories, currentCategories) + if err != nil { + return err + } + body += summary if err := os.WriteFile(outputPath, []byte(body), 0600); err != nil { return fmt.Errorf("write PR body: %w", err) } @@ -42,10 +56,9 @@ func generatePRBody(outputPath string) error { } func decodePreviousJSON(repositoryRoot, path string, destination any) error { - // #nosec G204 -- repositoryRoot is discovered by Git and path is an internal constant. - contents, err := exec.Command("git", "-C", repositoryRoot, "show", "HEAD:"+path).Output() + contents, err := readPreviousFile(repositoryRoot, path) if err != nil { - return fmt.Errorf("read previous %s: %w", path, err) + return err } if err := json.Unmarshal(contents, destination); err != nil { return fmt.Errorf("decode previous %s: %w", path, err) @@ -53,6 +66,38 @@ func decodePreviousJSON(repositoryRoot, path string, destination any) error { return nil } +func readPreviousFile(repositoryRoot, path string) ([]byte, error) { + // #nosec G204 -- repositoryRoot is discovered by Git and path is an internal constant. + contents, err := exec.Command("git", "-C", repositoryRoot, "show", "HEAD:"+path).Output() + if err != nil { + return nil, fmt.Errorf("read previous %s: %w", path, err) + } + return contents, nil +} + +func buildCategorySummary(previous, current []byte) (string, error) { + oldLines := strings.SplitN(string(previous), "\n", 5) + newLines := strings.SplitN(string(current), "\n", 5) + const prefix = "// scancode-licensedb commit: " + if len(oldLines) != 5 || len(newLines) != 5 || !strings.HasPrefix(oldLines[3], prefix) || !strings.HasPrefix(newLines[3], prefix) { + return "", fmt.Errorf("generated license categories are missing the ScanCode provenance header") + } + oldCommit := strings.TrimPrefix(oldLines[3], prefix) + newCommit := strings.TrimPrefix(newLines[3], prefix) + change := "Unchanged (any generated file changes are provenance-only)." + if oldLines[4] != newLines[4] { + change = "Changed; review license names, categories, overrides, exclusions, and generation warnings." + } + return fmt.Sprintf("\n## ScanCode and npm license categories\n\n"+ + "- **Previous ScanCode commit:** `%s`\n"+ + "- **New ScanCode commit:** `%s`\n"+ + "- **License category payload:** %s\n\n"+ + "ScanCode pins advance even for upstream documentation-only changes. This workflow does not bump versions, merge, or publish. "+ + "Only generated license category data changes trigger release publication. Provenance and other package changes wait for the next data update, "+ + "or an approved dispatch with a valid new npm version.\n", + oldCommit, newCommit, change), nil +} + func decodeJSONFile(path string, destination any) error { // #nosec G304 -- path is built from the Git repository root and an internal constant. contents, err := os.ReadFile(path) @@ -69,7 +114,7 @@ func buildPRBody(previousLicenses, currentLicenses LicenseData, previousExceptio var body bytes.Buffer body.WriteString("> [!WARNING]\n") body.WriteString("> Do not push commits directly to this PR's `auto-update-licenses` branch or manually edit this description. The nightly workflow updates both and may overwrite manual changes. For changes that depend on this update, open a stacked PR with `auto-update-licenses` as its base. For generator or workflow fixes that must persist, open a separate PR against `main`, merge it, and rerun this workflow.\n\n") - body.WriteString("This PR was generated by the [fetch-licenses](./.github/workflows/fetch-licenses.yaml) workflow from [spdx/license-list-data](https://github.com/spdx/license-list-data).\n\n") + body.WriteString("This PR was generated by the [fetch-licenses](./.github/workflows/fetch-licenses.yaml) workflow from [spdx/license-list-data](https://github.com/spdx/license-list-data) and [ScanCode LicenseDB](https://github.com/aboutcode-org/scancode-licensedb).\n\n") fmt.Fprintf(&body, "- **Previous SPDX License List Version:** `%s`\n", previousLicenses.Version) fmt.Fprintf(&body, "- **New SPDX License List Version:** `%s`\n\n", currentLicenses.Version) diff --git a/cmd/pr_body_test.go b/cmd/pr_body_test.go index 9fcdcf3..8044e93 100644 --- a/cmd/pr_body_test.go +++ b/cmd/pr_body_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestBuildPRBody(t *testing.T) { @@ -17,6 +18,7 @@ func TestBuildPRBody(t *testing.T) { {LicenseID: "Reactivated", IsDeprecated: true}, }, } + currentLicenses := LicenseData{ Version: "3.29.0", Licenses: []License{ @@ -44,3 +46,22 @@ func TestBuildPRBody(t *testing.T) { assert.Contains(t, body, "- **Added:** `Example-`") assert.Contains(t, body, "Human review required: license range family changed") } + +func TestBuildCategorySummary(t *testing.T) { + t.Parallel() + header := func(commit, payload string) []byte { + return []byte("// generated\n// go-spdx: (devel)\n// spdx-license-list-version: 3.29.0\n// scancode-licensedb commit: " + commit + "\n" + payload) + } + previous := header("old", "data") + summary, err := buildCategorySummary(previous, header("new", "data")) + require.NoError(t, err) + assert.Contains(t, summary, "**Previous ScanCode commit:** `old`") + assert.Contains(t, summary, "**New ScanCode commit:** `new`") + assert.Contains(t, summary, "provenance-only") + assert.Contains(t, summary, "Unchanged") + summary, err = buildCategorySummary(previous, header("new", "new data")) + require.NoError(t, err) + assert.Contains(t, summary, "Changed; review") + _, err = buildCategorySummary(previous, []byte("no header")) + require.ErrorContains(t, err, "missing the ScanCode provenance header") +} diff --git a/packages/license-categories/docs/operations.md b/packages/license-categories/docs/operations.md index 0b068ae..fdd64ad 100644 --- a/packages/license-categories/docs/operations.md +++ b/packages/license-categories/docs/operations.md @@ -76,6 +76,41 @@ overrides, warnings, and attribution in the package's [THIRD_PARTY_NOTICE.md](../THIRD_PARTY_NOTICE.md). Preserve upstream data licensing and any other third-party notices. +### Scheduled SPDX and ScanCode updates + +The [Fetch Licenses workflow](../../../.github/workflows/fetch-licenses.yaml) +runs daily at 04:05 UTC and checks SPDX and ScanCode independently. + +- **Immutable input:** Resolve `aboutcode-org/scancode-licensedb`'s `main` once + to a full commit, download its archive, compute SHA-256, and generate npm data + from those same bytes. Monitor the repository, not the potentially lagging website. +- **Local refresh:** Run from the repository root: + `go run ./cmd/license-categories -update-scancode -out packages/license-categories/src/generated.ts`. + This updates `scancodeCommit` and `scancodeZipSHA256`; `scancodeZipURL` is derived + from the commit, and `currentScancodeSource` holds the recorded source. +- **Pinned generation:** Ordinary generation and verification never advance the + pin. An unchanged upstream revision still regenerates against the recorded checksum. + `-verify` cannot be combined with `-update-scancode`. +- **Failures:** Network, API, archive, and unsupported-category errors stop the + update rather than substituting older data. Rendering succeeds before pins are + written. Manual overrides and exclusions remain intact; review warnings and notices. +- **One reviewable PR:** Serialized runs create or update `auto-update-licenses` + against `main`, including ScanCode-only and pin/provenance-only updates. + SPDX extraction is followed by npm regeneration. The PR includes both generated + outputs and the immutable ScanCode pin/checksum. +- **Validation:** Go tests, pinned generated-data verification, and npm tests run + before opening the PR. Its description distinguishes category payload changes + from provenance-only changes. Review the diff, warnings, overrides, and notices. +- **Dispatch:** `force_run=true` repeats SPDX extraction and validation even without + source changes. There are no automatic version bumps, merges, or npm publications. + Provenance-only updates do not trigger automatic release publication. + +GitHub may require enabling **Allow GitHub Actions to create and approve pull +requests** in repository Actions settings. PRs created using `GITHUB_TOKEN` may +not trigger other PR workflows, so the updater validates before opening its PR. + +## Validation workflow + [Validate License Categories](../../../.github/workflows/validate-license-categories.yaml) verifies committed generated data and runs npm tests on relevant pull requests and pushes to `main`. It has read-only permissions and cannot publish. The existing @@ -198,10 +233,8 @@ Retries recheck npm rather than using a prior failed or skipped run as a baselin Fix the reported problem and release or dispatch an approved corrected ref. A repeated attempt cannot reuse a published version for changed contents. -Scheduled ScanCode updates and npm regeneration in the existing fetch workflow -are separate; this layer does not change that workflow. The existing SPDX list -identifier behavior is unchanged; expanding a no-version identifier to a full SHA -is deferred. +The existing SPDX list identifier behavior is unchanged; expanding a no-version +identifier to a full SHA is deferred. ## Publishing permissions and versioning From 50de9b3c9cf4f8269c45b7c50b881234e2b521a6 Mon Sep 17 00:00:00 2001 From: "E. Lynette Rayle" Date: Wed, 11 Nov 2026 12:47:29 -0500 Subject: [PATCH 2/2] Document safe updater test fixture for CI gosec Scope G703's path-traversal exclusion to the fixed fixture write under t.TempDir. The CI-pinned golangci-lint v2.11.4 reports this controlled test path; keep production scanning unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- cmd/license-categories/update_scancode_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/license-categories/update_scancode_test.go b/cmd/license-categories/update_scancode_test.go index c3e47ad..9d10cac 100644 --- a/cmd/license-categories/update_scancode_test.go +++ b/cmd/license-categories/update_scancode_test.go @@ -114,6 +114,7 @@ func TestRunUpdateScancode(t *testing.T) { t.Run(tt.name, func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(dir, "cmd/license-categories"), 0o750)) + // #nosec G703 -- t.TempDir and the constant fixture path confine this write to the test directory. require.NoError(t, os.WriteFile(filepath.Join(dir, scancodePinPath), original, 0o600)) t.Chdir(dir) if tt.matchingChecksum {