Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions .github/workflows/publish-license-categories.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ jobs:
run: npm ci
working-directory: packages/license-categories

- name: Stamp publication provenance
run: node script/stamp-provenance.mjs
working-directory: packages/license-categories

- name: Build and test committed package source
run: npm test
working-directory: packages/license-categories
Expand All @@ -65,7 +69,5 @@ jobs:

- name: Publish compared tarball using trusted publishing
if: steps.prepare.outputs.changed == 'true'
run: npm publish "$RUNNER_TEMP/license-categories.tgz" --ignore-scripts --access public --registry=https://registry.npmjs.org
run: npm publish "$RUNNER_TEMP/license-categories.tgz" --ignore-scripts --access public --provenance --registry=https://registry.npmjs.org
Comment thread
elrayle marked this conversation as resolved.
Resolved
working-directory: packages/license-categories
env:
NPM_CONFIG_PROVENANCE: 'false'
5 changes: 1 addition & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,7 @@ license-categories:

.PHONY: verify-license-categories
verify-license-categories:
@tmp=$$(mktemp); \
trap 'rm -f "$$tmp"' 0; \
go run ./cmd/license-categories -out "$$tmp" && \
diff -u packages/license-categories/src/generated.ts "$$tmp"
go run ./cmd/license-categories -verify packages/license-categories/src/generated.ts

.PHONY: test-license-categories
test-license-categories:
Expand Down
92 changes: 91 additions & 1 deletion cmd/license-categories/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ import (
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"runtime/debug"
"slices"
"strings"
Expand Down Expand Up @@ -101,11 +104,16 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error {
fs.SetOutput(stderr)

outPath := fs.String("out", "-", "Output file path, or '-' for stdout")
verifyPath := fs.String("verify", "", "Verify an existing file, ignoring only its leading go-spdx provenance value")
err := fs.Parse(args)
if err != nil {
return err
}

if *verifyPath != "" && *outPath != "-" {
return errors.New("-verify cannot be combined with -out")
}

spdxIDs := spdxlicenses.GetLicenses()
records, err := loadScancodeBySpdx(context.Background(), client, defaultScancodeSource, maxZipBytes)
if err != nil {
Expand Down Expand Up @@ -135,6 +143,15 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error {
return err
}

if *verifyPath != "" {
// #nosec G304 -- the caller explicitly selects the generated file to verify.
expected, err := os.ReadFile(*verifyPath)
if err != nil {
return fmt.Errorf("read generated file for verification: %w", err)
}
return verifyGeneratedData(expected, output)
}

if *outPath == "-" {
_, err = stdout.Write(output)
return err
Expand Down Expand Up @@ -240,7 +257,80 @@ func moduleVersion() (string, error) {
if !ok {
return "", errors.New("read Go build information")
}
return buildInfo.Main.Version, nil
if version := buildInfo.Main.Version; version != "" && version != "(devel)" {
return version, nil
}
Comment on lines +260 to +262
_, source, _, ok := runtime.Caller(0)
if !ok {
return "", errors.New("locate generator source checkout for local provenance")
}
return localCheckoutVersion(source)
}

func localCheckoutVersion(source string) (string, error) {
if !filepath.IsAbs(source) {
return "", errors.New("local provenance requires an absolute generator source path; build without -trimpath or use a versioned installation")
}
directory := filepath.Dir(source)
rootCommand := exec.Command("git", "rev-parse", "--show-toplevel")
rootCommand.Dir = directory
root, err := rootCommand.Output()
if err != nil {
return "", fmt.Errorf("locate generator source Git checkout (use a checkout or versioned installation): %w", err)
}
expectedDirectory := filepath.Join(strings.TrimSpace(string(root)), "cmd", "license-categories")
resolvedDirectory, err := filepath.EvalSymlinks(directory)
if err != nil {
return "", fmt.Errorf("resolve generator source directory: %w", err)
}
if resolvedDirectory != expectedDirectory {
return "", errors.New("generator source is not in the expected go-spdx checkout directory")
}
revisionCommand := exec.Command("git", "rev-parse", "--verify", "HEAD")
revisionCommand.Dir = directory
revision, err := revisionCommand.Output()
if err != nil {
return "", fmt.Errorf("read generator checkout HEAD: %w", err)
}
sha := strings.TrimSpace(string(revision))
decoded, err := hex.DecodeString(sha)
if err != nil || len(decoded) != 20 {
return "", errors.New("generator checkout HEAD is not a full 40-character commit SHA")
}
return sha + " (local)", nil
}

func verifyGeneratedData(expected, actual []byte) error {
normalize := func(content []byte) ([]byte, error) {
lines := bytes.Split(content, []byte("\n"))
if len(lines) < 4 ||
string(lines[0]) != "// Code generated by cmd/license-categories in go-spdx. DO NOT EDIT." {
return nil, errors.New("generated data must have the expected leading generator header")
}
for i, prefix := range []string{"// go-spdx: ", "// spdx-license-list-version: ", "// scancode-licensedb commit: "} {
if bytes.ContainsAny(lines[i+1], "\r\u2028\u2029") {
return nil, fmt.Errorf("generated data provenance header on line %d must not contain TypeScript comment terminators", i+2)
}
if !bytes.HasPrefix(lines[i+1], []byte(prefix)) ||
len(bytes.TrimSpace(lines[i+1][len(prefix):])) == 0 {
return nil, fmt.Errorf("generated data must have a nonempty %q provenance header on line %d", prefix, i+2)
}
}
lines[1] = []byte("// go-spdx: <provenance>")
Comment thread
elrayle marked this conversation as resolved.
Resolved
return bytes.Join(lines, []byte("\n")), nil
}
normalizedExpected, err := normalize(expected)
if err != nil {
return err
}
normalizedActual, err := normalize(actual)
if err != nil {
return err
}
if !bytes.Equal(normalizedExpected, normalizedActual) {
return errors.New("generated license categories differ outside the leading go-spdx provenance value; run make license-categories and review the changes")
}
return nil
}

func scancodeGitHubURL(zipPath, commit string) string {
Expand Down
160 changes: 160 additions & 0 deletions cmd/license-categories/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,175 @@ package main

import (
"context"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime/debug"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestModuleVersion(t *testing.T) {
t.Parallel()

info, ok := debug.ReadBuildInfo()
require.True(t, ok)
got, err := moduleVersion()
require.NoError(t, err)
if info.Main.Version != "" && info.Main.Version != "(devel)" {
assert.Equal(t, info.Main.Version, got)
} else {
assert.Regexp(t, `^[a-f0-9]{40} \(local\)$`, got)
revision, err := exec.Command("git", "rev-parse", "HEAD").Output()
require.NoError(t, err)
assert.Equal(t, strings.TrimSpace(string(revision))+" (local)", got)
}
}

func TestLocalCheckoutVersionAndVerificationAcrossCommits(t *testing.T) {
t.Parallel()

root := t.TempDir()
source := filepath.Join(root, "cmd", "license-categories", "main.go")
require.NoError(t, os.MkdirAll(filepath.Dir(source), 0o750))
require.NoError(t, os.WriteFile(source, []byte("package main\n"), 0o600))
git := func(args ...string) string {
// #nosec G204 -- arguments are fixed test commands scoped to t.TempDir.
command := exec.Command("git", append([]string{"-C", root}, args...)...)
output, err := command.CombinedOutput()
require.NoError(t, err, string(output))
return strings.TrimSpace(string(output))
}
git("init", "--quiet")
git("add", ".")
commit := func() {
git("-c", "user.name=Test", "-c", "user.email=test@example.com",
"-c", "commit.gpgsign=false", "commit", "--quiet", "--allow-empty", "-m", "Test provenance")
}
commit()
first, err := localCheckoutVersion(source)
require.NoError(t, err)
assert.Equal(t, git("rev-parse", "HEAD")+" (local)", first)
commit()
second, err := localCheckoutVersion(source)
require.NoError(t, err)
assert.NotEqual(t, first, second)
assert.Regexp(t, `^[a-f0-9]{40} \(local\)$`, second)

render := func(version string) []byte {
output, err := renderTypeScript(map[string]licenseMetadata{"MIT": {Name: "MIT License", Category: "Permissive"}},
generationProvenance{GoSPDXVersion: version, SPDXLicenseListVersion: "3.29.0", ScancodeCommit: "abc123"})
require.NoError(t, err)
return output
}
require.NoError(t, verifyGeneratedData(render(first), render(second)))
require.NoError(t, verifyGeneratedData(render("(devel)"), render(second)))
}

func TestLocalCheckoutVersionErrors(t *testing.T) {
t.Parallel()

_, err := localCheckoutVersion("cmd/license-categories/main.go")
require.ErrorContains(t, err, "absolute generator source path")
_, err = localCheckoutVersion(filepath.Join(t.TempDir(), "main.go"))
require.ErrorContains(t, err, "Git checkout")
}

func TestVerifyGeneratedDataIgnoresOnlyGoSPDXProvenance(t *testing.T) {
t.Parallel()

render := func(version string) []byte {
output, err := renderTypeScript(map[string]licenseMetadata{"MIT": {Name: "MIT License", Category: "Permissive"}},
generationProvenance{GoSPDXVersion: version, SPDXLicenseListVersion: "3.29.0", ScancodeCommit: "abc123"})
require.NoError(t, err)
return output
}
output := render(strings.Repeat("a", 40) + " (local)")
for _, version := range []string{"(devel)", "v2.9.0 (release)", strings.Repeat("b", 40) + " (dispatch)"} {
actual := render(version)
require.NoError(t, verifyGeneratedData(output, actual))
require.NoError(t, verifyGeneratedData(actual, output))
}
}

func TestVerifyGeneratedDataRejectsOtherDifferences(t *testing.T) {
t.Parallel()

provenance := generationProvenance{GoSPDXVersion: strings.Repeat("a", 40) + " (local)",
SPDXLicenseListVersion: "3.29.0", ScancodeCommit: "abc123"}
output, err := renderTypeScript(map[string]licenseMetadata{"MIT": {Name: "MIT License", Category: "Permissive"}}, provenance)
require.NoError(t, err)
for _, change := range [][2]string{
{"MIT License", "changed-name"},
{"'MIT':", "'changed-ID':"},
{"Permissive", "Copyleft"},
{"Safe for most", "changed-description"},
{"// Code generated", "// Different owner"},
{"// go-spdx: ", "// wrong-header: "},
{"// spdx-license-list-version: ", "// wrong-header: "},
{"// scancode-licensedb commit: ", "// wrong-header: "},
{provenance.GoSPDXVersion, ""},
{"3.29.0", ""},
{"3.29.0", "3.30.0"},
{"abc123", "def456"},
{"abc123", " "},
{"// go-spdx: ", "// Extra comment\n// go-spdx: "},
{"// spdx-license-list-version: ", "// Extra comment\n// spdx-license-list-version: "},
{"// scancode-licensedb commit: ", "// Extra comment\n// scancode-licensedb commit: "},
{"// scancode-licensedb commit: abc123\n", ""},
{"export const", "// Extra comment\nexport const"},
{"export const", "// go-spdx: not leading provenance\nexport const"},
} {
changed := []byte(strings.Replace(string(output), change[0], change[1], 1))
require.NotEqual(t, output, changed, change)
require.Error(t, verifyGeneratedData(output, changed), change)
require.Error(t, verifyGeneratedData(changed, output), change)
}
require.Error(t, verifyGeneratedData([]byte("no headers"), output))
}

func TestVerifyGeneratedDataRejectsCommentTerminatorsInProvenance(t *testing.T) {
t.Parallel()

provenance := generationProvenance{GoSPDXVersion: "v2.9.0 (release)",
SPDXLicenseListVersion: "3.29.0", ScancodeCommit: "abc123"}
output, err := renderTypeScript(map[string]licenseMetadata{"MIT": {Name: "MIT License", Category: "Permissive"}}, provenance)
require.NoError(t, err)

for _, terminator := range []struct {
name string
value string
}{
{"CR", "\r"},
{"LF", "\n"},
{"line separator", "\u2028"},
{"paragraph separator", "\u2029"},
} {
t.Run(terminator.name, func(t *testing.T) {
t.Parallel()

changed := []byte(strings.Replace(string(output), provenance.GoSPDXVersion,
provenance.GoSPDXVersion+terminator.value+"export const injected = true;", 1))
require.Error(t, verifyGeneratedData(output, changed))
require.Error(t, verifyGeneratedData(changed, output))
require.Error(t, verifyGeneratedData(changed, changed))
})
}
}

func TestRunRejectsConflictingOutputAndVerification(t *testing.T) {
t.Parallel()

err := run([]string{"-verify", "generated.ts", "-out", "other.ts"}, io.Discard, io.Discard, nil)
require.ErrorContains(t, err, "-verify cannot be combined with -out")
}

func TestBuildLicenseData_ExclusionsAndUnknown(t *testing.T) {
t.Parallel()

Expand Down
Loading
Loading