Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 47 additions & 12 deletions .github/workflows/fetch-licenses.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,25 @@ on:
workflow_dispatch:
inputs:
force_run:
description: 'Force run license extraction'
description: 'Force SPDX extraction and license category regeneration'
type: boolean
required: false
default: 'false'
default: false
schedule:
- cron: '5 4 * * *' # Runs at 0405 UTC (0605 CET, 0005 ET, 2105 PT)

concurrency:
group: fetch-licenses
cancel-in-progress: false

jobs:
fetch-licenses:
runs-on: ubuntu-latest
env:
DT_STAMP: ''
GENFILES_CHANGED: 'false'
SRC_CHANGED: 'false'
FORCE_RUN: ${{ github.event.inputs.force_run || 'false' }}
permissions:
contents: write
pull-requests: write
Expand All @@ -30,6 +36,11 @@ jobs:
- name: Checkout this repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod

- name: Checkout official SPDX Repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand All @@ -52,32 +63,34 @@ jobs:
# --quiet: Exits with 1 if there are changes; otherwise, exits with 0
git diff --quiet -- cmd/licenses.json cmd/exceptions.json && src_changed=false
if [ $src_changed != 'true' ]; then
if [ ${{ github.event.inputs.force_run }} == 'true' ]; then
if [ "$FORCE_RUN" == 'true' ]; then
echo -e '***************\nNo changes in spdx json, but skipping abort due to force run\n***************'
else
echo -e '***************\nABORTING: No changes to spdx json data\n***************'
exit 0
echo 'No SPDX source changes; still checking ScanCode.'
fi
fi
echo "SRC_CHANGED=$src_changed" >> $GITHUB_ENV

- name: Run license extraction
if: ${{ env.SRC_CHANGED == 'true' || github.event.inputs.force_run == 'true' }}
if: ${{ env.SRC_CHANGED == 'true' || env.FORCE_RUN == 'true' }}
run: |
cd cmd
echo "Current branch: $(git branch)"
go run . extract -l -e
cd ..
git log --oneline -n 5

- name: Update ScanCode pin and generate npm license categories
run: go run ./cmd/license-categories -update-scancode -out packages/license-categories/src/generated.ts

- name: Check for changes in generated files
id: genfiles-changes
run: |
genfiles_changed=true
# --quiet: Exits with 1 if there are changes; otherwise, exits with 0
git diff --quiet -- spdxexp/spdxlicenses/*.go && genfiles_changed=false
git diff --quiet -- spdxexp/spdxlicenses/*.go cmd/license-categories/main.go packages/license-categories/src/generated.ts && genfiles_changed=false
if [ $genfiles_changed != 'true' ]; then
if [ ${{ github.event.inputs.force_run }} == 'true' ]; then
if [ "$FORCE_RUN" == 'true' ]; then
echo -e '***************\nNo changes to generated files, but skipping abort due to force run\n***************'
else
# This happens when the generated files are already up to date
Expand All @@ -88,18 +101,40 @@ jobs:
fi
echo "GENFILES_CHANGED=$genfiles_changed" >> $GITHUB_ENV

- name: Set up Node
if: ${{ env.GENFILES_CHANGED == 'true' || env.FORCE_RUN == 'true' }}
uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
with:
node-version: 24
cache: npm
cache-dependency-path: packages/license-categories/package-lock.json

- name: Validate update before opening PR
if: ${{ env.GENFILES_CHANGED == 'true' || env.FORCE_RUN == 'true' }}
run: |
go test ./...
make verify-license-categories
npm ci --prefix packages/license-categories
npm test --prefix packages/license-categories

- name: Generate Pull Request description
if: ${{ env.GENFILES_CHANGED == 'true' || github.event.inputs.force_run == 'true' }}
if: ${{ env.GENFILES_CHANGED == 'true' || env.FORCE_RUN == 'true' }}
run: go run ./cmd pr-body -o "${RUNNER_TEMP}/license-pr-body.md"

- name: Create Pull Request
if: ${{ env.GENFILES_CHANGED == 'true' || github.event.inputs.force_run == 'true' }}
if: ${{ env.GENFILES_CHANGED == 'true' || env.FORCE_RUN == 'true' }}
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: Add updated license files
commit-message: Update SPDX and ScanCode license data
branch: auto-update-licenses
base: main
title: "Update SPDX license files (${{ env.DT_STAMP }})"
title: "Update SPDX and ScanCode license data (${{ env.DT_STAMP }})"
body-path: ${{ runner.temp }}/license-pr-body.md
labels: 'auto-update,licenses'
add-paths: |
cmd/*.json
cmd/*.txt
cmd/license-categories/main.go
spdxexp/spdxlicenses/*.go
packages/license-categories/src/generated.ts
66 changes: 49 additions & 17 deletions cmd/license-categories/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ import (
)

const (
scancodeCommit = "5ee8765870cd2ed1cb345645be94a799ca584c90"
defaultScancodeZipURL = "https://codeload.github.com/aboutcode-org/scancode-licensedb/zip/" + scancodeCommit
defaultScancodeZipSHA256 = "6b4b05f985bd7a520b30e7f8547778ae4cdd6760acc577ab11cebbde41f165bb"
defaultTimeout = 60 * time.Second
maxZipBytes = 100 * 1024 * 1024 // 100MB upper bound for the scancode zip download.
maxZipEntryBytes = 1 * 1024 * 1024 // 1MB upper bound per zip entry to guard against zip bombs.
outputFileMode = 0o644 // World-readable since the generated file is manually copied into other projects.
scancodeCommit = "5ee8765870cd2ed1cb345645be94a799ca584c90"
scancodeZipURL = "https://codeload.github.com/aboutcode-org/scancode-licensedb/zip/" + scancodeCommit
scancodeZipSHA256 = "6b4b05f985bd7a520b30e7f8547778ae4cdd6760acc577ab11cebbde41f165bb"
defaultTimeout = 60 * time.Second
maxZipBytes = 100 * 1024 * 1024 // 100MB upper bound for the scancode zip download.
maxZipEntryBytes = 1 * 1024 * 1024 // 1MB upper bound per zip entry to guard against zip bombs.
outputFileMode = 0o644 // World-readable since the generated file is manually copied into other projects.
)

// excludedSpdxIDs contains SPDX IDs that should be ignored when generating license categories.
Expand Down Expand Up @@ -79,10 +79,10 @@ type scancodeSource struct {
zipSHA256 string
}

var defaultScancodeSource = scancodeSource{
var currentScancodeSource = scancodeSource{
commit: scancodeCommit,
zipURL: defaultScancodeZipURL,
zipSHA256: defaultScancodeZipSHA256,
zipURL: scancodeZipURL,
zipSHA256: scancodeZipSHA256,
}

type generationProvenance struct {
Expand All @@ -105,6 +105,7 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error {

outPath := fs.String("out", "-", "Output file path, or '-' for stdout")
verifyPath := fs.String("verify", "", "Verify an existing file, ignoring its leading go-spdx, SPDX list version, and ScanCode provenance values")
updateScancode := fs.Bool("update-scancode", false, "Update the ScanCode pin to upstream main and regenerate (run from repository root)")
err := fs.Parse(args)
if err != nil {
return err
Expand All @@ -113,13 +114,28 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error {
if *verifyPath != "" && *outPath != "-" {
return errors.New("-verify cannot be combined with -out")
}
if *verifyPath != "" && *updateScancode {
return errors.New("-verify cannot be combined with -update-scancode")
}

spdxIDs := spdxlicenses.GetLicenses()
records, err := loadScancodeBySpdx(context.Background(), client, defaultScancodeSource, maxZipBytes)
ctx := context.Background()
source := currentScancodeSource
var sourceFile []byte
var records map[string]licenseMetadata
if *updateScancode {
source, records, sourceFile, err = prepareScancodeUpdate(ctx, client, stderr)
if err != nil {
return fmt.Errorf("update scancode source: %w", err)
}
}
if records == nil {
records, err = loadScancodeBySpdx(ctx, client, source, maxZipBytes)
}
if err != nil {
return fmt.Errorf("load scancode data: %w", err)
}

spdxIDs := spdxlicenses.GetLicenses()
licenses, warnings, err := buildLicenseData(spdxIDs, records, excludedSpdxIDs, spdxCategoryOverrides)
if err != nil {
return err
Expand All @@ -137,7 +153,7 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error {
output, err := renderTypeScript(licenses, generationProvenance{
GoSPDXVersion: goSPDXVersion,
SPDXLicenseListVersion: spdxlicenses.LicenseListVersion(),
ScancodeCommit: defaultScancodeSource.commit,
ScancodeCommit: source.commit,
})
if err != nil {
return err
Expand All @@ -154,18 +170,30 @@ func run(args []string, stdout, stderr io.Writer, client *http.Client) error {

if *outPath == "-" {
_, err = stdout.Write(output)
return err
} else {
err = os.WriteFile(*outPath, output, outputFileMode)
}

err = os.WriteFile(*outPath, output, outputFileMode)
if err != nil {
return fmt.Errorf("write output: %w", err)
}
if sourceFile != nil {
if err := os.WriteFile(scancodePinPath, sourceFile, outputFileMode); err != nil {
return fmt.Errorf("write scancode pin: %w", err)
}
}
return nil
}

func loadScancodeBySpdx(ctx context.Context, client *http.Client, source scancodeSource, maxZipBytes int64) (map[string]licenseMetadata, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, source.zipURL, http.NoBody)
content, err := downloadScancodeArchive(ctx, client, source.zipURL, maxZipBytes)
if err != nil {
return nil, err
}
return parseScancodeArchive(content, source)
}

func downloadScancodeArchive(ctx context.Context, client *http.Client, url string, maxZipBytes int64) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
Expand All @@ -188,6 +216,10 @@ func loadScancodeBySpdx(ctx context.Context, client *http.Client, source scancod
if int64(len(content)) > maxZipBytes {
return nil, fmt.Errorf("zip response exceeds maximum size of %d bytes", maxZipBytes)
}
return content, nil
}

func parseScancodeArchive(content []byte, source scancodeSource) (map[string]licenseMetadata, error) {
if err := verifySHA256(content, source.zipSHA256); err != nil {
return nil, err
}
Expand Down
139 changes: 139 additions & 0 deletions cmd/license-categories/update_scancode.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package main

import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"go/ast"
"go/format"
"go/parser"
"go/token"
"io"
"net/http"
"os"
"regexp"
"strconv"
)

const (
scancodePinPath = "cmd/license-categories/main.go"
scancodeMainAPI = "https://api.github.com/repos/aboutcode-org/scancode-licensedb/commits/main"
)

var fullCommitSHA = regexp.MustCompile(`^[0-9a-f]{40}$`)

func prepareScancodeUpdate(ctx context.Context, client *http.Client, stderr io.Writer) (scancodeSource, map[string]licenseMetadata, []byte, error) {
commit, err := latestScancodeCommit(ctx, client)
if err != nil {
return scancodeSource{}, nil, nil, err
}
if commit == currentScancodeSource.commit {
fmt.Fprintln(stderr, "ScanCode pin is already current.") //nolint:errcheck,revive // status output
return currentScancodeSource, nil, nil, nil
}

source := scancodeSource{
commit: commit,
zipURL: "https://codeload.github.com/aboutcode-org/scancode-licensedb/zip/" + commit,
}
content, err := downloadScancodeArchive(ctx, client, source.zipURL, maxZipBytes)
if err != nil {
return scancodeSource{}, nil, nil, err
}
digest := sha256.Sum256(content)
source.zipSHA256 = hex.EncodeToString(digest[:])
records, err := parseScancodeArchive(content, source)
if err != nil {
return scancodeSource{}, nil, nil, err
}
pins, err := os.ReadFile(scancodePinPath)
if err != nil {
return scancodeSource{}, nil, nil, fmt.Errorf("read scancode pin: %w", err)
}
pins, err = replaceScancodePins(pins, source)
if err != nil {
return scancodeSource{}, nil, nil, err
}
fmt.Fprintf(stderr, "Updating ScanCode pin: %s -> %s\n", currentScancodeSource.commit, commit) //nolint:errcheck,revive // status output
return source, records, pins, nil
}

func latestScancodeCommit(ctx context.Context, client *http.Client) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, scancodeMainAPI, http.NoBody)
if err != nil {
return "", fmt.Errorf("create ScanCode main request: %w", err)
}
req.Header.Set("Accept", "application/vnd.github+json")
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("resolve ScanCode main: %w", err)
}
defer resp.Body.Close() //nolint:errcheck // best-effort close
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("resolve ScanCode main: unexpected status %s", resp.Status)
}
var response struct {
SHA string `json:"sha"`
}
if err := json.NewDecoder(io.LimitReader(resp.Body, maxZipEntryBytes)).Decode(&response); err != nil {
return "", fmt.Errorf("decode ScanCode main: %w", err)
}
if !fullCommitSHA.MatchString(response.SHA) {
return "", fmt.Errorf("ScanCode main returned invalid full commit SHA %q", response.SHA)
}
return response.SHA, nil
}

func replaceScancodePins(contents []byte, source scancodeSource) ([]byte, error) {
fset := token.NewFileSet()
file, err := parser.ParseFile(fset, scancodePinPath, contents, parser.ParseComments)
if err != nil {
return nil, fmt.Errorf("parse scancode pin: %w", err)
}
values := map[string]string{
"scancodeCommit": source.commit,
"scancodeZipSHA256": source.zipSHA256,
}
type replacement struct {
start, end int
text string
}
var replacements []replacement
for _, decl := range file.Decls {
gen, ok := decl.(*ast.GenDecl)
if !ok || gen.Tok != token.CONST {
continue
}
for _, spec := range gen.Specs {
value, ok := spec.(*ast.ValueSpec)
if !ok || len(value.Names) != 1 || len(value.Values) != 1 {
continue
}
name := value.Names[0].Name
text, ok := values[name]
if !ok {
continue
}
literal, ok := value.Values[0].(*ast.BasicLit)
if !ok || literal.Kind != token.STRING {
return nil, fmt.Errorf("scancode pin %s must be a string literal", name)
}
start, end := fset.Position(literal.Pos()).Offset, fset.Position(literal.End()).Offset
replacements = append(replacements, replacement{start, end, strconv.Quote(text)})
delete(values, name)
}
}
if len(values) != 0 {
return nil, fmt.Errorf("missing scancode pin constants: %v", values)
}
// Work backwards so changed literal lengths cannot shift later offsets.
for i := len(replacements) - 1; i >= 0; i-- {
edit := replacements[i]
updated := append([]byte{}, contents[:edit.start]...)
updated = append(updated, edit.text...)
contents = append(updated, contents[edit.end:]...)
}
return format.Source(contents)
}
Loading
Loading