Skip to content

Commit bdd84c1

Browse files
committed
[ci] Combine visual regression reports across per-platform runs
1 parent 93d13f2 commit bdd84c1

7 files changed

Lines changed: 181 additions & 52 deletions

File tree

‎.github/workflow-scripts/__tests__/visual-compare-test.js‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
const {compareCapture, compareImages, policyFor} = require('../visual/compare');
1212
const {
1313
comparePlatform,
14+
platformsForWorkflow,
1415
resolveContext,
1516
stageBaseline,
1617
} = require('../visual/compare-run');
@@ -389,6 +390,17 @@ describe('visual comparison', () => {
389390
});
390391
});
391392

393+
describe('platformsForWorkflow', () => {
394+
test.each([
395+
['Test Android', ['android']],
396+
['Test iOS', ['ios']],
397+
['Test All', ['android', 'ios']],
398+
[undefined, ['android', 'ios']],
399+
])('%s captures %j', (workflowName, platforms) => {
400+
expect(platformsForWorkflow(workflowName)).toEqual(platforms);
401+
});
402+
});
403+
392404
describe('resolveContext', () => {
393405
const ENV = {
394406
CAPTURE_HEAD_SHA: SHA_A,

‎.github/workflow-scripts/__tests__/visual-report-test.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ const {
2323
baselineImageKey,
2424
baselinePrefix,
2525
blobKey,
26+
newestPerPlatform,
2627
publicUrl,
2728
reportPrefix,
2829
} = require('../visual/storage');
@@ -88,6 +89,35 @@ describe('visual reports and approval', () => {
8889
'https://cdn.example/bucket/a%20b/c%23d.png',
8990
);
9091
});
92+
93+
test('keeps the newest comparison of each platform across runs', () => {
94+
const platforms = newestPerPlatform([
95+
{
96+
reportPrefix: 'reports/pull/12/x/36-1',
97+
platforms: [{platform: 'ios', results: ['ios, rerun']}],
98+
},
99+
{
100+
reportPrefix: 'reports/pull/12/x/35-1',
101+
platforms: [{platform: 'android', results: ['android']}],
102+
},
103+
{
104+
reportPrefix: 'reports/pull/12/x/34-1',
105+
platforms: [{platform: 'ios', results: ['ios, first run']}],
106+
},
107+
]);
108+
expect(platforms).toEqual([
109+
{
110+
platform: 'android',
111+
results: ['android'],
112+
reportPrefix: 'reports/pull/12/x/35-1',
113+
},
114+
{
115+
platform: 'ios',
116+
results: ['ios, rerun'],
117+
reportPrefix: 'reports/pull/12/x/36-1',
118+
},
119+
]);
120+
});
91121
});
92122

93123
describe('approval', () => {

‎.github/workflow-scripts/visual/approve-run.js‎

Lines changed: 5 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,14 @@ const {
2323
APPROVAL_LABEL,
2424
APPROVER_PERMISSIONS,
2525
GATE_CHECK_NAME,
26+
differenceFingerprint,
2627
gateDecision,
2728
} = require('./approval');
2829
const {createGitHubClient} = require('./github');
2930
const {
3031
approvalKey,
3132
fetchJsonOrNull,
32-
listPrefixes,
33+
latestPlatformResults,
3334
publicUrl,
3435
} = require('./storage');
3536
const fs = require('node:fs');
@@ -43,26 +44,6 @@ function requireEnv(name) {
4344
return value;
4445
}
4546

46-
/** The results of the most recent comparison for a PR head, or null. */
47-
async function latestResults({bucket, publicBaseUrl, prNumber, headSha}) {
48-
const runs = await listPrefixes(
49-
bucket,
50-
`reports/pull/${prNumber}/${headSha}/`,
51-
);
52-
const latest = runs
53-
.map(prefix => {
54-
const match = /\/(\d+)-(\d+)\/$/.exec(prefix);
55-
return match == null
56-
? null
57-
: {prefix, runId: Number(match[1]), attempt: Number(match[2])};
58-
})
59-
.filter(Boolean)
60-
.sort((a, b) => b.runId - a.runId || b.attempt - a.attempt)[0];
61-
return latest == null
62-
? null
63-
: fetchJsonOrNull(publicUrl(publicBaseUrl, `${latest.prefix}results.json`));
64-
}
65-
6647
async function prepare() {
6748
const github = createGitHubClient();
6849
const outDir = requireEnv('OUT_DIR');
@@ -83,13 +64,13 @@ async function prepare() {
8364

8465
const prNumber = event.pull_request.number;
8566
const headSha = event.pull_request.head.sha;
86-
const results = await latestResults({
67+
const platforms = await latestPlatformResults({
8768
bucket: requireEnv('VISUAL_BUCKET'),
8869
publicBaseUrl: requireEnv('VISUAL_PUBLIC_BASE_URL'),
8970
prNumber,
9071
headSha,
9172
});
92-
const fingerprint = results?.fingerprint ?? null;
73+
const fingerprint = differenceFingerprint(platforms);
9374

9475
if (event.action === 'labeled') {
9576
const approver = event.sender.login;
@@ -103,7 +84,7 @@ async function prepare() {
10384
writeState(null);
10485
return;
10586
}
106-
if (results == null) {
87+
if (platforms.length === 0) {
10788
await github.removeLabel(prNumber, APPROVAL_LABEL);
10889
await github.comment(
10990
prNumber,
@@ -179,5 +160,3 @@ if (require.main === module) {
179160
process.exitCode = 1;
180161
});
181162
}
182-
183-
module.exports = {latestResults};

‎.github/workflow-scripts/visual/compare-run.js‎

Lines changed: 70 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ const {
4242
fetchBuffer,
4343
fetchJsonOrNull,
4444
findBaseline,
45+
latestPlatformResults,
4546
publicUrl,
4647
reportPrefix,
4748
} = require('./storage');
@@ -50,6 +51,10 @@ const fs = require('node:fs');
5051
const path = require('node:path');
5152

5253
const PLATFORMS = ['android', 'ios'];
54+
const WORKFLOW_PLATFORMS = {
55+
'Test Android': ['android'],
56+
'Test iOS': ['ios'],
57+
};
5358
// The default Actions token posts as this bot. Only its comments are edited.
5459
const COMMENT_AUTHOR = 'github-actions[bot]';
5560

@@ -61,6 +66,10 @@ function requireEnv(name) {
6166
return value;
6267
}
6368

69+
function platformsForWorkflow(workflowName) {
70+
return WORKFLOW_PLATFORMS[workflowName] ?? PLATFORMS;
71+
}
72+
6473
function writeFile(filePath, contents) {
6574
fs.mkdirSync(path.dirname(filePath), {recursive: true});
6675
fs.writeFileSync(filePath, contents);
@@ -231,7 +240,7 @@ function missingCapture(platform, context) {
231240
results: [],
232241
error: {
233242
title: 'no capture',
234-
summary: `Test All run ${context.runId} has no ${platform} capture, so there is nothing to compare. Its capture job failed or didn't run: check the run for a build or capture failure.`,
243+
summary: `Run ${context.runId} has no ${platform} capture, so there is nothing to compare. Its capture job failed or didn't run: check the run for a build or capture failure.`,
235244
},
236245
};
237246
}
@@ -299,7 +308,9 @@ async function prepare() {
299308
const prefix = reportPrefix(context);
300309

301310
const platforms = [];
302-
for (const expectedPlatform of PLATFORMS) {
311+
for (const expectedPlatform of platformsForWorkflow(
312+
process.env.CAPTURE_WORKFLOW,
313+
)) {
303314
const captureDir = path.join(
304315
artifactsDir,
305316
`visual-capture-${expectedPlatform}`,
@@ -367,6 +378,27 @@ async function prepare() {
367378
}
368379
}
369380

381+
function imageUrlFor(publicBaseUrl, platform, prefix) {
382+
const expectedById = new Map(
383+
platform.results
384+
.filter(result => result.expected != null)
385+
.map(result => [result.id, result.expected]),
386+
);
387+
return (kind, id) =>
388+
kind === 'expected'
389+
? publicUrl(
390+
publicBaseUrl,
391+
baselineImageKey(
392+
{environmentKey: platform.environmentKey, ...platform.baseline},
393+
expectedById.get(id),
394+
),
395+
)
396+
: publicUrl(
397+
publicBaseUrl,
398+
`${prefix}/${platform.platform}/${kind}/${id}.png`,
399+
);
400+
}
401+
370402
async function report() {
371403
const github = createGitHubClient();
372404
const outDir = requireEnv('OUT_DIR');
@@ -377,30 +409,13 @@ async function report() {
377409
if (state == null) {
378410
return;
379411
}
380-
const {context, reportPrefix: prefix, fingerprint} = state;
412+
const {context, reportPrefix: prefix} = state;
381413
const detailsUrl = publicUrl(publicBaseUrl, `${prefix}/results.json`);
382414

383415
const summaryLines = [];
384-
const commentPlatforms = [];
416+
const checkUrls = new Map();
385417
for (const platform of state.platforms) {
386-
const expectedById = new Map(
387-
platform.results
388-
.filter(result => result.expected != null)
389-
.map(result => [result.id, result.expected]),
390-
);
391-
const imageUrl = (kind, id) =>
392-
kind === 'expected'
393-
? publicUrl(
394-
publicBaseUrl,
395-
baselineImageKey(
396-
{environmentKey: platform.environmentKey, ...platform.baseline},
397-
expectedById.get(id),
398-
),
399-
)
400-
: publicUrl(
401-
publicBaseUrl,
402-
`${prefix}/${platform.platform}/${kind}/${id}.png`,
403-
);
418+
const imageUrl = imageUrlFor(publicBaseUrl, platform, prefix);
404419
const output = renderCheckOutput({...platform, imageUrl});
405420
const {hasDifferences} = summarize(platform.results);
406421
const checkRun = await github.createCheckRun({
@@ -415,7 +430,7 @@ async function report() {
415430
output,
416431
detailsUrl,
417432
});
418-
commentPlatforms.push({...platform, imageUrl, checkUrl: checkRun.html_url});
433+
checkUrls.set(platform.platform, checkRun.html_url);
419434
summaryLines.push(
420435
`## ${output.title}`,
421436
'',
@@ -427,6 +442,32 @@ async function report() {
427442
}
428443

429444
if (context.kind === 'pull') {
445+
const platforms = await latestPlatformResults({
446+
bucket: requireEnv('VISUAL_BUCKET'),
447+
publicBaseUrl,
448+
prNumber: context.prNumber,
449+
headSha: context.headSha,
450+
current: state,
451+
});
452+
const fingerprint = differenceFingerprint(platforms);
453+
const commentPlatforms = [];
454+
for (const platform of platforms) {
455+
const checkUrl =
456+
checkUrls.get(platform.platform) ??
457+
(
458+
await github.findCheckRun(
459+
context.headSha,
460+
`${CHECK_NAME_PREFIX} (${platform.platform})`,
461+
)
462+
)?.html_url ??
463+
publicUrl(publicBaseUrl, `${platform.reportPrefix}/results.json`);
464+
commentPlatforms.push({
465+
...platform,
466+
imageUrl: imageUrlFor(publicBaseUrl, platform, platform.reportPrefix),
467+
checkUrl,
468+
});
469+
}
470+
430471
const labels = await github.labels(context.prNumber);
431472
const approval =
432473
fingerprint == null
@@ -511,4 +552,9 @@ if (require.main === module) {
511552
});
512553
}
513554

514-
module.exports = {comparePlatform, resolveContext, stageBaseline};
555+
module.exports = {
556+
comparePlatform,
557+
platformsForWorkflow,
558+
resolveContext,
559+
stageBaseline,
560+
};

‎.github/workflow-scripts/visual/github.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,14 @@ function createGitHubClient({
130130
return result?.permission ?? 'none';
131131
},
132132

133+
async findCheckRun(headSha, name) {
134+
const result = await request(
135+
'GET',
136+
`/commits/${headSha}/check-runs?check_name=${encodeURIComponent(name)}&filter=latest`,
137+
);
138+
return result?.check_runs?.[0] ?? null;
139+
},
140+
133141
createCheckRun({name, headSha, conclusion, output, detailsUrl}) {
134142
return request('POST', '/check-runs', {
135143
name,

‎.github/workflow-scripts/visual/storage.js‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,56 @@ async function listPrefixes(bucket, prefix) {
144144
return prefixes;
145145
}
146146

147+
function newestPerPlatform(statesNewestFirst) {
148+
const byPlatform = new Map();
149+
for (const state of statesNewestFirst) {
150+
for (const platform of state.platforms) {
151+
if (!byPlatform.has(platform.platform)) {
152+
byPlatform.set(platform.platform, {
153+
...platform,
154+
reportPrefix: state.reportPrefix,
155+
});
156+
}
157+
}
158+
}
159+
return [...byPlatform.values()].sort((a, b) =>
160+
a.platform.localeCompare(b.platform),
161+
);
162+
}
163+
164+
async function latestPlatformResults({
165+
bucket,
166+
publicBaseUrl,
167+
prNumber,
168+
headSha,
169+
current = null,
170+
}) {
171+
const runs = (
172+
await listPrefixes(bucket, `reports/pull/${prNumber}/${headSha}/`)
173+
)
174+
.map(prefix => {
175+
const match = /\/(\d+)-(\d+)\/$/.exec(prefix);
176+
return match == null
177+
? null
178+
: {prefix, runId: Number(match[1]), attempt: Number(match[2])};
179+
})
180+
.filter(Boolean)
181+
.sort((a, b) => b.runId - a.runId || b.attempt - a.attempt);
182+
const states = current == null ? [] : [current];
183+
for (const run of runs) {
184+
if (current != null && run.prefix === `${current.reportPrefix}/`) {
185+
continue;
186+
}
187+
const state = await fetchJsonOrNull(
188+
publicUrl(publicBaseUrl, `${run.prefix}results.json`),
189+
);
190+
if (state != null) {
191+
states.push(state);
192+
}
193+
}
194+
return newestPerPlatform(states);
195+
}
196+
147197
/**
148198
* Returns the first of `candidateShas` (newest first) that has a published
149199
* baseline for `environmentKey`, with its manifest, or null.
@@ -173,7 +223,9 @@ module.exports = {
173223
fetchBuffer,
174224
fetchJsonOrNull,
175225
findBaseline,
226+
latestPlatformResults,
176227
listPrefixes,
228+
newestPerPlatform,
177229
publicUrl,
178230
reportPrefix,
179231
};

0 commit comments

Comments
 (0)