diff --git a/.github/workflow-scripts/__tests__/__fixtures__/visual.js b/.github/workflow-scripts/__tests__/__fixtures__/visual.js new file mode 100644 index 000000000000..0d07ae0fcf66 --- /dev/null +++ b/.github/workflow-scripts/__tests__/__fixtures__/visual.js @@ -0,0 +1,27 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const SHA_A = 'a'.repeat(40); +const SHA_B = 'b'.repeat(40); + +const ANDROID_ENVIRONMENT = { + apiLevel: 35, + abi: 'x86_64', + nativeBridge: 'libndk_translation.so', + systemImage: + 'google/sdk_gphone64_x86_64/emu64xa:15/AE3A/1:userdebug/dev-keys', + screenSize: '1080x2400', + density: 420, + fontScale: '1.0', + locale: 'en-US', + animatorDurationScale: '0.0', + emulatorVersion: '37.2.12', +}; + +module.exports = {ANDROID_ENVIRONMENT, SHA_A, SHA_B}; diff --git a/.github/workflow-scripts/__tests__/visual-capture-test.js b/.github/workflow-scripts/__tests__/visual-capture-test.js new file mode 100644 index 000000000000..a5c870d5ce0e --- /dev/null +++ b/.github/workflow-scripts/__tests__/visual-capture-test.js @@ -0,0 +1,145 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const { + collectFlows, + failedFlowsFromJUnit, + maestroArgs, +} = require('../visual/capture'); +const {environmentKey} = require('../visual/manifest'); +const {ANDROID_ENVIRONMENT} = require('./__fixtures__/visual'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +describe('visual capture', () => { + let directory; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-test-')); + }); + + afterEach(() => { + fs.rmSync(directory, {recursive: true, force: true}); + }); + + describe('environmentKey', () => { + test('is readable and stable', () => { + expect(environmentKey('android', ANDROID_ENVIRONMENT)).toMatch( + /^android-api35-x86-64-1080x2400-420dpi-[0-9a-f]{8}$/, + ); + expect(environmentKey('android', {...ANDROID_ENVIRONMENT})).toBe( + environmentKey('android', ANDROID_ENVIRONMENT), + ); + }); + + test.each([ + ['fontScale', '1.15'], + ['emulatorVersion', '37.3.1'], + ])('changes when %s drifts', (property, value) => { + expect( + environmentKey('android', {...ANDROID_ENVIRONMENT, [property]: value}), + ).not.toBe(environmentKey('android', ANDROID_ENVIRONMENT)); + }); + + test('ignores properties that are only recorded', () => { + expect( + environmentKey('ios', { + runtime: 'iOS-26-2', + deviceType: 'iPhone-17-Pro', + appearance: 'light', + contentSizeCategory: 'large', + locale: 'en_US', + xcode: 'Xcode 26.2', + }), + ).toBe( + environmentKey('ios', { + runtime: 'iOS-26-2', + deviceType: 'iPhone-17-Pro', + appearance: 'light', + contentSizeCategory: 'large', + locale: 'en_US', + xcode: 'Xcode 26.3', + }), + ); + }); + test('names iOS keys by runtime and device', () => { + expect( + environmentKey('ios', { + runtime: 'iOS-26-2', + deviceType: 'iPhone-17-Pro', + }), + ).toMatch(/^ios-26-2-iphone-17-pro-[0-9a-f]{8}$/); + }); + }); + + describe('capture', () => { + test('collects top-level flows in order and ignores other files', () => { + fs.writeFileSync(path.join(directory, 'b.yml'), ''); + fs.writeFileSync(path.join(directory, 'a.yaml'), ''); + fs.writeFileSync(path.join(directory, 'notes.md'), ''); + fs.mkdirSync(path.join(directory, 'helpers')); + fs.writeFileSync(path.join(directory, 'helpers/open.yml'), ''); + expect(collectFlows(directory)).toEqual([ + path.join(directory, 'a.yaml'), + path.join(directory, 'b.yml'), + ]); + }); + + test('rejects flows Maestro would report under the same name', () => { + fs.writeFileSync(path.join(directory, 'a.yml'), ''); + fs.writeFileSync(path.join(directory, 'a.yaml'), ''); + expect(() => collectFlows(directory)).toThrow("both report as 'a'"); + }); + + test('reads failed flows from a Maestro JUnit report', () => { + const flows = ['/flows/text-a.yml', '/flows/text-b.yml', '/flows/c.yml']; + const report = [ + '', + '', + '', + ].join('\n'); + expect(failedFlowsFromJUnit(report, flows)).toEqual([ + '/flows/text-b.yml', + '/flows/c.yml', + ]); + expect(() => + failedFlowsFromJUnit('', flows), + ).toThrow("unknown flow 'other'"); + expect(failedFlowsFromJUnit('', flows)).toEqual([]); + }); + + test.each([ + [1, undefined, false], + [2, 'UDID', true], + ])('builds Maestro arguments for attempt %d', (attempt, udid, reuse) => { + const args = maestroArgs({ + flows: ['/f/a.yml', '/f/b.yml'], + appId: 'com.example', + outputDir: '/out', + reportPath: '/r.xml', + udid, + attempt, + }); + expect( + args.slice(args.indexOf('test'), args.indexOf('test') + 3), + ).toEqual(['test', '/f/a.yml', '/f/b.yml']); + expect(args).toEqual( + expect.arrayContaining([ + '--format=junit', + '--output=/r.xml', + 'APP_ID=com.example', + 'VISUAL_OUTPUT_DIR=/out', + ]), + ); + expect(args.includes('--no-reinstall-driver')).toBe(reuse); + expect(args.includes('--udid=UDID')).toBe(udid != null); + }); + }); +}); diff --git a/.github/workflow-scripts/__tests__/visual-compare-test.js b/.github/workflow-scripts/__tests__/visual-compare-test.js new file mode 100644 index 000000000000..170673ca4a96 --- /dev/null +++ b/.github/workflow-scripts/__tests__/visual-compare-test.js @@ -0,0 +1,261 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {compareCapture, compareImages, policyFor} = require('../visual/compare'); +const {buildManifest, environmentKey, sha256} = require('../visual/manifest'); +const {readPngDimensions} = require('../visual/png'); +const {validateCapture} = require('../visual/validate'); +const {ANDROID_ENVIRONMENT, SHA_A, SHA_B} = require('./__fixtures__/visual'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const {PNG} = require('pngjs'); + +function makePng(width, height, paint = () => [255, 255, 255, 255]) { + const png = new PNG({width, height}); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const offset = (y * width + x) * 4; + const [r, g, b, a] = paint(x, y); + png.data[offset] = r; + png.data[offset + 1] = g; + png.data[offset + 2] = b; + png.data[offset + 3] = a; + } + } + return PNG.sync.write(png); +} + +function writeCapture(directory, images, overrides = {}) { + for (const [id, buffer] of Object.entries(images)) { + const filePath = path.join(directory, `${id}.png`); + fs.mkdirSync(path.dirname(filePath), {recursive: true}); + fs.writeFileSync(filePath, buffer); + } + const manifest = buildManifest({ + captureDir: directory, + platform: 'android', + environment: ANDROID_ENVIRONMENT, + git: {testedSha: SHA_A, baseSha: SHA_B, headSha: SHA_A}, + }); + const finalManifest = {...manifest, ...overrides}; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(finalManifest), + ); + return finalManifest; +} + +describe('visual comparison', () => { + let directory; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-test-')); + }); + + afterEach(() => { + fs.rmSync(directory, {recursive: true, force: true}); + }); + + describe('readPngDimensions', () => { + test('reads the IHDR dimensions', () => { + expect(readPngDimensions(makePng(3, 2))).toEqual({width: 3, height: 2}); + }); + + test.each([ + [Buffer.from('not a png at all, but long enough')], + [Buffer.alloc(0)], + ])('rejects non-PNG data', buffer => { + expect(readPngDimensions(buffer)).toBeNull(); + }); + }); + + describe('validateCapture', () => { + test('accepts a well-formed capture', () => { + const manifest = writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + }); + expect(validateCapture(directory)).toEqual(manifest); + }); + + test.each([ + ['../escape/a/b'], + ['/absolute/a/b'], + ['Text/Upper/case'], + ['too/few'], + ['a/b/c/d'], + ])('rejects test ID %s', id => { + writeCapture(directory, {'text/width-mode/default': makePng(4, 4)}); + const manifest = JSON.parse( + fs.readFileSync(path.join(directory, 'manifest.json'), 'utf8'), + ); + manifest.images[0] = {...manifest.images[0], id, path: `${id}.png`}; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(manifest), + ); + expect(() => validateCapture(directory)).toThrow('Invalid test ID'); + }); + + test('rejects a path that does not match the ID', () => { + const manifest = writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + }); + manifest.images[0].path = '../../etc/passwd'; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(manifest), + ); + expect(() => validateCapture(directory)).toThrow('path must be'); + }); + + test('rejects duplicates, mismatched hashes and unlisted files', () => { + const manifest = writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + }); + manifest.images.push({...manifest.images[0]}); + manifest.images[0] = {...manifest.images[0], sha256: '0'.repeat(64)}; + fs.writeFileSync( + path.join(directory, 'manifest.json'), + JSON.stringify(manifest), + ); + fs.writeFileSync(path.join(directory, 'extra.sh'), 'echo hi'); + let message = ''; + try { + validateCapture(directory); + } catch (error) { + message = error.message; + } + expect(message).toContain('Duplicate test ID'); + expect(message).toContain('does not match its manifest entry'); + expect(message).toContain('Files not in the manifest: extra.sh'); + }); + + test('rejects non-PNG content and oversized images', () => { + writeCapture(directory, { + 'text/width-mode/default': makePng(4, 4), + 'text/width-mode/large': makePng(20, 4), + }); + fs.writeFileSync( + path.join(directory, 'text/width-mode/default.png'), + 'not a png', + ); + expect(() => + validateCapture(directory, { + maxImages: 10, + maxImageBytes: 1024 * 1024, + maxTotalBytes: 1024 * 1024, + maxDimension: 10, + maxManifestBytes: 1024 * 1024, + }), + ).toThrow(/default: not a PNG[\s\S]*large: 20x4 is outside/); + }); + + test('rejects symlinks', () => { + writeCapture(directory, {'text/width-mode/default': makePng(4, 4)}); + fs.rmSync(path.join(directory, 'text/width-mode/default.png')); + fs.symlinkSync( + '/etc/hosts', + path.join(directory, 'text/width-mode/default.png'), + ); + expect(() => validateCapture(directory)).toThrow( + 'file is missing or not a regular file', + ); + }); + + test('rejects invalid SHAs and environment keys', () => { + writeCapture( + directory, + {'text/width-mode/default': makePng(4, 4)}, + { + environmentKey: 'ios-../../x', + git: {testedSha: 'HEAD', baseSha: SHA_B, headSha: SHA_A}, + }, + ); + expect(() => validateCapture(directory)).toThrow( + /Invalid environment key[\s\S]*Invalid git.testedSha/, + ); + }); + }); + + describe('compareImages', () => { + const strict = policyFor('a/b/c', null); + + test('identical images are unchanged', () => { + expect(compareImages(makePng(8, 8), makePng(8, 8), strict)).toMatchObject( + {status: 'unchanged', diffPixels: 0, diff: null}, + ); + }); + + test('a single changed pixel fails the strict policy', () => { + const changed = makePng(8, 8, (x, y) => + x === 3 && y === 3 ? [254, 255, 255, 255] : [255, 255, 255, 255], + ); + const result = compareImages(makePng(8, 8), changed, strict); + expect(result).toMatchObject({status: 'changed', diffPixels: 1}); + expect(readPngDimensions(result.diff)).toEqual({width: 8, height: 8}); + }); + + test('a per-test tolerance can absorb a known amount of noise', () => { + const changed = makePng(8, 8, (x, y) => + x === 3 && y === 3 ? [0, 0, 0, 255] : [255, 255, 255, 255], + ); + const policy = policyFor('a/b/c', {tests: {'a/b/c': {maxDiffPixels: 1}}}); + expect(compareImages(makePng(8, 8), changed, policy).status).toBe( + 'unchanged', + ); + expect( + policyFor('a/b/d', {tests: {'a/b/c': {maxDiffPixels: 1}}}), + ).toEqual(strict); + }); + + test('different dimensions are reported, not compared', () => { + expect(compareImages(makePng(8, 8), makePng(8, 9), strict)).toMatchObject( + { + status: 'dimension-mismatch', + expectedSize: {width: 8, height: 8}, + capturedSize: {width: 8, height: 9}, + }, + ); + }); + }); + + describe('compareCapture', () => { + test('classifies unchanged, changed, added and removed tests', async () => { + const white = makePng(4, 4); + const black = makePng(4, 4, () => [0, 0, 0, 255]); + const image = (id, buffer) => ({id, sha256: sha256(buffer)}); + const results = await compareCapture({ + manifest: { + images: [ + image('a/a/same', white), + image('a/a/changed', black), + image('a/a/added', white), + ], + }, + baselineManifest: { + images: [ + image('a/a/same', white), + image('a/a/changed', white), + image('a/a/removed', white), + ], + }, + readExpected: async () => white, + readCaptured: async () => black, + config: null, + }); + expect(results.map(result => [result.id, result.status])).toEqual([ + ['a/a/added', 'added'], + ['a/a/changed', 'changed'], + ['a/a/removed', 'removed'], + ['a/a/same', 'unchanged'], + ]); + }); + }); +}); diff --git a/.github/workflow-scripts/__tests__/visual-report-test.js b/.github/workflow-scripts/__tests__/visual-report-test.js new file mode 100644 index 000000000000..28c2ffc9d4bc --- /dev/null +++ b/.github/workflow-scripts/__tests__/visual-report-test.js @@ -0,0 +1,243 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const { + APPROVAL_LABEL, + differenceFingerprint, + gateDecision, +} = require('../visual/approval'); +const {environmentKey, sha256} = require('../visual/manifest'); +const { + COMMENT_MARKER, + renderCheckOutput, + renderPullRequestComment, +} = require('../visual/report'); +const { + approvalKey, + baselinePrefix, + publicUrl, + reportPrefix, +} = require('../visual/storage'); +const {SHA_A, SHA_B} = require('./__fixtures__/visual'); + +describe('visual reports and approval', () => { + describe('storage keys', () => { + test('builds write-once keys', () => { + expect(baselinePrefix('ios-x-0123abcd', SHA_A)).toBe( + `baselines/ios-x-0123abcd/${SHA_A}`, + ); + expect( + reportPrefix({prNumber: 12, headSha: SHA_A, runId: 34, runAttempt: 2}), + ).toBe(`reports/pull/12/${SHA_A}/34-2`); + expect( + reportPrefix({ + prNumber: null, + headSha: SHA_A, + runId: 34, + runAttempt: 1, + }), + ).toBe(`reports/main/${SHA_A}/34-1`); + expect(approvalKey(12, '0123456789abcdef')).toBe( + 'approvals/pull/12/0123456789abcdef.json', + ); + }); + + test.each([ + [() => baselinePrefix('k', 'main')], + [ + () => + reportPrefix({prNumber: -1, headSha: SHA_A, runId: 1, runAttempt: 1}), + ], + [ + () => + reportPrefix({ + prNumber: 1, + headSha: SHA_A, + runId: 1.5, + runAttempt: 1, + }), + ], + [() => approvalKey(1, '../x')], + ])('rejects invalid input', build => { + expect(build).toThrow('Invalid'); + }); + + test('encodes each path segment of public URLs', () => { + expect(publicUrl('https://cdn.example/bucket/', 'a b/c#d.png')).toBe( + 'https://cdn.example/bucket/a%20b/c%23d.png', + ); + }); + }); + + describe('approval', () => { + const platform = results => ({ + platform: 'ios', + environmentKey: 'ios-x-0123abcd', + baselineSha: SHA_B, + results, + }); + const changed = sha => ({ + id: 'a/b/c', + status: 'changed', + expected: {sha256: 'e'}, + captured: {sha256: sha}, + }); + + test('has no fingerprint without differences', () => { + expect( + differenceFingerprint([platform([{id: 'a/b/c', status: 'unchanged'}])]), + ).toBeNull(); + }); + + test('fingerprints the exact captured images', () => { + const first = differenceFingerprint([platform([changed('1')])]); + expect(first).toMatch(/^[0-9a-f]{16}$/); + expect(differenceFingerprint([platform([changed('1')])])).toBe(first); + expect(differenceFingerprint([platform([changed('2')])])).not.toBe(first); + }); + + test.each([ + [null, false, null, 'success', false], + ['f', false, null, 'failure', false], + ['f', true, {fingerprint: 'f', approver: 'x'}, 'success', false], + ['f', true, {fingerprint: 'g', approver: 'x'}, 'failure', true], + ['f', true, null, 'failure', true], + ])( + 'gate for fingerprint %s, label %s, approval %j', + (fingerprint, labelPresent, approval, conclusion, staleLabel) => { + const decision = gateDecision({fingerprint, labelPresent, approval}); + expect(decision.conclusion).toBe(conclusion); + expect(decision.staleLabel ?? false).toBe(staleLabel); + if (conclusion === 'failure' && !staleLabel) { + expect(decision.summary).toContain(APPROVAL_LABEL); + } + }, + ); + }); + + describe('renderPullRequestComment', () => { + const changed = id => ({ + id, + status: 'changed', + diffPixels: 3, + diffRatio: 0.03, + expected: {}, + captured: {}, + hasDiffImage: true, + }); + const platform = (name, results, error = null) => ({ + platform: name, + environmentKey: `${name}-x-0123abcd`, + baseline: {sha: SHA_B, distance: 0}, + results, + error, + checkUrl: `https://github.example/checks/${name}`, + imageUrl: (kind, id) => `https://cdn.example/${name}/${kind}/${id}.png`, + }); + + test('shows each diff with expected and captured collapsed', () => { + const body = renderPullRequestComment({ + platforms: [ + platform('android', [ + {id: 'a/b/same', status: 'unchanged'}, + changed('a/b/changed'), + ]), + platform('ios', [{id: 'a/b/new', status: 'added', captured: {}}]), + ], + headSha: SHA_A, + gateSummary: 'Needs approval.', + }); + expect(body.startsWith(COMMENT_MARKER)).toBe(true); + expect(body).toContain( + '**2 visual difference(s)** at `aaaaaaaaaaaa`. Needs approval.', + ); + expect(body).toContain( + '### android `a/b/changed`: Changed, 3 px (3.000%)', + ); + expect(body).toContain( + 'diff', + ); + expect(body).toContain( + '
Expected and Captured', + ); + expect(body).toContain('### ios `a/b/new`: New'); + expect(body).toContain('
Captured'); + expect(body).not.toContain('a/b/same'); + }); + + test('lists tests beyond the limit by name', () => { + const results = Array.from({length: 23}, (_, index) => + changed(`a/b/test-${String(index).padStart(2, '0')}`), + ); + const body = renderPullRequestComment({ + platforms: [platform('android', results)], + headSha: SHA_A, + gateSummary: '', + }); + expect(body).toContain('3 more:'); + expect(body).toContain( + '- android `a/b/test-22`: Changed, 3 px (3.000%), [check run](https://github.example/checks/android)', + ); + expect(body).not.toContain('android/diff/a/b/test-20.png'); + }); + + test('reports errors and the absence of differences', () => { + const body = renderPullRequestComment({ + platforms: [ + platform('android', [{id: 'a/b/same', status: 'unchanged'}]), + platform('ios', [], {title: 'no baseline for this environment'}), + ], + headSha: SHA_A, + gateSummary: 'No visual differences.', + }); + expect(body).toContain('No visual differences at `aaaaaaaaaaaa`.'); + expect(body).toContain( + '**ios:** [no baseline for this environment](https://github.example/checks/ios)', + ); + }); + }); + + describe('renderCheckOutput', () => { + test('links expected, captured and diff images for each difference', () => { + const output = renderCheckOutput({ + platform: 'android', + environmentKey: 'android-x-0123abcd', + baseline: {sha: SHA_B, distance: 2}, + results: [ + {id: 'a/b/same', status: 'unchanged'}, + { + id: 'a/b/changed', + status: 'changed', + diffPixels: 3, + diffRatio: 0.03, + expected: {}, + captured: {}, + hasDiffImage: true, + }, + ], + imageUrl: (kind, id) => `https://cdn.example/${kind}/${id}.png`, + error: null, + }); + expect(output.title).toBe('android: 1 visual difference(s)'); + expect(output.summary).toContain('2 commit(s) earlier'); + expect(output.text).toContain( + 'https://cdn.example/expected/a/b/changed.png', + ); + expect(output.text).toContain( + 'https://cdn.example/captured/a/b/changed.png', + ); + expect(output.text).toContain('3 px (3.000%)'); + expect(output.images).toEqual([ + expect.objectContaining({ + image_url: 'https://cdn.example/diff/a/b/changed.png', + }), + ]); + }); + }); +}); diff --git a/.github/workflow-scripts/visual/approval.js b/.github/workflow-scripts/visual/approval.js new file mode 100644 index 000000000000..c9569332f384 --- /dev/null +++ b/.github/workflow-scripts/visual/approval.js @@ -0,0 +1,84 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {sha256} = require('./manifest'); + +const APPROVAL_LABEL = 'visual-change-approved'; +const CHECK_NAME_PREFIX = 'Visual regression'; +const GATE_CHECK_NAME = 'Visual regression approval'; +const APPROVER_PERMISSIONS = ['admin', 'maintain', 'write']; + +/** + * Identifies a set of visual differences: which tests differ, how, and the + * exact expected and captured images. An approval applies to one fingerprint, + * so a push that changes any difference needs a new approval. Returns null + * when there are no differences. + */ +function differenceFingerprint(platforms) { + const lines = []; + for (const {platform, environmentKey, baselineSha, results} of platforms) { + for (const result of results) { + if (result.status === 'unchanged') { + continue; + } + lines.push( + [ + platform, + environmentKey, + baselineSha ?? '', + result.id, + result.status, + result.expected?.sha256 ?? '', + result.captured?.sha256 ?? '', + ].join('\t'), + ); + } + } + if (lines.length === 0) { + return null; + } + return sha256(Buffer.from(lines.sort().join('\n'))).slice(0, 16); +} + +/** + * The approval gate passes when there is nothing to approve, or when the + * approval label is present and an approval was recorded for this exact + * fingerprint. + */ +function gateDecision({fingerprint, labelPresent, approval}) { + if (fingerprint == null) { + return {conclusion: 'success', summary: 'No visual differences.'}; + } + if (labelPresent && approval?.fingerprint === fingerprint) { + return { + conclusion: 'success', + summary: `Visual differences approved by @${approval.approver}.`, + }; + } + if (labelPresent) { + return { + conclusion: 'failure', + summary: `The visual differences changed after approval. A maintainer must review the new report and re-apply \`${APPROVAL_LABEL}\`.`, + staleLabel: true, + }; + } + return { + conclusion: 'failure', + summary: `Visual differences need maintainer approval. Review the report, then apply \`${APPROVAL_LABEL}\`.`, + }; +} + +module.exports = { + APPROVAL_LABEL, + APPROVER_PERMISSIONS, + CHECK_NAME_PREFIX, + GATE_CHECK_NAME, + differenceFingerprint, + gateDecision, +}; diff --git a/.github/workflow-scripts/visual/approve-run.js b/.github/workflow-scripts/visual/approve-run.js new file mode 100644 index 000000000000..81a80f18b092 --- /dev/null +++ b/.github/workflow-scripts/visual/approve-run.js @@ -0,0 +1,182 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * Records a maintainer's approval of a PR's visual differences and updates + * the approval gate. Runs from the default branch on `pull_request_target` + * when the approval label is added or removed. It never checks out or runs + * pull request code. + * + * node approve-run.js prepare Decide, and stage an approval record to + * upload under OUT_DIR. + * node approve-run.js report After the upload, update the gate check. + */ + +const { + APPROVAL_LABEL, + APPROVER_PERMISSIONS, + GATE_CHECK_NAME, + gateDecision, +} = require('./approval'); +const {createGitHubClient} = require('./github'); +const { + approvalKey, + fetchJsonOrNull, + listPrefixes, + publicUrl, +} = require('./storage'); +const fs = require('fs'); +const path = require('path'); + +function requireEnv(name) { + const value = process.env[name]; + if (value == null || value === '') { + throw new Error(`${name} is required`); + } + return value; +} + +/** The results of the most recent comparison for a PR head, or null. */ +async function latestResults({bucket, publicBaseUrl, prNumber, headSha}) { + const runs = await listPrefixes( + bucket, + `reports/pull/${prNumber}/${headSha}/`, + ); + const latest = runs + .map(prefix => { + const match = /\/(\d+)-(\d+)\/$/.exec(prefix); + return match == null + ? null + : {prefix, runId: Number(match[1]), attempt: Number(match[2])}; + }) + .filter(Boolean) + .sort((a, b) => b.runId - a.runId || b.attempt - a.attempt)[0]; + return latest == null + ? null + : fetchJsonOrNull(publicUrl(publicBaseUrl, `${latest.prefix}results.json`)); +} + +async function prepare() { + const github = createGitHubClient(); + const outDir = requireEnv('OUT_DIR'); + const event = JSON.parse( + fs.readFileSync(requireEnv('GITHUB_EVENT_PATH'), 'utf8'), + ); + const writeState = state => + fs.writeFileSync( + path.join(outDir, 'state.json'), + `${JSON.stringify(state)}\n`, + ); + fs.mkdirSync(outDir, {recursive: true}); + + if (event.label?.name !== APPROVAL_LABEL) { + writeState(null); + return; + } + + const prNumber = event.pull_request.number; + const headSha = event.pull_request.head.sha; + const results = await latestResults({ + bucket: requireEnv('VISUAL_BUCKET'), + publicBaseUrl: requireEnv('VISUAL_PUBLIC_BASE_URL'), + prNumber, + headSha, + }); + const fingerprint = results?.fingerprint ?? null; + + if (event.action === 'labeled') { + const approver = event.sender.login; + const permission = await github.permission(approver); + if (!APPROVER_PERMISSIONS.includes(permission)) { + await github.removeLabel(prNumber, APPROVAL_LABEL); + await github.comment( + prNumber, + `@${approver} \`${APPROVAL_LABEL}\` needs write access to this repository, so it was removed.`, + ); + writeState(null); + return; + } + if (results == null) { + await github.removeLabel(prNumber, APPROVAL_LABEL); + await github.comment( + prNumber, + `There is no visual report for \`${headSha.slice(0, 12)}\` yet, so \`${APPROVAL_LABEL}\` was removed. Apply it again once the report is posted.`, + ); + writeState(null); + return; + } + if (fingerprint != null) { + const key = approvalKey(prNumber, fingerprint); + const record = { + fingerprint, + approver, + approvedAt: new Date().toISOString(), + headSha, + }; + const recordPath = path.join(outDir, 'upload', key); + fs.mkdirSync(path.dirname(recordPath), {recursive: true}); + fs.writeFileSync(recordPath, `${JSON.stringify(record, null, 2)}\n`); + } + } + + writeState({ + prNumber, + headSha, + fingerprint, + labelPresent: event.action === 'labeled', + }); +} + +async function report() { + const github = createGitHubClient(); + const outDir = requireEnv('OUT_DIR'); + const state = JSON.parse( + fs.readFileSync(path.join(outDir, 'state.json'), 'utf8'), + ); + if (state == null) { + return; + } + const {prNumber, headSha, fingerprint, labelPresent} = state; + const approval = + fingerprint == null + ? null + : await fetchJsonOrNull( + publicUrl( + requireEnv('VISUAL_PUBLIC_BASE_URL'), + approvalKey(prNumber, fingerprint), + ), + ); + const decision = gateDecision({fingerprint, labelPresent, approval}); + await github.createCheckRun({ + name: GATE_CHECK_NAME, + headSha, + conclusion: decision.conclusion, + output: {title: decision.summary, summary: decision.summary}, + }); + console.info(decision.summary); +} + +async function main(command = process.argv[2]) { + if (command === 'prepare') { + await prepare(); + } else if (command === 'report') { + await report(); + } else { + throw new Error('Usage: node approve-run.js prepare|report'); + } +} + +if (require.main === module) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} + +module.exports = {latestResults}; diff --git a/.github/workflow-scripts/visual/capture.js b/.github/workflow-scripts/visual/capture.js new file mode 100644 index 000000000000..b19e08dad98b --- /dev/null +++ b/.github/workflow-scripts/visual/capture.js @@ -0,0 +1,205 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * Captures every visual flow in one Maestro invocation, so the JVM and + * driver startup (around two minutes on an iOS simulator) is paid once per + * job rather than once per flow. Flows that fail are rerun, and only those, + * on later attempts. + */ + +const childProcess = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +const MAESTRO = path.join(process.env.HOME ?? '', '.maestro/bin/maestro'); +const DEFAULT_ATTEMPTS = 3; +const ATTEMPT_TIMEOUT_MS = 30 * 60 * 1000; + +const usage = ` +node capture.js [ios_udid] + +Installs the app, then runs every flow in in one Maestro +invocation, writing screenshots under . Failed flows are retried +up to ${DEFAULT_ATTEMPTS} attempts in total (VISUAL_CAPTURE_ATTEMPTS). +`; + +/** The flows directly inside `flowsDir`, sorted. Helpers live elsewhere. */ +function collectFlows(flowsDir) { + const flows = fs + .readdirSync(flowsDir, {withFileTypes: true}) + .filter(entry => entry.isFile() && /\.ya?ml$/.test(entry.name)) + .map(entry => path.join(flowsDir, entry.name)) + .sort(); + flowsByReportName(flows); + return flows; +} + +/** + * Maps Maestro's report names to flow paths. Maestro reports a flow under its + * file name without the extension, so names must be unique. + */ +function flowsByReportName(flows) { + const byName = new Map(); + for (const flow of flows) { + const name = path.basename(flow).replace(/\.ya?ml$/, ''); + const existing = byName.get(name); + if (existing != null) { + throw new Error( + `${existing} and ${flow} would both report as '${name}'. Rename one.`, + ); + } + byName.set(name, flow); + } + return byName; +} + +/** + * The flows a Maestro JUnit report marks as anything but SUCCESS, in report + * order, matched by the report's file path or, failing that, its name. Throws + * on a report entry that matches none of `flows`. + */ +function failedFlowsFromJUnit(report, flows) { + const byName = flowsByReportName(flows); + const failed = []; + for (const [, attributes] of report.matchAll(/]*)>/g)) { + const name = /\bname="([^"]*)"/.exec(attributes)?.[1]; + const file = /\bfile="([^"]*)"/.exec(attributes)?.[1]; + const status = /\bstatus="([^"]*)"/.exec(attributes)?.[1]; + if (name == null || status === 'SUCCESS') { + continue; + } + const flow = + flows.find(candidate => path.resolve(candidate) === file) ?? + byName.get(name); + if (flow == null) { + throw new Error(`The Maestro report has an unknown flow '${name}'`); + } + failed.push(flow); + } + return failed; +} + +function maestroArgs({flows, appId, outputDir, reportPath, udid, attempt}) { + return [ + ...(udid != null ? [`--udid=${udid}`] : []), + 'test', + ...flows, + '--format=junit', + `--output=${reportPath}`, + '-e', + `APP_ID=${appId}`, + '-e', + `VISUAL_OUTPUT_DIR=${outputDir}`, + // The first attempt installs the driver. Retries reuse it. + ...(attempt > 1 ? ['--no-reinstall-driver'] : []), + ]; +} + +function installApp(platform, appPath, udid) { + if (platform === 'android') { + childProcess.execFileSync('adb', ['install', '-r', appPath], { + stdio: 'inherit', + }); + } else { + childProcess.execFileSync('xcrun', ['simctl', 'install', udid, appPath], { + stdio: 'inherit', + }); + } +} + +function runAttempt(args) { + const result = childProcess.spawnSync(MAESTRO, args, { + stdio: 'inherit', + timeout: ATTEMPT_TIMEOUT_MS, + env: { + ...process.env, + MAESTRO_CLI_NO_ANALYTICS: '1', + MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED: 'true', + MAESTRO_DRIVER_STARTUP_TIMEOUT: '180000', + }, + }); + if (result.error != null) { + throw result.error; + } + return result.status; +} + +function writeSummary(lines) { + const text = `${lines.join('\n')}\n`; + console.info(text); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, text); + } +} + +function main(args = process.argv.slice(2)) { + const [platform, appPath, appId, flowsDir, outputDir, udid] = args; + if ( + !['android', 'ios'].includes(platform) || + outputDir == null || + (platform === 'ios' && udid == null) + ) { + throw new Error(usage); + } + const attempts = Number( + process.env.VISUAL_CAPTURE_ATTEMPTS ?? DEFAULT_ATTEMPTS, + ); + const allFlows = collectFlows(flowsDir); + fs.mkdirSync(outputDir, {recursive: true}); + installApp(platform, appPath, udid); + + const summary = [`### Visual capture (${platform})`, '']; + let pending = allFlows; + for (let attempt = 1; attempt <= attempts && pending.length > 0; attempt++) { + const reportPath = path.resolve(`visual-capture-report-${attempt}.xml`); + const started = Date.now(); + console.info( + `Attempt ${attempt}/${attempts}: ${pending.length} flow(s) in one Maestro run`, + ); + const status = runAttempt( + maestroArgs({ + flows: pending, + appId, + outputDir, + reportPath, + udid, + attempt, + }), + ); + const report = fs.existsSync(reportPath) + ? fs.readFileSync(reportPath, 'utf8') + : ''; + const failed = failedFlowsFromJUnit(report, pending); + if (status !== 0 && failed.length === 0) { + // Maestro failed without attributing it to a flow, for example because + // it could not reach the device. Retrying won't help. + throw new Error(`Maestro exited with ${status} without a failed flow`); + } + summary.push( + `Attempt ${attempt}: ${pending.length - failed.length} of ${pending.length} flow(s) passed in ${Math.round((Date.now() - started) / 1000)} s`, + ); + pending = failed; + } + + if (pending.length > 0) { + summary.push('', `Failed after ${attempts} attempt(s):`); + summary.push(...pending.map(flow => `- \`${flow}\``)); + } + writeSummary(summary); + if (pending.length > 0) { + process.exitCode = 1; + } +} + +if (require.main === module) { + main(); +} + +module.exports = {collectFlows, failedFlowsFromJUnit, maestroArgs}; diff --git a/.github/workflow-scripts/visual/compare-run.js b/.github/workflow-scripts/visual/compare-run.js new file mode 100644 index 000000000000..5027ff729538 --- /dev/null +++ b/.github/workflow-scripts/visual/compare-run.js @@ -0,0 +1,465 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * Trusted half of the visual regression pipeline. Runs from the default + * branch on `workflow_run`, after the untrusted capture workflow finishes. + * + * node compare-run.js prepare Validate captures, compare with baselines, + * and lay out files to upload under OUT_DIR. + * node compare-run.js report After the upload, create check runs and + * evaluate the approval gate. + */ + +const { + APPROVAL_LABEL, + CHECK_NAME_PREFIX, + GATE_CHECK_NAME, + differenceFingerprint, + gateDecision, +} = require('./approval'); +const {compareCapture, summarize} = require('./compare'); +const {createGitHubClient} = require('./github'); +const { + COMMENT_MARKER, + renderCheckOutput, + renderPullRequestComment, +} = require('./report'); +const { + approvalKey, + baselinePrefix, + fetchBuffer, + fetchJsonOrNull, + findBaseline, + publicUrl, + reportPrefix, +} = require('./storage'); +const {validateCapture} = require('./validate'); +const fs = require('fs'); +const path = require('path'); + +const BASELINE_SEARCH_DEPTH = 20; +const PLATFORMS = ['android', 'ios']; +// The default Actions token posts as this bot. Only its comments are edited. +const COMMENT_AUTHOR = 'github-actions[bot]'; + +function requireEnv(name) { + const value = process.env[name]; + if (value == null || value === '') { + throw new Error(`${name} is required`); + } + return value; +} + +function writeFile(filePath, contents) { + fs.mkdirSync(path.dirname(filePath), {recursive: true}); + fs.writeFileSync(filePath, contents); +} + +async function resolveContext(github) { + const event = requireEnv('CAPTURE_EVENT'); + const headSha = requireEnv('CAPTURE_HEAD_SHA'); + const headBranch = requireEnv('CAPTURE_HEAD_BRANCH'); + const headRepository = requireEnv('CAPTURE_HEAD_REPOSITORY'); + const defaultBranch = requireEnv('DEFAULT_BRANCH'); + const runId = Number(requireEnv('CAPTURE_RUN_ID')); + const runAttempt = Number(requireEnv('CAPTURE_RUN_ATTEMPT')); + + if (event === 'pull_request') { + const pull = await github.findPullRequest(headSha, headRepository); + if (pull == null) { + return null; + } + return { + kind: 'pull', + prNumber: pull.number, + headSha, + runId, + runAttempt, + defaultBranch, + }; + } + if ( + event === 'push' && + headBranch === defaultBranch && + headRepository === github.repository + ) { + return { + kind: 'main', + prNumber: null, + headSha, + runId, + runAttempt, + defaultBranch, + }; + } + return null; +} + +async function comparePlatform({ + github, + context, + captureDir, + config, + publicBaseUrl, + outDir, + prefix, +}) { + const manifest = validateCapture(captureDir); + const {platform, environmentKey, git} = manifest; + if (git.headSha !== context.headSha) { + throw new Error( + `${platform}: manifest head ${git.headSha} does not match run head ${context.headSha}`, + ); + } + if (!(await github.isOnBranch(git.baseSha, context.defaultBranch))) { + return { + platform, + environmentKey, + manifest, + baseline: null, + results: [], + error: { + title: 'base commit is not on the default branch', + summary: `The capture's base commit \`${git.baseSha}\` is not on \`${context.defaultBranch}\`, so there is no baseline to compare with. Rebase or merge \`${context.defaultBranch}\` and push again.`, + }, + }; + } + + const candidateShas = await github.ancestry( + git.baseSha, + BASELINE_SEARCH_DEPTH, + ); + const baseline = await findBaseline({ + publicBaseUrl, + environmentKey, + candidateShas, + }); + if (baseline == null) { + return { + platform, + environmentKey, + manifest, + baseline: null, + results: [], + error: { + title: 'no baseline for this environment', + summary: `No baseline for environment \`${environmentKey}\` was published for \`${git.baseSha.slice(0, 12)}\` or its ${BASELINE_SEARCH_DEPTH - 1} preceding commits on \`${context.defaultBranch}\`. If the environment key is new, the CI device image changed: the next push to \`${context.defaultBranch}\` publishes a baseline for it. Otherwise check that baseline publication on \`${context.defaultBranch}\` is succeeding.`, + }, + }; + } + + const expectedUrl = id => + publicUrl( + publicBaseUrl, + `${baselinePrefix(environmentKey, baseline.sha)}/${id}.png`, + ); + const results = await compareCapture({ + manifest, + baselineManifest: baseline.manifest, + readExpected: id => fetchBuffer(expectedUrl(id)), + readCaptured: id => + fs.promises.readFile(path.join(captureDir, `${id}.png`)), + config, + }); + + for (const result of results) { + if (result.status === 'unchanged') { + continue; + } + const platformPrefix = `${prefix}/${platform}`; + if (result.captured != null) { + writeFile( + path.join( + outDir, + 'upload', + platformPrefix, + 'captured', + `${result.id}.png`, + ), + fs.readFileSync(path.join(captureDir, `${result.id}.png`)), + ); + } + if (result.diffImage != null) { + writeFile( + path.join(outDir, 'upload', platformPrefix, 'diff', `${result.id}.png`), + result.diffImage, + ); + } + } + + return { + platform, + environmentKey, + manifest, + baseline: {sha: baseline.sha, distance: baseline.distance}, + results, + error: null, + }; +} + +function missingCapture(platform, context) { + return { + platform, + environmentKey: null, + manifest: null, + baseline: null, + results: [], + error: { + title: 'no capture', + summary: `Test All run ${context.runId} has no ${platform} capture, so there is nothing to compare. Its capture job failed or didn't run: check the run for a build or capture failure.`, + }, + }; +} + +function stageBaseline({outDir, captureDir, manifest, sha}) { + const prefix = baselinePrefix(manifest.environmentKey, sha); + for (const image of manifest.images) { + writeFile( + path.join(outDir, 'baseline-images', prefix, `${image.id}.png`), + fs.readFileSync(path.join(captureDir, `${image.id}.png`)), + ); + } + // Uploaded after the images: a baseline exists once its manifest does. + writeFile( + path.join(outDir, 'baseline-manifests', prefix, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + ); +} + +function serializableResults(results) { + return results.map(({diffImage, ...result}) => ({ + ...result, + hasDiffImage: diffImage != null, + })); +} + +async function prepare() { + const github = createGitHubClient(); + const artifactsDir = requireEnv('VISUAL_ARTIFACTS_DIR'); + const outDir = requireEnv('OUT_DIR'); + const publicBaseUrl = requireEnv('VISUAL_PUBLIC_BASE_URL'); + const config = JSON.parse( + fs.readFileSync(requireEnv('VISUAL_CONFIG'), 'utf8'), + ); + + const context = await resolveContext(github); + if (context == null) { + console.info('Not an open pull request or a push to the default branch.'); + writeFile(path.join(outDir, 'state.json'), 'null\n'); + return; + } + const prefix = reportPrefix(context); + + const platforms = []; + for (const expectedPlatform of PLATFORMS) { + const captureDir = path.join( + artifactsDir, + `visual-capture-${expectedPlatform}`, + ); + if (!fs.existsSync(captureDir)) { + platforms.push(missingCapture(expectedPlatform, context)); + continue; + } + const result = await comparePlatform({ + github, + context, + captureDir, + config, + publicBaseUrl, + outDir, + prefix, + }); + if (result.platform !== expectedPlatform) { + throw new Error( + `visual-capture-${expectedPlatform}: manifest is for ${result.platform}`, + ); + } + if (context.kind === 'main') { + stageBaseline({ + outDir, + captureDir, + manifest: result.manifest, + sha: context.headSha, + }); + } + platforms.push(result); + } + + const fingerprint = differenceFingerprint( + platforms.map(platform => ({ + platform: platform.platform, + environmentKey: platform.environmentKey, + baselineSha: platform.baseline?.sha, + results: platform.results, + })), + ); + const state = { + context, + reportPrefix: prefix, + fingerprint, + platforms: platforms.map(({manifest, results, ...platform}) => ({ + ...platform, + git: manifest?.git ?? null, + results: serializableResults(results), + })), + }; + writeFile( + path.join(outDir, 'upload', prefix, 'results.json'), + `${JSON.stringify(state, null, 2)}\n`, + ); + writeFile(path.join(outDir, 'state.json'), `${JSON.stringify(state)}\n`); + + for (const platform of state.platforms) { + const {counts} = summarize(platform.results); + console.info( + `${platform.platform} (${platform.environmentKey}): ${ + platform.error?.title ?? JSON.stringify(counts) + }`, + ); + } +} + +async function report() { + const github = createGitHubClient(); + const outDir = requireEnv('OUT_DIR'); + const publicBaseUrl = requireEnv('VISUAL_PUBLIC_BASE_URL'); + const state = JSON.parse( + fs.readFileSync(path.join(outDir, 'state.json'), 'utf8'), + ); + if (state == null) { + return; + } + const {context, reportPrefix: prefix, fingerprint} = state; + const detailsUrl = publicUrl(publicBaseUrl, `${prefix}/results.json`); + + const summaryLines = []; + const commentPlatforms = []; + for (const platform of state.platforms) { + const imageUrl = (kind, id) => + kind === 'expected' + ? publicUrl( + publicBaseUrl, + `${baselinePrefix(platform.environmentKey, platform.baseline.sha)}/${id}.png`, + ) + : publicUrl( + publicBaseUrl, + `${prefix}/${platform.platform}/${kind}/${id}.png`, + ); + const output = renderCheckOutput({...platform, imageUrl}); + const {hasDifferences} = summarize(platform.results); + const checkRun = await github.createCheckRun({ + name: `${CHECK_NAME_PREFIX} (${platform.platform})`, + headSha: context.headSha, + conclusion: + platform.error != null + ? 'neutral' + : hasDifferences + ? 'failure' + : 'success', + output, + detailsUrl, + }); + commentPlatforms.push({...platform, imageUrl, checkUrl: checkRun.html_url}); + summaryLines.push( + `## ${output.title}`, + '', + output.summary, + '', + output.text, + '', + ); + } + + if (context.kind === 'pull') { + const labels = await github.labels(context.prNumber); + const approval = + fingerprint == null + ? null + : await fetchJsonOrNull( + publicUrl( + publicBaseUrl, + approvalKey(context.prNumber, fingerprint), + ), + ); + const decision = gateDecision({ + fingerprint, + labelPresent: labels.includes(APPROVAL_LABEL), + approval, + }); + if (decision.staleLabel) { + await github.removeLabel(context.prNumber, APPROVAL_LABEL); + await github.comment( + context.prNumber, + `The visual differences changed in \`${context.headSha.slice(0, 12)}\`, so \`${APPROVAL_LABEL}\` was removed. Review the new report and apply it again to approve.`, + ); + } + await github.createCheckRun({ + name: GATE_CHECK_NAME, + headSha: context.headSha, + conclusion: decision.conclusion, + output: {title: decision.summary, summary: decision.summary}, + detailsUrl, + }); + summaryLines.push(`## Approval`, '', decision.summary, ''); + await upsertComment({ + github, + prNumber: context.prNumber, + body: renderPullRequestComment({ + platforms: commentPlatforms, + headSha: context.headSha, + gateSummary: decision.summary, + }), + hasContent: + fingerprint != null || + commentPlatforms.some(platform => platform.error != null), + }); + } + + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summaryLines.join('\n')); + } +} + +/** + * Keeps one report comment per PR up to date. A PR that never had + * differences gets no comment. Once it has one, later runs edit it, including + * to say the differences are gone. + */ +async function upsertComment({github, prNumber, body, hasContent}) { + const existing = await github.findComment( + prNumber, + COMMENT_MARKER, + COMMENT_AUTHOR, + ); + if (existing != null) { + await github.updateComment(existing.id, body); + } else if (hasContent) { + await github.comment(prNumber, body); + } +} + +async function main(command = process.argv[2]) { + if (command === 'prepare') { + await prepare(); + } else if (command === 'report') { + await report(); + } else { + throw new Error('Usage: node compare-run.js prepare|report'); + } +} + +if (require.main === module) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} + +module.exports = {resolveContext}; diff --git a/.github/workflow-scripts/visual/compare.js b/.github/workflow-scripts/visual/compare.js new file mode 100644 index 000000000000..86694966dc4b --- /dev/null +++ b/.github/workflow-scripts/visual/compare.js @@ -0,0 +1,152 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const pixelmatch = require('pixelmatch'); +const {PNG} = require('pngjs'); + +// Strict by default: any changed pixel fails. Loosen per test in +// packages/rn-tester/visual-tests/config.json, with evidence. +const DEFAULT_POLICY = { + // pixelmatch's per-pixel color distance threshold, from 0 to 1. + threshold: 0, + // Count anti-aliased pixels as differences. + includeAA: true, + maxDiffPixels: 0, + // Optional: also fail when this share of the image changes. + maxDiffRatio: null, +}; + +function policyFor(testId, config) { + return {...DEFAULT_POLICY, ...config?.defaults, ...config?.tests?.[testId]}; +} + +/** + * Compares two PNG buffers. Returns the outcome and, when the images have + * the same dimensions and differ, a diff image highlighting changed pixels. + */ +function compareImages(expectedBuffer, capturedBuffer, policy) { + const expected = PNG.sync.read(expectedBuffer); + const captured = PNG.sync.read(capturedBuffer); + if ( + expected.width !== captured.width || + expected.height !== captured.height + ) { + return { + status: 'dimension-mismatch', + expectedSize: {width: expected.width, height: expected.height}, + capturedSize: {width: captured.width, height: captured.height}, + diffPixels: null, + diffRatio: null, + diff: null, + }; + } + + const {width, height} = expected; + const diff = new PNG({width, height}); + const diffPixels = pixelmatch( + expected.data, + captured.data, + diff.data, + width, + height, + {threshold: policy.threshold, includeAA: policy.includeAA}, + ); + const diffRatio = diffPixels / (width * height); + const changed = + diffPixels > policy.maxDiffPixels || + (policy.maxDiffRatio != null && diffRatio > policy.maxDiffRatio); + return { + status: changed ? 'changed' : 'unchanged', + diffPixels, + diffRatio, + diff: diffPixels > 0 ? PNG.sync.write(diff) : null, + }; +} + +/** + * Compares every captured image with the baseline. `readExpected(id)` and + * `readCaptured(id)` return PNG buffers. Results are sorted by test ID. + */ +async function compareCapture({ + manifest, + baselineManifest, + readExpected, + readCaptured, + config, +}) { + const expectedById = new Map( + (baselineManifest?.images ?? []).map(image => [image.id, image]), + ); + const results = []; + + for (const image of manifest.images) { + const expected = expectedById.get(image.id); + expectedById.delete(image.id); + if (expected == null) { + results.push({id: image.id, status: 'added', captured: image}); + continue; + } + if (expected.sha256 === image.sha256) { + results.push({ + id: image.id, + status: 'unchanged', + diffPixels: 0, + diffRatio: 0, + captured: image, + expected, + }); + continue; + } + const policy = policyFor(image.id, config); + const {diff, ...outcome} = compareImages( + await readExpected(image.id), + await readCaptured(image.id), + policy, + ); + results.push({ + id: image.id, + ...outcome, + policy, + captured: image, + expected, + diffImage: diff, + }); + } + + for (const expected of expectedById.values()) { + results.push({id: expected.id, status: 'removed', expected}); + } + + return results.sort((a, b) => a.id.localeCompare(b.id)); +} + +function summarize(results) { + const counts = { + unchanged: 0, + changed: 0, + 'dimension-mismatch': 0, + added: 0, + removed: 0, + }; + for (const result of results) { + counts[result.status] += 1; + } + return { + counts, + hasDifferences: results.some(result => result.status !== 'unchanged'), + }; +} + +module.exports = { + DEFAULT_POLICY, + compareCapture, + compareImages, + policyFor, + summarize, +}; diff --git a/.github/workflow-scripts/visual/github.js b/.github/workflow-scripts/visual/github.js new file mode 100644 index 000000000000..9dd1645207f9 --- /dev/null +++ b/.github/workflow-scripts/visual/github.js @@ -0,0 +1,145 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +function createGitHubClient({ + token = process.env.GITHUB_TOKEN, + repository = process.env.GITHUB_REPOSITORY, + apiUrl = process.env.GITHUB_API_URL ?? 'https://api.github.com', +} = {}) { + if (!token || !repository) { + throw new Error('GITHUB_TOKEN and GITHUB_REPOSITORY are required'); + } + + async function request(method, route, body) { + const response = await fetch(`${apiUrl}/repos/${repository}${route}`, { + method, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + ...(body != null ? {'Content-Type': 'application/json'} : null), + }, + body: body != null ? JSON.stringify(body) : undefined, + }); + if (response.status === 404 && method === 'GET') { + return null; + } + if (!response.ok) { + throw new Error( + `${method} ${route} failed with ${response.status}: ${await response.text()}`, + ); + } + return response.status === 204 ? null : response.json(); + } + + return { + repository, + + /** Finds the open PR whose head is `headSha` in `headRepository`. */ + async findPullRequest(headSha, headRepository) { + for (let page = 1; page <= 10; page++) { + const pulls = await request( + 'GET', + `/pulls?state=open&sort=updated&direction=desc&per_page=100&page=${page}`, + ); + const match = pulls.find( + pull => + pull.head.sha === headSha && + pull.head.repo?.full_name === headRepository, + ); + if (match != null || pulls.length < 100) { + return match ?? null; + } + } + return null; + }, + + /** Whether `sha` is an ancestor of, or equal to, `branch`. */ + async isOnBranch(sha, branch) { + const comparison = await request( + 'GET', + `/compare/${sha}...${encodeURIComponent(branch)}`, + ); + return ( + comparison != null && + (comparison.status === 'ahead' || comparison.status === 'identical') + ); + }, + + /** `sha` and up to `count - 1` of its ancestors, newest first. */ + async ancestry(sha, count) { + const commits = await request( + 'GET', + `/commits?sha=${sha}&per_page=${count}`, + ); + return (commits ?? []).map(commit => commit.sha); + }, + + async labels(prNumber) { + const labels = await request( + 'GET', + `/issues/${prNumber}/labels?per_page=100`, + ); + return (labels ?? []).map(label => label.name); + }, + + removeLabel(prNumber, label) { + return request( + 'DELETE', + `/issues/${prNumber}/labels/${encodeURIComponent(label)}`, + ); + }, + + comment(prNumber, body) { + return request('POST', `/issues/${prNumber}/comments`, {body}); + }, + + /** The first comment by `author` on the PR that contains `marker`. */ + async findComment(prNumber, marker, author) { + for (let page = 1; ; page++) { + const comments = await request( + 'GET', + `/issues/${prNumber}/comments?per_page=100&page=${page}`, + ); + const match = (comments ?? []).find( + comment => + comment.user?.login === author && comment.body?.includes(marker), + ); + if (match != null || (comments ?? []).length < 100) { + return match ?? null; + } + } + }, + + updateComment(commentId, body) { + return request('PATCH', `/issues/comments/${commentId}`, {body}); + }, + + async permission(login) { + const result = await request( + 'GET', + `/collaborators/${encodeURIComponent(login)}/permission`, + ); + return result?.permission ?? 'none'; + }, + + createCheckRun({name, headSha, conclusion, output, detailsUrl}) { + return request('POST', '/check-runs', { + name, + head_sha: headSha, + status: 'completed', + conclusion, + output, + details_url: detailsUrl, + }); + }, + }; +} + +module.exports = {createGitHubClient}; diff --git a/.github/workflow-scripts/visual/manifest.js b/.github/workflow-scripts/visual/manifest.js new file mode 100644 index 000000000000..6b7832ede2c4 --- /dev/null +++ b/.github/workflow-scripts/visual/manifest.js @@ -0,0 +1,282 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {readPngDimensions} = require('./png'); +const childProcess = require('child_process'); +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); + +const MANIFEST_VERSION = 1; +const PLATFORMS = ['android', 'ios']; + +// //, each segment lowercase kebab-case. +const TEST_ID_PATTERN = + /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*){2}$/; +const SHA_PATTERN = /^[0-9a-f]{40}$/; + +// Environment properties that select a baseline. Anything else in the +// environment is recorded for diagnosis but does not change the key. +const KEYED_PROPERTIES = { + android: [ + 'apiLevel', + 'abi', + 'nativeBridge', + 'systemImage', + 'screenSize', + 'density', + 'fontScale', + 'locale', + 'animatorDurationScale', + 'emulatorVersion', + ], + ios: ['runtime', 'deviceType', 'appearance', 'contentSizeCategory', 'locale'], +}; + +function sha256(buffer) { + return crypto.createHash('sha256').update(buffer).digest('hex'); +} + +function canonicalJson(value) { + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(',')}]`; + } + if (value != null && typeof value === 'object') { + return `{${Object.keys(value) + .sort() + .map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(',')}}`; + } + return JSON.stringify(value); +} + +function slug(value) { + return String(value) + .toLowerCase() + .replace(/[^a-z0-9.]+/g, '-') + .replace(/^-|-$/g, ''); +} + +/** + * A readable prefix for the properties people look for first, followed by a + * hash of every keyed property so that drift in any of them selects a + * different baseline instead of producing a spurious diff. + */ +function environmentKey(platform, environment) { + const keyed = {}; + for (const property of KEYED_PROPERTIES[platform]) { + keyed[property] = environment[property] ?? null; + } + const hash = sha256(Buffer.from(canonicalJson({platform, keyed}))).slice( + 0, + 8, + ); + const readable = + platform === 'android' + ? [ + 'android', + `api${environment.apiLevel}`, + environment.abi, + environment.screenSize, + `${environment.density}dpi`, + ] + : [ + 'ios', + String(environment.runtime).replace(/^iOS-/, ''), + environment.deviceType, + ]; + return [...readable.map(slug), hash].join('-'); +} + +function collectImages(captureDir) { + const images = []; + const visit = directory => { + for (const entry of fs.readdirSync(directory, {withFileTypes: true})) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(entryPath); + } else if (entry.isFile() && entry.name.endsWith('.png')) { + images.push(entryPath); + } + } + }; + visit(captureDir); + return images.sort(); +} + +function buildManifest({captureDir, platform, environment, git}) { + const images = collectImages(captureDir).map(imagePath => { + const relativePath = path + .relative(captureDir, imagePath) + .split(path.sep) + .join('/'); + const buffer = fs.readFileSync(imagePath); + const dimensions = readPngDimensions(buffer); + if (dimensions == null) { + throw new Error(`Not a PNG: ${relativePath}`); + } + return { + id: relativePath.replace(/\.png$/, ''), + path: relativePath, + ...dimensions, + bytes: buffer.length, + sha256: sha256(buffer), + }; + }); + + return { + version: MANIFEST_VERSION, + platform, + environmentKey: environmentKey(platform, environment), + environment, + git, + images, + }; +} + +function run(command, args) { + return childProcess + .execFileSync(command, args, {encoding: 'utf8', timeout: 30000}) + .trim(); +} + +/** The emulator package's version, from the Android SDK. */ +function readEmulatorVersion() { + const sdk = process.env.ANDROID_HOME ?? process.env.ANDROID_SDK_ROOT; + if (sdk == null) { + return null; + } + const properties = fs.readFileSync( + path.join(sdk, 'emulator', 'source.properties'), + 'utf8', + ); + return /^Pkg\.Revision=(\S+)$/m.exec(properties)?.[1] ?? null; +} + +function readAndroidEnvironment() { + const getprop = name => run('adb', ['shell', 'getprop', name]); + const setting = (namespace, name) => + run('adb', ['shell', 'settings', 'get', namespace, name]); + const wm = name => { + const output = run('adb', ['shell', 'wm', name]); + const override = /Override \w+: (\S+)/.exec(output); + return (override ?? /Physical \w+: (\S+)/.exec(output))?.[1] ?? output; + }; + return { + apiLevel: Number(getprop('ro.build.version.sdk')), + abi: getprop('ro.product.cpu.abi'), + nativeBridge: getprop('ro.dalvik.vm.native.bridge'), + systemImage: getprop('ro.build.fingerprint'), + screenSize: wm('size'), + density: Number(wm('density')), + fontScale: setting('system', 'font_scale'), + locale: getprop('persist.sys.locale') || getprop('ro.product.locale'), + animatorDurationScale: setting('global', 'animator_duration_scale'), + emulatorVersion: readEmulatorVersion(), + }; +} + +function readIosEnvironment(udid) { + const devices = JSON.parse( + run('xcrun', ['simctl', 'list', 'devices', '-j']), + ).devices; + let device = null; + let runtime = null; + for (const [runtimeId, runtimeDevices] of Object.entries(devices)) { + const match = runtimeDevices.find(candidate => candidate.udid === udid); + if (match != null) { + device = match; + runtime = runtimeId.replace('com.apple.CoreSimulator.SimRuntime.', ''); + } + } + if (device == null) { + throw new Error(`Simulator ${udid} not found`); + } + const ui = name => run('xcrun', ['simctl', 'ui', udid, name]); + return { + runtime, + deviceType: device.deviceTypeIdentifier.replace( + 'com.apple.CoreSimulator.SimDeviceType.', + '', + ), + appearance: ui('appearance'), + contentSizeCategory: ui('content_size'), + locale: run('xcrun', [ + 'simctl', + 'spawn', + udid, + 'defaults', + 'read', + '-g', + 'AppleLocale', + ]), + xcode: run('xcodebuild', ['-version']).split('\n').join(' '), + }; +} + +function readGitInfo() { + const git = args => run('git', args); + return { + // For pull_request events actions/checkout tests the merge commit, whose + // first parent is the base branch commit the PR was tested against. + testedSha: git(['rev-parse', 'HEAD']), + baseSha: git(['rev-parse', 'HEAD^1']), + headSha: process.env.VISUAL_HEAD_SHA || git(['rev-parse', 'HEAD']), + event: process.env.GITHUB_EVENT_NAME ?? null, + ref: process.env.GITHUB_REF ?? null, + }; +} + +const usage = ` +node manifest.js [ios_udid] + +Writes a manifest for the PNGs under , reading the environment +from the running emulator (adb) or the simulator with . +`; + +function main(args = process.argv.slice(2)) { + const [platform, captureDir, outputPath, udid] = args; + if ( + !PLATFORMS.includes(platform) || + captureDir == null || + outputPath == null + ) { + throw new Error(usage); + } + const environment = + platform === 'android' + ? readAndroidEnvironment() + : readIosEnvironment(udid); + const manifest = buildManifest({ + captureDir, + platform, + environment, + git: readGitInfo(), + }); + fs.writeFileSync(outputPath, `${JSON.stringify(manifest, null, 2)}\n`); + console.info( + `Wrote ${manifest.images.length} image(s) for ${manifest.environmentKey} to ${outputPath}`, + ); +} + +if (require.main === module) { + main(); +} + +module.exports = { + KEYED_PROPERTIES, + MANIFEST_VERSION, + PLATFORMS, + SHA_PATTERN, + TEST_ID_PATTERN, + buildManifest, + canonicalJson, + environmentKey, + sha256, +}; diff --git a/.github/workflow-scripts/visual/png.js b/.github/workflow-scripts/visual/png.js new file mode 100644 index 000000000000..73d9e5df978b --- /dev/null +++ b/.github/workflow-scripts/visual/png.js @@ -0,0 +1,30 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const PNG_SIGNATURE = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, +]); + +/** + * Reads width and height from a PNG's IHDR chunk without decoding pixels, so + * that untrusted files can be checked before any image library sees them. + * Returns null if the buffer does not start with a well-formed PNG header. + */ +function readPngDimensions(buffer) { + if ( + buffer.length < 24 || + !buffer.subarray(0, 8).equals(PNG_SIGNATURE) || + buffer.toString('ascii', 12, 16) !== 'IHDR' + ) { + return null; + } + return {width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20)}; +} + +module.exports = {readPngDimensions}; diff --git a/.github/workflow-scripts/visual/report.js b/.github/workflow-scripts/visual/report.js new file mode 100644 index 000000000000..7063847c2add --- /dev/null +++ b/.github/workflow-scripts/visual/report.js @@ -0,0 +1,217 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {summarize} = require('./compare'); + +// GitHub limits check run output text to 65535 characters and 50 images. +const MAX_TEXT_LENGTH = 60000; +const MAX_IMAGES = 50; + +const STATUS_LABELS = { + changed: 'Changed', + 'dimension-mismatch': 'Size changed', + added: 'New', + removed: 'Removed', + unchanged: 'Unchanged', +}; + +// GitHub limits comment bodies to 65536 characters. Each changed test costs +// about 1 KB of Markdown, so later tests are listed by name only. +const MAX_COMMENT_TESTS = 20; + +const COMMENT_MARKER = ''; + +function formatPercent(ratio) { + return ratio == null ? '' : `${(ratio * 100).toFixed(3)}%`; +} + +/** How a differing result changed, for example "9633 px (17.502%)". */ +function describeChange(result) { + if (result.status === 'dimension-mismatch') { + return `${result.expectedSize.width}×${result.expectedSize.height} → ${result.capturedSize.width}×${result.capturedSize.height}`; + } + return result.diffPixels != null + ? `${result.diffPixels} px (${formatPercent(result.diffRatio)})` + : ''; +} + +function heading(result) { + const change = describeChange(result); + return `\`${result.id}\`: ${STATUS_LABELS[result.status]}${change ? `, ${change}` : ''}`; +} + +/** + * Renders one platform's comparison as GitHub check run output. `imageUrl` + * maps (kind, test ID) to the public URL of an uploaded expected, captured + * or diff image. + */ +function renderCheckOutput({ + platform, + environmentKey, + baseline, + results, + imageUrl, + error, +}) { + if (error != null) { + return { + title: `${platform}: ${error.title}`, + summary: error.summary, + text: '', + images: [], + }; + } + + const {counts, hasDifferences} = summarize(results); + const differing = results.filter(result => result.status !== 'unchanged'); + const baselineLine = + baseline.distance === 0 + ? `Compared with the baseline for base commit \`${baseline.sha.slice(0, 12)}\`.` + : `No baseline was published for the base commit, so this compares with the nearest ancestor that has one, \`${baseline.sha.slice(0, 12)}\` (${baseline.distance} commit(s) earlier). Differences can come from commits in between.`; + + const summary = [ + hasDifferences + ? `${differing.length} of ${results.length} visual test(s) differ.` + : `All ${results.length} visual test(s) match.`, + '', + baselineLine, + '', + `Environment: \`${environmentKey}\``, + '', + Object.entries(counts) + .filter(([, count]) => count > 0) + .map(([status, count]) => `${STATUS_LABELS[status]}: ${count}`) + .join(' · '), + ].join('\n'); + + const sections = []; + const images = []; + for (const result of differing) { + const row = []; + const cell = kind => ``; + if (result.expected != null) { + row.push(['Expected', cell('expected')]); + } + if (result.captured != null) { + row.push(['Captured', cell('captured')]); + } + if (result.hasDiffImage) { + row.push(['Diff', cell('diff')]); + } + const details = describeChange(result); + sections.push( + [ + `### ${heading(result)}`, + '', + `| ${row.map(([label]) => label).join(' | ')} |`, + `|${row.map(() => '---').join('|')}|`, + `| ${row.map(([, html]) => html).join(' | ')} |`, + ].join('\n'), + ); + if (result.hasDiffImage && images.length < MAX_IMAGES) { + images.push({ + alt: `${result.id} diff`, + image_url: imageUrl('diff', result.id), + caption: `${result.id}: ${details}`, + }); + } + } + + let text = sections.join('\n\n'); + if (text.length > MAX_TEXT_LENGTH) { + text = `${text.slice(0, MAX_TEXT_LENGTH)}\n\n…truncated. See results.json for the full list.`; + } + + return { + title: hasDifferences + ? `${platform}: ${differing.length} visual difference(s)` + : `${platform}: no visual differences`, + summary, + text, + images, + }; +} + +/** + * Renders the pull request comment for all platforms. Each changed test + * shows its diff, with the expected and captured images in a collapsed + * section. `platforms` carry the same fields as `renderCheckOutput` takes, + * plus `checkUrl`, the platform's check run. + */ +function renderPullRequestComment({platforms, headSha, gateSummary}) { + const differing = platforms.flatMap(platform => + platform.error != null + ? [] + : platform.results + .filter(result => result.status !== 'unchanged') + .map(result => ({platform, result})), + ); + const lines = [COMMENT_MARKER, '## Visual regression', '']; + lines.push( + differing.length > 0 + ? `**${differing.length} visual difference(s)** at \`${headSha.slice(0, 12)}\`. ${gateSummary}` + : `No visual differences at \`${headSha.slice(0, 12)}\`.`, + '', + ); + for (const platform of platforms) { + if (platform.error != null) { + lines.push( + `**${platform.platform}:** [${platform.error.title}](${platform.checkUrl})`, + '', + ); + } + } + + for (const [index, {platform, result}] of differing.entries()) { + if (index === MAX_COMMENT_TESTS) { + lines.push( + `${differing.length - index} more:`, + '', + ...differing + .slice(index) + .map( + ({platform: more, result: rest}) => + `- ${more.platform} ${heading(rest)}, [check run](${more.checkUrl})`, + ), + '', + ); + break; + } + const image = (kind, width) => + `${kind}`; + lines.push(`### ${platform.platform} ${heading(result)}`, ''); + if (result.hasDiffImage) { + lines.push(image('diff', 360), ''); + } + const panels = []; + if (result.expected != null) { + panels.push(['Expected', image('expected', 250)]); + } + if (result.captured != null) { + panels.push(['Captured', image('captured', 250)]); + } + if (panels.length > 0) { + lines.push( + `${panels + .map(([label]) => label) + .join(' and ')}`, + '', + `| ${panels.map(([label]) => label).join(' | ')} |`, + `|${panels.map(() => '---').join('|')}|`, + `| ${panels.map(([, html]) => html).join(' | ')} |`, + '', + '
', + '', + ); + } + } + return lines.join('\n'); +} + +module.exports = {COMMENT_MARKER, renderCheckOutput, renderPullRequestComment}; diff --git a/.github/workflow-scripts/visual/storage.js b/.github/workflow-scripts/visual/storage.js new file mode 100644 index 000000000000..e84fd76e5da0 --- /dev/null +++ b/.github/workflow-scripts/visual/storage.js @@ -0,0 +1,140 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const {SHA_PATTERN} = require('./manifest'); + +// Object layout. Every key is write-once: reports and approvals include the +// run that produced them, and a baseline is only ever written for a new SHA. +// +// baselines///manifest.json +// baselines///.png +// reports/pull///-/results.json +// reports/pull///-//{expected,captured,diff}/.png +// reports/main//-/... +// approvals/pull//.json + +function assertSha(sha) { + if (!SHA_PATTERN.test(sha)) { + throw new Error(`Invalid SHA: ${String(sha)}`); + } +} + +function assertPositiveInteger(value, name) { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`Invalid ${name}: ${String(value)}`); + } +} + +function baselinePrefix(environmentKey, sha) { + assertSha(sha); + return `baselines/${environmentKey}/${sha}`; +} + +function reportPrefix({prNumber, headSha, runId, runAttempt}) { + assertSha(headSha); + assertPositiveInteger(runId, 'run ID'); + assertPositiveInteger(runAttempt, 'run attempt'); + if (prNumber == null) { + return `reports/main/${headSha}/${runId}-${runAttempt}`; + } + assertPositiveInteger(prNumber, 'PR number'); + return `reports/pull/${prNumber}/${headSha}/${runId}-${runAttempt}`; +} + +function approvalKey(prNumber, fingerprint) { + assertPositiveInteger(prNumber, 'PR number'); + if (!/^[0-9a-f]{16}$/.test(fingerprint)) { + throw new Error(`Invalid fingerprint: ${String(fingerprint)}`); + } + return `approvals/pull/${prNumber}/${fingerprint}.json`; +} + +function publicUrl(publicBaseUrl, key) { + return `${publicBaseUrl.replace(/\/$/, '')}/${key + .split('/') + .map(encodeURIComponent) + .join('/')}`; +} + +async function fetchOrNull(url) { + const response = await fetch(url, {cache: 'no-store'}); + if (response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`GET ${url} failed with ${response.status}`); + } + return response; +} + +async function fetchJsonOrNull(url) { + const response = await fetchOrNull(url); + return response == null ? null : response.json(); +} + +async function fetchBuffer(url) { + const response = await fetchOrNull(url); + if (response == null) { + throw new Error(`GET ${url} returned 404`); + } + return Buffer.from(await response.arrayBuffer()); +} + +/** + * Lists the immediate "subdirectories" under a prefix, using the public GCS + * JSON API. + */ +async function listPrefixes(bucket, prefix) { + const prefixes = []; + let pageToken = null; + do { + const url = new URL( + `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(bucket)}/o`, + ); + url.searchParams.set('prefix', prefix); + url.searchParams.set('delimiter', '/'); + if (pageToken != null) { + url.searchParams.set('pageToken', pageToken); + } + const page = await fetchJsonOrNull(url.toString()); + prefixes.push(...(page?.prefixes ?? [])); + pageToken = page?.nextPageToken ?? null; + } while (pageToken != null); + return prefixes; +} + +/** + * Returns the first of `candidateShas` (newest first) that has a published + * baseline for `environmentKey`, with its manifest, or null. + */ +async function findBaseline({publicBaseUrl, environmentKey, candidateShas}) { + for (const [distance, sha] of candidateShas.entries()) { + const manifest = await fetchJsonOrNull( + publicUrl( + publicBaseUrl, + `${baselinePrefix(environmentKey, sha)}/manifest.json`, + ), + ); + if (manifest != null) { + return {sha, distance, manifest}; + } + } + return null; +} + +module.exports = { + approvalKey, + baselinePrefix, + fetchBuffer, + fetchJsonOrNull, + findBaseline, + listPrefixes, + publicUrl, + reportPrefix, +}; diff --git a/.github/workflow-scripts/visual/validate.js b/.github/workflow-scripts/visual/validate.js new file mode 100644 index 000000000000..0b6361d420e5 --- /dev/null +++ b/.github/workflow-scripts/visual/validate.js @@ -0,0 +1,183 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const { + MANIFEST_VERSION, + PLATFORMS, + SHA_PATTERN, + TEST_ID_PATTERN, + sha256, +} = require('./manifest'); +const {readPngDimensions} = require('./png'); +const fs = require('fs'); +const path = require('path'); + +const LIMITS = { + maxImages: 200, + maxImageBytes: 5 * 1024 * 1024, + maxTotalBytes: 100 * 1024 * 1024, + maxDimension: 4096, + maxManifestBytes: 1024 * 1024, +}; + +const ENVIRONMENT_KEY_PATTERN = /^(android|ios)-[a-z0-9.-]{1,120}-[0-9a-f]{8}$/; + +/** + * Validates a capture artifact produced by untrusted pull request code. The + * artifact is data only: nothing in it is executed, and every path is checked + * before it is read. Returns the parsed manifest, or throws an Error listing + * every problem found. + */ +function validateCapture(captureDir, limits = LIMITS) { + const errors = []; + const manifestPath = path.join(captureDir, 'manifest.json'); + const manifestStat = lstatOrNull(manifestPath); + if (manifestStat == null || !manifestStat.isFile()) { + throw new Error('manifest.json is missing or not a regular file'); + } + if (manifestStat.size > limits.maxManifestBytes) { + throw new Error( + `manifest.json is larger than ${limits.maxManifestBytes} bytes`, + ); + } + + let manifest; + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + } catch (error) { + throw new Error(`manifest.json is not valid JSON: ${error.message}`); + } + + if (manifest?.version !== MANIFEST_VERSION) { + errors.push(`Unsupported manifest version: ${String(manifest?.version)}`); + } + if (!PLATFORMS.includes(manifest?.platform)) { + errors.push(`Unknown platform: ${String(manifest?.platform)}`); + } + if ( + typeof manifest?.environmentKey !== 'string' || + !ENVIRONMENT_KEY_PATTERN.test(manifest.environmentKey) || + !manifest.environmentKey.startsWith(`${manifest.platform}-`) + ) { + errors.push(`Invalid environment key: ${String(manifest?.environmentKey)}`); + } + for (const field of ['testedSha', 'baseSha', 'headSha']) { + if (!SHA_PATTERN.test(String(manifest?.git?.[field]))) { + errors.push(`Invalid git.${field}`); + } + } + + const images = Array.isArray(manifest?.images) ? manifest.images : null; + if (images == null) { + errors.push('images must be an array'); + } else if (images.length > limits.maxImages) { + errors.push(`Too many images: ${images.length} > ${limits.maxImages}`); + } else { + const seen = new Set(); + let totalBytes = 0; + for (const image of images) { + const id = String(image?.id); + if (!TEST_ID_PATTERN.test(id)) { + errors.push(`Invalid test ID: ${JSON.stringify(image?.id)}`); + continue; + } + if (seen.has(id)) { + errors.push(`Duplicate test ID: ${id}`); + continue; + } + seen.add(id); + // The path is derived from the validated ID, never taken from the + // manifest, so it cannot escape the capture directory. + if (image.path !== `${id}.png`) { + errors.push(`${id}: path must be ${id}.png`); + continue; + } + const imagePath = path.join(captureDir, ...id.split('/')) + '.png'; + const stat = lstatOrNull(imagePath); + if (stat == null || !stat.isFile()) { + errors.push(`${id}: file is missing or not a regular file`); + continue; + } + if (stat.size > limits.maxImageBytes) { + errors.push( + `${id}: ${stat.size} bytes exceeds ${limits.maxImageBytes}`, + ); + continue; + } + totalBytes += stat.size; + const buffer = fs.readFileSync(imagePath); + const dimensions = readPngDimensions(buffer); + if (dimensions == null) { + errors.push(`${id}: not a PNG`); + continue; + } + if ( + dimensions.width === 0 || + dimensions.height === 0 || + dimensions.width > limits.maxDimension || + dimensions.height > limits.maxDimension + ) { + errors.push( + `${id}: ${dimensions.width}x${dimensions.height} is outside 1-${limits.maxDimension} px`, + ); + continue; + } + if ( + image.width !== dimensions.width || + image.height !== dimensions.height || + image.bytes !== stat.size || + image.sha256 !== sha256(buffer) + ) { + errors.push(`${id}: file does not match its manifest entry`); + } + } + if (totalBytes > limits.maxTotalBytes) { + errors.push(`Total size ${totalBytes} exceeds ${limits.maxTotalBytes}`); + } + const unlisted = listFiles(captureDir).filter( + file => file !== 'manifest.json' && !seen.has(file.replace(/\.png$/, '')), + ); + if (unlisted.length > 0) { + errors.push( + `Files not in the manifest: ${unlisted.slice(0, 10).join(', ')}`, + ); + } + } + + if (errors.length > 0) { + throw new Error(`Invalid capture artifact:\n- ${errors.join('\n- ')}`); + } + return manifest; +} + +function lstatOrNull(filePath) { + try { + return fs.lstatSync(filePath); + } catch { + return null; + } +} + +function listFiles(root) { + const files = []; + const visit = (directory, prefix) => { + for (const entry of fs.readdirSync(directory, {withFileTypes: true})) { + const relative = prefix === '' ? entry.name : `${prefix}/${entry.name}`; + if (entry.isDirectory()) { + visit(path.join(directory, entry.name), relative); + } else { + files.push(relative); + } + } + }; + visit(root, ''); + return files; +} + +module.exports = {LIMITS, validateCapture}; diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index fe4f2a56633b..3f4d780d2b2e 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -209,6 +209,13 @@ jobs: fail-on-error: true secrets: inherit + test_visual_ios_rntester: + needs: test_ios_rntester + if: ${{ needs.test_ios_rntester.result == 'success' }} + uses: ./.github/workflows/visual-capture-rntester.yml + with: + platform: ios + test_e2e_ios_templateapp: needs: [build_npm_package, prebuild_apple_dependencies, check_code_changes] if: needs.check_code_changes.outputs.should_test_ios == 'true' @@ -406,6 +413,13 @@ jobs: retry-attempt: 2 secrets: inherit + test_visual_android_rntester: + needs: build_android + if: ${{ needs.build_android.result == 'success' }} + uses: ./.github/workflows/visual-capture-rntester.yml + with: + platform: android + build_npm_package: runs-on: 8-core-ubuntu needs: diff --git a/.github/workflows/visual-approve-rntester.yml b/.github/workflows/visual-approve-rntester.yml new file mode 100644 index 000000000000..9c6f05f7e1e6 --- /dev/null +++ b/.github/workflows/visual-approve-rntester.yml @@ -0,0 +1,67 @@ +# Records a maintainer's approval of a pull request's visual differences when +# the visual-change-approved label is added, and updates the approval gate +# when it is added or removed. Runs from the default branch on +# pull_request_target and never checks out pull request code. +name: Visual Approve RNTester + +on: + pull_request_target: + types: [labeled, unlabeled] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: false + +env: + OUT_DIR: /tmp/visual-approval + VISUAL_BUCKET: ${{ vars.VISUAL_BUCKET }} + VISUAL_PUBLIC_BASE_URL: ${{ vars.VISUAL_PUBLIC_BASE_URL }} + +jobs: + approve: + if: github.event.label.name == 'visual-change-approved' + runs-on: ubuntu-latest + permissions: + checks: write + contents: read + id-token: write + issues: write + pull-requests: write + steps: + - name: Checkout default branch + uses: actions/checkout@v6 + with: + # Never the pull request: this workflow has write permissions. + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Decide + env: + GITHUB_TOKEN: ${{ github.token }} + run: node .github/workflow-scripts/visual/approve-run.js prepare + - name: Check for an approval to record + id: staged + run: | + if [[ -d "$OUT_DIR/upload" ]]; then + echo "upload=true" >> "$GITHUB_OUTPUT" + fi + - name: Authenticate to Google Cloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: ${{ vars.VISUAL_GCP_PROJECT }} + workload_identity_provider: ${{ vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER }} + - name: Setup gcloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Record approval + if: steps.staged.outputs.upload == 'true' + run: gcloud storage cp --recursive --no-clobber "$OUT_DIR/upload/*" "gs://$VISUAL_BUCKET/" + - name: Update gate + env: + GITHUB_TOKEN: ${{ github.token }} + run: node .github/workflow-scripts/visual/approve-run.js report diff --git a/.github/workflows/visual-capture-rntester.yml b/.github/workflows/visual-capture-rntester.yml new file mode 100644 index 000000000000..cde456a4b221 --- /dev/null +++ b/.github/workflows/visual-capture-rntester.yml @@ -0,0 +1,166 @@ +# Untrusted half of RNTester visual regression testing. Called from Test All +# once the RNTester Release build for a platform is ready, it captures the +# visual flows on one canonical device and uploads the PNGs plus a manifest as +# an artifact. It has no secrets and no storage access: comparison and +# publication happen in visual-compare-rntester.yml, which runs from the +# default branch when Test All completes. +name: Visual Capture RNTester + +on: + workflow_call: + inputs: + platform: + description: 'The platform to capture, either android or ios' + required: true + type: string + +permissions: + contents: read + +env: + MAESTRO_VERSION: 2.6.1 + VISUAL_FLOWS: packages/rn-tester/visual-tests/flows + VISUAL_OUTPUT_DIR: /tmp/visual-capture + VISUAL_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + +jobs: + capture_android: + if: inputs.platform == 'android' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # The manifest records HEAD^1, the commit this capture is based on. + fetch-depth: 2 + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'zulu' + - name: Install Maestro + run: curl -Ls "https://get.maestro.mobile.dev" | bash + - name: Enable KVM group perms + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Download RNTester APK + uses: actions/download-artifact@v7 + with: + name: rntester-release + path: /tmp/rntester-android + - name: Capture + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 35 + target: google_apis + arch: x86_64 + # Pins screen size and density, which the action otherwise leaves + # to the AVD defaults. + profile: pixel_7 + # Emulator 37.2.12. Its version can change rendering, so it's + # pinned and is part of the environment key. + emulator-build: '16428233' + ram-size: '8192M' + heap-size: '4096M' + disk-size: '10G' + cores: '4' + disable-animations: true + avd-name: visual_emulator + script: | + adb shell settings put system font_scale 1.0 + # Pixel Launcher often stops responding during boot, and its dialog + # covers the app and anything captured beneath it. The setting + # suppresses new dialogs, and stopping the launcher dismisses one + # that's already showing. + adb shell settings put global hide_error_dialogs 1 + adb shell am force-stop "$(adb shell cmd package resolve-activity --brief -a android.intent.action.MAIN -c android.intent.category.HOME | tail -1 | cut -d/ -f1 | tr -d '\r')" + mkdir -p "$VISUAL_OUTPUT_DIR" + node .github/workflow-scripts/visual/capture.js android /tmp/rntester-android/app-arm64-v8a-release.apk com.facebook.react.uiapp "$VISUAL_FLOWS" "$VISUAL_OUTPUT_DIR" + node .github/workflow-scripts/visual/manifest.js android "$VISUAL_OUTPUT_DIR" "$VISUAL_OUTPUT_DIR/manifest.json" + - name: Upload captures + uses: ./.github/actions/upload-artifact + with: + name: visual-capture-android + path: ${{ env.VISUAL_OUTPUT_DIR }} + if-no-files-found: error + retention-days: 14 + - name: Upload Maestro logs + if: failure() + uses: ./.github/actions/upload-artifact + with: + name: visual-maestro-logs-android + path: | + ~/.maestro/tests + visual-capture-report-*.xml + if-no-files-found: ignore + + capture_ios: + if: inputs.platform == 'ios' + runs-on: macos-26 + env: + DEVICE_MODEL: iPhone-17-Pro + DEVICE_OS: iOS-26-2 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # The manifest records HEAD^1, the commit this capture is based on. + fetch-depth: 2 + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'zulu' + - name: Install Maestro + run: | + curl -Ls "https://get.maestro.mobile.dev" | bash + brew tap facebook/fb + brew install facebook/fb/idb-companion + - name: Download RNTester app + uses: actions/download-artifact@v7 + with: + name: RNTesterApp-NewArch-Release + path: /tmp/RNTesterBuild/RNTester.app + - name: Boot and configure simulator + id: simulator + run: | + UDID=$(xcrun simctl list devices available -j | jq -r --arg runtime "com.apple.CoreSimulator.SimRuntime.$DEVICE_OS" --arg name "${DEVICE_MODEL//-/ }" '.devices[$runtime][] | select(.name == $name) | .udid' | head -1) + if [[ -z "$UDID" ]]; then + echo "::error::No $DEVICE_MODEL simulator on $DEVICE_OS" + xcrun simctl list devices available + exit 1 + fi + xcrun simctl boot "$UDID" + xcrun simctl bootstatus "$UDID" + xcrun simctl ui "$UDID" appearance light + xcrun simctl ui "$UDID" content_size large + xcrun simctl spawn "$UDID" defaults write -g AppleLocale en_US + xcrun simctl spawn "$UDID" defaults write -g AppleLanguages -array en + echo "udid=$UDID" >> "$GITHUB_OUTPUT" + - name: Capture + run: | + mkdir -p "$VISUAL_OUTPUT_DIR" + node .github/workflow-scripts/visual/capture.js ios /tmp/RNTesterBuild/RNTester.app com.meta.RNTester.localDevelopment "$VISUAL_FLOWS" "$VISUAL_OUTPUT_DIR" "${{ steps.simulator.outputs.udid }}" + node .github/workflow-scripts/visual/manifest.js ios "$VISUAL_OUTPUT_DIR" "$VISUAL_OUTPUT_DIR/manifest.json" "${{ steps.simulator.outputs.udid }}" + - name: Upload captures + uses: ./.github/actions/upload-artifact + with: + name: visual-capture-ios + path: ${{ env.VISUAL_OUTPUT_DIR }} + if-no-files-found: error + retention-days: 14 + - name: Upload Maestro logs + if: failure() + uses: ./.github/actions/upload-artifact + with: + name: visual-maestro-logs-ios + path: | + ~/.maestro/tests + visual-capture-report-*.xml + if-no-files-found: ignore diff --git a/.github/workflows/visual-compare-rntester.yml b/.github/workflows/visual-compare-rntester.yml new file mode 100644 index 000000000000..f17981de34f5 --- /dev/null +++ b/.github/workflows/visual-compare-rntester.yml @@ -0,0 +1,135 @@ +# Trusted half of RNTester visual regression testing. Runs from the default +# branch after Test All completes, including for pull requests from forks, and +# compares whatever visual-capture-rntester.yml captured in that run. The +# capture artifacts are treated as inert data: they are validated before any +# image is decoded, and nothing in them is executed. +# +# Pull requests: compare with the baseline for the base commit (or its +# nearest ancestor with one), upload a report, and set check runs. +# Pushes to the default branch: as above, then publish the captures as the +# immutable baseline for that commit. +name: Visual Compare RNTester + +on: + workflow_run: + workflows: [Test All] + types: [completed] + +permissions: + contents: read + +concurrency: + # Serializes baseline publication on the default branch. + group: ${{ github.workflow }}-${{ github.event.workflow_run.event == 'push' && 'main' || github.event.workflow_run.id }} + cancel-in-progress: false + +env: + OUT_DIR: /tmp/visual-out + VISUAL_ARTIFACTS_DIR: /tmp/visual-artifacts + VISUAL_BUCKET: ${{ vars.VISUAL_BUCKET }} + VISUAL_PUBLIC_BASE_URL: ${{ vars.VISUAL_PUBLIC_BASE_URL }} + VISUAL_CONFIG: packages/rn-tester/visual-tests/config.json + +jobs: + find_captures: + # Test All can fail for reasons unrelated to rendering, so a failed run is + # still compared. A platform with no capture is reported as such. + if: contains(fromJSON('["success", "failure"]'), github.event.workflow_run.conclusion) + runs-on: ubuntu-latest + permissions: + actions: read + outputs: + found: ${{ steps.find.outputs.found }} + captures: ${{ steps.find.outputs.captures }} + steps: + - name: Find captures + id: find + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + CAPTURES=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" \ + --jq '.artifacts[] | select(.name | startswith("visual-capture-")) | .name' | tr '\n' ' ') + echo "Captures: ${CAPTURES:-none}" + echo "found=$([[ -n "$CAPTURES" ]] && echo true || echo false)" >> "$GITHUB_OUTPUT" + echo "captures=$CAPTURES" >> "$GITHUB_OUTPUT" + + compare: + needs: find_captures + if: needs.find_captures.outputs.found == 'true' + runs-on: ubuntu-latest + permissions: + actions: read + checks: write + contents: read + id-token: write + issues: write + pull-requests: write + steps: + - name: Checkout default branch + uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Setup node.js + uses: ./.github/actions/setup-node + - name: Install node dependencies + uses: ./.github/actions/yarn-install + # Downloaded by name: a pattern that matches one artifact extracts it + # without the per-artifact directory that compare-run.js reads. + - name: Download Android capture + if: contains(needs.find_captures.outputs.captures, 'visual-capture-android') + uses: actions/download-artifact@v7 + with: + name: visual-capture-android + path: ${{ env.VISUAL_ARTIFACTS_DIR }}/visual-capture-android + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Download iOS capture + if: contains(needs.find_captures.outputs.captures, 'visual-capture-ios') + uses: actions/download-artifact@v7 + with: + name: visual-capture-ios + path: ${{ env.VISUAL_ARTIFACTS_DIR }}/visual-capture-ios + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Compare + env: + GITHUB_TOKEN: ${{ github.token }} + CAPTURE_EVENT: ${{ github.event.workflow_run.event }} + CAPTURE_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + CAPTURE_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + CAPTURE_HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} + CAPTURE_RUN_ID: ${{ github.event.workflow_run.id }} + CAPTURE_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node .github/workflow-scripts/visual/compare-run.js prepare + - name: Check for files to upload + id: staged + run: | + if [[ -d "$OUT_DIR/upload" ]]; then + echo "upload=true" >> "$GITHUB_OUTPUT" + fi + - name: Authenticate to Google Cloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: ${{ vars.VISUAL_GCP_PROJECT }} + workload_identity_provider: ${{ vars.VISUAL_WORKLOAD_IDENTITY_PROVIDER }} + - name: Setup gcloud + if: steps.staged.outputs.upload == 'true' + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Upload report and baseline + if: steps.staged.outputs.upload == 'true' + run: | + # Objects are write-once; --no-clobber makes a rerun skip what an + # earlier attempt already uploaded. + gcloud storage cp --recursive --no-clobber "$OUT_DIR/upload/*" "gs://$VISUAL_BUCKET/" + if [[ -d "$OUT_DIR/baseline-images" ]]; then + gcloud storage cp --recursive --no-clobber "$OUT_DIR/baseline-images/*" "gs://$VISUAL_BUCKET/" + # A baseline exists once its manifest does, so the manifest goes last. + gcloud storage cp --recursive --no-clobber "$OUT_DIR/baseline-manifests/*" "gs://$VISUAL_BUCKET/" + fi + - name: Report + env: + GITHUB_TOKEN: ${{ github.token }} + run: node .github/workflow-scripts/visual/compare-run.js report diff --git a/package.json b/package.json index 1fa53cb0e539..a1743c2892cd 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,8 @@ "micromatch": "^4.0.4", "node-fetch": "^2.2.0", "nullthrows": "^1.1.1", + "pixelmatch": "5.3.0", + "pngjs": "7.0.0", "prettier": "3.9.4", "react": "19.3.0", "react-test-renderer": "19.3.0", diff --git a/packages/rn-tester/README.md b/packages/rn-tester/README.md index b0b3cdb367e7..47c6cc4508d1 100644 --- a/packages/rn-tester/README.md +++ b/packages/rn-tester/README.md @@ -67,3 +67,7 @@ This is different from apps created using `react-native init` which have a depen ## Running tests (iOS) RNTester provides both integration and unit tests. You can run them all by hitting `Cmd+U` in Xcode. Alternatively, it is possible to run just unit tests or integration tests using the keybinding after selecting the appropriate scheme (either `RNTesterUnitTests` or `RNTesterIntegrationTests`). + +## Visual tests + +CI captures screenshots of selected RNTester examples on Android and iOS and compares them with `main`. See [visual-tests/README.md](visual-tests/README.md) to add a test or review a failure. diff --git a/packages/rn-tester/js/examples/Text/TextSharedExamples.js b/packages/rn-tester/js/examples/Text/TextSharedExamples.js index 00ed6c2e27bb..f10cacc936fd 100644 --- a/packages/rn-tester/js/examples/Text/TextSharedExamples.js +++ b/packages/rn-tester/js/examples/Text/TextSharedExamples.js @@ -249,7 +249,7 @@ function LastLineClippingExample(): React.Node { // clipped by one physical pixel after layout rounding (issue #53450). The // border makes any clipping of the last line visible. return ( - + {[11, 12, 13, 14, 15, 16, 17].map(fontSize => ( /`. Wait for something that's only visible once the example has rendered, and crop to the example's `testID` so that the status bar and surrounding UI aren't part of the screenshot. + +3. Open a PR. The new test has no baseline yet, so the report lists it as new, and like any other difference it needs a maintainer's approval (see below). Once the PR lands, `main` is captured with it and later PRs are compared against that. + +To try a flow locally, build and install RNTester Release on a simulator or emulator and run it with Maestro: + +```sh +maestro test \ + -e APP_ID=com.meta.RNTester.localDevelopment \ + -e VISUAL_OUTPUT_DIR=/tmp/visual \ + packages/rn-tester/visual-tests/flows/text-width-mode.yml +``` + +Use `com.facebook.react.uiapp` as the `APP_ID` on Android. Local screenshots are only useful for checking the flow and the crop - they won't match CI's pixel for pixel unless you use the same device and OS image. + +## When a check fails + +Open the PR comment or the check run and compare the expected, captured and diff images. + +- If the change is unintended, fix it and push. +- If it's intended, a maintainer applies the `visual-change-approved` label. The approval covers the exact set of differences in the report: if a later push changes them, the label is removed and the PR needs a new review. The baseline itself only changes once the PR lands and `main` is captured again. + +## Tolerances + +Comparison is strict by default: any changed pixel is a difference. If a test is genuinely nondeterministic, loosen it for that test in `config.json`, keyed by test ID, and say why in the PR: + +```json +{ + "defaults": {"threshold": 0, "includeAA": true, "maxDiffPixels": 0}, + "tests": { + "text/width-mode/default": {"maxDiffPixels": 20} + } +} +``` + +`threshold` is pixelmatch's per-pixel color distance from 0 to 1, `includeAA` counts anti-aliased pixels, and `maxDiffPixels` and `maxDiffRatio` set how many changed pixels (or what share of the image) can pass. Prefer making the example deterministic over adding a tolerance. diff --git a/packages/rn-tester/visual-tests/config.json b/packages/rn-tester/visual-tests/config.json new file mode 100644 index 000000000000..e1e08c5047c6 --- /dev/null +++ b/packages/rn-tester/visual-tests/config.json @@ -0,0 +1,8 @@ +{ + "defaults": { + "threshold": 0, + "includeAA": true, + "maxDiffPixels": 0 + }, + "tests": {} +} diff --git a/packages/rn-tester/visual-tests/flows/text-last-line-clipping.yml b/packages/rn-tester/visual-tests/flows/text-last-line-clipping.yml new file mode 100644 index 000000000000..8cbdf153ca6b --- /dev/null +++ b/packages/rn-tester/visual-tests/flows/text-last-line-clipping.yml @@ -0,0 +1,15 @@ +appId: ${APP_ID} # iOS: com.meta.RNTester.localDevelopment | Android: com.facebook.react.uiapp +--- +- runFlow: + file: ../helpers/open-example.yml + env: + EXAMPLE_URL: rntester://example/TextExample/wrappedLastLineClipping +- extendedWaitUntil: + visible: + id: 'last-line-clipping-example' + timeout: 15000 +- assertVisible: '(17px) This sentence wraps across multiple lines so the final line sits near a pixel boundary and must render fully, without being cut off.' +- takeScreenshot: + path: ${VISUAL_OUTPUT_DIR}/text/last-line-clipping/default + cropOn: + id: 'last-line-clipping-example' diff --git a/packages/rn-tester/visual-tests/flows/text-width-mode.yml b/packages/rn-tester/visual-tests/flows/text-width-mode.yml new file mode 100644 index 000000000000..7a23ad2534d8 --- /dev/null +++ b/packages/rn-tester/visual-tests/flows/text-width-mode.yml @@ -0,0 +1,18 @@ +appId: ${APP_ID} # iOS: com.meta.RNTester.localDevelopment | Android: com.facebook.react.uiapp +--- +- runFlow: + file: ../helpers/open-example.yml + env: + EXAMPLE_URL: rntester://example/TextExample/textWidthMode +- extendedWaitUntil: + visible: + id: 'text-width-mode-example' + timeout: 15000 +- assertVisible: + id: 'text-width-mode-auto' +- assertVisible: + id: 'text-width-mode-longest-line' +- takeScreenshot: + path: ${VISUAL_OUTPUT_DIR}/text/width-mode/default + cropOn: + id: 'text-width-mode-example' diff --git a/packages/rn-tester/visual-tests/helpers/open-example.yml b/packages/rn-tester/visual-tests/helpers/open-example.yml new file mode 100644 index 000000000000..80d5d0f88961 --- /dev/null +++ b/packages/rn-tester/visual-tests/helpers/open-example.yml @@ -0,0 +1,15 @@ +appId: ${APP_ID} # iOS: com.meta.RNTester.localDevelopment | Android: com.facebook.react.uiapp +--- +- launchApp: + stopApp: true +- extendedWaitUntil: + visible: 'Components' + timeout: 30000 +- openLink: ${EXAMPLE_URL} +- runFlow: + when: + platform: iOS + commands: + - tapOn: + text: 'Open' + optional: true diff --git a/yarn.lock b/yarn.lock index 24979eb64692..50389318dcaf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7774,6 +7774,13 @@ pirates@^4.0.1, pirates@^4.0.4, pirates@^4.0.6: resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9" integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg== +pixelmatch@5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.3.0.tgz#5e5321a7abedfb7962d60dbf345deda87cb9560a" + integrity sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q== + dependencies: + pngjs "^6.0.0" + pkg-dir@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3" @@ -7809,6 +7816,16 @@ plist@^3.0.5, plist@^3.1.0: base64-js "^1.5.1" xmlbuilder "^15.1.1" +pngjs@7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-7.0.0.tgz#a8b7446020ebbc6ac739db6c5415a65d17090e26" + integrity sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow== + +pngjs@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-6.0.0.tgz#ca9e5d2aa48db0228a52c419c3308e87720da821" + integrity sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg== + possible-typed-array-names@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f"