diff --git a/.github/workflows/harness-validate.yml b/.github/workflows/harness-validate.yml index efdda5c..d3f4d26 100644 --- a/.github/workflows/harness-validate.yml +++ b/.github/workflows/harness-validate.yml @@ -52,13 +52,17 @@ jobs: echo "to=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - - name: Show selected harness suites - run: pnpm harness:select -- --from "${{ steps.range.outputs.from }}" --to "${{ steps.range.outputs.to }}" + - name: Run selected harness suites + run: pnpm harness:validate:changed -- --from "${{ steps.range.outputs.from }}" --to "${{ steps.range.outputs.to }}" - - name: Run PR harness suite on pull requests - if: github.event_name == 'pull_request' - run: pnpm harness:validate:pr - - - name: Run PR harness suite on main - if: github.event_name == 'push' - run: pnpm harness:validate:pr + - name: Upload harness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: harness-artifacts-${{ github.run_id }}-${{ github.run_attempt }} + if-no-files-found: ignore + path: | + .artifacts/harness + .artifacts/a11y + .artifacts/test-results + retention-days: 7 diff --git a/scripts/harness/core.mjs b/scripts/harness/core.mjs index 99abb31..db3dd9b 100644 --- a/scripts/harness/core.mjs +++ b/scripts/harness/core.mjs @@ -89,6 +89,9 @@ const suiteOrder = [ "release" ]; +const prSupersededSuites = ["static", "component", "docs", "a11y", "docs-smoke", "consumers"]; +const releaseSupersededSuites = [...prSupersededSuites, "pr"]; + const rootStaticFiles = new Set([ "eslint.config.mjs", "package.json", @@ -101,7 +104,7 @@ const rootStaticFiles = new Set([ ]); function normalizeFilePath(filePath) { - return filePath.split(path.sep).join(path.posix.sep); + return filePath.replace(/\\/g, path.posix.sep).split(path.sep).join(path.posix.sep); } function isExactMatch(filePath, exactPaths) { @@ -147,9 +150,12 @@ function isConsumerSurfaceChange(filePath) { ); } -function isHarnessCodeChange(filePath) { +function isHarnessControlPlaneChange(filePath) { return ( isWithin(filePath, "scripts/harness") || + filePath === "playwright.config.ts" || + filePath === "vitest.config.ts" || + isWithin(filePath, "tests/e2e/support") || filePath === "AGENTS.md" || filePath === "CONTRIBUTING.md" || filePath === "README.md" || @@ -159,6 +165,22 @@ function isHarnessCodeChange(filePath) { ); } +function isReleaseRiskChange(filePath) { + return ( + filePath === "package.json" || + filePath === "docs/releasing.md" || + filePath === "docs/registry.md" || + filePath === "scripts/release-metadata.mjs" || + filePath === "scripts/build-registry.mjs" || + filePath === "scripts/registry-install.mjs" || + filePath === "packages/ui/package.json" || + filePath === "packages/tokens/package.json" || + isWithin(filePath, ".github/workflows") || + isWithin(filePath, "tests/package-consumer") || + isWithin(filePath, "tests/registry") + ); +} + function isStaticSurfaceChange(filePath) { return ( isPackageContractChange(filePath) || @@ -167,7 +189,8 @@ function isStaticSurfaceChange(filePath) { isWithin(filePath, "scripts") || isWithin(filePath, ".github/workflows") || isExactMatch(filePath, rootStaticFiles) || - isHarnessCodeChange(filePath) + isHarnessControlPlaneChange(filePath) || + isReleaseRiskChange(filePath) ); } @@ -186,17 +209,22 @@ function runGitCommand(args) { .map((line) => normalizeFilePath(line)); } -function getWorkingTreeChangedFiles() { +function getWorkingTreeChangedFiles(runGitCommandFn = runGitCommand) { const files = new Set([ - ...runGitCommand(["diff", "--name-only", "--diff-filter=ACMR", "HEAD"]), - ...runGitCommand(["diff", "--cached", "--name-only", "--diff-filter=ACMR", "HEAD"]), - ...runGitCommand(["ls-files", "--others", "--exclude-standard"]) + ...runGitCommandFn(["diff", "--name-only", "--diff-filter=ACMR", "HEAD"]), + ...runGitCommandFn(["diff", "--cached", "--name-only", "--diff-filter=ACMR", "HEAD"]), + ...runGitCommandFn(["ls-files", "--others", "--exclude-standard"]) ]); return [...files].sort(); } -export function getChangedFiles({ changedFiles = [], from = null, to = null } = {}) { +export function getChangedFiles(options = {}) { + const changedFiles = options.changedFiles ?? []; + const from = options.from ?? null; + const to = options.to ?? null; + const runGitCommandFn = options.runGitCommandFn ?? runGitCommand; + if (changedFiles.length > 0) { return [...new Set(changedFiles.map((filePath) => normalizeFilePath(filePath)))].sort(); } @@ -207,8 +235,8 @@ export function getChangedFiles({ changedFiles = [], from = null, to = null } = if (from && zeroShaPattern.test(from)) { return [ ...new Set([ - ...runGitCommand(["ls-files"]), - ...runGitCommand(["ls-files", "--others", "--exclude-standard"]) + ...runGitCommandFn(["ls-files"]), + ...runGitCommandFn(["ls-files", "--others", "--exclude-standard"]) ]) ].sort(); } @@ -217,10 +245,10 @@ export function getChangedFiles({ changedFiles = [], from = null, to = null } = throw new Error("Expected --from when selecting changed files from git refs."); } - return runGitCommand(["diff", "--name-only", "--diff-filter=ACMR", `${from}...${toRef}`]); + return [...runGitCommandFn(["diff", "--name-only", "--diff-filter=ACMR", `${from}...${toRef}`])].sort(); } - return getWorkingTreeChangedFiles(); + return getWorkingTreeChangedFiles(runGitCommandFn); } function addReason(reasonMap, suiteName, filePath, reason) { @@ -238,6 +266,18 @@ export function selectSuitesForChangedFiles(changedFiles) { for (const filePath of normalizedFiles) { let matched = false; + if (isReleaseRiskChange(filePath)) { + selectedSuites.add("release"); + addReason(reasons, "release", filePath, "Release or consumer control plane changed."); + matched = true; + } + + if (isHarnessControlPlaneChange(filePath)) { + selectedSuites.add("pr"); + addReason(reasons, "pr", filePath, "Harness control plane changed."); + matched = true; + } + if (isPackageContractChange(filePath)) { selectedSuites.add("component"); selectedSuites.add("docs"); @@ -294,6 +334,20 @@ export function selectSuitesForChangedFiles(changedFiles) { reasons.delete("static"); } + if (selectedSuites.has("pr")) { + for (const suiteName of prSupersededSuites) { + selectedSuites.delete(suiteName); + reasons.delete(suiteName); + } + } + + if (selectedSuites.has("release")) { + for (const suiteName of releaseSupersededSuites) { + selectedSuites.delete(suiteName); + reasons.delete(suiteName); + } + } + if (selectedSuites.size === 0 && unmatchedFiles.some((filePath) => isLikelyCodeChange(filePath))) { selectedSuites.add("static"); @@ -354,9 +408,9 @@ export function ensureKnownSuite(name) { } } -export function writeHarnessArtifact(name, payload) { - fs.mkdirSync(artifactsDir, { recursive: true }); - const outputPath = path.join(artifactsDir, `${name}.json`); +export function writeHarnessArtifact(name, payload, outputDir = artifactsDir) { + fs.mkdirSync(outputDir, { recursive: true }); + const outputPath = path.join(outputDir, `${name}.json`); fs.writeFileSync(outputPath, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); return outputPath; } diff --git a/scripts/harness/core.test.ts b/scripts/harness/core.test.ts new file mode 100644 index 0000000..4f93980 --- /dev/null +++ b/scripts/harness/core.test.ts @@ -0,0 +1,109 @@ +// @vitest-environment node + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + getChangedFiles, + selectSuitesForChangedFiles, + writeHarnessArtifact +} from "./core.mjs"; + +const tempDirs: string[] = []; + +afterEach(() => { + for (const directory of tempDirs.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +function createTempDir(prefix: string) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(directory); + return directory; +} + +describe("harness changed-file helpers", () => { + it("normalizes and de-duplicates explicit changed files", () => { + expect(getChangedFiles({ changedFiles: ["scripts\\harness\\core.mjs", "scripts/harness/core.mjs"] })).toEqual([ + "scripts/harness/core.mjs" + ]); + }); + + it("uses the injected git reader for explicit diff ranges", () => { + const runGitCommandFn = vi.fn(() => ["scripts/harness/core.mjs", "README.md"]); + + expect(getChangedFiles({ from: "abc123", runGitCommandFn, to: "def456" })).toEqual([ + "README.md", + "scripts/harness/core.mjs" + ]); + expect(runGitCommandFn).toHaveBeenCalledWith([ + "diff", + "--name-only", + "--diff-filter=ACMR", + "abc123...def456" + ]); + }); + + it("falls back to a repo snapshot when the base ref is an all-zero sha", () => { + const runGitCommandFn = vi.fn((args: string[]) => { + if (args[0] === "ls-files" && args.length === 1) { + return ["tracked.ts"]; + } + + return ["untracked.ts"]; + }); + + expect( + getChangedFiles({ + from: "0000000000000000000000000000000000000000", + runGitCommandFn, + to: "HEAD" + }) + ).toEqual(["tracked.ts", "untracked.ts"]); + }); +}); + +describe("selectSuitesForChangedFiles", () => { + it("promotes harness control-plane changes to the PR gate", () => { + expect(selectSuitesForChangedFiles(["scripts/harness/core.mjs"]).suites).toEqual(["pr"]); + }); + + it("promotes release-risk changes to the release gate", () => { + expect(selectSuitesForChangedFiles([".github/workflows/publish-packages.yml"]).suites).toEqual([ + "release" + ]); + }); + + it("keeps docs stories on the docs review surfaces", () => { + expect(selectSuitesForChangedFiles(["apps/docs/src/components/button.stories.tsx"]).suites).toEqual([ + "static", + "docs", + "a11y", + "docs-smoke" + ]); + }); + + it("keeps package source changes on component, docs, browser, and consumer surfaces", () => { + expect(selectSuitesForChangedFiles(["packages/ui/src/components/button.tsx"]).suites).toEqual([ + "component", + "docs", + "a11y", + "docs-smoke", + "consumers" + ]); + }); +}); + +describe("writeHarnessArtifact", () => { + it("writes the requested report into a target directory", () => { + const outputDir = createTempDir("cadence-harness-artifacts-"); + const outputPath = writeHarnessArtifact("selection", { suites: ["pr"] }, outputDir); + + expect(path.dirname(outputPath)).toBe(outputDir); + expect(JSON.parse(fs.readFileSync(outputPath, "utf8"))).toEqual({ suites: ["pr"] }); + }); +}); diff --git a/scripts/harness/validate.mjs b/scripts/harness/validate.mjs index 9b36ba6..8a963b9 100644 --- a/scripts/harness/validate.mjs +++ b/scripts/harness/validate.mjs @@ -94,6 +94,40 @@ function runCommand(command) { }; } +function printSelection(selection) { + if (selection.suites.length === 0) { + process.stdout.write("[harness] No suites selected for the current diff.\n"); + + if (selection.unmatchedFiles.length > 0) { + process.stdout.write("[harness] Unmatched files:\n"); + + for (const filePath of selection.unmatchedFiles) { + process.stdout.write(` - ${filePath}\n`); + } + } + + return; + } + + process.stdout.write("[harness] Selected suites from changed files:\n"); + + for (const suiteName of selection.suites) { + process.stdout.write(`- ${suiteName}\n`); + + for (const reason of selection.reasons[suiteName] ?? []) { + process.stdout.write(` - ${reason.file}: ${reason.reason}\n`); + } + } + + if (selection.unmatchedFiles.length > 0) { + process.stdout.write("[harness] Unmatched files:\n"); + + for (const filePath of selection.unmatchedFiles) { + process.stdout.write(` - ${filePath}\n`); + } + } +} + const options = parseArgs(process.argv.slice(2)); if (options.list) { @@ -128,6 +162,7 @@ const report = { if (selection) { report.selection = selection; + printSelection(selection); } for (const command of commands) {