chore(harness): run changed suites in CI

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>

.github/workflows/harness-validate.yml

@@ -52,13 +52,17 @@ jobs:
 
           echo "to=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
 
-      - name: Show selected harness suites
-        run: pnpm harness:select -- --from "${{ steps.range.outputs.from }}" --to "${{ steps.range.outputs.to }}"
+      - name: Run selected harness suites
+        run: pnpm harness:validate:changed -- --from "${{ steps.range.outputs.from }}" --to "${{ steps.range.outputs.to }}"
 
-      - name: Run PR harness suite on pull requests
-        if: github.event_name == 'pull_request'
-        run: pnpm harness:validate:pr
-
-      - name: Run PR harness suite on main
-        if: github.event_name == 'push'
-        run: pnpm harness:validate:pr
+      - name: Upload harness artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: harness-artifacts-${{ github.run_id }}-${{ github.run_attempt }}
+          if-no-files-found: ignore
+          path: |
+            .artifacts/harness
+            .artifacts/a11y
+            .artifacts/test-results
+          retention-days: 7

scripts/harness/core.mjs

@@ -89,6 +89,9 @@ const suiteOrder = [
   "release"
 ];
 
+const prSupersededSuites = ["static", "component", "docs", "a11y", "docs-smoke", "consumers"];
+const releaseSupersededSuites = [...prSupersededSuites, "pr"];
+
 const rootStaticFiles = new Set([
   "eslint.config.mjs",
   "package.json",
@@ -101,7 +104,7 @@ const rootStaticFiles = new Set([
 ]);
 
 function normalizeFilePath(filePath) {
-  return filePath.split(path.sep).join(path.posix.sep);
+  return filePath.replace(/\\/g, path.posix.sep).split(path.sep).join(path.posix.sep);
 }
 
 function isExactMatch(filePath, exactPaths) {
@@ -147,9 +150,12 @@ function isConsumerSurfaceChange(filePath) {
   );
 }
 
-function isHarnessCodeChange(filePath) {
+function isHarnessControlPlaneChange(filePath) {
   return (
     isWithin(filePath, "scripts/harness") ||
+    filePath === "playwright.config.ts" ||
+    filePath === "vitest.config.ts" ||
+    isWithin(filePath, "tests/e2e/support") ||
     filePath === "AGENTS.md" ||
     filePath === "CONTRIBUTING.md" ||
     filePath === "README.md" ||
@@ -159,6 +165,22 @@ function isHarnessCodeChange(filePath) {
   );
 }
 
+function isReleaseRiskChange(filePath) {
+  return (
+    filePath === "package.json" ||
+    filePath === "docs/releasing.md" ||
+    filePath === "docs/registry.md" ||
+    filePath === "scripts/release-metadata.mjs" ||
+    filePath === "scripts/build-registry.mjs" ||
+    filePath === "scripts/registry-install.mjs" ||
+    filePath === "packages/ui/package.json" ||
+    filePath === "packages/tokens/package.json" ||
+    isWithin(filePath, ".github/workflows") ||
+    isWithin(filePath, "tests/package-consumer") ||
+    isWithin(filePath, "tests/registry")
+  );
+}
+
 function isStaticSurfaceChange(filePath) {
   return (
     isPackageContractChange(filePath) ||
@@ -167,7 +189,8 @@ function isStaticSurfaceChange(filePath) {
     isWithin(filePath, "scripts") ||
     isWithin(filePath, ".github/workflows") ||
     isExactMatch(filePath, rootStaticFiles) ||
-    isHarnessCodeChange(filePath)
+    isHarnessControlPlaneChange(filePath) ||
+    isReleaseRiskChange(filePath)
   );
 }
 
@@ -186,17 +209,22 @@ function runGitCommand(args) {
     .map((line) => normalizeFilePath(line));
 }
 
-function getWorkingTreeChangedFiles() {
+function getWorkingTreeChangedFiles(runGitCommandFn = runGitCommand) {
   const files = new Set([
-    ...runGitCommand(["diff", "--name-only", "--diff-filter=ACMR", "HEAD"]),
-    ...runGitCommand(["diff", "--cached", "--name-only", "--diff-filter=ACMR", "HEAD"]),
-    ...runGitCommand(["ls-files", "--others", "--exclude-standard"])
+    ...runGitCommandFn(["diff", "--name-only", "--diff-filter=ACMR", "HEAD"]),
+    ...runGitCommandFn(["diff", "--cached", "--name-only", "--diff-filter=ACMR", "HEAD"]),
+    ...runGitCommandFn(["ls-files", "--others", "--exclude-standard"])
   ]);
 
   return [...files].sort();
 }
 
-export function getChangedFiles({ changedFiles = [], from = null, to = null } = {}) {
+export function getChangedFiles(options = {}) {
+  const changedFiles = options.changedFiles ?? [];
+  const from = options.from ?? null;
+  const to = options.to ?? null;
+  const runGitCommandFn = options.runGitCommandFn ?? runGitCommand;
+
   if (changedFiles.length > 0) {
     return [...new Set(changedFiles.map((filePath) => normalizeFilePath(filePath)))].sort();
   }
@@ -207,8 +235,8 @@ export function getChangedFiles({ changedFiles = [], from = null, to = null } =
     if (from && zeroShaPattern.test(from)) {
       return [
         ...new Set([
-          ...runGitCommand(["ls-files"]),
-          ...runGitCommand(["ls-files", "--others", "--exclude-standard"])
+          ...runGitCommandFn(["ls-files"]),
+          ...runGitCommandFn(["ls-files", "--others", "--exclude-standard"])
         ])
       ].sort();
     }
@@ -217,10 +245,10 @@ export function getChangedFiles({ changedFiles = [], from = null, to = null } =
       throw new Error("Expected --from when selecting changed files from git refs.");
     }
 
-    return runGitCommand(["diff", "--name-only", "--diff-filter=ACMR", `${from}...${toRef}`]);
+    return [...runGitCommandFn(["diff", "--name-only", "--diff-filter=ACMR", `${from}...${toRef}`])].sort();
   }
 
-  return getWorkingTreeChangedFiles();
+  return getWorkingTreeChangedFiles(runGitCommandFn);
 }
 
 function addReason(reasonMap, suiteName, filePath, reason) {
@@ -238,6 +266,18 @@ export function selectSuitesForChangedFiles(changedFiles) {
   for (const filePath of normalizedFiles) {
     let matched = false;
 
+    if (isReleaseRiskChange(filePath)) {
+      selectedSuites.add("release");
+      addReason(reasons, "release", filePath, "Release or consumer control plane changed.");
+      matched = true;
+    }
+
+    if (isHarnessControlPlaneChange(filePath)) {
+      selectedSuites.add("pr");
+      addReason(reasons, "pr", filePath, "Harness control plane changed.");
+      matched = true;
+    }
+
     if (isPackageContractChange(filePath)) {
       selectedSuites.add("component");
       selectedSuites.add("docs");
@@ -294,6 +334,20 @@ export function selectSuitesForChangedFiles(changedFiles) {
     reasons.delete("static");
   }
 
+  if (selectedSuites.has("pr")) {
+    for (const suiteName of prSupersededSuites) {
+      selectedSuites.delete(suiteName);
+      reasons.delete(suiteName);
+    }
+  }
+
+  if (selectedSuites.has("release")) {
+    for (const suiteName of releaseSupersededSuites) {
+      selectedSuites.delete(suiteName);
+      reasons.delete(suiteName);
+    }
+  }
+
   if (selectedSuites.size === 0 && unmatchedFiles.some((filePath) => isLikelyCodeChange(filePath))) {
     selectedSuites.add("static");
 
@@ -354,9 +408,9 @@ export function ensureKnownSuite(name) {
   }
 }
 
-export function writeHarnessArtifact(name, payload) {
-  fs.mkdirSync(artifactsDir, { recursive: true });
-  const outputPath = path.join(artifactsDir, `${name}.json`);
+export function writeHarnessArtifact(name, payload, outputDir = artifactsDir) {
+  fs.mkdirSync(outputDir, { recursive: true });
+  const outputPath = path.join(outputDir, `${name}.json`);
   fs.writeFileSync(outputPath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
   return outputPath;
 }

scripts/harness/core.test.ts

@@ -0,0 +1,109 @@
+// @vitest-environment node
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import {
+  getChangedFiles,
+  selectSuitesForChangedFiles,
+  writeHarnessArtifact
+} from "./core.mjs";
+
+const tempDirs: string[] = [];
+
+afterEach(() => {
+  for (const directory of tempDirs.splice(0)) {
+    fs.rmSync(directory, { force: true, recursive: true });
+  }
+});
+
+function createTempDir(prefix: string) {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(directory);
+  return directory;
+}
+
+describe("harness changed-file helpers", () => {
+  it("normalizes and de-duplicates explicit changed files", () => {
+    expect(getChangedFiles({ changedFiles: ["scripts\\harness\\core.mjs", "scripts/harness/core.mjs"] })).toEqual([
+      "scripts/harness/core.mjs"
+    ]);
+  });
+
+  it("uses the injected git reader for explicit diff ranges", () => {
+    const runGitCommandFn = vi.fn(() => ["scripts/harness/core.mjs", "README.md"]);
+
+    expect(getChangedFiles({ from: "abc123", runGitCommandFn, to: "def456" })).toEqual([
+      "README.md",
+      "scripts/harness/core.mjs"
+    ]);
+    expect(runGitCommandFn).toHaveBeenCalledWith([
+      "diff",
+      "--name-only",
+      "--diff-filter=ACMR",
+      "abc123...def456"
+    ]);
+  });
+
+  it("falls back to a repo snapshot when the base ref is an all-zero sha", () => {
+    const runGitCommandFn = vi.fn((args: string[]) => {
+      if (args[0] === "ls-files" && args.length === 1) {
+        return ["tracked.ts"];
+      }
+
+      return ["untracked.ts"];
+    });
+
+    expect(
+      getChangedFiles({
+        from: "0000000000000000000000000000000000000000",
+        runGitCommandFn,
+        to: "HEAD"
+      })
+    ).toEqual(["tracked.ts", "untracked.ts"]);
+  });
+});
+
+describe("selectSuitesForChangedFiles", () => {
+  it("promotes harness control-plane changes to the PR gate", () => {
+    expect(selectSuitesForChangedFiles(["scripts/harness/core.mjs"]).suites).toEqual(["pr"]);
+  });
+
+  it("promotes release-risk changes to the release gate", () => {
+    expect(selectSuitesForChangedFiles([".github/workflows/publish-packages.yml"]).suites).toEqual([
+      "release"
+    ]);
+  });
+
+  it("keeps docs stories on the docs review surfaces", () => {
+    expect(selectSuitesForChangedFiles(["apps/docs/src/components/button.stories.tsx"]).suites).toEqual([
+      "static",
+      "docs",
+      "a11y",
+      "docs-smoke"
+    ]);
+  });
+
+  it("keeps package source changes on component, docs, browser, and consumer surfaces", () => {
+    expect(selectSuitesForChangedFiles(["packages/ui/src/components/button.tsx"]).suites).toEqual([
+      "component",
+      "docs",
+      "a11y",
+      "docs-smoke",
+      "consumers"
+    ]);
+  });
+});
+
+describe("writeHarnessArtifact", () => {
+  it("writes the requested report into a target directory", () => {
+    const outputDir = createTempDir("cadence-harness-artifacts-");
+    const outputPath = writeHarnessArtifact("selection", { suites: ["pr"] }, outputDir);
+
+    expect(path.dirname(outputPath)).toBe(outputDir);
+    expect(JSON.parse(fs.readFileSync(outputPath, "utf8"))).toEqual({ suites: ["pr"] });
+  });
+});

scripts/harness/validate.mjs

@@ -94,6 +94,40 @@ function runCommand(command) {
   };
 }
 
+function printSelection(selection) {
+  if (selection.suites.length === 0) {
+    process.stdout.write("[harness] No suites selected for the current diff.\n");
+
+    if (selection.unmatchedFiles.length > 0) {
+      process.stdout.write("[harness] Unmatched files:\n");
+
+      for (const filePath of selection.unmatchedFiles) {
+        process.stdout.write(`  - ${filePath}\n`);
+      }
+    }
+
+    return;
+  }
+
+  process.stdout.write("[harness] Selected suites from changed files:\n");
+
+  for (const suiteName of selection.suites) {
+    process.stdout.write(`- ${suiteName}\n`);
+
+    for (const reason of selection.reasons[suiteName] ?? []) {
+      process.stdout.write(`  - ${reason.file}: ${reason.reason}\n`);
+    }
+  }
+
+  if (selection.unmatchedFiles.length > 0) {
+    process.stdout.write("[harness] Unmatched files:\n");
+
+    for (const filePath of selection.unmatchedFiles) {
+      process.stdout.write(`  - ${filePath}\n`);
+    }
+  }
+}
+
 const options = parseArgs(process.argv.slice(2));
 
 if (options.list) {
@@ -128,6 +162,7 @@ const report = {
 
 if (selection) {
   report.selection = selection;
+  printSelection(selection);
 }
 
 for (const command of commands) {