Add spec-aware orch tasks and verification gates

This commit is contained in:
2026-03-23 14:05:10 +08:00
parent 4d8c90eb26
commit 9f9b66330c
22 changed files with 1696 additions and 55 deletions
@@ -1,6 +1,7 @@
package orch
import (
"os"
"path/filepath"
"strings"
"testing"
@@ -143,6 +144,98 @@ func TestOrchTaskAddRejectsInvalidPriority(t *testing.T) {
assertErrorMessageContains(t, stdout, "priority must be one of low, normal, high")
}
func TestOrchTaskAddSnapshotsSpecAndVerificationPolicy(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "coord.db")
specFile := filepath.Join(tempDir, "task.md")
if err := os.WriteFile(specFile, []byte("# Task\n\nImplement the first verifier slice.\n"), 0o644); err != nil {
t.Fatalf("write spec file: %v", err)
}
runOrchCommand(
t,
"--db", dbPath,
"--json",
"run", "init",
"--run", "run_blog_006",
"--goal", "Validate spec-aware task add",
)
taskOut := runOrchCommand(
t,
"--db", dbPath,
"--json",
"task", "add",
"--run", "run_blog_006",
"--task", "T1",
"--title", "Implement verifier",
"--spec-file", specFile,
"--check-profile", "cadence_component",
"--required-check", "lint",
"--required-check", "test",
"--allowed-path", "packages/ui",
"--blocked-path", "scripts/release-metadata.mjs",
"--metadata-json", `{"repo":"cadence-ui"}`,
)
var taskResp map[string]any
mustDecodeJSON(t, taskOut, &taskResp)
task := nestedValue(t, taskResp, "data", "task").(map[string]any)
spec := nestedValue(t, task, "spec").(map[string]any)
gate := nestedValue(t, task, "gate").(map[string]any)
if got, _ := spec["spec_file"].(string); got != specFile {
t.Fatalf("expected spec_file %q, got %#v", specFile, spec["spec_file"])
}
if got, _ := spec["check_profile"].(string); got != "cadence_component" {
t.Fatalf("expected check_profile cadence_component, got %#v", spec["check_profile"])
}
requiredChecks, ok := spec["required_checks"].([]any)
if !ok || len(requiredChecks) != 2 {
t.Fatalf("expected required_checks array, got %#v", spec["required_checks"])
}
if got, _ := gate["status"].(string); got != "pending" {
t.Fatalf("expected pending gate, got %#v", gate["status"])
}
}
func TestOrchTaskAddRejectsSpecSHAMismatch(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "coord.db")
specFile := filepath.Join(tempDir, "task.md")
if err := os.WriteFile(specFile, []byte("hello spec\n"), 0o644); err != nil {
t.Fatalf("write spec file: %v", err)
}
runOrchCommand(
t,
"--db", dbPath,
"--json",
"run", "init",
"--run", "run_blog_007",
"--goal", "Validate spec sha mismatch",
)
stdout, _, exitCode := executeOrchCommand(
"--db", dbPath,
"--json",
"task", "add",
"--run", "run_blog_007",
"--task", "T1",
"--title", "Implement verifier",
"--spec-file", specFile,
"--spec-sha", "deadbeef",
)
if exitCode != 30 {
t.Fatalf("expected invalid_input exit code 30, got %d\nstdout:\n%s", exitCode, stdout)
}
assertErrorJSON(t, stdout, "invalid_input")
assertErrorMessageContains(t, stdout, "spec-sha does not match spec-file contents")
}
func TestOrchReadyOrdersByPriorityAndRespectsLimit(t *testing.T) {
t.Parallel()
@@ -98,3 +98,20 @@ func TestOrchCleanupHelpExplainsScopeFlags(t *testing.T) {
t.Fatalf("expected cleanup help to include exact-attempt example, got:\n%s", combined)
}
}
func TestOrchVerifyHelpExplainsGateWorkflow(t *testing.T) {
t.Parallel()
stdout, stderr, exitCode := executeOrchCommand("verify", "--help")
if exitCode != 0 {
t.Fatalf("expected help exit 0, got %d\nstderr:\n%s\nstdout:\n%s", exitCode, stderr, stdout)
}
combined := stdout + stderr
if !strings.Contains(combined, "Verification gate commands") {
t.Fatalf("expected verify help to explain purpose, got:\n%s", combined)
}
if !strings.Contains(combined, "before the required checks pass") {
t.Fatalf("expected verify help to mention required checks, got:\n%s", combined)
}
}
@@ -182,6 +182,183 @@ func TestOrchRunDispatchReconcileLifecycle(t *testing.T) {
}
}
func TestOrchVerificationGateLifecycle(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "coord.db")
specFile := filepath.Join(tempDir, "task.md")
if err := os.WriteFile(specFile, []byte("# Task\n\nShip the verifier-backed component change.\n"), 0o644); err != nil {
t.Fatalf("write spec file: %v", err)
}
runOrchCommand(
t,
"--db", dbPath,
"--json",
"run", "init",
"--run", "run_verify_001",
"--goal", "Exercise verification gates",
)
taskOut := runOrchCommand(
t,
"--db", dbPath,
"--json",
"task", "add",
"--run", "run_verify_001",
"--task", "T1",
"--title", "Implement verifier-backed task",
"--default-to", "worker-a",
"--spec-file", specFile,
"--check-profile", "cadence_component",
"--required-check", "lint",
"--required-check", "test",
)
var taskResp map[string]any
mustDecodeJSON(t, taskOut, &taskResp)
if got := nestedString(t, taskResp, "data", "task", "status"); got != "ready" {
t.Fatalf("expected new task ready, got %q", got)
}
dispatchOut := runOrchCommand(
t,
"--db", dbPath,
"--json",
"dispatch",
"--run", "run_verify_001",
"--task", "T1",
"--execution-mode", "analysis",
"--body", "Implement the gated task.",
)
var dispatchResp map[string]any
mustDecodeJSON(t, dispatchOut, &dispatchResp)
threadID := nestedString(t, dispatchResp, "data", "attempt", "thread_id")
runInboxCommand(
t,
"--db", dbPath,
"--json",
"claim",
"--agent", "worker-a",
"--thread", threadID,
)
runInboxCommand(
t,
"--db", dbPath,
"--json",
"update",
"--agent", "worker-a",
"--thread", threadID,
"--status", "in_progress",
"--summary", "Implementation started",
)
runOrchCommand(
t,
"--db", dbPath,
"--json",
"reconcile",
"--run", "run_verify_001",
)
runInboxCommand(
t,
"--db", dbPath,
"--json",
"done",
"--agent", "worker-a",
"--thread", threadID,
"--summary", "Implementation finished",
"--body", "Ready for verification.",
)
reconcileOut := runOrchCommand(
t,
"--db", dbPath,
"--json",
"reconcile",
"--run", "run_verify_001",
)
var reconcileResp map[string]any
mustDecodeJSON(t, reconcileOut, &reconcileResp)
updatedTasks := nestedArray(t, reconcileResp, "data", "updated_tasks")
task := updatedTasks[0].(map[string]any)
if got, _ := task["status"].(string); got != "verifying" {
t.Fatalf("expected task verifying after done reconcile, got %#v", task["status"])
}
verifyStatusOut := runOrchCommand(
t,
"--db", dbPath,
"--json",
"verify", "status",
"--run", "run_verify_001",
"--task", "T1",
)
var verifyStatusResp map[string]any
mustDecodeJSON(t, verifyStatusOut, &verifyStatusResp)
if got := nestedString(t, verifyStatusResp, "data", "gate", "status"); got != "pending" {
t.Fatalf("expected pending gate, got %q", got)
}
verifyLintOut := runOrchCommand(
t,
"--db", dbPath,
"--json",
"verify", "record",
"--run", "run_verify_001",
"--task", "T1",
"--check", "lint",
"--status", "passed",
"--summary", "lint clean",
)
var verifyLintResp map[string]any
mustDecodeJSON(t, verifyLintOut, &verifyLintResp)
if got := nestedString(t, verifyLintResp, "data", "task", "status"); got != "verifying" {
t.Fatalf("expected task to stay verifying after first passed check, got %q", got)
}
verifyTestOut := runOrchCommand(
t,
"--db", dbPath,
"--json",
"verify", "record",
"--run", "run_verify_001",
"--task", "T1",
"--check", "test",
"--status", "passed",
"--summary", "tests clean",
)
var verifyTestResp map[string]any
mustDecodeJSON(t, verifyTestOut, &verifyTestResp)
if got := nestedString(t, verifyTestResp, "data", "task", "status"); got != "done" {
t.Fatalf("expected task done after all required checks pass, got %q", got)
}
if got := nestedString(t, verifyTestResp, "data", "gate", "status"); got != "passed" {
t.Fatalf("expected passed gate after all checks pass, got %q", got)
}
statusOut := runOrchCommand(
t,
"--db", dbPath,
"--json",
"status",
"--run", "run_verify_001",
)
var statusResp map[string]any
mustDecodeJSON(t, statusOut, &statusResp)
if got := nestedString(t, statusResp, "data", "run", "status"); got != "done" {
t.Fatalf("expected run status done after gate passes, got %q", got)
}
}
func TestOrchDependencyBlockedAndAnswerFlow(t *testing.T) {
t.Parallel()
@@ -16,7 +16,7 @@ func NewRootCmd() *cobra.Command {
Use: "orch",
Short: "Leader-facing scheduler and control plane",
Long: helpLong(
"Use orch to manage leader-side scheduling for runs, tasks, dependencies, dispatch, retries, reassignment, blocked-task answers, and worktree-backed code attempts.",
"Use orch to manage leader-side scheduling for runs, tasks, dependencies, dispatch, retries, reassignment, verification gates, blocked-task answers, and worktree-backed code attempts.",
"orch is the control plane; it creates durable handoff state in inbox but does not launch workers by itself.",
"After dispatch, a separate worker runtime or worker agent should claim the assigned inbox thread.",
"Use execution-mode analysis for thread-only work and execution-mode code for worktree-backed repository changes.",
@@ -48,6 +48,7 @@ func NewRootCmd() *cobra.Command {
cmd.AddCommand(newBlockedCmd(opts))
cmd.AddCommand(newAnswerCmd(opts))
cmd.AddCommand(newStatusCmd(opts))
cmd.AddCommand(newVerifyCmd(opts))
return cmd
}
@@ -1,7 +1,11 @@
package orch
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"strings"
"ai-workflow-skill/packages/coord-core/protocol"
"ai-workflow-skill/packages/coord-core/store"
@@ -17,6 +21,13 @@ type taskAddOptions struct {
defaultTo string
acceptanceJSON string
priority string
specFile string
specSHA string
checkProfile string
requiredChecks []string
allowedPaths []string
blockedPaths []string
metadataJSON string
}
func newTaskCmd(root *rootOptions) *cobra.Command {
@@ -42,10 +53,11 @@ func newTaskAddCmd(root *rootOptions) *cobra.Command {
Short: "Add a task to a run",
Long: helpLong(
"Use task add to register one schedulable task inside a run.",
"Tasks may include a default worker target, priority, and optional acceptance JSON that downstream tooling can inspect.",
"Tasks may include a default worker target, priority, optional acceptance JSON, and a task-spec snapshot with verification policy.",
"A task must belong to an existing run before it can become ready or be dispatched.",
),
Example: ` orch --db .agents/coord.db task add --run blog_mvp_001 --task T1 --title "Implement backend" --summary "Ship the first API slice" --default-to backend-worker --priority high`,
Example: ` orch --db .agents/coord.db task add --run blog_mvp_001 --task T1 --title "Implement backend" --summary "Ship the first API slice" --default-to backend-worker --priority high
orch --db .agents/coord.db task add --run blog_mvp_001 --task T2 --title "Polish release flow" --spec-file ./tasks/t2.md --check-profile cadence_component --required-check lint --required-check test:e2e`,
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
@@ -55,6 +67,11 @@ func newTaskAddCmd(root *rootOptions) *cobra.Command {
}
defer sqlDB.Close()
specBody, computedSpecSHA, err := loadTaskSpecSnapshot(opts.specFile, opts.specSHA)
if err != nil {
return err
}
task, err := store.NewOrchStore(sqlDB).AddTask(ctx, store.AddTaskInput{
RunID: opts.runID,
TaskID: opts.taskID,
@@ -63,6 +80,14 @@ func newTaskAddCmd(root *rootOptions) *cobra.Command {
DefaultTo: opts.defaultTo,
AcceptanceJSON: opts.acceptanceJSON,
Priority: opts.priority,
SpecFile: strings.TrimSpace(opts.specFile),
SpecSHA: computedSpecSHA,
SpecBody: specBody,
CheckProfile: strings.TrimSpace(opts.checkProfile),
RequiredChecks: opts.requiredChecks,
AllowedPaths: opts.allowedPaths,
BlockedPaths: opts.blockedPaths,
MetadataJSON: opts.metadataJSON,
})
if err != nil {
return err
@@ -91,9 +116,40 @@ func newTaskAddCmd(root *rootOptions) *cobra.Command {
cmd.Flags().StringVar(&opts.defaultTo, "default-to", "", "Default worker agent")
cmd.Flags().StringVar(&opts.acceptanceJSON, "acceptance-json", "", "Acceptance criteria JSON")
cmd.Flags().StringVar(&opts.priority, "priority", "normal", "Task priority")
cmd.Flags().StringVar(&opts.specFile, "spec-file", "", "Path to the task spec file to snapshot")
cmd.Flags().StringVar(&opts.specSHA, "spec-sha", "", "Optional expected SHA256 for --spec-file")
cmd.Flags().StringVar(&opts.checkProfile, "check-profile", "", "Verification check profile name")
cmd.Flags().StringSliceVar(&opts.requiredChecks, "required-check", nil, "Required verification check name; repeat for multiple checks")
cmd.Flags().StringSliceVar(&opts.allowedPaths, "allowed-path", nil, "Allowed path prefix for task scope; repeat for multiple paths")
cmd.Flags().StringSliceVar(&opts.blockedPaths, "blocked-path", nil, "Blocked path prefix for task scope; repeat for multiple paths")
cmd.Flags().StringVar(&opts.metadataJSON, "metadata-json", "", "Structured metadata JSON for task policy")
_ = cmd.MarkFlagRequired("run")
_ = cmd.MarkFlagRequired("task")
_ = cmd.MarkFlagRequired("title")
return cmd
}
func loadTaskSpecSnapshot(specFile, expectedSHA string) (string, string, error) {
specFile = strings.TrimSpace(specFile)
expectedSHA = strings.TrimSpace(expectedSHA)
if specFile == "" {
if expectedSHA != "" {
return "", "", fmt.Errorf("%w: spec-sha requires spec-file", store.ErrInvalidInput)
}
return "", "", nil
}
body, err := os.ReadFile(specFile)
if err != nil {
return "", "", protocol.InvalidInput("failed to read spec-file", err)
}
sum := sha256.Sum256(body)
computed := hex.EncodeToString(sum[:])
if expectedSHA != "" && !strings.EqualFold(expectedSHA, computed) {
return "", "", fmt.Errorf("%w: spec-sha does not match spec-file contents", store.ErrInvalidInput)
}
return string(body), computed, nil
}
@@ -0,0 +1,195 @@
package orch
import (
"fmt"
"ai-workflow-skill/packages/coord-core/protocol"
"ai-workflow-skill/packages/coord-core/store"
"github.com/spf13/cobra"
)
type verifyRecordOptions struct {
runID string
taskID string
attemptNo int
checkName string
status string
summary string
body string
bodyFile string
metadataJSON string
recordedBy string
}
type verifyStatusOptions struct {
runID string
taskID string
attemptNo int
}
func newVerifyCmd(root *rootOptions) *cobra.Command {
cmd := &cobra.Command{
Use: "verify",
Short: "Verification gate commands",
Long: helpLong(
"Verification gate commands record post-implementation checks and inspect task verification state.",
"Verification gates keep orch from treating a worker's done signal as final completion before the required checks pass.",
),
Example: ` orch --db .agents/coord.db verify record --run blog_mvp_001 --task T1 --check lint --status passed --summary "lint clean"
orch --db .agents/coord.db verify status --run blog_mvp_001 --task T1`,
}
cmd.AddCommand(newVerifyRecordCmd(root))
cmd.AddCommand(newVerifyStatusCmd(root))
return cmd
}
func newVerifyRecordCmd(root *rootOptions) *cobra.Command {
opts := &verifyRecordOptions{}
cmd := &cobra.Command{
Use: "record",
Short: "Record one verification check result for a task attempt",
Long: helpLong(
"Use verify record after reconcile has moved a task into verifying, or when a prior verification failure needs an updated check result.",
"record upserts one named check result for the selected attempt, then recomputes the task gate and task status.",
),
Example: ` orch --db .agents/coord.db verify record --run blog_mvp_001 --task T1 --check lint --status passed --summary "lint clean"
orch --db .agents/coord.db verify record --run blog_mvp_001 --task T1 --check package-consumer --status failed --summary "consumer smoke failed" --body-file ./artifacts/package-consumer.txt`,
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
sqlDB, err := openOrchDB(ctx, root.dbPath)
if err != nil {
return err
}
defer sqlDB.Close()
body, err := resolveBodyValue(opts.body, opts.bodyFile)
if err != nil {
return err
}
result, err := store.NewOrchStore(sqlDB).RecordCheck(ctx, store.VerifyRecordInput{
RunID: opts.runID,
TaskID: opts.taskID,
AttemptNo: opts.attemptNo,
CheckName: opts.checkName,
Status: opts.status,
Summary: opts.summary,
Body: body,
MetadataJSON: opts.metadataJSON,
RecordedBy: opts.recordedBy,
})
if err != nil {
return err
}
resp := protocol.Success{
OK: true,
Command: "verify record",
Data: map[string]any{
"task": result.Task,
"attempt": result.Attempt,
"check": result.Check,
"gate": result.Gate,
},
}
if root.json {
return protocol.WriteJSON(cmd.OutOrStdout(), resp)
}
_, err = fmt.Fprintf(
cmd.OutOrStdout(),
"recorded check %s=%s for %s/%s attempt %d\n",
result.Check.CheckName,
result.Check.Status,
result.Task.RunID,
result.Task.TaskID,
result.Attempt.AttemptNo,
)
return err
},
}
cmd.Flags().StringVar(&opts.runID, "run", "", "Run ID")
cmd.Flags().StringVar(&opts.taskID, "task", "", "Task ID")
cmd.Flags().IntVar(&opts.attemptNo, "attempt", 0, "Attempt number; defaults to the latest attempt")
cmd.Flags().StringVar(&opts.checkName, "check", "", "Check name")
cmd.Flags().StringVar(&opts.status, "status", "", "Check status: passed, failed, or skipped")
cmd.Flags().StringVar(&opts.summary, "summary", "", "Short verification summary")
cmd.Flags().StringVar(&opts.body, "body", "", "Optional verification details")
cmd.Flags().StringVar(&opts.bodyFile, "body-file", "", "Optional path to a verification details file")
cmd.Flags().StringVar(&opts.metadataJSON, "metadata-json", "", "Structured metadata JSON for the check result")
cmd.Flags().StringVar(&opts.recordedBy, "recorded-by", "orch", "Recorder identity for the check result")
_ = cmd.MarkFlagRequired("run")
_ = cmd.MarkFlagRequired("task")
_ = cmd.MarkFlagRequired("check")
_ = cmd.MarkFlagRequired("status")
return cmd
}
func newVerifyStatusCmd(root *rootOptions) *cobra.Command {
opts := &verifyStatusOptions{}
cmd := &cobra.Command{
Use: "status",
Short: "Show the current verification state for one task",
Long: helpLong(
"Use verify status to inspect the task spec snapshot, the selected attempt, and the current gate state in one response.",
"Prefer this when a task is stuck in verifying or failed because of gate results.",
),
Example: ` orch --db .agents/coord.db verify status --run blog_mvp_001 --task T1
orch --db .agents/coord.db --json verify status --run blog_mvp_001 --task T1 --attempt 2`,
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
sqlDB, err := openOrchDB(ctx, root.dbPath)
if err != nil {
return err
}
defer sqlDB.Close()
result, err := store.NewOrchStore(sqlDB).GetVerificationStatus(ctx, store.VerificationStatusInput{
RunID: opts.runID,
TaskID: opts.taskID,
AttemptNo: opts.attemptNo,
})
if err != nil {
return err
}
resp := protocol.Success{
OK: true,
Command: "verify status",
Data: map[string]any{
"task": result.Task,
"attempt": result.Attempt,
"spec": result.Spec,
"gate": result.Gate,
},
}
if root.json {
return protocol.WriteJSON(cmd.OutOrStdout(), resp)
}
if result.Gate == nil {
_, err = fmt.Fprintf(cmd.OutOrStdout(), "%s/%s has no verification gate\n", result.Task.RunID, result.Task.TaskID)
return err
}
_, err = fmt.Fprintf(cmd.OutOrStdout(), "%s/%s verification status %s\n", result.Task.RunID, result.Task.TaskID, result.Gate.Status)
return err
},
}
cmd.Flags().StringVar(&opts.runID, "run", "", "Run ID")
cmd.Flags().StringVar(&opts.taskID, "task", "", "Task ID")
cmd.Flags().IntVar(&opts.attemptNo, "attempt", 0, "Attempt number; defaults to the latest attempt")
_ = cmd.MarkFlagRequired("run")
_ = cmd.MarkFlagRequired("task")
return cmd
}