#!/usr/bin/env bash set -euo pipefail readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" readonly ORCH_BIN="${REPO_ROOT}/skills/orch/assets/orch" readonly INBOX_BIN="${REPO_ROOT}/skills/inbox/assets/inbox" OUTPUT_ROOT="" LAST_BG_PID="" usage() { cat <<'EOF' Usage: scripts/run_orch_skill_forward_tests.sh [--output-root PATH] Runs the documented orch-skill forward scenarios as direct bundled-CLI replays. Each case gets its own temporary workspace, SQLite DB, and JSON evidence files. Options: --output-root PATH Write results under PATH instead of a temporary directory -h, --help Show this help text EOF } require_command() { local cmd="$1" if ! command -v "${cmd}" >/dev/null 2>&1; then printf 'missing required command: %s\n' "${cmd}" >&2 exit 1 fi } run_json() { local output_path="$1" shift "$@" >"${output_path}" } json_get() { local file_path="$1" local filter="$2" jq -r "${filter}" "${file_path}" } json_check() { local file_path="$1" local filter="$2" local label="$3" if jq -e "${filter}" "${file_path}" >/dev/null; then printf 'PASS: %s\n' "${label}" else printf 'FAIL: %s\n' "${label}" >&2 printf ' file: %s\n' "${file_path}" >&2 printf ' jq: %s\n' "${filter}" >&2 exit 1 fi } start_wait() { local output_path="$1" shift "$@" >"${output_path}" & LAST_BG_PID="$!" } wait_for_pid() { local pid="$1" local label="$2" if ! wait "${pid}"; then printf 'background command failed: %s\n' "${label}" >&2 exit 1 fi } init_case_dir() { local case_slug="$1" local base_dir="${OUTPUT_ROOT}/${case_slug}" mkdir -p "${base_dir}" printf '%s\n' "${base_dir}" } init_db() { local db_path="$1" run_json "${db_path%.db}.init.json" "${INBOX_BIN}" --db "${db_path}" --json init } init_git_repo() { local repo_path="$1" mkdir -p "${repo_path}" git -C "${repo_path}" init >/dev/null git -C "${repo_path}" config user.name "Orch Skill Replay" git -C "${repo_path}" config user.email "orch-skill-replay@example.com" printf '# Replay Fixture\n' >"${repo_path}/README.md" git -C "${repo_path}" add README.md git -C "${repo_path}" commit -m "Initial commit" >/dev/null } join_json_array() { if [ "$#" -eq 0 ]; then printf '[]' return fi printf '%s\n' "$@" | jq -R . | jq -s . } write_result_json() { local case_dir="$1" local case_slug="$2" local db_path="$3" local run_id="$4" local result="$5" local duration_seconds="$6" local thread_ids_json="$7" local worktree_paths_json="$8" local notes="$9" jq -n \ --arg case "${case_slug}" \ --arg db_path "${db_path}" \ --arg run_id "${run_id}" \ --arg result "${result}" \ --arg notes "${notes}" \ --arg mode "direct_cli_replay" \ --arg runner_script "scripts/run_orch_skill_forward_tests.sh" \ --argjson duration_seconds "${duration_seconds}" \ --argjson thread_ids "${thread_ids_json}" \ --argjson worktree_paths "${worktree_paths_json}" \ '{ case: $case, db_path: $db_path, run_id: $run_id, thread_ids: $thread_ids, worktree_paths: $worktree_paths, result: $result, duration_seconds: $duration_seconds, execution_mode: $mode, runner_script: $runner_script, notes: $notes }' >"${case_dir}/result.json" } print_case_summary() { local case_slug="$1" local case_dir="$2" printf '%s\t%s\n' "${case_slug}" "$(json_get "${case_dir}/result.json" '.result')" } run_case_happy_path() { local case_slug="leader-run-dispatch-reconcile-through-bundled-cli" local run_id="run_blog_skill_001" local case_dir case_dir="$(init_case_dir "${case_slug}")" local db_path="${case_dir}/coord.db" local started_at started_at="$(date +%s)" init_db "${db_path}" run_json "${case_dir}/run.json" \ "${ORCH_BIN}" --db "${db_path}" --json run init \ --run "${run_id}" --goal "Build blog MVP" --summary "Public blog plus admin CRUD" run_json "${case_dir}/task.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T1 --title "Implement retry policy" \ --summary "Add retry policy to HTTP client" --default-to worker-a run_json "${case_dir}/dispatch.json" \ "${ORCH_BIN}" --db "${db_path}" --json dispatch \ --run "${run_id}" --task T1 --to worker-a \ --body "Implement retry handling for the HTTP client." local thread_id thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')" local wait_pid start_wait "${case_dir}/wait-task-done.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_done --timeout-seconds 15 wait_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/fetch.json" \ "${INBOX_BIN}" --db "${db_path}" --json fetch \ --agent worker-a --limit 1 run_json "${case_dir}/claim.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-a --thread "${thread_id}" run_json "${case_dir}/update.json" \ "${INBOX_BIN}" --db "${db_path}" --json update \ --agent worker-a --thread "${thread_id}" \ --status in_progress --summary "Implementation started" run_json "${case_dir}/done.json" \ "${INBOX_BIN}" --db "${db_path}" --json done \ --agent worker-a --thread "${thread_id}" \ --summary "Retry policy implemented" \ --body "The HTTP client now retries transient failures." wait_for_pid "${wait_pid}" "${case_slug}: wait task_done" run_json "${case_dir}/reconcile.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/status.json" \ "${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}" run_json "${case_dir}/show.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}" json_check "${case_dir}/wait-task-done.json" '.data.woke == true' "wait woke on task_done" json_check "${case_dir}/status.json" '.data.run.status == "done"' "run status done" json_check "${case_dir}/status.json" '.data.tasks | length == 1 and .[0].task_id == "T1" and .[0].status == "done"' "single done task" json_check "${case_dir}/show.json" '.data.thread.status == "done"' "thread status done" json_check "${case_dir}/show.json" '[.data.messages[].kind] | index("task") != null and index("progress") != null and index("result") != null' "thread history includes task progress result" local duration_seconds duration_seconds="$(( $(date +%s) - started_at ))" write_result_json \ "${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \ "$(join_json_array "${thread_id}")" \ "$(join_json_array)" \ "Direct CLI replay of the documented happy path." } run_case_blocked_answer() { local case_slug="leader-blocked-answer-resume-through-bundled-cli" local run_id="run_blog_skill_002" local case_dir case_dir="$(init_case_dir "${case_slug}")" local db_path="${case_dir}/coord.db" local started_at started_at="$(date +%s)" init_db "${db_path}" run_json "${case_dir}/run.json" \ "${ORCH_BIN}" --db "${db_path}" --json run init \ --run "${run_id}" --goal "Build dependency-aware workflow" \ --summary "Exercise blocked question handling" run_json "${case_dir}/task.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T1 --title "Build frontend" \ --summary "Implement frontend flow" --default-to worker-a run_json "${case_dir}/dispatch.json" \ "${ORCH_BIN}" --db "${db_path}" --json dispatch \ --run "${run_id}" --task T1 --to worker-a \ --body "Implement the worker flow and stop if a logging decision is needed." local thread_id thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')" local wait_blocked_pid start_wait "${case_dir}/wait-task-blocked.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_blocked --timeout-seconds 15 wait_blocked_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/claim.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-a --thread "${thread_id}" run_json "${case_dir}/progress.json" \ "${INBOX_BIN}" --db "${db_path}" --json update \ --agent worker-a --thread "${thread_id}" \ --status in_progress --summary "Implementation started" run_json "${case_dir}/blocked.json" \ "${INBOX_BIN}" --db "${db_path}" --json update \ --agent worker-a --thread "${thread_id}" \ --status blocked --summary "Need logging decision" \ --payload-json '{"question":"Should logging go to stdout or stderr?"}' wait_for_pid "${wait_blocked_pid}" "${case_slug}: wait task_blocked" local blocked_message_id blocked_message_id="$(json_get "${case_dir}/blocked.json" '.data.message.message_id')" run_json "${case_dir}/orch-blocked.json" \ "${ORCH_BIN}" --db "${db_path}" --json blocked --run "${run_id}" local wait_reply_pid start_wait "${case_dir}/wait-reply.json" \ "${INBOX_BIN}" --db "${db_path}" --agent worker-a --json wait-reply \ --thread "${thread_id}" --after-message "${blocked_message_id}" --timeout-seconds 15 wait_reply_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/answer.json" \ "${ORCH_BIN}" --db "${db_path}" --json answer \ --run "${run_id}" --task T1 --body "Use stdout for MVP." wait_for_pid "${wait_reply_pid}" "${case_slug}: inbox wait-reply" local wait_done_pid start_wait "${case_dir}/wait-task-done.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_done --timeout-seconds 15 wait_done_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/resume.json" \ "${INBOX_BIN}" --db "${db_path}" --json update \ --agent worker-a --thread "${thread_id}" \ --status in_progress --summary "Decision applied" run_json "${case_dir}/done.json" \ "${INBOX_BIN}" --db "${db_path}" --json done \ --agent worker-a --thread "${thread_id}" \ --summary "Frontend complete" \ --body "Worker resumed after the leader answer." wait_for_pid "${wait_done_pid}" "${case_slug}: wait task_done" run_json "${case_dir}/reconcile.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/status.json" \ "${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}" run_json "${case_dir}/show.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}" json_check "${case_dir}/wait-task-blocked.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_blocked"' "wait woke on task_blocked" json_check "${case_dir}/orch-blocked.json" '.data.blocked | length == 1 and .[0].task.task_id == "T1"' "blocked queue lists task" json_check "${case_dir}/wait-reply.json" '.data.woke == true and .data.message.kind == "answer"' "wait-reply woke on answer" json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "blocked flow run completes" json_check "${case_dir}/show.json" '[.data.messages[].kind] | index("question") != null and index("answer") != null and index("result") != null' "history includes question answer result" json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "question" and .payload_json.question == "Should logging go to stdout or stderr?")' "question payload matches" json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "answer" and .body == "Use stdout for MVP.")' "answer body matches" local duration_seconds duration_seconds="$(( $(date +%s) - started_at ))" write_result_json \ "${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \ "$(join_json_array "${thread_id}")" \ "$(join_json_array)" \ "Direct CLI replay of the blocked-question workflow, including orch wait and inbox wait-reply." } run_case_strict_worktree_cleanup() { local case_slug="strict-worktree-dispatch-to-cleanup-through-bundled-cli" local run_id="run_blog_skill_worktree_001" local case_dir case_dir="$(init_case_dir "${case_slug}")" local db_path="${case_dir}/coord.db" local repo_path="${case_dir}/repo" local started_at started_at="$(date +%s)" init_db "${db_path}" init_git_repo "${repo_path}" run_json "${case_dir}/run.json" \ "${ORCH_BIN}" --db "${db_path}" --json run init \ --run "${run_id}" --goal "Validate strict worktree dispatch" \ --summary "Exercise worktree allocation and cleanup" run_json "${case_dir}/task.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T1 --title "Implement backend" \ --summary "Implement inside an isolated worktree" --default-to worker-a run_json "${case_dir}/dispatch.json" \ "${ORCH_BIN}" --db "${db_path}" --json dispatch \ --run "${run_id}" --task T1 --to worker-a \ --repo-path "${repo_path}" --workspace-root .orch/worktrees --strict-worktree \ --body "Implement inside isolated worktree." local thread_id local worktree_path thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')" worktree_path="$(json_get "${case_dir}/dispatch.json" '.data.attempt.worktree_path')" local wait_pid start_wait "${case_dir}/wait-task-done.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_done --timeout-seconds 15 wait_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/claim.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-a --thread "${thread_id}" run_json "${case_dir}/thread-before-done.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}" run_json "${case_dir}/done.json" \ "${INBOX_BIN}" --db "${db_path}" --json done \ --agent worker-a --thread "${thread_id}" \ --summary "Backend complete" \ --body "Confirmed the assigned worktree path from the task payload." wait_for_pid "${wait_pid}" "${case_slug}: wait task_done" run_json "${case_dir}/reconcile.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/cleanup.json" \ "${ORCH_BIN}" --db "${db_path}" --json cleanup --run "${run_id}" --task T1 run_json "${case_dir}/status.json" \ "${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}" run_json "${case_dir}/show.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}" json_check "${case_dir}/dispatch.json" '.data.attempt.worktree_path != ""' "dispatch returns worktree path" if jq -e --arg worktree "${worktree_path}" 'any(.data.messages[]; .kind == "task" and .payload_json.worktree_path == $worktree)' "${case_dir}/thread-before-done.json" >/dev/null; then printf 'PASS: task payload exposes worktree path\n' else printf 'FAIL: task payload exposes worktree path\n' >&2 exit 1 fi json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "worktree flow run completes" json_check "${case_dir}/cleanup.json" '.data.cleaned | length == 1' "cleanup removes one attempt" if [ -d "${worktree_path}" ]; then printf 'FAIL: worktree path still exists after cleanup: %s\n' "${worktree_path}" >&2 exit 1 fi printf 'PASS: cleaned worktree removed\n' local duration_seconds duration_seconds="$(( $(date +%s) - started_at ))" write_result_json \ "${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \ "$(join_json_array "${thread_id}")" \ "$(join_json_array "${worktree_path}")" \ "Direct CLI replay of strict worktree dispatch, completion, and cleanup." } run_case_dependency_ready_dispatch() { local case_slug="leader-dispatches-dependent-task-after-prerequisite-through-bundled-cli" local run_id="run_blog_skill_deps_001" local case_dir case_dir="$(init_case_dir "${case_slug}")" local db_path="${case_dir}/coord.db" local started_at started_at="$(date +%s)" init_db "${db_path}" run_json "${case_dir}/run.json" \ "${ORCH_BIN}" --db "${db_path}" --json run init \ --run "${run_id}" --goal "Validate dependency-gated dispatch" \ --summary "Exercise dep add, ready, and staged dispatch" run_json "${case_dir}/task-1.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T1 --title "Implement backend" \ --summary "Prerequisite task" --default-to worker-a run_json "${case_dir}/task-2.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T2 --title "Implement frontend" \ --summary "Dependent task" --default-to worker-b run_json "${case_dir}/dep.json" \ "${ORCH_BIN}" --db "${db_path}" --json dep add \ --run "${run_id}" --task T2 --depends-on T1 run_json "${case_dir}/ready-initial.json" \ "${ORCH_BIN}" --db "${db_path}" --json ready --run "${run_id}" run_json "${case_dir}/dispatch-1.json" \ "${ORCH_BIN}" --db "${db_path}" --json dispatch \ --run "${run_id}" --task T1 --to worker-a \ --body "Complete the prerequisite task first." local thread_id_1 thread_id_1="$(json_get "${case_dir}/dispatch-1.json" '.data.attempt.thread_id')" local wait_done_pid start_wait "${case_dir}/wait-task-done-1.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_done --timeout-seconds 15 wait_done_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/claim-1.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-a --thread "${thread_id_1}" run_json "${case_dir}/progress-1.json" \ "${INBOX_BIN}" --db "${db_path}" --json update \ --agent worker-a --thread "${thread_id_1}" \ --status in_progress --summary "Prerequisite started" run_json "${case_dir}/done-1.json" \ "${INBOX_BIN}" --db "${db_path}" --json done \ --agent worker-a --thread "${thread_id_1}" \ --summary "Prerequisite complete" \ --body "T1 completed so T2 can be dispatched." wait_for_pid "${wait_done_pid}" "${case_slug}: wait first task_done" run_json "${case_dir}/reconcile-1.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/ready-after-prerequisite.json" \ "${ORCH_BIN}" --db "${db_path}" --json ready --run "${run_id}" run_json "${case_dir}/dispatch-2.json" \ "${ORCH_BIN}" --db "${db_path}" --json dispatch \ --run "${run_id}" --task T2 --to worker-b \ --body "Dependent task is now ready after T1." local thread_id_2 thread_id_2="$(json_get "${case_dir}/dispatch-2.json" '.data.attempt.thread_id')" run_json "${case_dir}/claim-2.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-b --thread "${thread_id_2}" run_json "${case_dir}/done-2.json" \ "${INBOX_BIN}" --db "${db_path}" --json done \ --agent worker-b --thread "${thread_id_2}" \ --summary "Dependent task complete" \ --body "T2 completed after the dependency cleared." run_json "${case_dir}/reconcile-2.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/status.json" \ "${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}" run_json "${case_dir}/show-1.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_1}" run_json "${case_dir}/show-2.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_2}" json_check "${case_dir}/dep.json" '.data.dependency.task_id == "T2" and .data.dependency.depends_on_task_id == "T1"' "dep add stores T2 -> T1" json_check "${case_dir}/ready-initial.json" '.data.tasks | length == 1 and .[0].task_id == "T1"' "initial ready lists only prerequisite" json_check "${case_dir}/wait-task-done-1.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_done"' "wait woke on prerequisite completion" json_check "${case_dir}/ready-after-prerequisite.json" '.data.tasks | length == 1 and .[0].task_id == "T2"' "dependent task becomes ready after reconcile" json_check "${case_dir}/status.json" '.data.run.status == "done" and ([.data.tasks[] | select(.task_id == "T1")][0].status == "done") and ([.data.tasks[] | select(.task_id == "T2")][0].status == "done")' "dependency-gated run completes with both tasks done" json_check "${case_dir}/show-1.json" '.data.thread.status == "done"' "prerequisite thread done" json_check "${case_dir}/show-2.json" '.data.thread.status == "done"' "dependent thread done" if [ "${thread_id_1}" = "${thread_id_2}" ]; then printf 'FAIL: dependency flow reused thread ID %s\n' "${thread_id_1}" >&2 exit 1 fi printf 'PASS: dependency flow created distinct thread IDs\n' local duration_seconds duration_seconds="$(( $(date +%s) - started_at ))" write_result_json \ "${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \ "$(join_json_array "${thread_id_1}" "${thread_id_2}")" \ "$(join_json_array)" \ "Direct CLI replay of dependency-gated ready sequencing from prerequisite completion to dependent dispatch." } run_case_cancel_active_task() { local case_slug="leader-cancels-active-task-through-bundled-cli" local run_id="run_blog_skill_cancel_001" local case_dir case_dir="$(init_case_dir "${case_slug}")" local db_path="${case_dir}/coord.db" local started_at started_at="$(date +%s)" init_db "${db_path}" run_json "${case_dir}/run.json" \ "${ORCH_BIN}" --db "${db_path}" --json run init \ --run "${run_id}" --goal "Validate active task cancellation" \ --summary "Exercise direct orch cancel on an active attempt" run_json "${case_dir}/task-1.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T1 --title "Implement backend" \ --summary "Task that will be cancelled mid-flight" --default-to worker-a run_json "${case_dir}/task-2.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T2 --title "Implement frontend" \ --summary "Task that should remain ready" --default-to worker-b run_json "${case_dir}/dispatch.json" \ "${ORCH_BIN}" --db "${db_path}" --json dispatch \ --run "${run_id}" --task T1 --to worker-a \ --body "Start work so the leader can cancel an active task." local thread_id thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')" run_json "${case_dir}/claim.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-a --thread "${thread_id}" run_json "${case_dir}/progress.json" \ "${INBOX_BIN}" --db "${db_path}" --json update \ --agent worker-a --thread "${thread_id}" \ --status in_progress --summary "Active work in progress" run_json "${case_dir}/cancel.json" \ "${ORCH_BIN}" --db "${db_path}" --json cancel \ --run "${run_id}" --task T1 \ --reason "Task superseded by dependency review." run_json "${case_dir}/status.json" \ "${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}" run_json "${case_dir}/ready.json" \ "${ORCH_BIN}" --db "${db_path}" --json ready --run "${run_id}" run_json "${case_dir}/show.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}" json_check "${case_dir}/cancel.json" '.data.cancelled_tasks | length == 1 and .[0].task_id == "T1" and .[0].status == "cancelled"' "cancel returns cancelled T1" json_check "${case_dir}/status.json" '.data.run.status == "ready"' "run remains ready after cancelling one active task" json_check "${case_dir}/status.json" '([.data.tasks[] | select(.task_id == "T1")][0].status == "cancelled") and ([.data.tasks[] | select(.task_id == "T2")][0].status == "ready")' "status keeps T2 ready while T1 is cancelled" json_check "${case_dir}/ready.json" '.data.tasks | length == 1 and .[0].task_id == "T2"' "ready queue still exposes untouched T2" json_check "${case_dir}/show.json" '.data.thread.status == "cancelled"' "cancelled thread reaches terminal state" json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "progress")' "cancelled thread preserves worker progress history" local duration_seconds duration_seconds="$(( $(date +%s) - started_at ))" write_result_json \ "${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \ "$(join_json_array "${thread_id}")" \ "$(join_json_array)" \ "Direct CLI replay of cancelling an active task while leaving unrelated ready work untouched." } run_case_payload_only_answer() { local case_slug="leader-answers-blocked-task-with-payload-json-through-bundled-cli" local run_id="run_blog_skill_payload_answer_001" local case_dir case_dir="$(init_case_dir "${case_slug}")" local db_path="${case_dir}/coord.db" local started_at started_at="$(date +%s)" init_db "${db_path}" run_json "${case_dir}/run.json" \ "${ORCH_BIN}" --db "${db_path}" --json run init \ --run "${run_id}" --goal "Validate payload-only answers" \ --summary "Exercise answer --payload-json on a blocked task" run_json "${case_dir}/task.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T1 --title "Build frontend" \ --summary "Resume after structured leader decision" --default-to worker-a run_json "${case_dir}/dispatch.json" \ "${ORCH_BIN}" --db "${db_path}" --json dispatch \ --run "${run_id}" --task T1 --to worker-a \ --body "Pause if a structured logging decision is needed." local thread_id thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')" local wait_blocked_pid start_wait "${case_dir}/wait-task-blocked.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_blocked --timeout-seconds 15 wait_blocked_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/claim.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-a --thread "${thread_id}" run_json "${case_dir}/blocked.json" \ "${INBOX_BIN}" --db "${db_path}" --json update \ --agent worker-a --thread "${thread_id}" \ --status blocked --summary "Need structured logging decision" \ --payload-json '{"question":"Use stdout or stderr for structured logs?"}' wait_for_pid "${wait_blocked_pid}" "${case_slug}: wait task_blocked" local blocked_message_id blocked_message_id="$(json_get "${case_dir}/blocked.json" '.data.message.message_id')" run_json "${case_dir}/orch-blocked.json" \ "${ORCH_BIN}" --db "${db_path}" --json blocked --run "${run_id}" local wait_reply_pid start_wait "${case_dir}/wait-reply.json" \ "${INBOX_BIN}" --db "${db_path}" --agent worker-a --json wait-reply \ --thread "${thread_id}" --after-message "${blocked_message_id}" --timeout-seconds 15 wait_reply_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/answer.json" \ "${ORCH_BIN}" --db "${db_path}" --json answer \ --run "${run_id}" --task T1 \ --payload-json '{"decision":"stdout","source":"leader","format":"structured"}' wait_for_pid "${wait_reply_pid}" "${case_slug}: inbox wait-reply" local wait_done_pid start_wait "${case_dir}/wait-task-done.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_done --timeout-seconds 15 wait_done_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/resume.json" \ "${INBOX_BIN}" --db "${db_path}" --json update \ --agent worker-a --thread "${thread_id}" \ --status in_progress --summary "Structured decision applied" run_json "${case_dir}/done.json" \ "${INBOX_BIN}" --db "${db_path}" --json done \ --agent worker-a --thread "${thread_id}" \ --summary "Frontend complete" \ --body "Worker resumed after reading the structured leader decision." wait_for_pid "${wait_done_pid}" "${case_slug}: wait task_done" run_json "${case_dir}/reconcile.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/status.json" \ "${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}" run_json "${case_dir}/show.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}" json_check "${case_dir}/wait-task-blocked.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_blocked"' "payload-answer flow woke on task_blocked" json_check "${case_dir}/orch-blocked.json" '.data.blocked | length == 1 and .[0].task.task_id == "T1"' "blocked queue lists payload-answer task" json_check "${case_dir}/answer.json" '.data.message.kind == "answer" and .data.message.payload_json.decision == "stdout" and .data.message.payload_json.source == "leader" and .data.message.payload_json.format == "structured"' "answer writes payload-json without body" json_check "${case_dir}/wait-reply.json" '.data.woke == true and .data.message.kind == "answer" and .data.message.payload_json.decision == "stdout"' "wait-reply exposes structured answer payload" json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "payload-answer flow run completes" json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "question" and .payload_json.question == "Use stdout or stderr for structured logs?")' "payload-answer question recorded" json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "answer" and .payload_json.decision == "stdout" and .payload_json.source == "leader" and .payload_json.format == "structured")' "payload-answer message preserved in history" local duration_seconds duration_seconds="$(( $(date +%s) - started_at ))" write_result_json \ "${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \ "$(join_json_array "${thread_id}")" \ "$(join_json_array)" \ "Direct CLI replay of answering a blocked task with payload-json only." } run_case_retry() { local case_slug="leader-retries-failed-task-through-bundled-cli" local run_id="run_blog_skill_retry_001" local case_dir case_dir="$(init_case_dir "${case_slug}")" local db_path="${case_dir}/coord.db" local started_at started_at="$(date +%s)" init_db "${db_path}" run_json "${case_dir}/run.json" \ "${ORCH_BIN}" --db "${db_path}" --json run init \ --run "${run_id}" --goal "Validate retry behavior" \ --summary "Exercise failed attempt retry" run_json "${case_dir}/task.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T1 --title "Implement backend" \ --summary "Retry after a simulated failure" --default-to worker-a run_json "${case_dir}/dispatch.json" \ "${ORCH_BIN}" --db "${db_path}" --json dispatch \ --run "${run_id}" --task T1 --to worker-a \ --body "Initial attempt expected to fail for retry validation." local thread_id_1 thread_id_1="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')" local wait_failed_pid start_wait "${case_dir}/wait-task-failed.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_failed --timeout-seconds 15 wait_failed_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/claim-1.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-a --thread "${thread_id_1}" run_json "${case_dir}/fail.json" \ "${INBOX_BIN}" --db "${db_path}" --json fail \ --agent worker-a --thread "${thread_id_1}" \ --summary "Build failed" --body "Simulated first-attempt failure." wait_for_pid "${wait_failed_pid}" "${case_slug}: wait task_failed" run_json "${case_dir}/reconcile-failed.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/retry.json" \ "${ORCH_BIN}" --db "${db_path}" --json retry \ --run "${run_id}" --task T1 --to worker-a \ --body "Retry after fixing the failure." local thread_id_2 thread_id_2="$(json_get "${case_dir}/retry.json" '.data.attempt.thread_id')" local wait_done_pid start_wait "${case_dir}/wait-task-done.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_done --timeout-seconds 15 wait_done_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/fetch-retry.json" \ "${INBOX_BIN}" --db "${db_path}" --json fetch \ --agent worker-a --status pending --limit 5 run_json "${case_dir}/claim-2.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-a --thread "${thread_id_2}" run_json "${case_dir}/done-2.json" \ "${INBOX_BIN}" --db "${db_path}" --json done \ --agent worker-a --thread "${thread_id_2}" \ --summary "Retry completed" --body "Second attempt succeeded." wait_for_pid "${wait_done_pid}" "${case_slug}: wait task_done" run_json "${case_dir}/reconcile-done.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/status.json" \ "${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}" run_json "${case_dir}/show-1.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_1}" run_json "${case_dir}/show-2.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_2}" json_check "${case_dir}/wait-task-failed.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_failed"' "wait woke on task_failed" json_check "${case_dir}/retry.json" '.data.attempt.attempt_no == 2 and .data.task.status == "dispatched"' "retry creates second dispatched attempt" json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "retry flow run completes" json_check "${case_dir}/show-1.json" '.data.thread.status == "failed"' "first thread failed" json_check "${case_dir}/show-2.json" '.data.thread.status == "done"' "second thread done" if [ "${thread_id_1}" = "${thread_id_2}" ]; then printf 'FAIL: retry reused thread ID %s\n' "${thread_id_1}" >&2 exit 1 fi printf 'PASS: retry created distinct thread IDs\n' local duration_seconds duration_seconds="$(( $(date +%s) - started_at ))" write_result_json \ "${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \ "$(join_json_array "${thread_id_1}" "${thread_id_2}")" \ "$(join_json_array)" \ "Direct CLI replay of failed attempt reconciliation followed by retry." } run_case_reassign() { local case_slug="leader-reassigns-blocked-task-through-bundled-cli" local run_id="run_blog_skill_reassign_001" local case_dir case_dir="$(init_case_dir "${case_slug}")" local db_path="${case_dir}/coord.db" local started_at started_at="$(date +%s)" init_db "${db_path}" run_json "${case_dir}/run.json" \ "${ORCH_BIN}" --db "${db_path}" --json run init \ --run "${run_id}" --goal "Validate reassign behavior" \ --summary "Exercise blocked-task reassignment" run_json "${case_dir}/task.json" \ "${ORCH_BIN}" --db "${db_path}" --json task add \ --run "${run_id}" --task T1 --title "Implement backend" \ --summary "Reassign after worker-a blocks" --default-to worker-a run_json "${case_dir}/dispatch.json" \ "${ORCH_BIN}" --db "${db_path}" --json dispatch \ --run "${run_id}" --task T1 --to worker-a \ --body "Initial attempt expected to be reassigned." local thread_id_1 thread_id_1="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')" local wait_blocked_pid start_wait "${case_dir}/wait-task-blocked.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_blocked --timeout-seconds 15 wait_blocked_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/claim-1.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-a --thread "${thread_id_1}" run_json "${case_dir}/blocked.json" \ "${INBOX_BIN}" --db "${db_path}" --json update \ --agent worker-a --thread "${thread_id_1}" \ --status blocked --summary "Need product decision" \ --payload-json '{"question":"Proceed with v1 scope?"}' wait_for_pid "${wait_blocked_pid}" "${case_slug}: wait task_blocked" run_json "${case_dir}/reconcile-blocked.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/orch-blocked.json" \ "${ORCH_BIN}" --db "${db_path}" --json blocked --run "${run_id}" run_json "${case_dir}/reassign.json" \ "${ORCH_BIN}" --db "${db_path}" --json reassign \ --run "${run_id}" --task T1 --to worker-b \ --reason "Try another worker with clearer ownership." local thread_id_2 thread_id_2="$(json_get "${case_dir}/reassign.json" '.data.attempt.thread_id')" local wait_done_pid start_wait "${case_dir}/wait-task-done.json" \ "${ORCH_BIN}" --db "${db_path}" --json wait \ --run "${run_id}" --for task_done --timeout-seconds 15 wait_done_pid="${LAST_BG_PID}" sleep 0.2 run_json "${case_dir}/fetch-worker-b.json" \ "${INBOX_BIN}" --db "${db_path}" --json fetch \ --agent worker-b --status pending --limit 5 run_json "${case_dir}/claim-2.json" \ "${INBOX_BIN}" --db "${db_path}" --json claim \ --agent worker-b --thread "${thread_id_2}" run_json "${case_dir}/done-2.json" \ "${INBOX_BIN}" --db "${db_path}" --json done \ --agent worker-b --thread "${thread_id_2}" \ --summary "Reassigned work complete" --body "Worker-b completed the reassigned attempt." wait_for_pid "${wait_done_pid}" "${case_slug}: wait task_done" run_json "${case_dir}/reconcile-done.json" \ "${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}" run_json "${case_dir}/status.json" \ "${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}" run_json "${case_dir}/show-1.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_1}" run_json "${case_dir}/show-2.json" \ "${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_2}" json_check "${case_dir}/wait-task-blocked.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_blocked"' "wait woke on blocked event for reassign" json_check "${case_dir}/orch-blocked.json" '.data.blocked | length == 1 and .[0].task.task_id == "T1"' "blocked queue lists reassigned task" json_check "${case_dir}/reassign.json" '.data.attempt.attempt_no == 2 and .data.attempt.assigned_to == "worker-b"' "reassign creates attempt 2 for worker-b" json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "reassign flow run completes" json_check "${case_dir}/show-1.json" '.data.thread.status == "cancelled"' "original thread cancelled" json_check "${case_dir}/show-2.json" '.data.thread.status == "done"' "new thread done" json_check "${case_dir}/show-1.json" 'any(.data.messages[]; .kind == "question" and .payload_json.question == "Proceed with v1 scope?")' "original blocked question preserved" if [ "${thread_id_1}" = "${thread_id_2}" ]; then printf 'FAIL: reassign reused thread ID %s\n' "${thread_id_1}" >&2 exit 1 fi printf 'PASS: reassign created distinct thread IDs\n' local duration_seconds duration_seconds="$(( $(date +%s) - started_at ))" write_result_json \ "${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \ "$(join_json_array "${thread_id_1}" "${thread_id_2}")" \ "$(join_json_array)" \ "Direct CLI replay of blocked-task reassignment from worker-a to worker-b." } main() { while [ "$#" -gt 0 ]; do case "$1" in --output-root) OUTPUT_ROOT="$2" shift 2 ;; -h|--help) usage exit 0 ;; *) printf 'unknown argument: %s\n' "$1" >&2 usage >&2 exit 1 ;; esac done require_command jq require_command git require_command mktemp if [ ! -x "${ORCH_BIN}" ]; then printf 'orch skill binary is not executable: %s\n' "${ORCH_BIN}" >&2 exit 1 fi if [ ! -x "${INBOX_BIN}" ]; then printf 'inbox skill binary is not executable: %s\n' "${INBOX_BIN}" >&2 exit 1 fi if [ -z "${OUTPUT_ROOT}" ]; then OUTPUT_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/orch-skill-forward.XXXXXX")" else mkdir -p "${OUTPUT_ROOT}" fi run_case_happy_path run_case_blocked_answer run_case_strict_worktree_cleanup run_case_dependency_ready_dispatch run_case_cancel_active_task run_case_payload_only_answer run_case_retry run_case_reassign printf '\nResults written to %s\n' "${OUTPUT_ROOT}" printf 'case\tresult\n' print_case_summary "leader-run-dispatch-reconcile-through-bundled-cli" "${OUTPUT_ROOT}/leader-run-dispatch-reconcile-through-bundled-cli" print_case_summary "leader-blocked-answer-resume-through-bundled-cli" "${OUTPUT_ROOT}/leader-blocked-answer-resume-through-bundled-cli" print_case_summary "strict-worktree-dispatch-to-cleanup-through-bundled-cli" "${OUTPUT_ROOT}/strict-worktree-dispatch-to-cleanup-through-bundled-cli" print_case_summary "leader-dispatches-dependent-task-after-prerequisite-through-bundled-cli" "${OUTPUT_ROOT}/leader-dispatches-dependent-task-after-prerequisite-through-bundled-cli" print_case_summary "leader-cancels-active-task-through-bundled-cli" "${OUTPUT_ROOT}/leader-cancels-active-task-through-bundled-cli" print_case_summary "leader-answers-blocked-task-with-payload-json-through-bundled-cli" "${OUTPUT_ROOT}/leader-answers-blocked-task-with-payload-json-through-bundled-cli" print_case_summary "leader-retries-failed-task-through-bundled-cli" "${OUTPUT_ROOT}/leader-retries-failed-task-through-bundled-cli" print_case_summary "leader-reassigns-blocked-task-through-bundled-cli" "${OUTPUT_ROOT}/leader-reassigns-blocked-task-through-bundled-cli" } main "$@"