Add orch skill forward test evidence
This commit is contained in:
Executable
+739
@@ -0,0 +1,739 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
readonly REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
readonly ORCH_BIN="${REPO_ROOT}/skills/orch/assets/orch"
|
||||
readonly INBOX_BIN="${REPO_ROOT}/skills/inbox/assets/inbox"
|
||||
|
||||
OUTPUT_ROOT=""
|
||||
LAST_BG_PID=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: scripts/run_orch_skill_forward_tests.sh [--output-root PATH]
|
||||
|
||||
Runs the documented orch-skill forward scenarios as direct bundled-CLI replays.
|
||||
Each case gets its own temporary workspace, SQLite DB, and JSON evidence files.
|
||||
|
||||
Options:
|
||||
--output-root PATH Write results under PATH instead of a temporary directory
|
||||
-h, --help Show this help text
|
||||
EOF
|
||||
}
|
||||
|
||||
require_command() {
|
||||
local cmd="$1"
|
||||
if ! command -v "${cmd}" >/dev/null 2>&1; then
|
||||
printf 'missing required command: %s\n' "${cmd}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_json() {
|
||||
local output_path="$1"
|
||||
shift
|
||||
|
||||
"$@" >"${output_path}"
|
||||
}
|
||||
|
||||
json_get() {
|
||||
local file_path="$1"
|
||||
local filter="$2"
|
||||
|
||||
jq -r "${filter}" "${file_path}"
|
||||
}
|
||||
|
||||
json_check() {
|
||||
local file_path="$1"
|
||||
local filter="$2"
|
||||
local label="$3"
|
||||
|
||||
if jq -e "${filter}" "${file_path}" >/dev/null; then
|
||||
printf 'PASS: %s\n' "${label}"
|
||||
else
|
||||
printf 'FAIL: %s\n' "${label}" >&2
|
||||
printf ' file: %s\n' "${file_path}" >&2
|
||||
printf ' jq: %s\n' "${filter}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
start_wait() {
|
||||
local output_path="$1"
|
||||
shift
|
||||
|
||||
"$@" >"${output_path}" &
|
||||
LAST_BG_PID="$!"
|
||||
}
|
||||
|
||||
wait_for_pid() {
|
||||
local pid="$1"
|
||||
local label="$2"
|
||||
|
||||
if ! wait "${pid}"; then
|
||||
printf 'background command failed: %s\n' "${label}" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
init_case_dir() {
|
||||
local case_slug="$1"
|
||||
local base_dir="${OUTPUT_ROOT}/${case_slug}"
|
||||
mkdir -p "${base_dir}"
|
||||
printf '%s\n' "${base_dir}"
|
||||
}
|
||||
|
||||
init_db() {
|
||||
local db_path="$1"
|
||||
run_json "${db_path%.db}.init.json" "${INBOX_BIN}" --db "${db_path}" --json init
|
||||
}
|
||||
|
||||
init_git_repo() {
|
||||
local repo_path="$1"
|
||||
|
||||
mkdir -p "${repo_path}"
|
||||
git -C "${repo_path}" init >/dev/null
|
||||
git -C "${repo_path}" config user.name "Orch Skill Replay"
|
||||
git -C "${repo_path}" config user.email "orch-skill-replay@example.com"
|
||||
printf '# Replay Fixture\n' >"${repo_path}/README.md"
|
||||
git -C "${repo_path}" add README.md
|
||||
git -C "${repo_path}" commit -m "Initial commit" >/dev/null
|
||||
}
|
||||
|
||||
join_json_array() {
|
||||
if [ "$#" -eq 0 ]; then
|
||||
printf '[]'
|
||||
return
|
||||
fi
|
||||
printf '%s\n' "$@" | jq -R . | jq -s .
|
||||
}
|
||||
|
||||
write_result_json() {
|
||||
local case_dir="$1"
|
||||
local case_slug="$2"
|
||||
local db_path="$3"
|
||||
local run_id="$4"
|
||||
local result="$5"
|
||||
local duration_seconds="$6"
|
||||
local thread_ids_json="$7"
|
||||
local worktree_paths_json="$8"
|
||||
local notes="$9"
|
||||
|
||||
jq -n \
|
||||
--arg case "${case_slug}" \
|
||||
--arg db_path "${db_path}" \
|
||||
--arg run_id "${run_id}" \
|
||||
--arg result "${result}" \
|
||||
--arg notes "${notes}" \
|
||||
--arg mode "direct_cli_replay" \
|
||||
--arg runner_script "scripts/run_orch_skill_forward_tests.sh" \
|
||||
--argjson duration_seconds "${duration_seconds}" \
|
||||
--argjson thread_ids "${thread_ids_json}" \
|
||||
--argjson worktree_paths "${worktree_paths_json}" \
|
||||
'{
|
||||
case: $case,
|
||||
db_path: $db_path,
|
||||
run_id: $run_id,
|
||||
thread_ids: $thread_ids,
|
||||
worktree_paths: $worktree_paths,
|
||||
result: $result,
|
||||
duration_seconds: $duration_seconds,
|
||||
execution_mode: $mode,
|
||||
runner_script: $runner_script,
|
||||
notes: $notes
|
||||
}' >"${case_dir}/result.json"
|
||||
}
|
||||
|
||||
print_case_summary() {
|
||||
local case_slug="$1"
|
||||
local case_dir="$2"
|
||||
|
||||
printf '%s\t%s\n' "${case_slug}" "$(json_get "${case_dir}/result.json" '.result')"
|
||||
}
|
||||
|
||||
run_case_happy_path() {
|
||||
local case_slug="leader-run-dispatch-reconcile-through-bundled-cli"
|
||||
local run_id="run_blog_skill_001"
|
||||
local case_dir
|
||||
case_dir="$(init_case_dir "${case_slug}")"
|
||||
local db_path="${case_dir}/coord.db"
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
|
||||
init_db "${db_path}"
|
||||
|
||||
run_json "${case_dir}/run.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json run init \
|
||||
--run "${run_id}" --goal "Build blog MVP" --summary "Public blog plus admin CRUD"
|
||||
|
||||
run_json "${case_dir}/task.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T1 --title "Implement retry policy" \
|
||||
--summary "Add retry policy to HTTP client" --default-to worker-a
|
||||
|
||||
run_json "${case_dir}/dispatch.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dispatch \
|
||||
--run "${run_id}" --task T1 --to worker-a \
|
||||
--body "Implement retry handling for the HTTP client."
|
||||
|
||||
local thread_id
|
||||
thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')"
|
||||
|
||||
local wait_pid
|
||||
start_wait "${case_dir}/wait-task-done.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_done --timeout-seconds 15
|
||||
wait_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/fetch.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json fetch \
|
||||
--agent worker-a --limit 1
|
||||
|
||||
run_json "${case_dir}/claim.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-a --thread "${thread_id}"
|
||||
|
||||
run_json "${case_dir}/update.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json update \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--status in_progress --summary "Implementation started"
|
||||
|
||||
run_json "${case_dir}/done.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json done \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--summary "Retry policy implemented" \
|
||||
--body "The HTTP client now retries transient failures."
|
||||
|
||||
wait_for_pid "${wait_pid}" "${case_slug}: wait task_done"
|
||||
|
||||
run_json "${case_dir}/reconcile.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/status.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/show.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}"
|
||||
|
||||
json_check "${case_dir}/wait-task-done.json" '.data.woke == true' "wait woke on task_done"
|
||||
json_check "${case_dir}/status.json" '.data.run.status == "done"' "run status done"
|
||||
json_check "${case_dir}/status.json" '.data.tasks | length == 1 and .[0].task_id == "T1" and .[0].status == "done"' "single done task"
|
||||
json_check "${case_dir}/show.json" '.data.thread.status == "done"' "thread status done"
|
||||
json_check "${case_dir}/show.json" '[.data.messages[].kind] | index("task") != null and index("progress") != null and index("result") != null' "thread history includes task progress result"
|
||||
|
||||
local duration_seconds
|
||||
duration_seconds="$(( $(date +%s) - started_at ))"
|
||||
write_result_json \
|
||||
"${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \
|
||||
"$(join_json_array "${thread_id}")" \
|
||||
"$(join_json_array)" \
|
||||
"Direct CLI replay of the documented happy path."
|
||||
}
|
||||
|
||||
run_case_blocked_answer() {
|
||||
local case_slug="leader-blocked-answer-resume-through-bundled-cli"
|
||||
local run_id="run_blog_skill_002"
|
||||
local case_dir
|
||||
case_dir="$(init_case_dir "${case_slug}")"
|
||||
local db_path="${case_dir}/coord.db"
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
|
||||
init_db "${db_path}"
|
||||
|
||||
run_json "${case_dir}/run.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json run init \
|
||||
--run "${run_id}" --goal "Build dependency-aware workflow" \
|
||||
--summary "Exercise blocked question handling"
|
||||
|
||||
run_json "${case_dir}/task.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T1 --title "Build frontend" \
|
||||
--summary "Implement frontend flow" --default-to worker-a
|
||||
|
||||
run_json "${case_dir}/dispatch.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dispatch \
|
||||
--run "${run_id}" --task T1 --to worker-a \
|
||||
--body "Implement the worker flow and stop if a logging decision is needed."
|
||||
|
||||
local thread_id
|
||||
thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')"
|
||||
|
||||
local wait_blocked_pid
|
||||
start_wait "${case_dir}/wait-task-blocked.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_blocked --timeout-seconds 15
|
||||
wait_blocked_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/claim.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-a --thread "${thread_id}"
|
||||
|
||||
run_json "${case_dir}/progress.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json update \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--status in_progress --summary "Implementation started"
|
||||
|
||||
run_json "${case_dir}/blocked.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json update \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--status blocked --summary "Need logging decision" \
|
||||
--payload-json '{"question":"Should logging go to stdout or stderr?"}'
|
||||
|
||||
wait_for_pid "${wait_blocked_pid}" "${case_slug}: wait task_blocked"
|
||||
|
||||
local blocked_message_id
|
||||
blocked_message_id="$(json_get "${case_dir}/blocked.json" '.data.message.message_id')"
|
||||
|
||||
run_json "${case_dir}/orch-blocked.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json blocked --run "${run_id}"
|
||||
|
||||
local wait_reply_pid
|
||||
start_wait "${case_dir}/wait-reply.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --agent worker-a --json wait-reply \
|
||||
--thread "${thread_id}" --after-message "${blocked_message_id}" --timeout-seconds 15
|
||||
wait_reply_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/answer.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json answer \
|
||||
--run "${run_id}" --task T1 --body "Use stdout for MVP."
|
||||
|
||||
wait_for_pid "${wait_reply_pid}" "${case_slug}: inbox wait-reply"
|
||||
|
||||
local wait_done_pid
|
||||
start_wait "${case_dir}/wait-task-done.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_done --timeout-seconds 15
|
||||
wait_done_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/resume.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json update \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--status in_progress --summary "Decision applied"
|
||||
|
||||
run_json "${case_dir}/done.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json done \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--summary "Frontend complete" \
|
||||
--body "Worker resumed after the leader answer."
|
||||
|
||||
wait_for_pid "${wait_done_pid}" "${case_slug}: wait task_done"
|
||||
|
||||
run_json "${case_dir}/reconcile.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/status.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/show.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}"
|
||||
|
||||
json_check "${case_dir}/wait-task-blocked.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_blocked"' "wait woke on task_blocked"
|
||||
json_check "${case_dir}/orch-blocked.json" '.data.blocked | length == 1 and .[0].task.task_id == "T1"' "blocked queue lists task"
|
||||
json_check "${case_dir}/wait-reply.json" '.data.woke == true and .data.message.kind == "answer"' "wait-reply woke on answer"
|
||||
json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "blocked flow run completes"
|
||||
json_check "${case_dir}/show.json" '[.data.messages[].kind] | index("question") != null and index("answer") != null and index("result") != null' "history includes question answer result"
|
||||
json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "question" and .payload_json.question == "Should logging go to stdout or stderr?")' "question payload matches"
|
||||
json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "answer" and .body == "Use stdout for MVP.")' "answer body matches"
|
||||
|
||||
local duration_seconds
|
||||
duration_seconds="$(( $(date +%s) - started_at ))"
|
||||
write_result_json \
|
||||
"${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \
|
||||
"$(join_json_array "${thread_id}")" \
|
||||
"$(join_json_array)" \
|
||||
"Direct CLI replay of the blocked-question workflow, including orch wait and inbox wait-reply."
|
||||
}
|
||||
|
||||
run_case_strict_worktree_cleanup() {
|
||||
local case_slug="strict-worktree-dispatch-to-cleanup-through-bundled-cli"
|
||||
local run_id="run_blog_skill_worktree_001"
|
||||
local case_dir
|
||||
case_dir="$(init_case_dir "${case_slug}")"
|
||||
local db_path="${case_dir}/coord.db"
|
||||
local repo_path="${case_dir}/repo"
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
|
||||
init_db "${db_path}"
|
||||
init_git_repo "${repo_path}"
|
||||
|
||||
run_json "${case_dir}/run.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json run init \
|
||||
--run "${run_id}" --goal "Validate strict worktree dispatch" \
|
||||
--summary "Exercise worktree allocation and cleanup"
|
||||
|
||||
run_json "${case_dir}/task.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T1 --title "Implement backend" \
|
||||
--summary "Implement inside an isolated worktree" --default-to worker-a
|
||||
|
||||
run_json "${case_dir}/dispatch.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dispatch \
|
||||
--run "${run_id}" --task T1 --to worker-a \
|
||||
--repo-path "${repo_path}" --workspace-root .orch/worktrees --strict-worktree \
|
||||
--body "Implement inside isolated worktree."
|
||||
|
||||
local thread_id
|
||||
local worktree_path
|
||||
thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')"
|
||||
worktree_path="$(json_get "${case_dir}/dispatch.json" '.data.attempt.worktree_path')"
|
||||
|
||||
local wait_pid
|
||||
start_wait "${case_dir}/wait-task-done.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_done --timeout-seconds 15
|
||||
wait_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/claim.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-a --thread "${thread_id}"
|
||||
|
||||
run_json "${case_dir}/thread-before-done.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}"
|
||||
|
||||
run_json "${case_dir}/done.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json done \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--summary "Backend complete" \
|
||||
--body "Confirmed the assigned worktree path from the task payload."
|
||||
|
||||
wait_for_pid "${wait_pid}" "${case_slug}: wait task_done"
|
||||
|
||||
run_json "${case_dir}/reconcile.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/cleanup.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json cleanup --run "${run_id}" --task T1
|
||||
|
||||
run_json "${case_dir}/status.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/show.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}"
|
||||
|
||||
json_check "${case_dir}/dispatch.json" '.data.attempt.worktree_path != ""' "dispatch returns worktree path"
|
||||
if jq -e --arg worktree "${worktree_path}" 'any(.data.messages[]; .kind == "task" and .payload_json.worktree_path == $worktree)' "${case_dir}/thread-before-done.json" >/dev/null; then
|
||||
printf 'PASS: task payload exposes worktree path\n'
|
||||
else
|
||||
printf 'FAIL: task payload exposes worktree path\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "worktree flow run completes"
|
||||
json_check "${case_dir}/cleanup.json" '.data.cleaned | length == 1' "cleanup removes one attempt"
|
||||
if [ -d "${worktree_path}" ]; then
|
||||
printf 'FAIL: worktree path still exists after cleanup: %s\n' "${worktree_path}" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'PASS: cleaned worktree removed\n'
|
||||
|
||||
local duration_seconds
|
||||
duration_seconds="$(( $(date +%s) - started_at ))"
|
||||
write_result_json \
|
||||
"${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \
|
||||
"$(join_json_array "${thread_id}")" \
|
||||
"$(join_json_array "${worktree_path}")" \
|
||||
"Direct CLI replay of strict worktree dispatch, completion, and cleanup."
|
||||
}
|
||||
|
||||
run_case_retry() {
|
||||
local case_slug="leader-retries-failed-task-through-bundled-cli"
|
||||
local run_id="run_blog_skill_retry_001"
|
||||
local case_dir
|
||||
case_dir="$(init_case_dir "${case_slug}")"
|
||||
local db_path="${case_dir}/coord.db"
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
|
||||
init_db "${db_path}"
|
||||
|
||||
run_json "${case_dir}/run.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json run init \
|
||||
--run "${run_id}" --goal "Validate retry behavior" \
|
||||
--summary "Exercise failed attempt retry"
|
||||
|
||||
run_json "${case_dir}/task.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T1 --title "Implement backend" \
|
||||
--summary "Retry after a simulated failure" --default-to worker-a
|
||||
|
||||
run_json "${case_dir}/dispatch.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dispatch \
|
||||
--run "${run_id}" --task T1 --to worker-a \
|
||||
--body "Initial attempt expected to fail for retry validation."
|
||||
|
||||
local thread_id_1
|
||||
thread_id_1="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')"
|
||||
|
||||
local wait_failed_pid
|
||||
start_wait "${case_dir}/wait-task-failed.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_failed --timeout-seconds 15
|
||||
wait_failed_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/claim-1.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-a --thread "${thread_id_1}"
|
||||
|
||||
run_json "${case_dir}/fail.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json fail \
|
||||
--agent worker-a --thread "${thread_id_1}" \
|
||||
--summary "Build failed" --body "Simulated first-attempt failure."
|
||||
|
||||
wait_for_pid "${wait_failed_pid}" "${case_slug}: wait task_failed"
|
||||
|
||||
run_json "${case_dir}/reconcile-failed.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/retry.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json retry \
|
||||
--run "${run_id}" --task T1 --to worker-a \
|
||||
--body "Retry after fixing the failure."
|
||||
|
||||
local thread_id_2
|
||||
thread_id_2="$(json_get "${case_dir}/retry.json" '.data.attempt.thread_id')"
|
||||
|
||||
local wait_done_pid
|
||||
start_wait "${case_dir}/wait-task-done.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_done --timeout-seconds 15
|
||||
wait_done_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/fetch-retry.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json fetch \
|
||||
--agent worker-a --status pending --limit 5
|
||||
|
||||
run_json "${case_dir}/claim-2.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-a --thread "${thread_id_2}"
|
||||
|
||||
run_json "${case_dir}/done-2.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json done \
|
||||
--agent worker-a --thread "${thread_id_2}" \
|
||||
--summary "Retry completed" --body "Second attempt succeeded."
|
||||
|
||||
wait_for_pid "${wait_done_pid}" "${case_slug}: wait task_done"
|
||||
|
||||
run_json "${case_dir}/reconcile-done.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/status.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/show-1.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_1}"
|
||||
|
||||
run_json "${case_dir}/show-2.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_2}"
|
||||
|
||||
json_check "${case_dir}/wait-task-failed.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_failed"' "wait woke on task_failed"
|
||||
json_check "${case_dir}/retry.json" '.data.attempt.attempt_no == 2 and .data.task.status == "dispatched"' "retry creates second dispatched attempt"
|
||||
json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "retry flow run completes"
|
||||
json_check "${case_dir}/show-1.json" '.data.thread.status == "failed"' "first thread failed"
|
||||
json_check "${case_dir}/show-2.json" '.data.thread.status == "done"' "second thread done"
|
||||
if [ "${thread_id_1}" = "${thread_id_2}" ]; then
|
||||
printf 'FAIL: retry reused thread ID %s\n' "${thread_id_1}" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'PASS: retry created distinct thread IDs\n'
|
||||
|
||||
local duration_seconds
|
||||
duration_seconds="$(( $(date +%s) - started_at ))"
|
||||
write_result_json \
|
||||
"${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \
|
||||
"$(join_json_array "${thread_id_1}" "${thread_id_2}")" \
|
||||
"$(join_json_array)" \
|
||||
"Direct CLI replay of failed attempt reconciliation followed by retry."
|
||||
}
|
||||
|
||||
run_case_reassign() {
|
||||
local case_slug="leader-reassigns-blocked-task-through-bundled-cli"
|
||||
local run_id="run_blog_skill_reassign_001"
|
||||
local case_dir
|
||||
case_dir="$(init_case_dir "${case_slug}")"
|
||||
local db_path="${case_dir}/coord.db"
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
|
||||
init_db "${db_path}"
|
||||
|
||||
run_json "${case_dir}/run.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json run init \
|
||||
--run "${run_id}" --goal "Validate reassign behavior" \
|
||||
--summary "Exercise blocked-task reassignment"
|
||||
|
||||
run_json "${case_dir}/task.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T1 --title "Implement backend" \
|
||||
--summary "Reassign after worker-a blocks" --default-to worker-a
|
||||
|
||||
run_json "${case_dir}/dispatch.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dispatch \
|
||||
--run "${run_id}" --task T1 --to worker-a \
|
||||
--body "Initial attempt expected to be reassigned."
|
||||
|
||||
local thread_id_1
|
||||
thread_id_1="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')"
|
||||
|
||||
local wait_blocked_pid
|
||||
start_wait "${case_dir}/wait-task-blocked.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_blocked --timeout-seconds 15
|
||||
wait_blocked_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/claim-1.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-a --thread "${thread_id_1}"
|
||||
|
||||
run_json "${case_dir}/blocked.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json update \
|
||||
--agent worker-a --thread "${thread_id_1}" \
|
||||
--status blocked --summary "Need product decision" \
|
||||
--payload-json '{"question":"Proceed with v1 scope?"}'
|
||||
|
||||
wait_for_pid "${wait_blocked_pid}" "${case_slug}: wait task_blocked"
|
||||
|
||||
run_json "${case_dir}/reconcile-blocked.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/orch-blocked.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json blocked --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/reassign.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reassign \
|
||||
--run "${run_id}" --task T1 --to worker-b \
|
||||
--reason "Try another worker with clearer ownership."
|
||||
|
||||
local thread_id_2
|
||||
thread_id_2="$(json_get "${case_dir}/reassign.json" '.data.attempt.thread_id')"
|
||||
|
||||
local wait_done_pid
|
||||
start_wait "${case_dir}/wait-task-done.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_done --timeout-seconds 15
|
||||
wait_done_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/fetch-worker-b.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json fetch \
|
||||
--agent worker-b --status pending --limit 5
|
||||
|
||||
run_json "${case_dir}/claim-2.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-b --thread "${thread_id_2}"
|
||||
|
||||
run_json "${case_dir}/done-2.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json done \
|
||||
--agent worker-b --thread "${thread_id_2}" \
|
||||
--summary "Reassigned work complete" --body "Worker-b completed the reassigned attempt."
|
||||
|
||||
wait_for_pid "${wait_done_pid}" "${case_slug}: wait task_done"
|
||||
|
||||
run_json "${case_dir}/reconcile-done.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/status.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/show-1.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_1}"
|
||||
|
||||
run_json "${case_dir}/show-2.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_2}"
|
||||
|
||||
json_check "${case_dir}/wait-task-blocked.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_blocked"' "wait woke on blocked event for reassign"
|
||||
json_check "${case_dir}/orch-blocked.json" '.data.blocked | length == 1 and .[0].task.task_id == "T1"' "blocked queue lists reassigned task"
|
||||
json_check "${case_dir}/reassign.json" '.data.attempt.attempt_no == 2 and .data.attempt.assigned_to == "worker-b"' "reassign creates attempt 2 for worker-b"
|
||||
json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "reassign flow run completes"
|
||||
json_check "${case_dir}/show-1.json" '.data.thread.status == "cancelled"' "original thread cancelled"
|
||||
json_check "${case_dir}/show-2.json" '.data.thread.status == "done"' "new thread done"
|
||||
json_check "${case_dir}/show-1.json" 'any(.data.messages[]; .kind == "question" and .payload_json.question == "Proceed with v1 scope?")' "original blocked question preserved"
|
||||
if [ "${thread_id_1}" = "${thread_id_2}" ]; then
|
||||
printf 'FAIL: reassign reused thread ID %s\n' "${thread_id_1}" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'PASS: reassign created distinct thread IDs\n'
|
||||
|
||||
local duration_seconds
|
||||
duration_seconds="$(( $(date +%s) - started_at ))"
|
||||
write_result_json \
|
||||
"${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \
|
||||
"$(join_json_array "${thread_id_1}" "${thread_id_2}")" \
|
||||
"$(join_json_array)" \
|
||||
"Direct CLI replay of blocked-task reassignment from worker-a to worker-b."
|
||||
}
|
||||
|
||||
main() {
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--output-root)
|
||||
OUTPUT_ROOT="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
printf 'unknown argument: %s\n' "$1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
require_command jq
|
||||
require_command git
|
||||
require_command mktemp
|
||||
|
||||
if [ ! -x "${ORCH_BIN}" ]; then
|
||||
printf 'orch skill binary is not executable: %s\n' "${ORCH_BIN}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -x "${INBOX_BIN}" ]; then
|
||||
printf 'inbox skill binary is not executable: %s\n' "${INBOX_BIN}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${OUTPUT_ROOT}" ]; then
|
||||
OUTPUT_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/orch-skill-forward.XXXXXX")"
|
||||
else
|
||||
mkdir -p "${OUTPUT_ROOT}"
|
||||
fi
|
||||
|
||||
run_case_happy_path
|
||||
run_case_blocked_answer
|
||||
run_case_strict_worktree_cleanup
|
||||
run_case_retry
|
||||
run_case_reassign
|
||||
|
||||
printf '\nResults written to %s\n' "${OUTPUT_ROOT}"
|
||||
printf 'case\tresult\n'
|
||||
print_case_summary "leader-run-dispatch-reconcile-through-bundled-cli" "${OUTPUT_ROOT}/leader-run-dispatch-reconcile-through-bundled-cli"
|
||||
print_case_summary "leader-blocked-answer-resume-through-bundled-cli" "${OUTPUT_ROOT}/leader-blocked-answer-resume-through-bundled-cli"
|
||||
print_case_summary "strict-worktree-dispatch-to-cleanup-through-bundled-cli" "${OUTPUT_ROOT}/strict-worktree-dispatch-to-cleanup-through-bundled-cli"
|
||||
print_case_summary "leader-retries-failed-task-through-bundled-cli" "${OUTPUT_ROOT}/leader-retries-failed-task-through-bundled-cli"
|
||||
print_case_summary "leader-reassigns-blocked-task-through-bundled-cli" "${OUTPUT_ROOT}/leader-reassigns-blocked-task-through-bundled-cli"
|
||||
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user