docs: add orch skill gap-fill cases
This commit is contained in:
@@ -448,6 +448,320 @@ run_case_strict_worktree_cleanup() {
|
||||
"Direct CLI replay of strict worktree dispatch, completion, and cleanup."
|
||||
}
|
||||
|
||||
run_case_dependency_ready_dispatch() {
|
||||
local case_slug="leader-dispatches-dependent-task-after-prerequisite-through-bundled-cli"
|
||||
local run_id="run_blog_skill_deps_001"
|
||||
local case_dir
|
||||
case_dir="$(init_case_dir "${case_slug}")"
|
||||
local db_path="${case_dir}/coord.db"
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
|
||||
init_db "${db_path}"
|
||||
|
||||
run_json "${case_dir}/run.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json run init \
|
||||
--run "${run_id}" --goal "Validate dependency-gated dispatch" \
|
||||
--summary "Exercise dep add, ready, and staged dispatch"
|
||||
|
||||
run_json "${case_dir}/task-1.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T1 --title "Implement backend" \
|
||||
--summary "Prerequisite task" --default-to worker-a
|
||||
|
||||
run_json "${case_dir}/task-2.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T2 --title "Implement frontend" \
|
||||
--summary "Dependent task" --default-to worker-b
|
||||
|
||||
run_json "${case_dir}/dep.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dep add \
|
||||
--run "${run_id}" --task T2 --depends-on T1
|
||||
|
||||
run_json "${case_dir}/ready-initial.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json ready --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/dispatch-1.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dispatch \
|
||||
--run "${run_id}" --task T1 --to worker-a \
|
||||
--body "Complete the prerequisite task first."
|
||||
|
||||
local thread_id_1
|
||||
thread_id_1="$(json_get "${case_dir}/dispatch-1.json" '.data.attempt.thread_id')"
|
||||
|
||||
local wait_done_pid
|
||||
start_wait "${case_dir}/wait-task-done-1.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_done --timeout-seconds 15
|
||||
wait_done_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/claim-1.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-a --thread "${thread_id_1}"
|
||||
|
||||
run_json "${case_dir}/progress-1.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json update \
|
||||
--agent worker-a --thread "${thread_id_1}" \
|
||||
--status in_progress --summary "Prerequisite started"
|
||||
|
||||
run_json "${case_dir}/done-1.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json done \
|
||||
--agent worker-a --thread "${thread_id_1}" \
|
||||
--summary "Prerequisite complete" \
|
||||
--body "T1 completed so T2 can be dispatched."
|
||||
|
||||
wait_for_pid "${wait_done_pid}" "${case_slug}: wait first task_done"
|
||||
|
||||
run_json "${case_dir}/reconcile-1.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/ready-after-prerequisite.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json ready --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/dispatch-2.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dispatch \
|
||||
--run "${run_id}" --task T2 --to worker-b \
|
||||
--body "Dependent task is now ready after T1."
|
||||
|
||||
local thread_id_2
|
||||
thread_id_2="$(json_get "${case_dir}/dispatch-2.json" '.data.attempt.thread_id')"
|
||||
|
||||
run_json "${case_dir}/claim-2.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-b --thread "${thread_id_2}"
|
||||
|
||||
run_json "${case_dir}/done-2.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json done \
|
||||
--agent worker-b --thread "${thread_id_2}" \
|
||||
--summary "Dependent task complete" \
|
||||
--body "T2 completed after the dependency cleared."
|
||||
|
||||
run_json "${case_dir}/reconcile-2.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/status.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/show-1.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_1}"
|
||||
|
||||
run_json "${case_dir}/show-2.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id_2}"
|
||||
|
||||
json_check "${case_dir}/dep.json" '.data.dependency.task_id == "T2" and .data.dependency.depends_on_task_id == "T1"' "dep add stores T2 -> T1"
|
||||
json_check "${case_dir}/ready-initial.json" '.data.tasks | length == 1 and .[0].task_id == "T1"' "initial ready lists only prerequisite"
|
||||
json_check "${case_dir}/wait-task-done-1.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_done"' "wait woke on prerequisite completion"
|
||||
json_check "${case_dir}/ready-after-prerequisite.json" '.data.tasks | length == 1 and .[0].task_id == "T2"' "dependent task becomes ready after reconcile"
|
||||
json_check "${case_dir}/status.json" '.data.run.status == "done" and ([.data.tasks[] | select(.task_id == "T1")][0].status == "done") and ([.data.tasks[] | select(.task_id == "T2")][0].status == "done")' "dependency-gated run completes with both tasks done"
|
||||
json_check "${case_dir}/show-1.json" '.data.thread.status == "done"' "prerequisite thread done"
|
||||
json_check "${case_dir}/show-2.json" '.data.thread.status == "done"' "dependent thread done"
|
||||
if [ "${thread_id_1}" = "${thread_id_2}" ]; then
|
||||
printf 'FAIL: dependency flow reused thread ID %s\n' "${thread_id_1}" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'PASS: dependency flow created distinct thread IDs\n'
|
||||
|
||||
local duration_seconds
|
||||
duration_seconds="$(( $(date +%s) - started_at ))"
|
||||
write_result_json \
|
||||
"${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \
|
||||
"$(join_json_array "${thread_id_1}" "${thread_id_2}")" \
|
||||
"$(join_json_array)" \
|
||||
"Direct CLI replay of dependency-gated ready sequencing from prerequisite completion to dependent dispatch."
|
||||
}
|
||||
|
||||
run_case_cancel_active_task() {
|
||||
local case_slug="leader-cancels-active-task-through-bundled-cli"
|
||||
local run_id="run_blog_skill_cancel_001"
|
||||
local case_dir
|
||||
case_dir="$(init_case_dir "${case_slug}")"
|
||||
local db_path="${case_dir}/coord.db"
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
|
||||
init_db "${db_path}"
|
||||
|
||||
run_json "${case_dir}/run.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json run init \
|
||||
--run "${run_id}" --goal "Validate active task cancellation" \
|
||||
--summary "Exercise direct orch cancel on an active attempt"
|
||||
|
||||
run_json "${case_dir}/task-1.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T1 --title "Implement backend" \
|
||||
--summary "Task that will be cancelled mid-flight" --default-to worker-a
|
||||
|
||||
run_json "${case_dir}/task-2.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T2 --title "Implement frontend" \
|
||||
--summary "Task that should remain ready" --default-to worker-b
|
||||
|
||||
run_json "${case_dir}/dispatch.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dispatch \
|
||||
--run "${run_id}" --task T1 --to worker-a \
|
||||
--body "Start work so the leader can cancel an active task."
|
||||
|
||||
local thread_id
|
||||
thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')"
|
||||
|
||||
run_json "${case_dir}/claim.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-a --thread "${thread_id}"
|
||||
|
||||
run_json "${case_dir}/progress.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json update \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--status in_progress --summary "Active work in progress"
|
||||
|
||||
run_json "${case_dir}/cancel.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json cancel \
|
||||
--run "${run_id}" --task T1 \
|
||||
--reason "Task superseded by dependency review."
|
||||
|
||||
run_json "${case_dir}/status.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/ready.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json ready --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/show.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}"
|
||||
|
||||
json_check "${case_dir}/cancel.json" '.data.cancelled_tasks | length == 1 and .[0].task_id == "T1" and .[0].status == "cancelled"' "cancel returns cancelled T1"
|
||||
json_check "${case_dir}/status.json" '.data.run.status == "ready"' "run remains ready after cancelling one active task"
|
||||
json_check "${case_dir}/status.json" '([.data.tasks[] | select(.task_id == "T1")][0].status == "cancelled") and ([.data.tasks[] | select(.task_id == "T2")][0].status == "ready")' "status keeps T2 ready while T1 is cancelled"
|
||||
json_check "${case_dir}/ready.json" '.data.tasks | length == 1 and .[0].task_id == "T2"' "ready queue still exposes untouched T2"
|
||||
json_check "${case_dir}/show.json" '.data.thread.status == "cancelled"' "cancelled thread reaches terminal state"
|
||||
json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "progress")' "cancelled thread preserves worker progress history"
|
||||
|
||||
local duration_seconds
|
||||
duration_seconds="$(( $(date +%s) - started_at ))"
|
||||
write_result_json \
|
||||
"${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \
|
||||
"$(join_json_array "${thread_id}")" \
|
||||
"$(join_json_array)" \
|
||||
"Direct CLI replay of cancelling an active task while leaving unrelated ready work untouched."
|
||||
}
|
||||
|
||||
run_case_payload_only_answer() {
|
||||
local case_slug="leader-answers-blocked-task-with-payload-json-through-bundled-cli"
|
||||
local run_id="run_blog_skill_payload_answer_001"
|
||||
local case_dir
|
||||
case_dir="$(init_case_dir "${case_slug}")"
|
||||
local db_path="${case_dir}/coord.db"
|
||||
local started_at
|
||||
started_at="$(date +%s)"
|
||||
|
||||
init_db "${db_path}"
|
||||
|
||||
run_json "${case_dir}/run.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json run init \
|
||||
--run "${run_id}" --goal "Validate payload-only answers" \
|
||||
--summary "Exercise answer --payload-json on a blocked task"
|
||||
|
||||
run_json "${case_dir}/task.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json task add \
|
||||
--run "${run_id}" --task T1 --title "Build frontend" \
|
||||
--summary "Resume after structured leader decision" --default-to worker-a
|
||||
|
||||
run_json "${case_dir}/dispatch.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json dispatch \
|
||||
--run "${run_id}" --task T1 --to worker-a \
|
||||
--body "Pause if a structured logging decision is needed."
|
||||
|
||||
local thread_id
|
||||
thread_id="$(json_get "${case_dir}/dispatch.json" '.data.attempt.thread_id')"
|
||||
|
||||
local wait_blocked_pid
|
||||
start_wait "${case_dir}/wait-task-blocked.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_blocked --timeout-seconds 15
|
||||
wait_blocked_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/claim.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json claim \
|
||||
--agent worker-a --thread "${thread_id}"
|
||||
|
||||
run_json "${case_dir}/blocked.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json update \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--status blocked --summary "Need structured logging decision" \
|
||||
--payload-json '{"question":"Use stdout or stderr for structured logs?"}'
|
||||
|
||||
wait_for_pid "${wait_blocked_pid}" "${case_slug}: wait task_blocked"
|
||||
|
||||
local blocked_message_id
|
||||
blocked_message_id="$(json_get "${case_dir}/blocked.json" '.data.message.message_id')"
|
||||
|
||||
run_json "${case_dir}/orch-blocked.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json blocked --run "${run_id}"
|
||||
|
||||
local wait_reply_pid
|
||||
start_wait "${case_dir}/wait-reply.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --agent worker-a --json wait-reply \
|
||||
--thread "${thread_id}" --after-message "${blocked_message_id}" --timeout-seconds 15
|
||||
wait_reply_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/answer.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json answer \
|
||||
--run "${run_id}" --task T1 \
|
||||
--payload-json '{"decision":"stdout","source":"leader","format":"structured"}'
|
||||
|
||||
wait_for_pid "${wait_reply_pid}" "${case_slug}: inbox wait-reply"
|
||||
|
||||
local wait_done_pid
|
||||
start_wait "${case_dir}/wait-task-done.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json wait \
|
||||
--run "${run_id}" --for task_done --timeout-seconds 15
|
||||
wait_done_pid="${LAST_BG_PID}"
|
||||
|
||||
sleep 0.2
|
||||
|
||||
run_json "${case_dir}/resume.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json update \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--status in_progress --summary "Structured decision applied"
|
||||
|
||||
run_json "${case_dir}/done.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json done \
|
||||
--agent worker-a --thread "${thread_id}" \
|
||||
--summary "Frontend complete" \
|
||||
--body "Worker resumed after reading the structured leader decision."
|
||||
|
||||
wait_for_pid "${wait_done_pid}" "${case_slug}: wait task_done"
|
||||
|
||||
run_json "${case_dir}/reconcile.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json reconcile --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/status.json" \
|
||||
"${ORCH_BIN}" --db "${db_path}" --json status --run "${run_id}"
|
||||
|
||||
run_json "${case_dir}/show.json" \
|
||||
"${INBOX_BIN}" --db "${db_path}" --json show --thread "${thread_id}"
|
||||
|
||||
json_check "${case_dir}/wait-task-blocked.json" '.data.woke == true and (.data.events | length) >= 1 and .data.events[0].type == "task_blocked"' "payload-answer flow woke on task_blocked"
|
||||
json_check "${case_dir}/orch-blocked.json" '.data.blocked | length == 1 and .[0].task.task_id == "T1"' "blocked queue lists payload-answer task"
|
||||
json_check "${case_dir}/answer.json" '.data.message.kind == "answer" and .data.message.payload_json.decision == "stdout" and .data.message.payload_json.source == "leader" and .data.message.payload_json.format == "structured"' "answer writes payload-json without body"
|
||||
json_check "${case_dir}/wait-reply.json" '.data.woke == true and .data.message.kind == "answer" and .data.message.payload_json.decision == "stdout"' "wait-reply exposes structured answer payload"
|
||||
json_check "${case_dir}/status.json" '.data.run.status == "done" and .data.tasks[0].status == "done"' "payload-answer flow run completes"
|
||||
json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "question" and .payload_json.question == "Use stdout or stderr for structured logs?")' "payload-answer question recorded"
|
||||
json_check "${case_dir}/show.json" 'any(.data.messages[]; .kind == "answer" and .payload_json.decision == "stdout" and .payload_json.source == "leader" and .payload_json.format == "structured")' "payload-answer message preserved in history"
|
||||
|
||||
local duration_seconds
|
||||
duration_seconds="$(( $(date +%s) - started_at ))"
|
||||
write_result_json \
|
||||
"${case_dir}" "${case_slug}" "${db_path}" "${run_id}" pass "${duration_seconds}" \
|
||||
"$(join_json_array "${thread_id}")" \
|
||||
"$(join_json_array)" \
|
||||
"Direct CLI replay of answering a blocked task with payload-json only."
|
||||
}
|
||||
|
||||
run_case_retry() {
|
||||
local case_slug="leader-retries-failed-task-through-bundled-cli"
|
||||
local run_id="run_blog_skill_retry_001"
|
||||
@@ -723,6 +1037,9 @@ main() {
|
||||
run_case_happy_path
|
||||
run_case_blocked_answer
|
||||
run_case_strict_worktree_cleanup
|
||||
run_case_dependency_ready_dispatch
|
||||
run_case_cancel_active_task
|
||||
run_case_payload_only_answer
|
||||
run_case_retry
|
||||
run_case_reassign
|
||||
|
||||
@@ -731,6 +1048,9 @@ main() {
|
||||
print_case_summary "leader-run-dispatch-reconcile-through-bundled-cli" "${OUTPUT_ROOT}/leader-run-dispatch-reconcile-through-bundled-cli"
|
||||
print_case_summary "leader-blocked-answer-resume-through-bundled-cli" "${OUTPUT_ROOT}/leader-blocked-answer-resume-through-bundled-cli"
|
||||
print_case_summary "strict-worktree-dispatch-to-cleanup-through-bundled-cli" "${OUTPUT_ROOT}/strict-worktree-dispatch-to-cleanup-through-bundled-cli"
|
||||
print_case_summary "leader-dispatches-dependent-task-after-prerequisite-through-bundled-cli" "${OUTPUT_ROOT}/leader-dispatches-dependent-task-after-prerequisite-through-bundled-cli"
|
||||
print_case_summary "leader-cancels-active-task-through-bundled-cli" "${OUTPUT_ROOT}/leader-cancels-active-task-through-bundled-cli"
|
||||
print_case_summary "leader-answers-blocked-task-with-payload-json-through-bundled-cli" "${OUTPUT_ROOT}/leader-answers-blocked-task-with-payload-json-through-bundled-cli"
|
||||
print_case_summary "leader-retries-failed-task-through-bundled-cli" "${OUTPUT_ROOT}/leader-retries-failed-task-through-bundled-cli"
|
||||
print_case_summary "leader-reassigns-blocked-task-through-bundled-cli" "${OUTPUT_ROOT}/leader-reassigns-blocked-task-through-bundled-cli"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user