Add council review report command
This commit is contained in:
@@ -513,6 +513,15 @@ CREATE TABLE IF NOT EXISTS council_groups (
|
||||
source_finding_ids_json TEXT NOT NULL DEFAULT '[]',
|
||||
PRIMARY KEY (run_id, group_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS council_reports (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
show_json TEXT NOT NULL DEFAULT '[]',
|
||||
summary_json TEXT NOT NULL DEFAULT '{}',
|
||||
markdown_path TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
```
|
||||
|
||||
## Embedded Skill Draft
|
||||
|
||||
@@ -31,9 +31,10 @@ As of now:
|
||||
- `orch council start` now creates a dedicated council run, persists council target input metadata, and dispatches the three fixed reviewer roles through the existing scheduler
|
||||
- `orch council wait` now blocks until the three reviewer tasks reach terminal states or a timeout is reached
|
||||
- `orch council tally` now parses completed reviewer outputs, persists `council_findings`, groups recommendations into `consensus`, `majority`, and `minority`, and persists `council_groups`
|
||||
- automated integration tests now cover the main `orch` scheduler slice, including dependency gating, dispatch, blocked-answer flow, retry, reassign, cancel, cleanup, strict worktree creation, automatic code-task worktree enablement, dirty-repo rejection rules, wait wake/timeout behavior, and council start/wait/tally behavior
|
||||
- `orch council report` now reads persisted `council_groups`, renders human-readable markdown reports, writes markdown artifacts, and persists final report metadata in `council_reports`
|
||||
- automated integration tests now cover the main `orch` scheduler slice, including dependency gating, dispatch, blocked-answer flow, retry, reassign, cancel, cleanup, strict worktree creation, automatic code-task worktree enablement, dirty-repo rejection rules, wait wake/timeout behavior, and council start/wait/tally/report behavior
|
||||
|
||||
This means the project now has a working `orch` core scheduler with automatic worktree selection for code-like tasks, strict worktree-backed dispatch, the main leader-side control loop, and the first three council workflow slices.
|
||||
This means the project now has a working `orch` core scheduler with automatic worktree selection for code-like tasks, strict worktree-backed dispatch, the main leader-side control loop, and the full v1 council workflow from start through final report generation.
|
||||
|
||||
## Source Of Truth
|
||||
|
||||
@@ -76,9 +77,9 @@ Current implementation status:
|
||||
- `Milestone 4: Orch Core Scheduling` is complete for the current non-worktree scheduler scope
|
||||
- `Milestone 5: Strict Worktree Support` is complete
|
||||
- `Milestone 6: Waiting Primitives` is complete
|
||||
- `Milestone 7: Council Review` is partially complete through `orch council start`, `orch council wait`, and `orch council tally`
|
||||
- `Milestone 7: Council Review` is complete
|
||||
|
||||
The next practical coding target is the final `Milestone 7` slice: `orch council report`.
|
||||
The council review v1 surface is now complete, including final report rendering and metadata persistence.
|
||||
|
||||
### Milestone 1: Go Skeleton
|
||||
|
||||
@@ -337,32 +338,34 @@ Definition of done:
|
||||
|
||||
Status:
|
||||
|
||||
- partially complete through `orch council start`, `orch council wait`, and `orch council tally`
|
||||
- completed
|
||||
|
||||
Completed so far:
|
||||
|
||||
- council-specific storage now includes run metadata, reviewer assignment rows, reviewer findings/groups tables, and persisted council input references
|
||||
- council-specific storage now includes run metadata, reviewer assignment rows, reviewer findings/groups tables, persisted council input references, and final report metadata
|
||||
- `orch council start`
|
||||
- `orch council wait`
|
||||
- `orch council tally`
|
||||
- `orch council report`
|
||||
- council start creates a dedicated run, stores council target input metadata, creates reviewer tasks `CR1` through `CR3`, and dispatches the fixed reviewer roles `architecture-reviewer`, `implementation-reviewer`, and `risk-reviewer`
|
||||
- council wait blocks until all three reviewer tasks reach terminal states or timeout
|
||||
- council tally parses structured reviewer outputs from completed reviewer result messages and persists grouped recommendations
|
||||
- CLI integration tests cover council start dispatch, metadata persistence, council wait wake/timeout behavior, and council tally grouping in `normal` and `strict` modes
|
||||
- council report reads grouped recommendations from persisted `council_groups`, supports `--show` bucket filtering, renders markdown report artifacts, and persists report metadata plus artifact paths
|
||||
- CLI integration tests cover council start dispatch, metadata persistence, council wait wake/timeout behavior, council tally grouping in `normal` and `strict` modes, and council report default/all/JSON rendering behavior
|
||||
|
||||
Remaining:
|
||||
|
||||
- `orch council report`
|
||||
- none for the v1 council workflow
|
||||
|
||||
## Immediate Next Task
|
||||
|
||||
If a new agent is taking over now, the next concrete step should be:
|
||||
|
||||
1. continue `Milestone 7: Council Review` with `orch council report`
|
||||
2. define the persisted report artifact shape and how markdown output should be rendered from grouped recommendations
|
||||
3. keep the authored inbox test-plan set in `docs/tests/inbox/` synchronized if CLI behavior changes during further `orch` work
|
||||
1. treat `Milestone 7: Council Review` as complete unless a new user request introduces a new council capability
|
||||
2. keep the authored inbox test-plan set in `docs/tests/inbox/` synchronized if future `orch` work changes shared CLI behavior
|
||||
3. choose the next milestone explicitly instead of reopening the completed council v1 slice
|
||||
|
||||
The inbox implementation and its human-readable test-plan set are already in place, and `orch` now supports the main scheduler loop plus council start/wait/tally, so the next meaningful project step is rendering final council reports.
|
||||
The inbox implementation and its human-readable test-plan set are already in place, and `orch` now supports the main scheduler loop plus the complete council start/wait/tally/report workflow, so any next step should be a new milestone rather than unfinished council v1 work.
|
||||
|
||||
## Recommended Driver Choices
|
||||
|
||||
@@ -391,11 +394,11 @@ Completed so far:
|
||||
- orch council start dispatch and persistence coverage
|
||||
- orch council wait wake and timeout coverage
|
||||
- orch council tally grouping coverage
|
||||
- orch council report default markdown, `--show all`, and JSON shape coverage
|
||||
|
||||
Still recommended before the codebase grows too much:
|
||||
|
||||
- worktree path generation test
|
||||
- council tally grouping test
|
||||
|
||||
## Inbox Test Documentation Roadmap
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# Orch Council Report
|
||||
|
||||
## Status
|
||||
|
||||
- `completed`
|
||||
|
||||
## Owner
|
||||
|
||||
- codex
|
||||
|
||||
## Started At
|
||||
|
||||
- `2026-03-19`
|
||||
|
||||
## Goal
|
||||
|
||||
- implement `orch council report` so persisted grouped recommendations can be rendered as a final council report in human-readable markdown and stable JSON output
|
||||
|
||||
## Scope
|
||||
|
||||
- add `orch council report` with `--run`, `--show`, and `--json`
|
||||
- read grouped recommendations from persisted `council_groups`
|
||||
- render a markdown report for the requested buckets and persist report metadata if needed by the existing design
|
||||
- add integration coverage for default output, `--show all`, and JSON shape
|
||||
- run `go test ./...`, update the implementation roadmap, and archive this workstream when complete
|
||||
|
||||
## Checklist
|
||||
|
||||
- [x] inspect council report requirements, current council store, and CLI/test patterns
|
||||
- [x] implement council report store and CLI command
|
||||
- [x] add integration coverage for default buckets, `--show all`, and JSON output
|
||||
- [x] run `go test ./...`
|
||||
- [x] update `docs/implementation-roadmap.md`
|
||||
- [x] archive this roadmap with a completion summary
|
||||
|
||||
## Files
|
||||
|
||||
- `docs/roadmaps/archive/orch-council-report.md`
|
||||
- `docs/implementation-roadmap.md`
|
||||
- `docs/council-review.md`
|
||||
- `internal/store/council.go`
|
||||
- `internal/cli/orch/council.go`
|
||||
- `internal/cli/orch/council_report.go`
|
||||
- `internal/cli/orch/integration_test.go`
|
||||
- `internal/db/schema/007_council_reports.sql`
|
||||
|
||||
## Decisions
|
||||
|
||||
- keep the scope limited to report rendering on top of existing persisted council data
|
||||
- persist final report metadata in a dedicated `council_reports` table so the last rendered report artifact path can be recovered without re-reading files
|
||||
- place markdown artifacts under a `.orch/reports/` tree rooted next to the active database context so tests and non-default databases do not dirty the repository root
|
||||
|
||||
## Blockers
|
||||
|
||||
- none
|
||||
|
||||
## Next Step
|
||||
|
||||
- none
|
||||
|
||||
## Completion Summary
|
||||
|
||||
- added `orch council report` with `--run`, `--show`, and `--json` on top of persisted `council_groups`
|
||||
- report rendering now produces human-readable markdown, writes a markdown artifact, and persists final report metadata in `council_reports`
|
||||
- integration coverage now verifies default `consensus,majority` output, `--show all`, and the JSON response shape
|
||||
@@ -11,5 +11,6 @@ func newCouncilCmd(root *rootOptions) *cobra.Command {
|
||||
cmd.AddCommand(newCouncilStartCmd(root))
|
||||
cmd.AddCommand(newCouncilWaitCmd(root))
|
||||
cmd.AddCommand(newCouncilTallyCmd(root))
|
||||
cmd.AddCommand(newCouncilReportCmd(root))
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package orch
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"ai-workflow-skill/internal/protocol"
|
||||
"ai-workflow-skill/internal/store"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
type councilReportOptions struct {
|
||||
runID string
|
||||
show string
|
||||
}
|
||||
|
||||
func newCouncilReportCmd(root *rootOptions) *cobra.Command {
|
||||
opts := &councilReportOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "report",
|
||||
Short: "Render the final grouped council output",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
ctx := cmd.Context()
|
||||
|
||||
sqlDB, err := openOrchDB(ctx, root.dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sqlDB.Close()
|
||||
|
||||
orchStore := store.NewOrchStore(sqlDB)
|
||||
result, err := orchStore.BuildCouncilReport(ctx, store.CouncilReportInput{
|
||||
RunID: opts.runID,
|
||||
Show: opts.show,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reportPath := councilReportArtifactPath(root.dbPath, result.RunID)
|
||||
if err := os.MkdirAll(filepath.Dir(reportPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create council report directory: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(reportPath, []byte(result.Markdown), 0o644); err != nil {
|
||||
return fmt.Errorf("write council report artifact: %w", err)
|
||||
}
|
||||
|
||||
result.ReportArtifacts = []store.CouncilReportArtifact{
|
||||
{
|
||||
Kind: "markdown",
|
||||
Path: reportPath,
|
||||
},
|
||||
}
|
||||
if err := orchStore.PersistCouncilReport(ctx, store.CouncilPersistReportInput{
|
||||
RunID: result.RunID,
|
||||
Show: result.Show,
|
||||
Summary: result.Summary,
|
||||
MarkdownPath: reportPath,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp := protocol.Success{
|
||||
OK: true,
|
||||
Command: "council report",
|
||||
Data: map[string]any{
|
||||
"run_id": result.RunID,
|
||||
"show": result.Show,
|
||||
"summary": result.Summary,
|
||||
"grouped_recommendations": result.GroupedRecommendations,
|
||||
"report_artifacts": result.ReportArtifacts,
|
||||
},
|
||||
}
|
||||
if root.json {
|
||||
return protocol.WriteJSON(cmd.OutOrStdout(), resp)
|
||||
}
|
||||
|
||||
_, err = fmt.Fprint(cmd.OutOrStdout(), result.Markdown)
|
||||
return err
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&opts.runID, "run", "", "Council run ID")
|
||||
cmd.Flags().StringVar(&opts.show, "show", "", "Buckets to show: consensus,majority,minority,all")
|
||||
_ = cmd.MarkFlagRequired("run")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func councilReportArtifactPath(dbPath, runID string) string {
|
||||
baseDir := "."
|
||||
dbDir := filepath.Dir(dbPath)
|
||||
switch {
|
||||
case dbDir == "", dbDir == ".":
|
||||
baseDir = "."
|
||||
case filepath.Base(dbDir) == ".agents":
|
||||
baseDir = filepath.Dir(dbDir)
|
||||
default:
|
||||
baseDir = dbDir
|
||||
}
|
||||
if baseDir == "" {
|
||||
baseDir = "."
|
||||
}
|
||||
|
||||
fileName := strings.ReplaceAll(strings.TrimSpace(runID), string(os.PathSeparator), "_") + ".md"
|
||||
return filepath.Join(baseDir, ".orch", "reports", fileName)
|
||||
}
|
||||
@@ -1870,6 +1870,186 @@ func TestOrchCouncilTallyStrictKeepsDistinctProposals(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrchCouncilReportDefaultShowsConsensusAndMajority(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dbPath := filepath.Join(t.TempDir(), "coord.db")
|
||||
runID := "council_blog_report_001"
|
||||
seedCouncilReportRun(t, dbPath, runID)
|
||||
|
||||
reportOut := runOrchCommand(
|
||||
t,
|
||||
"--db", dbPath,
|
||||
"council", "report",
|
||||
"--run", runID,
|
||||
)
|
||||
|
||||
if !strings.Contains(reportOut, "# Council Review Report") {
|
||||
t.Fatalf("expected markdown report header, got %q", reportOut)
|
||||
}
|
||||
if !strings.Contains(reportOut, "## Consensus") {
|
||||
t.Fatalf("expected consensus section, got %q", reportOut)
|
||||
}
|
||||
if !strings.Contains(reportOut, "## Majority") {
|
||||
t.Fatalf("expected majority section, got %q", reportOut)
|
||||
}
|
||||
if strings.Contains(reportOut, "## Minority") {
|
||||
t.Fatalf("did not expect minority section in default report, got %q", reportOut)
|
||||
}
|
||||
|
||||
reportPath := councilReportArtifactPath(dbPath, runID)
|
||||
reportBytes, err := os.ReadFile(reportPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read council report artifact: %v", err)
|
||||
}
|
||||
if string(reportBytes) != reportOut {
|
||||
t.Fatalf("expected stdout markdown to match artifact contents")
|
||||
}
|
||||
|
||||
sqlDB, err := openOrchDB(t.Context(), dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open orch db: %v", err)
|
||||
}
|
||||
defer sqlDB.Close()
|
||||
|
||||
var showJSON string
|
||||
var summaryJSON string
|
||||
var markdownPath string
|
||||
if err := sqlDB.QueryRowContext(
|
||||
t.Context(),
|
||||
`SELECT show_json, summary_json, markdown_path
|
||||
FROM council_reports
|
||||
WHERE run_id = ?`,
|
||||
runID,
|
||||
).Scan(&showJSON, &summaryJSON, &markdownPath); err != nil {
|
||||
t.Fatalf("query council report metadata: %v", err)
|
||||
}
|
||||
if markdownPath != reportPath {
|
||||
t.Fatalf("expected report path %q, got %q", reportPath, markdownPath)
|
||||
}
|
||||
|
||||
var show []string
|
||||
mustDecodeJSON(t, showJSON, &show)
|
||||
if len(show) != 2 || show[0] != "consensus" || show[1] != "majority" {
|
||||
t.Fatalf("expected default show buckets [consensus majority], got %#v", show)
|
||||
}
|
||||
|
||||
var summary map[string]any
|
||||
mustDecodeJSON(t, summaryJSON, &summary)
|
||||
if got, _ := summary["consensus"].(float64); got != 1 {
|
||||
t.Fatalf("expected one consensus group, got %#v", summary["consensus"])
|
||||
}
|
||||
if got, _ := summary["majority"].(float64); got != 1 {
|
||||
t.Fatalf("expected one majority group, got %#v", summary["majority"])
|
||||
}
|
||||
if got, _ := summary["minority"].(float64); got != 1 {
|
||||
t.Fatalf("expected one minority group, got %#v", summary["minority"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrchCouncilReportShowAllIncludesMinority(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dbPath := filepath.Join(t.TempDir(), "coord.db")
|
||||
runID := "council_blog_report_002"
|
||||
seedCouncilReportRun(t, dbPath, runID)
|
||||
|
||||
reportOut := runOrchCommand(
|
||||
t,
|
||||
"--db", dbPath,
|
||||
"council", "report",
|
||||
"--run", runID,
|
||||
"--show", "all",
|
||||
)
|
||||
|
||||
if !strings.Contains(reportOut, "## Consensus") {
|
||||
t.Fatalf("expected consensus section, got %q", reportOut)
|
||||
}
|
||||
if !strings.Contains(reportOut, "## Majority") {
|
||||
t.Fatalf("expected majority section, got %q", reportOut)
|
||||
}
|
||||
if !strings.Contains(reportOut, "## Minority") {
|
||||
t.Fatalf("expected minority section when --show all is used, got %q", reportOut)
|
||||
}
|
||||
if !strings.Contains(reportOut, "Add regression tests for council report JSON output.") {
|
||||
t.Fatalf("expected minority proposal in report output, got %q", reportOut)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrchCouncilReportJSONShape(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dbPath := filepath.Join(t.TempDir(), "coord.db")
|
||||
runID := "council_blog_report_003"
|
||||
seedCouncilReportRun(t, dbPath, runID)
|
||||
|
||||
reportOut := runOrchCommand(
|
||||
t,
|
||||
"--db", dbPath,
|
||||
"--json",
|
||||
"council", "report",
|
||||
"--run", runID,
|
||||
)
|
||||
|
||||
var reportResp map[string]any
|
||||
mustDecodeJSON(t, reportOut, &reportResp)
|
||||
if ok, _ := reportResp["ok"].(bool); !ok {
|
||||
t.Fatalf("expected ok=true, got %#v", reportResp)
|
||||
}
|
||||
if got, _ := reportResp["command"].(string); got != "council report" {
|
||||
t.Fatalf("expected command council report, got %#v", reportResp["command"])
|
||||
}
|
||||
if got := nestedString(t, reportResp, "data", "run_id"); got != runID {
|
||||
t.Fatalf("expected run id %q, got %q", runID, got)
|
||||
}
|
||||
|
||||
show := nestedArray(t, reportResp, "data", "show")
|
||||
if len(show) != 2 || show[0] != "consensus" || show[1] != "majority" {
|
||||
t.Fatalf("expected default show buckets in JSON output, got %#v", show)
|
||||
}
|
||||
|
||||
summary, ok := nestedValue(t, reportResp, "data", "summary").(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected summary object, got %#v", nestedValue(t, reportResp, "data", "summary"))
|
||||
}
|
||||
if got, _ := summary["consensus"].(float64); got != 1 {
|
||||
t.Fatalf("expected one consensus group, got %#v", summary["consensus"])
|
||||
}
|
||||
if got, _ := summary["majority"].(float64); got != 1 {
|
||||
t.Fatalf("expected one majority group, got %#v", summary["majority"])
|
||||
}
|
||||
if got, _ := summary["minority"].(float64); got != 1 {
|
||||
t.Fatalf("expected one minority group, got %#v", summary["minority"])
|
||||
}
|
||||
|
||||
artifacts := nestedArray(t, reportResp, "data", "report_artifacts")
|
||||
if len(artifacts) != 1 {
|
||||
t.Fatalf("expected one report artifact, got %#v", artifacts)
|
||||
}
|
||||
artifact, ok := artifacts[0].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected report artifact object, got %#v", artifacts[0])
|
||||
}
|
||||
if got, _ := artifact["kind"].(string); got != "markdown" {
|
||||
t.Fatalf("expected markdown artifact, got %#v", artifact["kind"])
|
||||
}
|
||||
if got, _ := artifact["path"].(string); got != councilReportArtifactPath(dbPath, runID) {
|
||||
t.Fatalf("expected report artifact path %q, got %#v", councilReportArtifactPath(dbPath, runID), artifact["path"])
|
||||
}
|
||||
|
||||
groups := nestedArray(t, reportResp, "data", "grouped_recommendations")
|
||||
if len(groups) != 2 {
|
||||
t.Fatalf("expected two grouped recommendations in default JSON report, got %#v", groups)
|
||||
}
|
||||
firstGroup, ok := groups[0].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected first group object, got %#v", groups[0])
|
||||
}
|
||||
if got, _ := firstGroup["bucket"].(string); got != "consensus" {
|
||||
t.Fatalf("expected first reported group to be consensus, got %#v", firstGroup["bucket"])
|
||||
}
|
||||
}
|
||||
|
||||
func completeCouncilReviewer(t *testing.T, dbPath, runID, reviewerRole, bodyJSON string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -1916,6 +2096,50 @@ func completeCouncilReviewer(t *testing.T, dbPath, runID, reviewerRole, bodyJSON
|
||||
)
|
||||
}
|
||||
|
||||
func seedCouncilReportRun(t *testing.T, dbPath, runID string) {
|
||||
t.Helper()
|
||||
|
||||
runOrchCommand(
|
||||
t,
|
||||
"--db", dbPath,
|
||||
"--json",
|
||||
"council", "start",
|
||||
"--run", runID,
|
||||
"--target", "Review the council reporting flow.",
|
||||
)
|
||||
|
||||
completeCouncilReviewer(
|
||||
t,
|
||||
dbPath,
|
||||
runID,
|
||||
"architecture-reviewer",
|
||||
`{"reviewer_role":"architecture-reviewer","findings":[{"title":"Split contracts","summary":"Transport contracts are mixed into UI code.","proposal":"Move API contract definitions into a dedicated module.","rationale":"This lowers coupling.","confidence":"high","tags":["architecture"],"target_refs":{"repo_path":"."}},{"title":"Share helpers","summary":"Council report rendering paths are repeated.","proposal":"Introduce shared council coordinator helpers for report rendering.","rationale":"This keeps report assembly consistent.","confidence":"medium","tags":["reporting"],"target_refs":{"repo_path":"."}}]}`,
|
||||
)
|
||||
completeCouncilReviewer(
|
||||
t,
|
||||
dbPath,
|
||||
runID,
|
||||
"implementation-reviewer",
|
||||
`{"reviewer_role":"implementation-reviewer","findings":[{"title":"Extract contracts","summary":"Shared transport shapes are duplicated.","proposal":"Move API contract definitions into dedicated module","rationale":"This reduces duplication.","confidence":"high","tags":["maintainability"],"target_refs":{"repo_path":"."}},{"title":"Reuse report helpers","summary":"Formatting logic should stay shared.","proposal":"Introduce shared council coordinator helpers for report rendering","rationale":"This avoids formatter drift.","confidence":"medium","tags":["reporting"],"target_refs":{"repo_path":"."}}]}`,
|
||||
)
|
||||
completeCouncilReviewer(
|
||||
t,
|
||||
dbPath,
|
||||
runID,
|
||||
"risk-reviewer",
|
||||
`{"reviewer_role":"risk-reviewer","findings":[{"title":"Lock contracts","summary":"Contract drift becomes risky over time.","proposal":"Move API contract definitions into a dedicated module.","rationale":"This reduces integration regressions.","confidence":"high","tags":["risk"],"target_refs":{"repo_path":"."}},{"title":"Cover JSON output","summary":"The council report response should stay stable.","proposal":"Add regression tests for council report JSON output.","rationale":"This catches contract regressions earlier.","confidence":"high","tags":["testing"],"target_refs":{"repo_path":"."}}]}`,
|
||||
)
|
||||
|
||||
runOrchCommand(
|
||||
t,
|
||||
"--db", dbPath,
|
||||
"--json",
|
||||
"council", "tally",
|
||||
"--run", runID,
|
||||
"--similarity", "normal",
|
||||
)
|
||||
}
|
||||
|
||||
func runInboxCommandEventually(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
CREATE TABLE IF NOT EXISTS council_reports (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
show_json TEXT NOT NULL DEFAULT '[]',
|
||||
summary_json TEXT NOT NULL DEFAULT '{}',
|
||||
markdown_path TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
@@ -107,6 +107,32 @@ type CouncilTallyResult struct {
|
||||
GroupedRecommendations []CouncilGroup `json:"grouped_recommendations"`
|
||||
}
|
||||
|
||||
type CouncilReportInput struct {
|
||||
RunID string
|
||||
Show string
|
||||
}
|
||||
|
||||
type CouncilReportArtifact struct {
|
||||
Kind string `json:"kind"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
type CouncilReportResult struct {
|
||||
RunID string `json:"run_id"`
|
||||
Show []string `json:"show"`
|
||||
Summary map[string]int `json:"summary"`
|
||||
GroupedRecommendations []CouncilGroup `json:"grouped_recommendations"`
|
||||
Markdown string `json:"markdown,omitempty"`
|
||||
ReportArtifacts []CouncilReportArtifact `json:"report_artifacts,omitempty"`
|
||||
}
|
||||
|
||||
type CouncilPersistReportInput struct {
|
||||
RunID string
|
||||
Show []string
|
||||
Summary map[string]int
|
||||
MarkdownPath string
|
||||
}
|
||||
|
||||
type councilReviewerOutput struct {
|
||||
ReviewerRole string `json:"reviewer_role"`
|
||||
Findings []councilFindingOutput `json:"findings"`
|
||||
@@ -797,6 +823,107 @@ func (s *OrchStore) TallyCouncil(ctx context.Context, input CouncilTallyInput) (
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *OrchStore) BuildCouncilReport(ctx context.Context, input CouncilReportInput) (CouncilReportResult, error) {
|
||||
runID := strings.TrimSpace(input.RunID)
|
||||
if runID == "" {
|
||||
return CouncilReportResult{}, fmt.Errorf("%w: run id is required", ErrInvalidInput)
|
||||
}
|
||||
|
||||
run, err := s.GetCouncilRun(ctx, runID)
|
||||
if err != nil {
|
||||
return CouncilReportResult{}, err
|
||||
}
|
||||
councilInput, err := s.GetCouncilInput(ctx, runID)
|
||||
if err != nil {
|
||||
return CouncilReportResult{}, err
|
||||
}
|
||||
|
||||
show, err := normalizeCouncilReportShow(input.Show, run.OnlyUnanimous)
|
||||
if err != nil {
|
||||
return CouncilReportResult{}, err
|
||||
}
|
||||
|
||||
groups, tallied, err := s.ListCouncilGroups(ctx, runID)
|
||||
if err != nil {
|
||||
return CouncilReportResult{}, err
|
||||
}
|
||||
if !tallied {
|
||||
return CouncilReportResult{}, fmt.Errorf("%w: council groups are not available; run council tally first", ErrInvalidState)
|
||||
}
|
||||
|
||||
summary := councilGroupSummary(groups)
|
||||
selectedGroups := selectCouncilGroupsForReport(groups, show)
|
||||
markdown := renderCouncilReportMarkdown(run, councilInput, show, summary, selectedGroups)
|
||||
|
||||
return CouncilReportResult{
|
||||
RunID: runID,
|
||||
Show: show,
|
||||
Summary: summary,
|
||||
GroupedRecommendations: selectedGroups,
|
||||
Markdown: markdown,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *OrchStore) PersistCouncilReport(ctx context.Context, input CouncilPersistReportInput) error {
|
||||
runID := strings.TrimSpace(input.RunID)
|
||||
if runID == "" {
|
||||
return fmt.Errorf("%w: run id is required", ErrInvalidInput)
|
||||
}
|
||||
if _, err := s.GetCouncilRun(ctx, runID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
showJSON := marshalJSON(input.Show)
|
||||
summaryJSON := marshalJSON(normalizeCouncilSummary(input.Summary))
|
||||
markdownPath := strings.TrimSpace(input.MarkdownPath)
|
||||
now := nowUTC()
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin council report transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
if _, err := tx.ExecContext(
|
||||
ctx,
|
||||
`INSERT INTO council_reports (
|
||||
run_id, show_json, summary_json, markdown_path, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(run_id) DO UPDATE SET
|
||||
show_json = excluded.show_json,
|
||||
summary_json = excluded.summary_json,
|
||||
markdown_path = excluded.markdown_path,
|
||||
updated_at = excluded.updated_at`,
|
||||
runID,
|
||||
showJSON,
|
||||
summaryJSON,
|
||||
markdownPath,
|
||||
formatTime(now),
|
||||
formatTime(now),
|
||||
); err != nil {
|
||||
return fmt.Errorf("persist council report metadata: %w", err)
|
||||
}
|
||||
|
||||
if err := insertEvent(ctx, tx, eventInput{
|
||||
RunID: runID,
|
||||
Source: "orch",
|
||||
EventType: "council_reported",
|
||||
Summary: "council report generated",
|
||||
PayloadJSON: marshalJSON(map[string]any{
|
||||
"show": input.Show,
|
||||
"markdown_path": markdownPath,
|
||||
}),
|
||||
CreatedAt: now,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit council report transaction: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *OrchStore) collectCouncilFindings(ctx context.Context, runID string, reviewers []CouncilReviewer) ([]CouncilFinding, error) {
|
||||
findings := make([]CouncilFinding, 0)
|
||||
for _, reviewer := range reviewers {
|
||||
@@ -847,6 +974,110 @@ func (s *OrchStore) collectCouncilFindings(ctx context.Context, runID string, re
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
func (s *OrchStore) GetCouncilInput(ctx context.Context, runID string) (CouncilInput, error) {
|
||||
row := s.db.QueryRowContext(
|
||||
ctx,
|
||||
`SELECT run_id, prompt, target_file, repo_path, target_task_id
|
||||
FROM council_inputs
|
||||
WHERE run_id = ?`,
|
||||
runID,
|
||||
)
|
||||
|
||||
var input CouncilInput
|
||||
if err := row.Scan(
|
||||
&input.RunID,
|
||||
&input.Prompt,
|
||||
&input.TargetFile,
|
||||
&input.RepoPath,
|
||||
&input.TargetTaskID,
|
||||
); errors.Is(err, sql.ErrNoRows) {
|
||||
return CouncilInput{RunID: runID}, nil
|
||||
} else if err != nil {
|
||||
return CouncilInput{}, fmt.Errorf("scan council input: %w", err)
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
|
||||
func (s *OrchStore) ListCouncilGroups(ctx context.Context, runID string) ([]CouncilGroup, bool, error) {
|
||||
rows, err := s.db.QueryContext(
|
||||
ctx,
|
||||
`SELECT
|
||||
run_id, group_id, proposal, bucket, support_count, supporters_json,
|
||||
dissenters_json, rationale_summary, tags_json, source_finding_ids_json
|
||||
FROM council_groups
|
||||
WHERE run_id = ?
|
||||
ORDER BY
|
||||
CASE bucket
|
||||
WHEN 'consensus' THEN 1
|
||||
WHEN 'majority' THEN 2
|
||||
WHEN 'minority' THEN 3
|
||||
ELSE 4
|
||||
END,
|
||||
support_count DESC,
|
||||
group_id ASC`,
|
||||
runID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("query council groups: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
groups := make([]CouncilGroup, 0)
|
||||
for rows.Next() {
|
||||
var group CouncilGroup
|
||||
var supportersJSON string
|
||||
var dissentersJSON string
|
||||
var tagsJSON string
|
||||
var sourceFindingIDsJSON string
|
||||
if err := rows.Scan(
|
||||
&group.RunID,
|
||||
&group.GroupID,
|
||||
&group.Proposal,
|
||||
&group.Bucket,
|
||||
&group.SupportCount,
|
||||
&supportersJSON,
|
||||
&dissentersJSON,
|
||||
&group.RationaleSummary,
|
||||
&tagsJSON,
|
||||
&sourceFindingIDsJSON,
|
||||
); err != nil {
|
||||
return nil, false, fmt.Errorf("scan council group: %w", err)
|
||||
}
|
||||
group.SupportersJSON = json.RawMessage(supportersJSON)
|
||||
group.DissentersJSON = json.RawMessage(dissentersJSON)
|
||||
group.TagsJSON = json.RawMessage(tagsJSON)
|
||||
group.SourceFindingIDsJSON = json.RawMessage(sourceFindingIDsJSON)
|
||||
groups = append(groups, group)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, fmt.Errorf("iterate council groups: %w", err)
|
||||
}
|
||||
|
||||
if len(groups) > 0 {
|
||||
return groups, true, nil
|
||||
}
|
||||
|
||||
tallied, err := s.hasCouncilTallyEvent(ctx, runID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return groups, tallied, nil
|
||||
}
|
||||
|
||||
func (s *OrchStore) hasCouncilTallyEvent(ctx context.Context, runID string) (bool, error) {
|
||||
var count int
|
||||
if err := s.db.QueryRowContext(
|
||||
ctx,
|
||||
`SELECT COUNT(*)
|
||||
FROM events
|
||||
WHERE run_id = ? AND event_type = 'council_tallied'`,
|
||||
runID,
|
||||
).Scan(&count); err != nil {
|
||||
return false, fmt.Errorf("query council tally events: %w", err)
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
func (s *OrchStore) loadCouncilReviewerResultMessage(ctx context.Context, runID, taskID string) (Message, error) {
|
||||
task, err := selectTask(ctx, s.db, runID, taskID)
|
||||
if err != nil {
|
||||
@@ -1040,6 +1271,192 @@ func groupCouncilFindings(runID string, findings []CouncilFinding, reviewers []C
|
||||
return result
|
||||
}
|
||||
|
||||
func normalizeCouncilReportShow(raw string, onlyUnanimous bool) ([]string, error) {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
if onlyUnanimous {
|
||||
return []string{"consensus"}, nil
|
||||
}
|
||||
return []string{"consensus", "majority"}, nil
|
||||
}
|
||||
|
||||
parts := strings.Split(raw, ",")
|
||||
show := make([]string, 0, len(parts))
|
||||
seen := make(map[string]struct{}, len(parts))
|
||||
for _, part := range parts {
|
||||
value := strings.ToLower(strings.TrimSpace(part))
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
if value == "all" {
|
||||
return []string{"consensus", "majority", "minority"}, nil
|
||||
}
|
||||
switch value {
|
||||
case "consensus", "majority", "minority":
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: show must contain consensus, majority, minority, or all", ErrInvalidInput)
|
||||
}
|
||||
if _, ok := seen[value]; ok {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
show = append(show, value)
|
||||
}
|
||||
if len(show) == 0 {
|
||||
return nil, fmt.Errorf("%w: show must contain at least one bucket", ErrInvalidInput)
|
||||
}
|
||||
return show, nil
|
||||
}
|
||||
|
||||
func councilGroupSummary(groups []CouncilGroup) map[string]int {
|
||||
summary := normalizeCouncilSummary(nil)
|
||||
for _, group := range groups {
|
||||
summary[group.Bucket]++
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
func normalizeCouncilSummary(summary map[string]int) map[string]int {
|
||||
result := map[string]int{
|
||||
"consensus": 0,
|
||||
"majority": 0,
|
||||
"minority": 0,
|
||||
}
|
||||
for key, value := range summary {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func selectCouncilGroupsForReport(groups []CouncilGroup, show []string) []CouncilGroup {
|
||||
groupedByBucket := make(map[string][]CouncilGroup, len(show))
|
||||
for _, group := range groups {
|
||||
groupedByBucket[group.Bucket] = append(groupedByBucket[group.Bucket], group)
|
||||
}
|
||||
|
||||
selected := make([]CouncilGroup, 0, len(groups))
|
||||
for _, bucket := range show {
|
||||
selected = append(selected, groupedByBucket[bucket]...)
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
func renderCouncilReportMarkdown(run CouncilRun, input CouncilInput, show []string, summary map[string]int, groups []CouncilGroup) string {
|
||||
var builder strings.Builder
|
||||
|
||||
builder.WriteString("# Council Review Report\n\n")
|
||||
builder.WriteString(fmt.Sprintf("- Run ID: `%s`\n", run.RunID))
|
||||
builder.WriteString(fmt.Sprintf("- Mode: `%s`\n", run.Mode))
|
||||
builder.WriteString(fmt.Sprintf("- Target Type: `%s`\n", run.TargetType))
|
||||
builder.WriteString(fmt.Sprintf("- Report Buckets: `%s`\n\n", strings.Join(show, "`, `")))
|
||||
|
||||
builder.WriteString("## Target\n\n")
|
||||
if strings.TrimSpace(input.Prompt) != "" {
|
||||
builder.WriteString(fmt.Sprintf("- Prompt: %s\n", input.Prompt))
|
||||
}
|
||||
if strings.TrimSpace(input.TargetFile) != "" {
|
||||
builder.WriteString(fmt.Sprintf("- Target File: `%s`\n", input.TargetFile))
|
||||
}
|
||||
if strings.TrimSpace(input.RepoPath) != "" {
|
||||
builder.WriteString(fmt.Sprintf("- Repo Path: `%s`\n", input.RepoPath))
|
||||
}
|
||||
if strings.TrimSpace(input.TargetTaskID) != "" {
|
||||
builder.WriteString(fmt.Sprintf("- Task ID: `%s`\n", input.TargetTaskID))
|
||||
}
|
||||
if strings.TrimSpace(input.Prompt) == "" &&
|
||||
strings.TrimSpace(input.TargetFile) == "" &&
|
||||
strings.TrimSpace(input.RepoPath) == "" &&
|
||||
strings.TrimSpace(input.TargetTaskID) == "" {
|
||||
builder.WriteString("- No explicit target metadata was recorded.\n")
|
||||
}
|
||||
builder.WriteString("\n")
|
||||
|
||||
builder.WriteString("## Summary\n\n")
|
||||
builder.WriteString(fmt.Sprintf("- Consensus: %d\n", summary["consensus"]))
|
||||
builder.WriteString(fmt.Sprintf("- Majority: %d\n", summary["majority"]))
|
||||
builder.WriteString(fmt.Sprintf("- Minority: %d\n\n", summary["minority"]))
|
||||
|
||||
groupedByBucket := make(map[string][]CouncilGroup, len(show))
|
||||
for _, group := range groups {
|
||||
groupedByBucket[group.Bucket] = append(groupedByBucket[group.Bucket], group)
|
||||
}
|
||||
|
||||
for _, bucket := range show {
|
||||
builder.WriteString(fmt.Sprintf("## %s\n\n", councilBucketHeading(bucket)))
|
||||
bucketGroups := groupedByBucket[bucket]
|
||||
if len(bucketGroups) == 0 {
|
||||
builder.WriteString(fmt.Sprintf("No %s recommendations.\n\n", bucket))
|
||||
continue
|
||||
}
|
||||
|
||||
for _, group := range bucketGroups {
|
||||
supporters := decodeCouncilStringSlice(group.SupportersJSON)
|
||||
dissenters := decodeCouncilStringSlice(group.DissentersJSON)
|
||||
tags := decodeCouncilStringSlice(group.TagsJSON)
|
||||
sourceFindingIDs := decodeCouncilStringSlice(group.SourceFindingIDsJSON)
|
||||
|
||||
builder.WriteString(fmt.Sprintf("### %s\n\n", group.GroupID))
|
||||
builder.WriteString(group.Proposal)
|
||||
builder.WriteString("\n\n")
|
||||
builder.WriteString(fmt.Sprintf("- Support: %d of 3 reviewers", group.SupportCount))
|
||||
if len(supporters) > 0 {
|
||||
builder.WriteString(fmt.Sprintf(" (`%s`)", strings.Join(supporters, "`, `")))
|
||||
}
|
||||
builder.WriteString("\n")
|
||||
if len(dissenters) > 0 {
|
||||
builder.WriteString(fmt.Sprintf("- Dissenters: `%s`\n", strings.Join(dissenters, "`, `")))
|
||||
}
|
||||
if strings.TrimSpace(group.RationaleSummary) != "" {
|
||||
builder.WriteString(fmt.Sprintf("- Rationale: %s\n", group.RationaleSummary))
|
||||
}
|
||||
if len(tags) > 0 {
|
||||
builder.WriteString(fmt.Sprintf("- Tags: `%s`\n", strings.Join(tags, "`, `")))
|
||||
}
|
||||
if len(sourceFindingIDs) > 0 {
|
||||
builder.WriteString(fmt.Sprintf("- Source Findings: `%s`\n", strings.Join(sourceFindingIDs, "`, `")))
|
||||
}
|
||||
builder.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func councilBucketHeading(bucket string) string {
|
||||
switch bucket {
|
||||
case "consensus":
|
||||
return "Consensus"
|
||||
case "majority":
|
||||
return "Majority"
|
||||
case "minority":
|
||||
return "Minority"
|
||||
default:
|
||||
if bucket == "" {
|
||||
return "Recommendations"
|
||||
}
|
||||
return strings.ToUpper(bucket[:1]) + bucket[1:]
|
||||
}
|
||||
}
|
||||
|
||||
func decodeCouncilStringSlice(raw json.RawMessage) []string {
|
||||
if len(raw) == 0 || strings.TrimSpace(string(raw)) == "" || strings.TrimSpace(string(raw)) == "null" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var values []string
|
||||
if err := json.Unmarshal(raw, &values); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value != "" {
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func councilProposalGroupKey(proposal, similarity string) string {
|
||||
tokens := proposalTokens(proposal)
|
||||
if similarity == "strict" {
|
||||
|
||||
Reference in New Issue
Block a user