chore(repo): reinitialize repository
This commit is contained in:
@@ -0,0 +1,698 @@
|
||||
package taskexec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"inbox/internal/app/lanegit"
|
||||
"inbox/internal/app/lanematerialize"
|
||||
"inbox/internal/app/lanesnapshot"
|
||||
"inbox/internal/app/runtimeconfig"
|
||||
"inbox/internal/base/timeutil"
|
||||
"inbox/internal/domain/lane"
|
||||
"inbox/internal/domain/message"
|
||||
"inbox/internal/domain/task"
|
||||
"inbox/internal/domain/topic"
|
||||
"inbox/internal/domain/workflow"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
GetLane(ctx context.Context, laneID string) (lane.Record, error)
|
||||
UpdateLane(ctx context.Context, value lane.Record) (lane.Record, error)
|
||||
GetTask(ctx context.Context, taskID string) (task.Record, error)
|
||||
ListTasksByLane(ctx context.Context, laneID string) ([]task.Record, error)
|
||||
UpdateTask(ctx context.Context, value task.Record) (task.Record, error)
|
||||
ListTaskDependencies(ctx context.Context, taskID string) ([]task.Dependency, error)
|
||||
AppendTaskEvent(ctx context.Context, value task.Event) (task.Event, error)
|
||||
GetTopic(ctx context.Context, topicID string) (topic.Record, error)
|
||||
UpdateTopic(ctx context.Context, value topic.Record) (topic.Record, error)
|
||||
ListMessagesByTopic(ctx context.Context, topicID string) ([]message.Record, error)
|
||||
ListLanesByTopic(ctx context.Context, topicID string) ([]lane.Record, error)
|
||||
ListTasksByTopic(ctx context.Context, topicID string) ([]task.Record, error)
|
||||
CreateMessage(ctx context.Context, value message.Record) (message.Record, error)
|
||||
CreateWorkflowRun(ctx context.Context, value workflow.Run) (workflow.Run, error)
|
||||
GetWorkflowRun(ctx context.Context, runID string) (workflow.Run, error)
|
||||
ClaimTaskExecution(ctx context.Context, run workflow.Run, taskID, startedAt string) (workflow.Run, task.Record, error)
|
||||
UpdateWorkflowRun(ctx context.Context, value workflow.Run) (workflow.Run, error)
|
||||
CompleteTaskExecution(ctx context.Context, runID, taskID, laneID string, status workflow.RunStatus, exitCode int, resultMarkdown, errorMessage, completedAt string) (workflow.Run, task.Record, lane.Record, []task.Record, error)
|
||||
AppendWorkflowRunLog(ctx context.Context, value workflow.RunLog) (workflow.RunLog, error)
|
||||
}
|
||||
|
||||
type RuntimeResolver interface {
|
||||
ResolveRole(ctx context.Context, workspaceID, roleName string) (runtimeconfig.ResolvedRole, error)
|
||||
}
|
||||
|
||||
type Snapshotter interface {
|
||||
Capture(ctx context.Context, item lane.Record, taskRecord task.Record) (string, error)
|
||||
}
|
||||
|
||||
type Materializer interface {
|
||||
Materialize(ctx context.Context, downstream lane.Record, taskID string, upstreams []lanematerialize.Upstream) error
|
||||
}
|
||||
|
||||
type LaneRuntimeReleaser interface {
|
||||
ReleaseLaneRuntime(ctx context.Context, laneID string) (lane.Record, error)
|
||||
}
|
||||
|
||||
type Option func(*Service)
|
||||
|
||||
type Assignment struct {
|
||||
Run workflow.Run `json:"run"`
|
||||
Lane lane.Record `json:"lane"`
|
||||
Task task.Record `json:"task"`
|
||||
Role runtimeconfig.ResolvedRole `json:"role"`
|
||||
Prompt string `json:"prompt"`
|
||||
Model string `json:"model,omitempty"`
|
||||
OutputMode string `json:"output_mode"`
|
||||
}
|
||||
|
||||
type Completion struct {
|
||||
RunID string `json:"run_id"`
|
||||
Status workflow.RunStatus `json:"status"`
|
||||
ExitCode int `json:"exit_code"`
|
||||
ResultMarkdown string `json:"result_markdown"`
|
||||
ErrorMessage string `json:"error_message"`
|
||||
}
|
||||
|
||||
type commandMetadata struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
TaskID string `json:"task_id"`
|
||||
RunnerID string `json:"runner_id,omitempty"`
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
repo Repository
|
||||
resolver RuntimeResolver
|
||||
clock timeutil.Clock
|
||||
snapshotter Snapshotter
|
||||
materializer Materializer
|
||||
runtime LaneRuntimeReleaser
|
||||
}
|
||||
|
||||
func NewService(repo Repository, resolver RuntimeResolver, clock timeutil.Clock, opts ...Option) *Service {
|
||||
if clock == nil {
|
||||
clock = timeutil.SystemClock{}
|
||||
}
|
||||
svc := &Service{
|
||||
repo: repo,
|
||||
resolver: resolver,
|
||||
clock: clock,
|
||||
snapshotter: lanesnapshot.NewService(lanegit.ExecRunner{}, clock),
|
||||
}
|
||||
if recorder, ok := repo.(lanematerialize.SyncRecorder); ok {
|
||||
svc.materializer = lanematerialize.NewService(recorder, lanegit.ExecRunner{}, clock)
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(svc)
|
||||
}
|
||||
return svc
|
||||
}
|
||||
|
||||
func WithSnapshotter(snapshotter Snapshotter) Option {
|
||||
return func(s *Service) {
|
||||
s.snapshotter = snapshotter
|
||||
}
|
||||
}
|
||||
|
||||
func WithMaterializer(materializer Materializer) Option {
|
||||
return func(s *Service) {
|
||||
s.materializer = materializer
|
||||
}
|
||||
}
|
||||
|
||||
func WithLaneRuntimeReleaser(runtime LaneRuntimeReleaser) Option {
|
||||
return func(s *Service) {
|
||||
s.runtime = runtime
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) ClaimNext(ctx context.Context, laneID, runnerID string) (Assignment, error) {
|
||||
laneRecord, err := s.repo.GetLane(ctx, laneID)
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
topicRecord, err := s.repo.GetTopic(ctx, laneRecord.TopicID)
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
if strings.TrimSpace(topicRecord.Status) != "execution" {
|
||||
return Assignment{}, sql.ErrNoRows
|
||||
}
|
||||
tasks, err := s.repo.ListTasksByLane(ctx, laneID)
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
if hasRunningTask(tasks) {
|
||||
return Assignment{}, sql.ErrNoRows
|
||||
}
|
||||
candidate, err := s.nextReadyTask(ctx, tasks)
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
if candidate.ID == "" {
|
||||
return Assignment{}, sql.ErrNoRows
|
||||
}
|
||||
if err := s.materializeTaskInputs(ctx, laneRecord, candidate); err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
|
||||
resolved, err := s.resolver.ResolveRole(ctx, laneRecord.WorkspaceID, "worker")
|
||||
if err != nil {
|
||||
return Assignment{}, fmt.Errorf("resolve worker config: %w", err)
|
||||
}
|
||||
snapshot, err := resolved.SnapshotJSON()
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
commandJSON, err := marshalCommandMetadata(commandMetadata{
|
||||
LaneID: laneID,
|
||||
TaskID: candidate.ID,
|
||||
RunnerID: strings.TrimSpace(runnerID),
|
||||
})
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
runTemplate := workflow.Run{
|
||||
WorkspaceID: candidate.WorkspaceID,
|
||||
TopicID: candidate.TopicID,
|
||||
RoleName: "worker",
|
||||
Stage: workflow.StageExecution,
|
||||
Mode: "task",
|
||||
Status: workflow.RunStatusRunning,
|
||||
ConfigSnapshotJSON: snapshot,
|
||||
CommandJSON: commandJSON,
|
||||
}
|
||||
startedAt := timeutil.FormatRFC3339(s.clock.Now())
|
||||
run, candidate, err := s.repo.ClaimTaskExecution(ctx, runTemplate, candidate.ID, startedAt)
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
if _, err := s.repo.AppendTaskEvent(ctx, task.Event{
|
||||
TaskID: candidate.ID,
|
||||
EventType: "started",
|
||||
BodyMarkdown: "Worker started execution.",
|
||||
CreatedByRoleName: "worker",
|
||||
CreatedAt: startedAt,
|
||||
}); err != nil {
|
||||
_, _ = s.repo.AppendWorkflowRunLog(ctx, workflow.RunLog{
|
||||
RunID: run.ID,
|
||||
Stream: workflow.LogStreamSystem,
|
||||
Content: "claim side effects failed after main state commit:\n- append started event for task " + candidate.ID + ": " + err.Error(),
|
||||
CreatedAt: startedAt,
|
||||
})
|
||||
}
|
||||
|
||||
topicRecord, err = s.repo.GetTopic(ctx, candidate.TopicID)
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
messages, err := s.repo.ListMessagesByTopic(ctx, candidate.TopicID)
|
||||
if err != nil {
|
||||
return Assignment{}, err
|
||||
}
|
||||
|
||||
assignment := Assignment{
|
||||
Run: run,
|
||||
Lane: laneRecord,
|
||||
Task: candidate,
|
||||
Role: resolved,
|
||||
Prompt: buildWorkerPrompt(topicRecord, laneRecord, candidate, resolved, messages),
|
||||
OutputMode: "markdown",
|
||||
}
|
||||
return assignment, nil
|
||||
}
|
||||
|
||||
func (s *Service) Complete(ctx context.Context, value Completion) (workflow.Run, error) {
|
||||
if err := workflow.ValidateRunStatus(value.Status); err != nil {
|
||||
return workflow.Run{}, err
|
||||
}
|
||||
if value.Status == workflow.RunStatusRunning {
|
||||
return workflow.Run{}, fmt.Errorf("completion status must be terminal")
|
||||
}
|
||||
run, err := s.repo.GetWorkflowRun(ctx, value.RunID)
|
||||
if err != nil {
|
||||
return workflow.Run{}, err
|
||||
}
|
||||
command, err := parseCommandMetadata(run.CommandJSON)
|
||||
if err != nil {
|
||||
return workflow.Run{}, err
|
||||
}
|
||||
topicRecord, err := s.repo.GetTopic(ctx, run.TopicID)
|
||||
if err != nil {
|
||||
return workflow.Run{}, err
|
||||
}
|
||||
if strings.TrimSpace(topicRecord.Status) == "cancelled" {
|
||||
run.Status = workflow.RunStatusCancelled
|
||||
run.ExitCode = 130
|
||||
run.CompletedAt = timeutil.FormatRFC3339(s.clock.Now())
|
||||
run.ErrorMessage = "Topic was stopped before task completion could be applied."
|
||||
return s.repo.UpdateWorkflowRun(ctx, run)
|
||||
}
|
||||
|
||||
now := timeutil.FormatRFC3339(s.clock.Now())
|
||||
value.ResultMarkdown = strings.TrimSpace(value.ResultMarkdown)
|
||||
value.ErrorMessage = strings.TrimSpace(value.ErrorMessage)
|
||||
taskRecord, err := s.repo.GetTask(ctx, command.TaskID)
|
||||
if err != nil {
|
||||
return workflow.Run{}, err
|
||||
}
|
||||
laneRecord, err := s.repo.GetLane(ctx, command.LaneID)
|
||||
if err != nil {
|
||||
return workflow.Run{}, err
|
||||
}
|
||||
if value.Status == workflow.RunStatusSucceeded && shouldSnapshotTask(taskRecord) && s.snapshotter != nil {
|
||||
headCommit, snapshotErr := s.snapshotter.Capture(ctx, laneRecord, taskRecord)
|
||||
if snapshotErr != nil {
|
||||
value.Status = workflow.RunStatusFailed
|
||||
value.ExitCode = failedExitCode(value.ExitCode)
|
||||
value.ResultMarkdown = ""
|
||||
value.ErrorMessage = "Lane snapshot failed: " + strings.TrimSpace(snapshotErr.Error())
|
||||
} else if headCommit != "" && laneRecord.HeadCommit != headCommit {
|
||||
laneRecord.HeadCommit = headCommit
|
||||
if _, err := s.repo.UpdateLane(ctx, laneRecord); err != nil {
|
||||
value.Status = workflow.RunStatusFailed
|
||||
value.ExitCode = failedExitCode(value.ExitCode)
|
||||
value.ResultMarkdown = ""
|
||||
value.ErrorMessage = "Persist lane head commit: " + strings.TrimSpace(err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
updatedRun, taskRecord, chainRecord, promotedTasks, err := s.repo.CompleteTaskExecution(
|
||||
ctx,
|
||||
run.ID,
|
||||
command.TaskID,
|
||||
command.LaneID,
|
||||
value.Status,
|
||||
value.ExitCode,
|
||||
value.ResultMarkdown,
|
||||
value.ErrorMessage,
|
||||
now,
|
||||
)
|
||||
if err != nil {
|
||||
return workflow.Run{}, err
|
||||
}
|
||||
if err := s.syncTopicStatus(ctx, run.TopicID); err != nil {
|
||||
return workflow.Run{}, err
|
||||
}
|
||||
s.emitCompletionSideEffects(ctx, updatedRun, taskRecord, chainRecord, promotedTasks, value, now)
|
||||
return updatedRun, nil
|
||||
}
|
||||
|
||||
func (s *Service) AppendLog(ctx context.Context, runID string, stream workflow.LogStream, content string) (workflow.RunLog, error) {
|
||||
return s.repo.AppendWorkflowRunLog(ctx, workflow.RunLog{
|
||||
RunID: runID,
|
||||
Stream: stream,
|
||||
Content: strings.TrimSpace(content),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) nextReadyTask(ctx context.Context, items []task.Record) (task.Record, error) {
|
||||
for _, item := range items {
|
||||
if item.Status != task.StatusReady {
|
||||
continue
|
||||
}
|
||||
if item.Kind == task.KindMilestone {
|
||||
continue
|
||||
}
|
||||
ready, err := s.dependenciesSatisfied(ctx, item.ID)
|
||||
if err != nil {
|
||||
return task.Record{}, err
|
||||
}
|
||||
if ready {
|
||||
return item, nil
|
||||
}
|
||||
}
|
||||
return task.Record{}, nil
|
||||
}
|
||||
|
||||
func (s *Service) materializeTaskInputs(ctx context.Context, laneRecord lane.Record, taskRecord task.Record) error {
|
||||
if s.materializer == nil {
|
||||
return nil
|
||||
}
|
||||
upstreams, err := s.upstreamLanesForTask(ctx, laneRecord, taskRecord)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(upstreams) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := s.materializer.Materialize(ctx, laneRecord, taskRecord.ID, upstreams); err != nil {
|
||||
if blockErr := s.blockTaskForMaterializationFailure(ctx, laneRecord, taskRecord, err); blockErr != nil {
|
||||
return fmt.Errorf("%v; block task: %w", err, blockErr)
|
||||
}
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) upstreamLanesForTask(ctx context.Context, downstream lane.Record, taskRecord task.Record) ([]lanematerialize.Upstream, error) {
|
||||
deps, err := s.repo.ListTaskDependencies(ctx, taskRecord.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]lanematerialize.Upstream, 0, len(deps))
|
||||
for _, dep := range deps {
|
||||
upstreamTask, err := s.repo.GetTask(ctx, dep.DependsOnTaskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if upstreamTask.LaneID == downstream.ID {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[upstreamTask.LaneID]; ok {
|
||||
continue
|
||||
}
|
||||
upstreamLane, err := s.repo.GetLane(ctx, upstreamTask.LaneID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen[upstreamTask.LaneID] = struct{}{}
|
||||
out = append(out, lanematerialize.Upstream{TaskID: upstreamTask.ID, Lane: upstreamLane})
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
return out[i].Lane.ID < out[j].Lane.ID
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Service) blockTaskForMaterializationFailure(ctx context.Context, laneRecord lane.Record, taskRecord task.Record, cause error) error {
|
||||
now := timeutil.FormatRFC3339(s.clock.Now())
|
||||
reason := "Lane input materialization failed.\n\n" + strings.TrimSpace(cause.Error())
|
||||
|
||||
taskRecord.Status = task.StatusBlocked
|
||||
taskRecord.BlockingReasonMarkdown = reason
|
||||
taskRecord.ResultSummaryMarkdown = ""
|
||||
taskRecord.AssignedRunID = ""
|
||||
taskRecord.UpdatedAt = now
|
||||
taskRecord.StartedAt = ""
|
||||
if _, err := s.repo.UpdateTask(ctx, taskRecord); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
laneRecord.Status = lane.StatusBlocked
|
||||
laneRecord.ErrorMessage = strings.TrimSpace(cause.Error())
|
||||
laneRecord.UpdatedAt = now
|
||||
if _, err := s.repo.UpdateLane(ctx, laneRecord); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := s.repo.AppendTaskEvent(ctx, task.Event{
|
||||
TaskID: taskRecord.ID,
|
||||
EventType: "blocked",
|
||||
BodyMarkdown: reason,
|
||||
CreatedByRoleName: "worker",
|
||||
CreatedAt: now,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.repo.CreateMessage(ctx, message.Record{
|
||||
WorkspaceID: taskRecord.WorkspaceID,
|
||||
TopicID: taskRecord.TopicID,
|
||||
FromRoleName: "worker",
|
||||
ToExpr: "leader",
|
||||
Type: message.TypeSummary,
|
||||
Stage: string(workflow.StageExecution),
|
||||
BodyMarkdown: fmt.Sprintf("Lane `%s` blocked before task `%s` could start.\n\n%s", laneRecord.Name, taskRecord.Title, reason),
|
||||
CreatedAt: now,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.syncTopicStatus(ctx, taskRecord.TopicID)
|
||||
}
|
||||
|
||||
func shouldSnapshotTask(taskRecord task.Record) bool {
|
||||
switch taskRecord.Kind {
|
||||
case task.KindExecution, task.KindVerification:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func failedExitCode(exitCode int) int {
|
||||
if exitCode != 0 {
|
||||
return exitCode
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func (s *Service) dependenciesSatisfied(ctx context.Context, taskID string) (bool, error) {
|
||||
deps, err := s.repo.ListTaskDependencies(ctx, taskID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, dep := range deps {
|
||||
item, err := s.repo.GetTask(ctx, dep.DependsOnTaskID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if item.Status != task.StatusSucceeded {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func hasRunningTask(items []task.Record) bool {
|
||||
for _, item := range items {
|
||||
if item.Status == task.StatusRunning {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Service) emitCompletionSideEffects(
|
||||
ctx context.Context,
|
||||
run workflow.Run,
|
||||
taskRecord task.Record,
|
||||
chainRecord lane.Record,
|
||||
promotedTasks []task.Record,
|
||||
value Completion,
|
||||
now string,
|
||||
) {
|
||||
var failures []string
|
||||
|
||||
eventType := "completed"
|
||||
eventBody := taskRecord.ResultSummaryMarkdown
|
||||
if taskRecord.Status == task.StatusFailed {
|
||||
eventType = "failed"
|
||||
eventBody = taskRecord.BlockingReasonMarkdown
|
||||
}
|
||||
if _, err := s.repo.AppendTaskEvent(ctx, task.Event{
|
||||
TaskID: taskRecord.ID,
|
||||
EventType: eventType,
|
||||
BodyMarkdown: eventBody,
|
||||
CreatedByRoleName: "worker",
|
||||
CreatedAt: now,
|
||||
}); err != nil {
|
||||
failures = append(failures, fmt.Sprintf("append %s event for task %s: %v", eventType, taskRecord.ID, err))
|
||||
}
|
||||
|
||||
for _, item := range promotedTasks {
|
||||
if _, err := s.repo.AppendTaskEvent(ctx, task.Event{
|
||||
TaskID: item.ID,
|
||||
EventType: "ready",
|
||||
BodyMarkdown: "Dependencies satisfied. Task is ready for execution.",
|
||||
CreatedByRoleName: "leader",
|
||||
CreatedAt: now,
|
||||
}); err != nil {
|
||||
failures = append(failures, fmt.Sprintf("append ready event for task %s: %v", item.ID, err))
|
||||
}
|
||||
}
|
||||
|
||||
topicRecord, err := s.repo.GetTopic(ctx, taskRecord.TopicID)
|
||||
if err == nil && strings.TrimSpace(topicRecord.Status) == "cancelled" {
|
||||
if len(failures) == 0 {
|
||||
return
|
||||
}
|
||||
_, _ = s.repo.AppendWorkflowRunLog(ctx, workflow.RunLog{
|
||||
RunID: run.ID,
|
||||
Stream: workflow.LogStreamSystem,
|
||||
Content: "completion side effects failed after main state commit:\n- " + strings.Join(failures, "\n- "),
|
||||
CreatedAt: now,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := s.repo.CreateMessage(ctx, message.Record{
|
||||
WorkspaceID: taskRecord.WorkspaceID,
|
||||
TopicID: taskRecord.TopicID,
|
||||
FromRoleName: "worker",
|
||||
ToExpr: "leader",
|
||||
Type: message.TypeSummary,
|
||||
Stage: string(workflow.StageExecution),
|
||||
BodyMarkdown: workerSummaryMarkdown(chainRecord, taskRecord, value),
|
||||
ReplyToMessageID: "",
|
||||
CreatedAt: now,
|
||||
}); err != nil {
|
||||
failures = append(failures, fmt.Sprintf("create summary message for run %s: %v", run.ID, err))
|
||||
}
|
||||
|
||||
if len(failures) == 0 {
|
||||
return
|
||||
}
|
||||
_, _ = s.repo.AppendWorkflowRunLog(ctx, workflow.RunLog{
|
||||
RunID: run.ID,
|
||||
Stream: workflow.LogStreamSystem,
|
||||
Content: "completion side effects failed after main state commit:\n- " + strings.Join(failures, "\n- "),
|
||||
CreatedAt: now,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) syncTopicStatus(ctx context.Context, topicID string) error {
|
||||
record, err := s.repo.GetTopic(ctx, topicID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch strings.TrimSpace(record.Status) {
|
||||
case "cancelled", "awaiting_confirmation":
|
||||
return nil
|
||||
}
|
||||
tasks, err := s.repo.ListTasksByTopic(ctx, topicID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chains, err := s.repo.ListLanesByTopic(ctx, topicID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
nextStatus := "execution"
|
||||
if len(tasks) > 0 && graphCompleted(tasks, chains) {
|
||||
nextStatus = "completed"
|
||||
} else if graphBlocked(tasks, chains) {
|
||||
nextStatus = "blocked"
|
||||
}
|
||||
if record.Status == nextStatus {
|
||||
return nil
|
||||
}
|
||||
record.Status = nextStatus
|
||||
if nextStatus == "completed" && record.ClosedAt == "" {
|
||||
record.ClosedAt = timeutil.FormatRFC3339(s.clock.Now())
|
||||
}
|
||||
if _, err := s.repo.UpdateTopic(ctx, record); err != nil {
|
||||
return err
|
||||
}
|
||||
if nextStatus == "completed" && s.runtime != nil {
|
||||
s.releaseTopicLaneRuntimes(ctx, chains)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) releaseTopicLaneRuntimes(ctx context.Context, lanes []lane.Record) {
|
||||
for _, item := range lanes {
|
||||
if strings.TrimSpace(item.ContainerName) == "" && strings.TrimSpace(item.RuntimeEndpoint) == "" {
|
||||
continue
|
||||
}
|
||||
_, _ = s.runtime.ReleaseLaneRuntime(ctx, item.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func graphBlocked(tasks []task.Record, chains []lane.Record) bool {
|
||||
for _, item := range tasks {
|
||||
if item.Status == task.StatusFailed || item.Status == task.StatusBlocked {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, item := range chains {
|
||||
if item.Status == lane.StatusBlocked || item.Status == lane.StatusFailed {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func graphCompleted(tasks []task.Record, chains []lane.Record) bool {
|
||||
hasTasks := false
|
||||
for _, item := range tasks {
|
||||
hasTasks = true
|
||||
switch item.Status {
|
||||
case task.StatusSucceeded, task.StatusCancelled:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
if !hasTasks {
|
||||
return false
|
||||
}
|
||||
for _, item := range chains {
|
||||
switch item.Status {
|
||||
case lane.StatusSucceeded, lane.StatusCancelled:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func workerSummaryMarkdown(chainRecord lane.Record, taskRecord task.Record, value Completion) string {
|
||||
if value.Status == workflow.RunStatusSucceeded {
|
||||
body := strings.TrimSpace(value.ResultMarkdown)
|
||||
if body == "" {
|
||||
body = "Task completed without an explicit summary."
|
||||
}
|
||||
return fmt.Sprintf("Lane `%s` completed task `%s`.\n\n%s", chainRecord.Name, taskRecord.Title, body)
|
||||
}
|
||||
body := strings.TrimSpace(value.ErrorMessage)
|
||||
if body == "" {
|
||||
body = "Task failed without an explicit error message."
|
||||
}
|
||||
return fmt.Sprintf("Lane `%s` failed task `%s`.\n\n%s", chainRecord.Name, taskRecord.Title, body)
|
||||
}
|
||||
|
||||
func buildWorkerPrompt(topicRecord topic.Record, chainRecord lane.Record, taskRecord task.Record, resolved runtimeconfig.ResolvedRole, messages []message.Record) string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString("## Role\n")
|
||||
builder.WriteString(runtimeconfig.RenderInstructions(resolved, ""))
|
||||
builder.WriteString("\n")
|
||||
builder.WriteString("\n## Execution Context\n")
|
||||
builder.WriteString(fmt.Sprintf("- topic: %s\n- lane: %s\n- task: %s\n- task kind: %s\n- branch: %s\n", topicRecord.Slug, chainRecord.Name, taskRecord.Title, taskRecord.Kind, chainRecord.BranchName))
|
||||
builder.WriteString("\n## Task\n")
|
||||
builder.WriteString(taskRecord.BodyMarkdown)
|
||||
if strings.TrimSpace(taskRecord.AcceptanceMarkdown) != "" {
|
||||
builder.WriteString("\n\n## Acceptance\n")
|
||||
builder.WriteString(taskRecord.AcceptanceMarkdown)
|
||||
}
|
||||
if len(messages) > 0 {
|
||||
builder.WriteString("\n## Recent Messages\n")
|
||||
start := 0
|
||||
if len(messages) > 6 {
|
||||
start = len(messages) - 6
|
||||
}
|
||||
for _, item := range messages[start:] {
|
||||
builder.WriteString(fmt.Sprintf("### %s -> %s (%s)\n%s\n", item.FromRoleName, item.ToExpr, item.Stage, item.BodyMarkdown))
|
||||
}
|
||||
}
|
||||
builder.WriteString("\n## Blockers\n")
|
||||
builder.WriteString("If you are blocked, send exactly one concrete question to the leader via the inbox skill with the current execution stage, then stop and make the final markdown summary describe the blocker and the decision you need.\n")
|
||||
builder.WriteString("\nReturn a concise markdown execution summary suitable to send back to the leader. Do not add front matter.\n")
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func marshalCommandMetadata(value commandMetadata) (string, error) {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal task command metadata: %w", err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func parseCommandMetadata(value string) (commandMetadata, error) {
|
||||
var item commandMetadata
|
||||
if err := json.Unmarshal([]byte(value), &item); err != nil {
|
||||
return commandMetadata{}, fmt.Errorf("decode task command metadata: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(item.TaskID) == "" {
|
||||
return commandMetadata{}, fmt.Errorf("task command metadata missing task id")
|
||||
}
|
||||
if strings.TrimSpace(item.LaneID) == "" {
|
||||
return commandMetadata{}, fmt.Errorf("task command metadata missing lane id")
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
@@ -0,0 +1,819 @@
|
||||
package taskexec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"inbox/internal/app/lanematerialize"
|
||||
"inbox/internal/app/runtimeconfig"
|
||||
"inbox/internal/base/timeutil"
|
||||
"inbox/internal/domain/lane"
|
||||
"inbox/internal/domain/message"
|
||||
"inbox/internal/domain/task"
|
||||
"inbox/internal/domain/topic"
|
||||
"inbox/internal/domain/workflow"
|
||||
sqlitestore "inbox/internal/store/sqlite"
|
||||
)
|
||||
|
||||
func TestCompletePromotesDependentTaskAndNotifiesLeader(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := timeutil.FixedClock{Time: time.Date(2026, 3, 16, 10, 0, 0, 0, time.UTC)}
|
||||
store, err := sqlitestore.OpenInMemory(clock)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenInMemory() error = %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
_, chainRecord := seedTaskExecGraph(t, ctx, store)
|
||||
svc := NewService(store, newResolvedRoleResolver(store, clock), clock, WithSnapshotter(noopSnapshotter{}), WithMaterializer(noopMaterializer{}))
|
||||
|
||||
assignment, err := svc.ClaimNext(ctx, chainRecord.ID, "runner-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ClaimNext() error = %v", err)
|
||||
}
|
||||
if assignment.Task.Title != "Task A" {
|
||||
t.Fatalf("expected Task A to be claimed first, got %#v", assignment.Task)
|
||||
}
|
||||
if !strings.Contains(assignment.Prompt, "Task A") {
|
||||
t.Fatalf("expected task prompt to include task title, got %q", assignment.Prompt)
|
||||
}
|
||||
if !strings.Contains(assignment.Prompt, "## Skills") || !strings.Contains(assignment.Prompt, "Use Inbox V2") {
|
||||
t.Fatalf("expected task prompt to include bound skills, got %q", assignment.Prompt)
|
||||
}
|
||||
|
||||
updatedRun, err := svc.Complete(ctx, Completion{
|
||||
RunID: assignment.Run.ID,
|
||||
Status: workflow.RunStatusSucceeded,
|
||||
ExitCode: 0,
|
||||
ResultMarkdown: "Implemented Task A.",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Complete() error = %v", err)
|
||||
}
|
||||
if updatedRun.Status != workflow.RunStatusSucceeded {
|
||||
t.Fatalf("unexpected run status: %#v", updatedRun)
|
||||
}
|
||||
|
||||
tasks, err := store.ListTasksByLane(ctx, chainRecord.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTasksByLane() error = %v", err)
|
||||
}
|
||||
byTitle := make(map[string]task.Record, len(tasks))
|
||||
for _, item := range tasks {
|
||||
byTitle[item.Title] = item
|
||||
}
|
||||
if byTitle["Task A"].Status != task.StatusSucceeded {
|
||||
t.Fatalf("expected Task A succeeded, got %#v", byTitle["Task A"])
|
||||
}
|
||||
if byTitle["Task B"].Status != task.StatusReady {
|
||||
t.Fatalf("expected Task B promoted to ready, got %#v", byTitle["Task B"])
|
||||
}
|
||||
|
||||
messages, err := store.ListMessagesByTopic(ctx, chainRecord.TopicID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListMessagesByTopic() error = %v", err)
|
||||
}
|
||||
var workerSummary *message.Record
|
||||
for _, item := range messages {
|
||||
if item.FromRoleName == "worker" && item.ToExpr == "leader" {
|
||||
workerSummary = &item
|
||||
}
|
||||
}
|
||||
if workerSummary == nil {
|
||||
t.Fatalf("expected worker summary message, got %#v", messages)
|
||||
}
|
||||
if !strings.Contains(workerSummary.BodyMarkdown, "Implemented Task A.") {
|
||||
t.Fatalf("unexpected worker summary body: %q", workerSummary.BodyMarkdown)
|
||||
}
|
||||
|
||||
chainState, err := store.GetLane(ctx, chainRecord.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetLane() error = %v", err)
|
||||
}
|
||||
if chainState.Status != lane.StatusReady {
|
||||
t.Fatalf("expected chain to stay ready for next task, got %#v", chainState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteAutoCompletesMilestoneAndPromotesDownstreamTask(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := timeutil.FixedClock{Time: time.Date(2026, 3, 16, 11, 0, 0, 0, time.UTC)}
|
||||
store, err := sqlitestore.OpenInMemory(clock)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenInMemory() error = %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
ws, chainRecord := seedTaskExecGraph(t, ctx, store)
|
||||
items, err := store.ListTasksByLane(ctx, chainRecord.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTasksByLane() error = %v", err)
|
||||
}
|
||||
var taskAID string
|
||||
for _, item := range items {
|
||||
if item.Title == "Task A" {
|
||||
taskAID = item.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if taskAID == "" {
|
||||
t.Fatal("expected Task A in seed graph")
|
||||
}
|
||||
milestone, err := store.CreateTask(ctx, task.Record{
|
||||
WorkspaceID: ws.ID,
|
||||
TopicID: chainRecord.TopicID,
|
||||
LaneID: chainRecord.ID,
|
||||
Title: "Ready for Verification",
|
||||
BodyMarkdown: "Milestone node.",
|
||||
Kind: task.KindMilestone,
|
||||
Status: task.StatusDraft,
|
||||
Priority: 8,
|
||||
TaskOrder: 3,
|
||||
CreatedByRoleName: "leader",
|
||||
}, []task.Dependency{{DependsOnTaskID: taskAID}})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTask(milestone) error = %v", err)
|
||||
}
|
||||
if _, err := store.CreateTask(ctx, task.Record{
|
||||
WorkspaceID: ws.ID,
|
||||
TopicID: chainRecord.TopicID,
|
||||
LaneID: chainRecord.ID,
|
||||
Title: "Task C",
|
||||
BodyMarkdown: "Run final verification.",
|
||||
Kind: task.KindVerification,
|
||||
Status: task.StatusDraft,
|
||||
Priority: 7,
|
||||
TaskOrder: 4,
|
||||
CreatedByRoleName: "leader",
|
||||
}, []task.Dependency{{DependsOnTaskID: milestone.ID}}); err != nil {
|
||||
t.Fatalf("CreateTask(task C) error = %v", err)
|
||||
}
|
||||
|
||||
svc := NewService(store, newResolvedRoleResolver(store, clock), clock, WithSnapshotter(noopSnapshotter{}), WithMaterializer(noopMaterializer{}))
|
||||
assignment, err := svc.ClaimNext(ctx, chainRecord.ID, "runner-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ClaimNext() error = %v", err)
|
||||
}
|
||||
if assignment.Task.Title != "Task A" {
|
||||
t.Fatalf("expected Task A first, got %#v", assignment.Task)
|
||||
}
|
||||
|
||||
if _, err := svc.Complete(ctx, Completion{
|
||||
RunID: assignment.Run.ID,
|
||||
Status: workflow.RunStatusSucceeded,
|
||||
ExitCode: 0,
|
||||
ResultMarkdown: "Implemented Task A.",
|
||||
}); err != nil {
|
||||
t.Fatalf("Complete() error = %v", err)
|
||||
}
|
||||
|
||||
items, err = store.ListTasksByLane(ctx, chainRecord.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTasksByLane() error = %v", err)
|
||||
}
|
||||
byTitle := make(map[string]task.Record, len(items))
|
||||
for _, item := range items {
|
||||
byTitle[item.Title] = item
|
||||
}
|
||||
if byTitle["Ready for Verification"].Status != task.StatusSucceeded {
|
||||
t.Fatalf("expected milestone auto-succeeded, got %#v", byTitle["Ready for Verification"])
|
||||
}
|
||||
if byTitle["Task C"].Status != task.StatusReady {
|
||||
t.Fatalf("expected downstream task promoted after milestone, got %#v", byTitle["Task C"])
|
||||
}
|
||||
next, err := svc.ClaimNext(ctx, chainRecord.ID, "runner-2")
|
||||
if err != nil {
|
||||
t.Fatalf("ClaimNext(second) error = %v", err)
|
||||
}
|
||||
if next.Task.Title != "Task B" {
|
||||
t.Fatalf("expected Task B next due ordering before Task C, got %#v", next.Task)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimNextCommitsMainStateWhenStartedEventFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := timeutil.FixedClock{Time: time.Date(2026, 3, 16, 10, 0, 0, 0, time.UTC)}
|
||||
store, err := sqlitestore.OpenInMemory(clock)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenInMemory() error = %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
_, chainRecord := seedTaskExecGraph(t, ctx, store)
|
||||
repo := &flakyTaskExecRepo{
|
||||
Store: store,
|
||||
failAppendTaskEvent: true,
|
||||
}
|
||||
svc := NewService(repo, newResolvedRoleResolver(store, clock), clock, WithSnapshotter(noopSnapshotter{}), WithMaterializer(noopMaterializer{}))
|
||||
|
||||
assignment, err := svc.ClaimNext(ctx, chainRecord.ID, "runner-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ClaimNext() should succeed even if started event fails, got %v", err)
|
||||
}
|
||||
if assignment.Run.Status != workflow.RunStatusRunning {
|
||||
t.Fatalf("expected running run, got %#v", assignment.Run)
|
||||
}
|
||||
if assignment.Task.Status != task.StatusRunning {
|
||||
t.Fatalf("expected running task, got %#v", assignment.Task)
|
||||
}
|
||||
|
||||
runs, err := store.ListWorkflowRunsByTopic(ctx, chainRecord.TopicID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListWorkflowRunsByTopic() error = %v", err)
|
||||
}
|
||||
if len(runs) != 1 || runs[0].ID != assignment.Run.ID || runs[0].Status != workflow.RunStatusRunning {
|
||||
t.Fatalf("unexpected workflow runs after claim: %#v", runs)
|
||||
}
|
||||
|
||||
persistedTask, err := store.GetTask(ctx, assignment.Task.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetTask() error = %v", err)
|
||||
}
|
||||
if persistedTask.Status != task.StatusRunning || persistedTask.AssignedRunID != assignment.Run.ID {
|
||||
t.Fatalf("expected persisted task running with assigned run, got %#v", persistedTask)
|
||||
}
|
||||
|
||||
events, err := store.ListTaskEvents(ctx, assignment.Task.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTaskEvents() error = %v", err)
|
||||
}
|
||||
for _, item := range events {
|
||||
if item.EventType == "started" {
|
||||
t.Fatalf("expected no started event when side effect fails, got %#v", events)
|
||||
}
|
||||
}
|
||||
|
||||
logs, err := store.ListWorkflowRunLogs(ctx, assignment.Run.ID, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListWorkflowRunLogs() error = %v", err)
|
||||
}
|
||||
if len(logs) == 0 {
|
||||
t.Fatalf("expected warning log after started-event failure")
|
||||
}
|
||||
if !strings.Contains(logs[len(logs)-1].Content, "claim side effects failed after main state commit") {
|
||||
t.Fatalf("unexpected warning log: %#v", logs[len(logs)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteCommitsMainStateWhenNotificationSideEffectsFail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := timeutil.FixedClock{Time: time.Date(2026, 3, 16, 10, 0, 0, 0, time.UTC)}
|
||||
store, err := sqlitestore.OpenInMemory(clock)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenInMemory() error = %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
_, chainRecord := seedTaskExecGraph(t, ctx, store)
|
||||
repo := &flakyTaskExecRepo{
|
||||
Store: store,
|
||||
failCreateMessage: true,
|
||||
}
|
||||
svc := NewService(repo, newResolvedRoleResolver(store, clock), clock, WithSnapshotter(noopSnapshotter{}), WithMaterializer(noopMaterializer{}))
|
||||
|
||||
assignment, err := svc.ClaimNext(ctx, chainRecord.ID, "runner-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ClaimNext() error = %v", err)
|
||||
}
|
||||
|
||||
updatedRun, err := svc.Complete(ctx, Completion{
|
||||
RunID: assignment.Run.ID,
|
||||
Status: workflow.RunStatusSucceeded,
|
||||
ExitCode: 0,
|
||||
ResultMarkdown: "Implemented Task A.",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Complete() should succeed even if side effects fail, got %v", err)
|
||||
}
|
||||
if updatedRun.Status != workflow.RunStatusSucceeded {
|
||||
t.Fatalf("unexpected run status: %#v", updatedRun)
|
||||
}
|
||||
|
||||
tasks, err := store.ListTasksByLane(ctx, chainRecord.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTasksByLane() error = %v", err)
|
||||
}
|
||||
byTitle := make(map[string]task.Record, len(tasks))
|
||||
for _, item := range tasks {
|
||||
byTitle[item.Title] = item
|
||||
}
|
||||
if byTitle["Task A"].Status != task.StatusSucceeded {
|
||||
t.Fatalf("expected Task A succeeded, got %#v", byTitle["Task A"])
|
||||
}
|
||||
if byTitle["Task B"].Status != task.StatusReady {
|
||||
t.Fatalf("expected Task B promoted to ready, got %#v", byTitle["Task B"])
|
||||
}
|
||||
|
||||
chainState, err := store.GetLane(ctx, chainRecord.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetLane() error = %v", err)
|
||||
}
|
||||
if chainState.Status != lane.StatusReady {
|
||||
t.Fatalf("expected chain ready after commit, got %#v", chainState)
|
||||
}
|
||||
|
||||
messages, err := store.ListMessagesByTopic(ctx, chainRecord.TopicID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListMessagesByTopic() error = %v", err)
|
||||
}
|
||||
for _, item := range messages {
|
||||
if item.FromRoleName == "worker" && item.ToExpr == "leader" {
|
||||
t.Fatalf("expected no worker summary message when notification fails, got %#v", item)
|
||||
}
|
||||
}
|
||||
|
||||
logs, err := store.ListWorkflowRunLogs(ctx, assignment.Run.ID, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ListWorkflowRunLogs() error = %v", err)
|
||||
}
|
||||
if len(logs) == 0 {
|
||||
t.Fatalf("expected warning log after side-effect failure")
|
||||
}
|
||||
if !strings.Contains(logs[len(logs)-1].Content, "completion side effects failed after main state commit") {
|
||||
t.Fatalf("unexpected warning log: %#v", logs[len(logs)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteSnapshotsLaneHeadCommit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := timeutil.FixedClock{Time: time.Date(2026, 3, 16, 12, 0, 0, 0, time.UTC)}
|
||||
store, err := sqlitestore.OpenInMemory(clock)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenInMemory() error = %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
repoDir := createGitRepo(t)
|
||||
initialHead := gitHead(t, repoDir)
|
||||
seedTaskExecRepoGraph(t, ctx, store, repoDir, initialHead)
|
||||
|
||||
svc := NewService(store, newResolvedRoleResolver(store, clock), clock, WithMaterializer(noopMaterializer{}))
|
||||
assignment, err := svc.ClaimNext(ctx, "chain_1", "runner-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ClaimNext() error = %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(repoDir, "todo.txt"), []byte("hello lane snapshot\n"), 0644); err != nil {
|
||||
t.Fatalf("WriteFile() error = %v", err)
|
||||
}
|
||||
|
||||
if _, err := svc.Complete(ctx, Completion{
|
||||
RunID: assignment.Run.ID,
|
||||
Status: workflow.RunStatusSucceeded,
|
||||
ExitCode: 0,
|
||||
ResultMarkdown: "Implemented Task A.",
|
||||
}); err != nil {
|
||||
t.Fatalf("Complete() error = %v", err)
|
||||
}
|
||||
|
||||
laneRecord, err := store.GetLane(ctx, "chain_1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetLane() error = %v", err)
|
||||
}
|
||||
if laneRecord.HeadCommit == "" || laneRecord.HeadCommit == initialHead {
|
||||
t.Fatalf("expected updated lane head commit, got %#v", laneRecord)
|
||||
}
|
||||
if status := gitStatusShort(t, repoDir); strings.TrimSpace(status) != "" {
|
||||
t.Fatalf("expected clean repo after snapshot, got %q", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimNextMaterializesUpstreamLaneCommit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := timeutil.FixedClock{Time: time.Date(2026, 3, 16, 13, 0, 0, 0, time.UTC)}
|
||||
store, err := sqlitestore.OpenInMemory(clock)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenInMemory() error = %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
rootRepo := createGitRepo(t)
|
||||
runGit(t, rootRepo, "checkout", "-b", "upstream")
|
||||
if err := os.WriteFile(filepath.Join(rootRepo, "backend.txt"), []byte("backend\n"), 0644); err != nil {
|
||||
t.Fatalf("WriteFile(upstream) error = %v", err)
|
||||
}
|
||||
runGit(t, rootRepo, "add", "-A")
|
||||
runGit(t, rootRepo, "commit", "-m", "upstream change")
|
||||
upstreamHead := gitHead(t, rootRepo)
|
||||
runGit(t, rootRepo, "checkout", "main")
|
||||
|
||||
downstreamDir := filepath.Join(t.TempDir(), "downstream")
|
||||
runGit(t, rootRepo, "worktree", "add", "-b", "downstream", downstreamDir, "main")
|
||||
seedMaterializationGraph(t, ctx, store, rootRepo, downstreamDir, upstreamHead)
|
||||
|
||||
svc := NewService(store, newResolvedRoleResolver(store, clock), clock, WithSnapshotter(noopSnapshotter{}))
|
||||
assignment, err := svc.ClaimNext(ctx, "lane_downstream", "runner-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ClaimNext() error = %v", err)
|
||||
}
|
||||
if assignment.Task.ID != "task_downstream" {
|
||||
t.Fatalf("unexpected task claimed: %#v", assignment.Task)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(downstreamDir, "backend.txt")); err != nil {
|
||||
t.Fatalf("expected upstream file materialized into downstream lane: %v", err)
|
||||
}
|
||||
|
||||
var count int
|
||||
if err := store.DB().QueryRow(`SELECT COUNT(*) FROM lane_syncs WHERE downstream_lane_id = ? AND upstream_lane_id = ? AND status = ?`,
|
||||
"lane_downstream", "lane_upstream", "applied").Scan(&count); err != nil {
|
||||
t.Fatalf("count lane_syncs: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("expected one applied lane sync, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimNextBlocksTaskWhenUpstreamLaneHasNoHeadCommit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := timeutil.FixedClock{Time: time.Date(2026, 3, 16, 14, 0, 0, 0, time.UTC)}
|
||||
store, err := sqlitestore.OpenInMemory(clock)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenInMemory() error = %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
repoDir := createGitRepo(t)
|
||||
downstreamDir := filepath.Join(t.TempDir(), "downstream")
|
||||
runGit(t, repoDir, "worktree", "add", "-b", "downstream", downstreamDir, "main")
|
||||
seedMaterializationGraph(t, ctx, store, repoDir, downstreamDir, "")
|
||||
|
||||
svc := NewService(store, newResolvedRoleResolver(store, clock), clock, WithSnapshotter(noopSnapshotter{}))
|
||||
assignment, err := svc.ClaimNext(ctx, "lane_downstream", "runner-1")
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("expected sql.ErrNoRows after block, got assignment=%#v err=%v", assignment, err)
|
||||
}
|
||||
|
||||
taskRecord, err := store.GetTask(ctx, "task_downstream")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTask() error = %v", err)
|
||||
}
|
||||
if taskRecord.Status != task.StatusBlocked {
|
||||
t.Fatalf("expected blocked task, got %#v", taskRecord)
|
||||
}
|
||||
laneRecord, err := store.GetLane(ctx, "lane_downstream")
|
||||
if err != nil {
|
||||
t.Fatalf("GetLane() error = %v", err)
|
||||
}
|
||||
if laneRecord.Status != lane.StatusBlocked {
|
||||
t.Fatalf("expected blocked lane, got %#v", laneRecord)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteReleasesLaneRuntimeWhenTopicCompletes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := timeutil.FixedClock{Time: time.Date(2026, 3, 16, 15, 0, 0, 0, time.UTC)}
|
||||
store, err := sqlitestore.OpenInMemory(clock)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenInMemory() error = %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
_, chainRecord := seedTaskExecGraph(t, ctx, store)
|
||||
releaser := &recordingLaneRuntimeReleaser{}
|
||||
svc := NewService(
|
||||
store,
|
||||
newResolvedRoleResolver(store, clock),
|
||||
clock,
|
||||
WithSnapshotter(noopSnapshotter{}),
|
||||
WithMaterializer(noopMaterializer{}),
|
||||
WithLaneRuntimeReleaser(releaser),
|
||||
)
|
||||
|
||||
first, err := svc.ClaimNext(ctx, chainRecord.ID, "runner-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ClaimNext(first) error = %v", err)
|
||||
}
|
||||
if _, err := svc.Complete(ctx, Completion{
|
||||
RunID: first.Run.ID,
|
||||
Status: workflow.RunStatusSucceeded,
|
||||
ExitCode: 0,
|
||||
ResultMarkdown: "Implemented Task A.",
|
||||
}); err != nil {
|
||||
t.Fatalf("Complete(first) error = %v", err)
|
||||
}
|
||||
|
||||
second, err := svc.ClaimNext(ctx, chainRecord.ID, "runner-2")
|
||||
if err != nil {
|
||||
t.Fatalf("ClaimNext(second) error = %v", err)
|
||||
}
|
||||
if second.Task.Title != "Task B" {
|
||||
t.Fatalf("expected Task B second, got %#v", second.Task)
|
||||
}
|
||||
if _, err := svc.Complete(ctx, Completion{
|
||||
RunID: second.Run.ID,
|
||||
Status: workflow.RunStatusSucceeded,
|
||||
ExitCode: 0,
|
||||
ResultMarkdown: "Implemented Task B.",
|
||||
}); err != nil {
|
||||
t.Fatalf("Complete(second) error = %v", err)
|
||||
}
|
||||
|
||||
topicRecord, err := store.GetTopic(ctx, chainRecord.TopicID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetTopic() error = %v", err)
|
||||
}
|
||||
if topicRecord.Status != "completed" {
|
||||
t.Fatalf("expected completed topic, got %#v", topicRecord)
|
||||
}
|
||||
if len(releaser.laneIDs) != 1 || releaser.laneIDs[0] != chainRecord.ID {
|
||||
t.Fatalf("expected runtime release for completed lane, got %#v", releaser.laneIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func seedTaskExecGraph(t *testing.T, ctx context.Context, store *sqlitestore.Store) (roleWorkspace, lane.Record) {
|
||||
t.Helper()
|
||||
now := timeutil.FormatRFC3339(time.Date(2026, 3, 16, 10, 0, 0, 0, time.UTC))
|
||||
if _, err := store.DB().Exec(`
|
||||
INSERT INTO projects(id, slug, name, root_path, default_branch, status, created_at, updated_at)
|
||||
VALUES('proj_1', 'proj', 'Project', '/tmp/project', 'main', 'active', ?, ?)
|
||||
`, now, now); err != nil {
|
||||
t.Fatalf("insert project: %v", err)
|
||||
}
|
||||
if _, err := store.DB().Exec(`
|
||||
INSERT INTO workspaces(id, project_id, slug, name, root_path, base_branch, worktree_branch, runtime_backend, status, created_at, updated_at)
|
||||
VALUES('ws_1', 'proj_1', 'main', 'Main', '/tmp/workspace', 'main', 'worktree/main', 'host', 'active', ?, ?)
|
||||
`, now, now); err != nil {
|
||||
t.Fatalf("insert workspace: %v", err)
|
||||
}
|
||||
if _, err := store.DB().Exec(`
|
||||
INSERT INTO topics(id, workspace_id, slug, title, space, status, created_at, updated_at)
|
||||
VALUES('topic_1', 'ws_1', 'sample', 'Sample', ?, 'execution', ?, ?)
|
||||
`, string(topic.SpaceWorkflow), now, now); err != nil {
|
||||
t.Fatalf("insert topic: %v", err)
|
||||
}
|
||||
if _, err := store.CreateLane(ctx, lane.Record{
|
||||
ID: "chain_1",
|
||||
WorkspaceID: "ws_1",
|
||||
TopicID: "topic_1",
|
||||
Name: "Backend Chain",
|
||||
Slug: "backend-chain",
|
||||
Status: lane.StatusReady,
|
||||
BaseBranch: "main",
|
||||
BranchName: "lane/main/backend-lane",
|
||||
WorktreePath: "/tmp/workspace--backend-chain",
|
||||
ContainerName: "lane-main-backend-lane",
|
||||
CreatedByRoleName: "leader",
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateLane() error = %v", err)
|
||||
}
|
||||
taskA, err := store.CreateTask(ctx, task.Record{
|
||||
WorkspaceID: "ws_1",
|
||||
TopicID: "topic_1",
|
||||
LaneID: "chain_1",
|
||||
Title: "Task A",
|
||||
BodyMarkdown: "Implement backend changes.",
|
||||
Status: task.StatusReady,
|
||||
TaskOrder: 1,
|
||||
Priority: 10,
|
||||
CreatedByRoleName: "leader",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTask(Task A) error = %v", err)
|
||||
}
|
||||
if _, err := store.CreateTask(ctx, task.Record{
|
||||
WorkspaceID: "ws_1",
|
||||
TopicID: "topic_1",
|
||||
LaneID: "chain_1",
|
||||
Title: "Task B",
|
||||
BodyMarkdown: "Add verification.",
|
||||
Status: task.StatusDraft,
|
||||
TaskOrder: 2,
|
||||
Priority: 5,
|
||||
CreatedByRoleName: "leader",
|
||||
}, []task.Dependency{{DependsOnTaskID: taskA.ID}}); err != nil {
|
||||
t.Fatalf("CreateTask(Task B) error = %v", err)
|
||||
}
|
||||
return roleWorkspace{ID: "ws_1"}, lane.Record{ID: "chain_1", TopicID: "topic_1"}
|
||||
}
|
||||
|
||||
func seedTaskExecRepoGraph(t *testing.T, ctx context.Context, store *sqlitestore.Store, repoDir, headCommit string) {
|
||||
t.Helper()
|
||||
now := timeutil.FormatRFC3339(time.Date(2026, 3, 16, 12, 0, 0, 0, time.UTC))
|
||||
mustExec(t, store, `
|
||||
INSERT INTO projects(id, slug, name, root_path, default_branch, status, created_at, updated_at)
|
||||
VALUES('proj_git', 'proj-git', 'Project Git', ?, 'main', 'active', ?, ?)
|
||||
`, repoDir, now, now)
|
||||
mustExec(t, store, `
|
||||
INSERT INTO workspaces(id, project_id, slug, name, root_path, base_branch, worktree_branch, runtime_backend, status, created_at, updated_at)
|
||||
VALUES('ws_git', 'proj_git', 'main', 'Main', ?, 'main', 'worktree/main', 'host', 'active', ?, ?)
|
||||
`, repoDir, now, now)
|
||||
mustExec(t, store, `
|
||||
INSERT INTO topics(id, workspace_id, slug, title, space, status, created_at, updated_at)
|
||||
VALUES('topic_git', 'ws_git', 'sample-git', 'Sample Git', ?, 'execution', ?, ?)
|
||||
`, string(topic.SpaceWorkflow), now, now)
|
||||
if _, err := store.CreateLane(ctx, lane.Record{
|
||||
ID: "chain_1",
|
||||
WorkspaceID: "ws_git",
|
||||
TopicID: "topic_git",
|
||||
Name: "Snapshot Lane",
|
||||
Slug: "snapshot-lane",
|
||||
Status: lane.StatusReady,
|
||||
BaseBranch: "main",
|
||||
BranchName: "main",
|
||||
HeadCommit: headCommit,
|
||||
WorktreePath: repoDir,
|
||||
ContainerName: "lane-snapshot",
|
||||
CreatedByRoleName: "leader",
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateLane(snapshot) error = %v", err)
|
||||
}
|
||||
if _, err := store.CreateTask(ctx, task.Record{
|
||||
ID: "task_snapshot",
|
||||
WorkspaceID: "ws_git",
|
||||
TopicID: "topic_git",
|
||||
LaneID: "chain_1",
|
||||
Title: "Task A",
|
||||
BodyMarkdown: "Implement backend changes.",
|
||||
Status: task.StatusReady,
|
||||
TaskOrder: 1,
|
||||
Priority: 10,
|
||||
CreatedByRoleName: "leader",
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("CreateTask(snapshot) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func seedMaterializationGraph(t *testing.T, ctx context.Context, store *sqlitestore.Store, projectRoot, downstreamDir, upstreamHead string) {
|
||||
t.Helper()
|
||||
now := timeutil.FormatRFC3339(time.Date(2026, 3, 16, 13, 0, 0, 0, time.UTC))
|
||||
mustExec(t, store, `
|
||||
INSERT INTO projects(id, slug, name, root_path, default_branch, status, created_at, updated_at)
|
||||
VALUES('proj_mat', 'proj-mat', 'Project Mat', ?, 'main', 'active', ?, ?)
|
||||
`, projectRoot, now, now)
|
||||
mustExec(t, store, `
|
||||
INSERT INTO workspaces(id, project_id, slug, name, root_path, base_branch, worktree_branch, runtime_backend, status, created_at, updated_at)
|
||||
VALUES('ws_mat', 'proj_mat', 'Main', 'Main', ?, 'main', 'worktree/main', 'host', 'active', ?, ?)
|
||||
`, projectRoot, now, now)
|
||||
mustExec(t, store, `
|
||||
INSERT INTO topics(id, workspace_id, slug, title, space, status, created_at, updated_at)
|
||||
VALUES('topic_mat', 'ws_mat', 'sample-mat', 'Sample Mat', ?, 'execution', ?, ?)
|
||||
`, string(topic.SpaceWorkflow), now, now)
|
||||
if _, err := store.CreateLane(ctx, lane.Record{
|
||||
ID: "lane_upstream",
|
||||
WorkspaceID: "ws_mat",
|
||||
TopicID: "topic_mat",
|
||||
Name: "Upstream Lane",
|
||||
Slug: "upstream-lane",
|
||||
Status: lane.StatusSucceeded,
|
||||
BaseBranch: "main",
|
||||
BranchName: "upstream",
|
||||
HeadCommit: upstreamHead,
|
||||
WorktreePath: projectRoot,
|
||||
ContainerName: "lane-upstream",
|
||||
CreatedByRoleName: "leader",
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateLane(upstream) error = %v", err)
|
||||
}
|
||||
if _, err := store.CreateLane(ctx, lane.Record{
|
||||
ID: "lane_downstream",
|
||||
WorkspaceID: "ws_mat",
|
||||
TopicID: "topic_mat",
|
||||
Name: "Downstream Lane",
|
||||
Slug: "downstream-lane",
|
||||
Status: lane.StatusReady,
|
||||
BaseBranch: "main",
|
||||
BranchName: "downstream",
|
||||
WorktreePath: downstreamDir,
|
||||
ContainerName: "lane-downstream",
|
||||
CreatedByRoleName: "leader",
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateLane(downstream) error = %v", err)
|
||||
}
|
||||
upstreamTask, err := store.CreateTask(ctx, task.Record{
|
||||
ID: "task_upstream",
|
||||
WorkspaceID: "ws_mat",
|
||||
TopicID: "topic_mat",
|
||||
LaneID: "lane_upstream",
|
||||
Title: "Task Upstream",
|
||||
BodyMarkdown: "Implement upstream work.",
|
||||
Status: task.StatusSucceeded,
|
||||
TaskOrder: 1,
|
||||
Priority: 10,
|
||||
CreatedByRoleName: "leader",
|
||||
ResultSummaryMarkdown: "Done.",
|
||||
BlockingReasonMarkdown: "",
|
||||
CompletedAt: now,
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTask(upstream) error = %v", err)
|
||||
}
|
||||
if _, err := store.CreateTask(ctx, task.Record{
|
||||
ID: "task_downstream",
|
||||
WorkspaceID: "ws_mat",
|
||||
TopicID: "topic_mat",
|
||||
LaneID: "lane_downstream",
|
||||
Title: "Task Downstream",
|
||||
BodyMarkdown: "Integrate upstream work.",
|
||||
Status: task.StatusReady,
|
||||
TaskOrder: 1,
|
||||
Priority: 5,
|
||||
CreatedByRoleName: "leader",
|
||||
}, []task.Dependency{{DependsOnTaskID: upstreamTask.ID}}); err != nil {
|
||||
t.Fatalf("CreateTask(downstream) error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustExec(t *testing.T, store *sqlitestore.Store, query string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := store.DB().Exec(query, args...); err != nil {
|
||||
t.Fatalf("Exec(%q) error = %v", query, err)
|
||||
}
|
||||
}
|
||||
|
||||
func createGitRepo(t *testing.T) string {
|
||||
t.Helper()
|
||||
repoDir := filepath.Join(t.TempDir(), "repo")
|
||||
if err := os.MkdirAll(repoDir, 0755); err != nil {
|
||||
t.Fatalf("MkdirAll() error = %v", err)
|
||||
}
|
||||
runGit(t, repoDir, "init", "-b", "main")
|
||||
runGit(t, repoDir, "config", "user.name", "Test User")
|
||||
runGit(t, repoDir, "config", "user.email", "test@example.com")
|
||||
runGit(t, repoDir, "commit", "--allow-empty", "-m", "initial commit")
|
||||
return repoDir
|
||||
}
|
||||
|
||||
func gitHead(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
return strings.TrimSpace(runGit(t, dir, "rev-parse", "HEAD"))
|
||||
}
|
||||
|
||||
func gitStatusShort(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
return runGit(t, dir, "status", "--short")
|
||||
}
|
||||
|
||||
func runGit(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
type roleWorkspace struct {
|
||||
ID string
|
||||
}
|
||||
|
||||
type resolvedRoleResolver struct {
|
||||
store *sqlitestore.Store
|
||||
clock timeutil.Clock
|
||||
}
|
||||
|
||||
func newResolvedRoleResolver(store *sqlitestore.Store, clock timeutil.Clock) *resolvedRoleResolver {
|
||||
return &resolvedRoleResolver{store: store, clock: clock}
|
||||
}
|
||||
|
||||
func (r *resolvedRoleResolver) ResolveRole(ctx context.Context, workspaceID, roleName string) (runtimeconfig.ResolvedRole, error) {
|
||||
return runtimeconfig.NewService(r.store, r.store, r.clock).ResolveRole(ctx, workspaceID, roleName)
|
||||
}
|
||||
|
||||
type noopSnapshotter struct{}
|
||||
|
||||
func (noopSnapshotter) Capture(_ context.Context, item lane.Record, _ task.Record) (string, error) {
|
||||
return item.HeadCommit, nil
|
||||
}
|
||||
|
||||
type noopMaterializer struct{}
|
||||
|
||||
func (noopMaterializer) Materialize(_ context.Context, _ lane.Record, _ string, _ []lanematerialize.Upstream) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type recordingLaneRuntimeReleaser struct {
|
||||
laneIDs []string
|
||||
}
|
||||
|
||||
func (r *recordingLaneRuntimeReleaser) ReleaseLaneRuntime(_ context.Context, laneID string) (lane.Record, error) {
|
||||
r.laneIDs = append(r.laneIDs, laneID)
|
||||
return lane.Record{ID: laneID}, nil
|
||||
}
|
||||
|
||||
type flakyTaskExecRepo struct {
|
||||
*sqlitestore.Store
|
||||
failCreateMessage bool
|
||||
failAppendTaskEvent bool
|
||||
}
|
||||
|
||||
func (r *flakyTaskExecRepo) CreateMessage(ctx context.Context, value message.Record) (message.Record, error) {
|
||||
if r.failCreateMessage {
|
||||
return message.Record{}, errors.New("forced create message failure")
|
||||
}
|
||||
return r.Store.CreateMessage(ctx, value)
|
||||
}
|
||||
|
||||
func (r *flakyTaskExecRepo) AppendTaskEvent(ctx context.Context, value task.Event) (task.Event, error) {
|
||||
if r.failAppendTaskEvent {
|
||||
return task.Event{}, errors.New("forced append task event failure")
|
||||
}
|
||||
return r.Store.AppendTaskEvent(ctx, value)
|
||||
}
|
||||
Reference in New Issue
Block a user