From aa02cef3403a9b35b09f3ddfc48e0ba814257004 Mon Sep 17 00:00:00 2001 From: Jason Lernerman <43508353+u9g@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:44:16 -0500 Subject: [PATCH 1/2] refactor(simulate): return the whole create response from createSimulationRun --- cmd/lk/simulate.go | 6 +++--- cmd/lk/simulate_ci.go | 6 +++--- cmd/lk/simulate_tui.go | 15 +++++++-------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index efe654871..5b533db31 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -867,7 +867,7 @@ func ciFromEnv() *livekit.SimulationRun_CI { } } -func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livekit.PresignedPostRequest, error) { +func createSimulationRun(ctx context.Context, c *simulateConfig) (*livekit.SimulationRun_Create_Response, error) { req := &livekit.SimulationRun_Create_Request{ AgentName: c.agentName, Deployment: c.deployment, @@ -893,9 +893,9 @@ func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livek resp, err := c.client.CreateSimulationRun(ctx, req) if err != nil { - return "", nil, fmt.Errorf("failed to create simulation: %w", err) + return nil, fmt.Errorf("failed to create simulation: %w", err) } - return resp.SimulationRunId, resp.PresignedPostRequest, nil + return resp, nil } func uploadSource(ctx context.Context, client *lksdk.AgentSimulationClient, runID string, presigned *livekit.PresignedPostRequest, projectDir, entrypoint string) error { diff --git a/cmd/lk/simulate_ci.go b/cmd/lk/simulate_ci.go index 0fab86da0..0830bdaaa 100644 --- a/cmd/lk/simulate_ci.go +++ b/cmd/lk/simulate_ci.go @@ -112,18 +112,18 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { } start := time.Now() - var presigned *livekit.PresignedPostRequest - runID, presigned, err = createSimulationRun(ctx, config) + created, err := createSimulationRun(ctx, config) if err != nil { report.SetupFailed(err) report.EndSetup() return err } + runID = created.SimulationRunId report.SimulationCreated(time.Since(start)) if config.mode == modeGenerateFromSource { start = time.Now() - if err := uploadSource(ctx, config.client, runID, presigned, config.projectDir, config.entrypoint); err != nil { + if err := uploadSource(ctx, config.client, runID, created.PresignedPostRequest, config.projectDir, config.entrypoint); err != nil { report.SetupFailed(err) report.EndSetup() return err diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index b2e91ab0d..37c686f56 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -427,10 +427,9 @@ type agentReadyMsg struct { } type simulationCreatedMsg struct { - runID string - presigned *livekit.PresignedPostRequest - elapsed time.Duration - err error + resp *livekit.SimulationRun_Create_Response + elapsed time.Duration + err error } type sourceUploadedMsg struct { @@ -565,8 +564,8 @@ func (m *simulateModel) waitAgentReadyCmd() tea.Cmd { func (m *simulateModel) createSimulationCmd() tea.Cmd { return func() tea.Msg { start := time.Now() - runID, presigned, err := createSimulationRun(m.setupCtx, m.config) - return simulationCreatedMsg{runID: runID, presigned: presigned, elapsed: time.Since(start), err: err} + resp, err := createSimulationRun(m.setupCtx, m.config) + return simulationCreatedMsg{resp: resp, elapsed: time.Since(start), err: err} } } @@ -670,11 +669,11 @@ func (m *simulateModel) update(msg tea.Msg) (tea.Model, tea.Cmd) { m.failSetupStep(msg.err) return m, nil } - m.runID = msg.runID + m.runID = msg.resp.SimulationRunId m.reporter.SimulationCreated(msg.elapsed) if m.config.mode == modeGenerateFromSource { m.advanceSetupStep(msg.elapsed) - return m, m.uploadSourceCmd(msg.presigned) + return m, m.uploadSourceCmd(msg.resp.PresignedPostRequest) } m.completeSetup(msg.elapsed) return m, tea.Batch(m.pollSimulation(), m.waitSubprocess()) From f4996ffc97bd6826189afcb2cb22ba33aabf26a0 Mon Sep 17 00:00:00 2001 From: Jason Lernerman <43508353+u9g@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:44:17 -0500 Subject: [PATCH 2/2] feat(simulate): say when a run starts that Agent Observability is off --- cmd/lk/simulate.go | 19 +++++++++++++++++++ cmd/lk/simulate_ci.go | 3 +++ cmd/lk/simulate_host_test.go | 11 +++++++++++ cmd/lk/simulate_report.go | 8 +++++++- cmd/lk/simulate_tui.go | 19 ++++++++++++++++++- 5 files changed, 58 insertions(+), 2 deletions(-) diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index 5b533db31..68e29387a 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -898,6 +898,25 @@ func createSimulationRun(ctx context.Context, c *simulateConfig) (*livekit.Simul return resp, nil } +// observabilitySettingsURL is where a project's Agent Observability is turned +// on, or "" when the project ID is unknown. +func observabilitySettingsURL(projectID string) string { + if projectID == "" { + return "" + } + return dashboardBaseURL() + "/projects/" + projectID + "/settings/observability" +} + +// observabilityNotice explains a run created while the project's Agent +// Observability is off, and where to turn it on. +func observabilityNotice(projectID string) string { + where := "in the project's Observability settings on the LiveKit Cloud dashboard" + if url := observabilitySettingsURL(projectID); url != "" { + where = "at " + url + } + return "Agent Observability is off for this project, so this run gets no summary and its sessions aren't recorded. Turn it on " + where +} + func uploadSource(ctx context.Context, client *lksdk.AgentSimulationClient, runID string, presigned *livekit.PresignedPostRequest, projectDir, entrypoint string) error { if presigned == nil { return fmt.Errorf("server did not return upload URL") diff --git a/cmd/lk/simulate_ci.go b/cmd/lk/simulate_ci.go index 0830bdaaa..a1f15911c 100644 --- a/cmd/lk/simulate_ci.go +++ b/cmd/lk/simulate_ci.go @@ -120,6 +120,9 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { } runID = created.SimulationRunId report.SimulationCreated(time.Since(start)) + if created.ObservabilityDisabled { + report.ObservabilityDisabled(observabilityNotice(config.pc.ProjectId)) + } if config.mode == modeGenerateFromSource { start = time.Now() diff --git a/cmd/lk/simulate_host_test.go b/cmd/lk/simulate_host_test.go index e403cb8c7..106632bd6 100644 --- a/cmd/lk/simulate_host_test.go +++ b/cmd/lk/simulate_host_test.go @@ -71,3 +71,14 @@ func TestSimulateCommandHintKeepsUserAuth(t *testing.T) { } assert.Contains(t, simulateCommandHint("view", "SR_1"), "--project one --user-auth") } + +// The notice links the settings page on the project's own dashboard, and still +// says where to go when the project ID is unknown. +func TestObservabilityNotice(t *testing.T) { + origPC := simulateProjectConfig + t.Cleanup(func() { simulateProjectConfig = origPC }) + + simulateProjectConfig = &config.ProjectConfig{Name: "stg", URL: "wss://stg.staging.livekit.cloud"} + assert.Contains(t, observabilityNotice("p_123"), "at https://cloud.staging.livekit.io/projects/p_123/settings/observability") + assert.Contains(t, observabilityNotice(""), "in the project's Observability settings") +} diff --git a/cmd/lk/simulate_report.go b/cmd/lk/simulate_report.go index b4557b753..f9700102c 100644 --- a/cmd/lk/simulate_report.go +++ b/cmd/lk/simulate_report.go @@ -78,6 +78,12 @@ func (l *simLog) ConfigWarning(msg string) { fmt.Fprintf(l.out, "⚠ %s\n", msg) } +// ObservabilityDisabled is an annotation rather than a setup line: GitHub +// collapses the setup group, and the notice carries the link to act on. +func (l *simLog) ObservabilityDisabled(notice string) { + fmt.Fprintf(l.out, "::warning::%s\n", notice) +} + func (l *simLog) QuotaExceeded(desc string, suggested int) { l.quotaNote = fmt.Sprintf( "This run hit the project's %s (inference 429s). Try re-running with --concurrency %d.", @@ -242,7 +248,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) msg = run.Error } fmt.Fprintln(w) - fmt.Fprintln(w, "⚠ "+msg) + fmt.Fprintln(w, "::warning::"+msg) } } diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index 37c686f56..9c508daeb 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -262,6 +262,9 @@ type simulateModel struct { confirmQuit bool confirmQuitSel int + // the project's Agent Observability is off, as the run's create response said + observabilityOff bool + // inference-quota (429) dialog; shows at most once per run quotaWarning *quotaInfo quotaDismissed bool @@ -671,6 +674,10 @@ func (m *simulateModel) update(msg tea.Msg) (tea.Model, tea.Cmd) { } m.runID = msg.resp.SimulationRunId m.reporter.SimulationCreated(msg.elapsed) + if msg.resp.ObservabilityDisabled { + m.observabilityOff = true + m.reporter.ObservabilityDisabled(observabilityNotice(m.config.pc.ProjectId)) + } if m.config.mode == modeGenerateFromSource { m.advanceSetupStep(msg.elapsed) return m, m.uploadSourceCmd(msg.resp.PresignedPostRequest) @@ -1233,6 +1240,9 @@ func (m *simulateModel) viewSetup() string { for _, w := range m.config.warnings { writeWrappedLines(&b, yellowStyle(), " ", "⚠ "+w, m.width) } + if m.observabilityOff { + writeWrappedLines(&b, yellowStyle(), " ", "⚠ "+observabilityNotice(m.config.pc.ProjectId), m.width) + } // in file mode the scenarios are already known, nothing is generated if m.setupDone && m.err == nil && m.config.mode == modeGenerateFromSource { @@ -1423,7 +1433,14 @@ func (m *simulateModel) viewRunning() string { if m.run.Error != "" { msg = m.run.Error } - fmt.Fprintf(&b, "\n %s %s\n", yellowStyle().Render("⚠"), yellowStyle().Render(msg)) + b.WriteString("\n") + writeWrappedLines(&b, yellowStyle(), " ", "⚠ "+msg, m.width) + if url := observabilitySettingsURL(m.config.pc.ProjectId); m.observabilityOff && url != "" { + fmt.Fprintf(&b, " %s\n", dimStyle.Render("Turn on Agent Observability: "+url)) + } + } else if m.observabilityOff { + b.WriteString("\n") + writeWrappedLines(&b, yellowStyle(), " ", "⚠ "+observabilityNotice(m.config.pc.ProjectId), m.width) } }