Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 22 additions & 3 deletions cmd/lk/simulate.go
Original file line number Diff line number Diff line change
Expand Up @@ -867,7 +867,7 @@ func ciFromEnv() *livekit.SimulationRun_CI {
}
}

func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livekit.PresignedPostRequest, error) {
func createSimulationRun(ctx context.Context, c *simulateConfig) (*livekit.SimulationRun_Create_Response, error) {
req := &livekit.SimulationRun_Create_Request{
AgentName: c.agentName,
Deployment: c.deployment,
Expand All @@ -893,9 +893,28 @@ func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livek

resp, err := c.client.CreateSimulationRun(ctx, req)
if err != nil {
return "", nil, fmt.Errorf("failed to create simulation: %w", err)
return nil, fmt.Errorf("failed to create simulation: %w", err)
}
return resp.SimulationRunId, resp.PresignedPostRequest, nil
return resp, nil
}

// observabilitySettingsURL is where a project's Agent Observability is turned
// on, or "" when the project ID is unknown.
func observabilitySettingsURL(projectID string) string {
if projectID == "" {
return ""
}
return dashboardBaseURL() + "/projects/" + projectID + "/settings/observability"
}

// observabilityNotice explains a run created while the project's Agent
// Observability is off, and where to turn it on.
func observabilityNotice(projectID string) string {
where := "in the project's Observability settings on the LiveKit Cloud dashboard"
if url := observabilitySettingsURL(projectID); url != "" {
where = "at " + url
}
return "Agent Observability is off for this project, so this run gets no summary and its sessions aren't recorded. Turn it on " + where
}

func uploadSource(ctx context.Context, client *lksdk.AgentSimulationClient, runID string, presigned *livekit.PresignedPostRequest, projectDir, entrypoint string) error {
Expand Down
9 changes: 6 additions & 3 deletions cmd/lk/simulate_ci.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,18 +112,21 @@
}

start := time.Now()
var presigned *livekit.PresignedPostRequest
runID, presigned, err = createSimulationRun(ctx, config)
created, err := createSimulationRun(ctx, config)
if err != nil {
report.SetupFailed(err)
report.EndSetup()
return err
}
runID = created.SimulationRunId
report.SimulationCreated(time.Since(start))
if created.ObservabilityDisabled {

Check failure on line 123 in cmd/lk/simulate_ci.go

View workflow job for this annotation

GitHub Actions / Cross-build Windows artifacts (zig)

created.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 123 in cmd/lk/simulate_ci.go

View workflow job for this annotation

GitHub Actions / Cross-build Windows artifacts (zig)

created.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 123 in cmd/lk/simulate_ci.go

View workflow job for this annotation

GitHub Actions / lint

created.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 123 in cmd/lk/simulate_ci.go

View workflow job for this annotation

GitHub Actions / lk agent with python agent on ubuntu-latest

created.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 123 in cmd/lk/simulate_ci.go

View workflow job for this annotation

GitHub Actions / lk agent with node agent on ubuntu-latest

created.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 123 in cmd/lk/simulate_ci.go

View workflow job for this annotation

GitHub Actions / Build darwin (native)

created.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 123 in cmd/lk/simulate_ci.go

View workflow job for this annotation

GitHub Actions / test (ubuntu-latest)

created.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 123 in cmd/lk/simulate_ci.go

View workflow job for this annotation

GitHub Actions / lk agent with node agent on macos-latest

created.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 123 in cmd/lk/simulate_ci.go

View workflow job for this annotation

GitHub Actions / lk agent with python agent on macos-latest

created.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)
report.ObservabilityDisabled(observabilityNotice(config.pc.ProjectId))
}

if config.mode == modeGenerateFromSource {
start = time.Now()
if err := uploadSource(ctx, config.client, runID, presigned, config.projectDir, config.entrypoint); err != nil {
if err := uploadSource(ctx, config.client, runID, created.PresignedPostRequest, config.projectDir, config.entrypoint); err != nil {
report.SetupFailed(err)
report.EndSetup()
return err
Expand Down
11 changes: 11 additions & 0 deletions cmd/lk/simulate_host_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,14 @@ func TestSimulateCommandHintKeepsUserAuth(t *testing.T) {
}
assert.Contains(t, simulateCommandHint("view", "SR_1"), "--project one --user-auth")
}

// The notice links the settings page on the project's own dashboard, and still
// says where to go when the project ID is unknown.
func TestObservabilityNotice(t *testing.T) {
origPC := simulateProjectConfig
t.Cleanup(func() { simulateProjectConfig = origPC })

simulateProjectConfig = &config.ProjectConfig{Name: "stg", URL: "wss://stg.staging.livekit.cloud"}
assert.Contains(t, observabilityNotice("p_123"), "at https://cloud.staging.livekit.io/projects/p_123/settings/observability")
assert.Contains(t, observabilityNotice(""), "in the project's Observability settings")
}
8 changes: 7 additions & 1 deletion cmd/lk/simulate_report.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ func (l *simLog) ConfigWarning(msg string) {
fmt.Fprintf(l.out, "⚠ %s\n", msg)
}

// ObservabilityDisabled is an annotation rather than a setup line: GitHub
// collapses the setup group, and the notice carries the link to act on.
func (l *simLog) ObservabilityDisabled(notice string) {
fmt.Fprintf(l.out, "::warning::%s\n", notice)
}

func (l *simLog) QuotaExceeded(desc string, suggested int) {
l.quotaNote = fmt.Sprintf(
"This run hit the project's %s (inference 429s). Try re-running with --concurrency %d.",
Expand Down Expand Up @@ -242,7 +248,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess)
msg = run.Error
}
fmt.Fprintln(w)
fmt.Fprintln(w, "⚠ "+msg)
fmt.Fprintln(w, "::warning::"+msg)
}
}

Expand Down
34 changes: 25 additions & 9 deletions cmd/lk/simulate_tui.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,9 @@
confirmQuit bool
confirmQuitSel int

// the project's Agent Observability is off, as the run's create response said
observabilityOff bool

// inference-quota (429) dialog; shows at most once per run
quotaWarning *quotaInfo
quotaDismissed bool
Expand Down Expand Up @@ -427,10 +430,9 @@
}

type simulationCreatedMsg struct {
runID string
presigned *livekit.PresignedPostRequest
elapsed time.Duration
err error
resp *livekit.SimulationRun_Create_Response
elapsed time.Duration
err error
}

type sourceUploadedMsg struct {
Expand Down Expand Up @@ -565,8 +567,8 @@
func (m *simulateModel) createSimulationCmd() tea.Cmd {
return func() tea.Msg {
start := time.Now()
runID, presigned, err := createSimulationRun(m.setupCtx, m.config)
return simulationCreatedMsg{runID: runID, presigned: presigned, elapsed: time.Since(start), err: err}
resp, err := createSimulationRun(m.setupCtx, m.config)
return simulationCreatedMsg{resp: resp, elapsed: time.Since(start), err: err}
}
}

Expand Down Expand Up @@ -670,11 +672,15 @@
m.failSetupStep(msg.err)
return m, nil
}
m.runID = msg.runID
m.runID = msg.resp.SimulationRunId
m.reporter.SimulationCreated(msg.elapsed)
if msg.resp.ObservabilityDisabled {

Check failure on line 677 in cmd/lk/simulate_tui.go

View workflow job for this annotation

GitHub Actions / Cross-build Windows artifacts (zig)

msg.resp.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 677 in cmd/lk/simulate_tui.go

View workflow job for this annotation

GitHub Actions / Cross-build Windows artifacts (zig)

msg.resp.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 677 in cmd/lk/simulate_tui.go

View workflow job for this annotation

GitHub Actions / lint

msg.resp.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled) (typecheck)

Check failure on line 677 in cmd/lk/simulate_tui.go

View workflow job for this annotation

GitHub Actions / lk agent with python agent on ubuntu-latest

msg.resp.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 677 in cmd/lk/simulate_tui.go

View workflow job for this annotation

GitHub Actions / lk agent with node agent on ubuntu-latest

msg.resp.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 677 in cmd/lk/simulate_tui.go

View workflow job for this annotation

GitHub Actions / Build darwin (native)

msg.resp.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 677 in cmd/lk/simulate_tui.go

View workflow job for this annotation

GitHub Actions / test (ubuntu-latest)

msg.resp.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 677 in cmd/lk/simulate_tui.go

View workflow job for this annotation

GitHub Actions / lk agent with node agent on macos-latest

msg.resp.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)

Check failure on line 677 in cmd/lk/simulate_tui.go

View workflow job for this annotation

GitHub Actions / lk agent with python agent on macos-latest

msg.resp.ObservabilityDisabled undefined (type *livekit.SimulationRun_Create_Response has no field or method ObservabilityDisabled)
m.observabilityOff = true
m.reporter.ObservabilityDisabled(observabilityNotice(m.config.pc.ProjectId))
}
if m.config.mode == modeGenerateFromSource {
m.advanceSetupStep(msg.elapsed)
return m, m.uploadSourceCmd(msg.presigned)
return m, m.uploadSourceCmd(msg.resp.PresignedPostRequest)
}
m.completeSetup(msg.elapsed)
return m, tea.Batch(m.pollSimulation(), m.waitSubprocess())
Expand Down Expand Up @@ -1234,6 +1240,9 @@
for _, w := range m.config.warnings {
writeWrappedLines(&b, yellowStyle(), " ", "⚠ "+w, m.width)
}
if m.observabilityOff {
writeWrappedLines(&b, yellowStyle(), " ", "⚠ "+observabilityNotice(m.config.pc.ProjectId), m.width)
}

// in file mode the scenarios are already known, nothing is generated
if m.setupDone && m.err == nil && m.config.mode == modeGenerateFromSource {
Expand Down Expand Up @@ -1424,7 +1433,14 @@
if m.run.Error != "" {
msg = m.run.Error
}
fmt.Fprintf(&b, "\n %s %s\n", yellowStyle().Render("⚠"), yellowStyle().Render(msg))
b.WriteString("\n")
writeWrappedLines(&b, yellowStyle(), " ", "⚠ "+msg, m.width)
if url := observabilitySettingsURL(m.config.pc.ProjectId); m.observabilityOff && url != "" {
fmt.Fprintf(&b, " %s\n", dimStyle.Render("Turn on Agent Observability: "+url))
}
} else if m.observabilityOff {
b.WriteString("\n")
writeWrappedLines(&b, yellowStyle(), " ", "⚠ "+observabilityNotice(m.config.pc.ProjectId), m.width)
}
}

Expand Down
Loading