From 0ac666b59708d11efb434ae7f59a9e0afd125c82 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Thu, 5 Nov 2026 13:37:40 +0545 Subject: [PATCH 1/5] Add concurrency sweeps to presets --- mkdocs/docs/concepts/presets.md | 6 +- .../_internal/cli/models/preset_agent.py | 15 +- .../_internal/cli/services/presets/build.py | 2 + .../_internal/cli/services/presets/output.py | 62 +++++-- .../presets/resources/system_prompt.md | 65 +++++--- .../_internal/cli/services/presets/session.py | 93 +++++++++-- .../_internal/cli/services/presets/verify.py | 30 +++- .../cli/services/presets/workspace.py | 51 +++++- .../_internal/core/models/configurations.py | 53 +++++- src/dstack/_internal/core/models/presets.py | 37 ++++- src/tests/_internal/cli/common.py | 3 +- .../cli/services/presets/test_agent.py | 129 +++++++++++++++ .../cli/services/presets/test_create.py | 6 +- .../cli/services/presets/test_output.py | 155 +++++++++++++++++- .../services/presets/test_report_schema.py | 67 ++++---- .../cli/services/presets/test_store.py | 4 +- .../cli/services/presets/test_verify.py | 134 +++++++++++++-- .../cli/services/presets/test_workspace.py | 59 +++++++ .../core/models/test_configurations.py | 43 +++++ .../_internal/core/models/test_presets.py | 30 ++++ 20 files changed, 911 insertions(+), 133 deletions(-) diff --git a/mkdocs/docs/concepts/presets.md b/mkdocs/docs/concepts/presets.md index b01c63f62b..4096f11226 100644 --- a/mkdocs/docs/concepts/presets.md +++ b/mkdocs/docs/concepts/presets.md @@ -33,8 +33,9 @@ trials: 5 min_context_length: 1048576 max_ttft: 675 -# The number of simultaneous requests every benchmark uses -concurrency: 1 +# The numbers of simultaneous requests every benchmark is run at: +# a value, a list, `2,3`, or a doubling range `2..16` +concurrency: 2..16 # The request shape every benchmark uses (defaults to 1024 and 1024) input_tokens: 10000 @@ -390,7 +391,6 @@ At the same time, it's recommended to create presets using your own agent — ei ## Limitations * Currently, the agent doesn't upload compiled binaries anywhere; patches compile at runtime -* Doesn't support ranges for `concurrency` > Report bugs and request features on [GitHub](https://github.com/dstackai/dstack/issues), and ask questions on [Discord](https://discord.gg/u8SmfwPpMd). diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index 812ddce17e..918e622737 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -9,6 +9,7 @@ ValidationError, ValidatorFunctionWrapHandler, WrapValidator, + model_validator, ) from dstack._internal.core.models.common import CoreModel @@ -53,7 +54,19 @@ class PresetAgentSuccess(CoreModel): base: Annotated[str, Field(min_length=1)] model: Annotated[str, Field(min_length=1)] context_length: PositiveInt - benchmark: PresetBenchmark + benchmarks: Annotated[list[PresetBenchmark], Field(min_length=1)] + + @model_validator(mode="before") + @classmethod + def wrap_legacy_single_benchmark(cls, data: Any) -> Any: + # An agent process started by a pre-sweep CLI reports one `benchmark`, and + # a session detached before the upgrade is attached and verified by the + # new CLI. The one point is its whole sweep; the sweep check in `verify` + # still requires it to be the requested concurrency. + if isinstance(data, dict) and "benchmark" in data and "benchmarks" not in data: + data = dict(data) + data["benchmarks"] = [data.pop("benchmark")] + return data class PresetAgentFailure(CoreModel): diff --git a/src/dstack/_internal/cli/services/presets/build.py b/src/dstack/_internal/cli/services/presets/build.py index 1c3137f7aa..1fa4971158 100644 --- a/src/dstack/_internal/cli/services/presets/build.py +++ b/src/dstack/_internal/cli/services/presets/build.py @@ -32,6 +32,7 @@ def build_preset( base_model: str, repo: str, context_length: int, + benchmarks: list[PresetBenchmark], benchmark: PresetBenchmark, configuration: PresetConfiguration, best_trial: int, @@ -53,6 +54,7 @@ def build_preset( configuration=configuration, created_at=created_at, service=service, + benchmarks=benchmarks, benchmark=benchmark, verified_on=verification_replica_groups, ) diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 86f39d54c7..f5cf9fde07 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Any, Optional, Sequence +from typing import Any, Callable, Optional, Sequence from rich.table import Table @@ -198,9 +198,9 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False share = round(100 * shared_prefix_tokens / input_tokens) if share: objective.append(f"prefix={share}%") - concurrency = (best or {}).get("concurrency") or constraints.get("concurrency") - if concurrency: - objective.append(f"c={concurrency}") + sweep = constraints.get("concurrency") or (best or {}).get("concurrency") + if sweep: + objective.append(f"c={_format_concurrency(sweep)}") min_context_length = constraints.get("min_context_length") if verbose and isinstance(min_context_length, int): objective.append(f"ctx>={_format_token_count(min_context_length)}") @@ -211,14 +211,18 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False tps = _format_number(best["tok_s"]) if objective: # Lead with per-user tps: comparable across rows regardless of concurrency. - tpot_ms = best.get("tpot_ms") - if isinstance(tpot_ms, (int, float)) and tpot_ms > 0: - parts.append(f"tps/user={_format_number(1000 / tpot_ms)}") + per_user = best.get("per_user_tok_s_range") + if per_user: + parts.append(f"tps/user={_format_range(per_user, _format_number)}") + # Which point of a sweep the numbers are from; redundant for one value. + if isinstance(sweep, list) and len(sweep) > 1 and best.get("concurrency"): + parts.append(f"c={best['concurrency']}") if verbose: - parts.append(f"tps={tps}") - ttft_ms = best.get("ttft_ms") - if isinstance(ttft_ms, (int, float)): - parts.append(f"ttft={_format_duration_ms(ttft_ms)}") + tok_s_range = best.get("tok_s_range") or [best["tok_s"]] + parts.append(f"tps={_format_range(tok_s_range, _format_number)}") + ttft_range = best.get("ttft_ms_range") + if ttft_range: + parts.append(f"ttft={_format_range(ttft_range, _format_duration_ms)}") context_length = best.get("context_length") if isinstance(context_length, int): parts.append(f"ctx={_format_token_count(context_length)}") @@ -304,7 +308,7 @@ def format_preset_objective( share = round(100 * workload.shared_prefix_tokens / workload.input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={workload.concurrency}") + parts.append(f"c={_format_concurrency([b.workload.concurrency for b in preset.benchmarks])}") return f"[secondary]{' '.join(parts)}[/]" @@ -323,7 +327,7 @@ def _format_creation_objective( share = round(100 * (configuration.shared_prefix_tokens or 0) / input_tokens) if share: parts.append(f"prefix={share}%") - parts.append(f"c={configuration.concurrency or workload.concurrency}") + parts.append(f"c={_format_concurrency(configuration.concurrency or [workload.concurrency])}") if verbose and configuration.min_context_length is not None: parts.append(f"ctx>={_format_token_count(configuration.min_context_length)}") if verbose and configuration.max_ttft is not None: @@ -345,15 +349,22 @@ def _breaches_constraints(preset: AnyStoredPreset) -> bool: def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) -> str: - benchmark = preset.benchmark - metrics = benchmark.metrics + # A sweep shows each metric as its range across concurrencies, and `c=` names + # the selected point. + benchmarks = preset.benchmarks parts = [ - f"tps/user={_format_number(benchmark.effective_per_user_tok_per_s)}", + "tps/user=" + + _format_range([b.effective_per_user_tok_per_s for b in benchmarks], _format_number) ] + if len(benchmarks) > 1: + parts.append(f"c={preset.benchmark.workload.concurrency}") if verbose: - parts.append(f"tps={_format_number(benchmark.effective_output_tok_per_s)}") + parts.append( + "tps=" + + _format_range([b.effective_output_tok_per_s for b in benchmarks], _format_number) + ) parts += [ - f"ttft={_format_duration_ms(metrics.ttft_ms.p50)}", + "ttft=" + _format_range([b.metrics.ttft_ms.p50 for b in benchmarks], _format_duration_ms), f"ctx={_format_token_count(preset.context_length)}", ] text = " ".join(parts) @@ -362,6 +373,21 @@ def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) - return text +def _format_range(values: list[float], fmt: Callable[[float], str]) -> str: + low, high = fmt(min(values)), fmt(max(values)) + return low if low == high else f"{low}..{high}" + + +def _format_concurrency(value: Any) -> str: + """A sweep in the shape the user wrote it: `8`, `2,3`, or `2..16` when every + step doubles. A pre-sweep record carries a bare int.""" + if not isinstance(value, list): + return str(value) + if len(value) > 1 and all(b == a * 2 for a, b in zip(value, value[1:])): + return f"{value[0]}..{value[-1]}" + return ",".join(str(v) for v in value) + + def _format_duration_ms(value: float) -> str: # 999.6 rounds to 1000, which must read as 1s rather than 1000ms. if value < 999.5: diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 691df34b33..79605eeec3 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -5,9 +5,21 @@ hardware, through sequential experimental trials. ## Performance -Performance is `output_tok_per_s`, the output token throughput of a benchmark -run at the `concurrency` from `constraints.json` (see `## Benchmark`). Benchmark -tools report several throughput numbers; performance is the output one. +Performance is the highest `output_tok_per_s` a configuration reaches across +the `concurrency` sweep from `constraints.json` (see `## Benchmark`), counting +only sweep points that meet `max_ttft`, the maximum p50 time to first token, in +milliseconds (see `# Constraints`). Points that breach `max_ttft` are ignored. +For example, with `max_ttft` of 2000: + +| `concurrency` | `output_tok_per_s` | `ttft_ms.p50` | +|---|---|---| +| 4 | 130 | 400 | +| 8 | 240 | 1500 | +| 16 | 380 | 2400 | + +The point at 16 breaches `max_ttft`, so it is ignored. Performance is 240, the +highest throughput among the remaining points. Benchmark tools report several +throughput numbers; performance is the output one. Significantly better performance goes beyond what conventional configurations and published benchmarks reach; they are not the ceiling. Estimate the @@ -35,10 +47,9 @@ Field semantics: - `max_ttft`: the maximum p50 time to first token, in milliseconds, that any benchmark in this session may report. - `trials_num`: the number of trials in this session. -- `concurrency`: the number of simultaneous requests for every benchmark in - this session. It is fixed so that benchmark results are comparable. - +- `concurrency`: the numbers of simultaneous requests every benchmark in + this session is run at, one benchmark per value. The list is fixed so + that benchmark results are comparable across trials. - `dataset`: the benchmark dataset for every benchmark in this session; see `## Benchmark`. @@ -257,7 +268,7 @@ mindful of which specific change was the root cause. others: ``` -{"resources": {...}, "context_length": ..., "benchmark": {...}, "learned": ..., "failed": ...} +{"resources": {...}, "context_length": ..., "benchmarks": [{...}, ...], "learned": ..., "failed": ...} ``` 2. In case the task is using node groups (a router group plus worker @@ -265,7 +276,7 @@ mindful of which specific change was the root cause. includes `groups`, and the fields are these and no others: ``` -{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": ..., "benchmark": {...}, "learned": ..., "failed": ...} +{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": ..., "benchmarks": [{...}, ...], "learned": ..., "failed": ...} ``` - `resources`: the exact resources of the instance the task ran on, in @@ -296,12 +307,14 @@ mindful of which specific change was the root cause. - `context_length`: the largest context the trial's configuration handles, found as described in `## Benchmark`; `null` only when the benchmark couldn't be done at all. -- `benchmark`: the trial benchmark (see `## Benchmark` for the structure); - `null` only when the benchmark couldn't be done at all. +- `benchmarks`: one trial benchmark per value of `concurrency`, in that + order (see `## Benchmark` for the structure); `null` only when no benchmark + could be done at all. - `learned`: the major things this trial taught you that you did not know before it ran. Required for every trial, including a failed one. - `failed`: `true` if the configuration never ran or the benchmark broke a - constraint such as `max_ttft` or `min_context_length`, absent otherwise. + constraint such as `min_context_length`, or no sweep point met `max_ttft`; + absent otherwise. Breaching `max_ttft` at some points only is not a failure. You're expected to do exactly `trials_num` trials (see `# Constraints`). @@ -317,9 +330,11 @@ no trials remain. In that case, log the failure to `final_report.json` (see ## Benchmark During trials, run benchmarks via SSH inside the task, directly against the -serving engine: use `dataset` and `concurrency``concurrency`, `input_tokens`, `output_tokens`, and -`shared_prefix_tokens` from `constraints.json` and measure all trials the same -way so that their results are comparable with each other. +serving engine: run one benchmark per value of `concurrency`, using +`dataset``input_tokens`, `output_tokens`, and +`shared_prefix_tokens` from `constraints.json`, and measure all trials the same +way so that their results are comparable with each other. Use a different +seed for each point of the sweep. In case the task is using a router (see `## Cluster Placement`), run benchmarks via SSH inside the router node, directly against the router engine, never against a @@ -375,9 +390,10 @@ never part of the measured metrics. All verification and benchmark requests must succeed. -Record every benchmark using the following structure and field names — -trial benchmarks in `trials//trial.json`, the final benchmark as -`final_report.json.benchmark` (values are illustrative): +Record every benchmark using the following structure and field names, one +object per value of `concurrency` — trial benchmarks in +`trials//trial.json.benchmarks`, the final benchmarks as +`final_report.json.benchmarks` (values are illustrative): ```json { @@ -407,9 +423,9 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free -invocation. For the final benchmark, run it with streaming responses, set -`workload.concurrency` to `concurrency` from `constraints.json`, produce -every field of the structure, and calculate all metrics from the +invocation. For the final benchmarks, run each with streaming responses, one +per value of `concurrency` from `constraints.json` with `workload.concurrency` +set to that value, produce every field of the structure, and calculate all metrics from the `num_requests` measured requests only — exclude setup, health-check, and warmup requests. Never invent missing values. @@ -594,12 +610,12 @@ variable listed under `env` in `constraints.json`. Do not put secret values in `final_report.json`, or print them. Use env references in `final_report.json.service_yaml`; use environment variable names or redacted values in -`final_report.json.benchmark.command`. +`final_report.json.benchmarks[].command`. # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmark`, +`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmarks`, and `failure_summary`. On success (even if you had to pick a failed trial because no non-failed @@ -614,7 +630,8 @@ trial remained), include exactly: - `model`: the exact repo/path loaded by the final service command - `context_length`: the largest context verified for the final service, as described in `## Benchmark` -- `benchmark`: the final service benchmark described in `## Benchmark` +- `benchmarks`: the final service benchmarks described in `## Benchmark`, + one per value of `concurrency` Set `final_report.json.base` as follows: diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index 9ce77213e3..dc09cc2b5d 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -529,9 +529,12 @@ def list_preset_sessions() -> list[dict[str, Any]]: entry["id"] = path.name entry["name"] = claimed_session_name(state) entry["status"] = status - entry["trials"] = _summarize_session_trials(path / TRIALS_DIRNAME) + constraints = _read_session_constraints(path) + entry["trials"] = _summarize_session_trials( + path / TRIALS_DIRNAME, max_ttft=constraints.get("max_ttft") + ) entry["verification"] = _read_last_session_verification(path / SERVICE_DIRNAME) - entry["constraints"] = _read_session_constraints(path) + entry["constraints"] = constraints entries.append(entry) return entries @@ -584,7 +587,9 @@ def _read_last_session_verification(path: Path) -> Optional[dict[str, Any]]: return {"status": "verifying"} -def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: +def _summarize_session_trials( + path: Path, *, max_ttft: Optional[float] = None +) -> Optional[dict[str, Any]]: # TODO: Refactor this crap - must be explicit what is this and where is it used; also dicts are prohibited in dstack repo """A trial directory without `trial.json` is still in flight and is not counted.""" @@ -605,35 +610,25 @@ def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: gpu: Optional[str] = None for record in records: count += 1 - benchmark = record.get("benchmark") failed.append(bool(record.get("failed"))) record_gpu = _format_trial_gpu(record) if record_gpu: gpu = record_gpu - if not isinstance(benchmark, dict): - series.append(None) - continue - metrics = benchmark.get("metrics") or {} - workload = benchmark.get("workload") or {} - duration = metrics.get("duration_seconds") - tokens = metrics.get("total_output_tokens") - if not isinstance(duration, (int, float)) or duration <= 0: - series.append(None) - continue - if not isinstance(tokens, (int, float)): + point = _select_trial_point(record, max_ttft) + if point is None: series.append(None) continue - tok_s = tokens / duration + tok_s, metrics, workload, sweep = point series.append(tok_s) # A failed trial keeps its benchmark — it is what the next trial learns # from — but it is not a candidate for best, and promoting one would put # a configuration that broke a constraint at the top of the listing. if record.get("failed"): if best_failed is None or tok_s > best_failed["tok_s"]: - best_failed = _trial_entry(tok_s, record, metrics, workload, record_gpu) + best_failed = _trial_entry(tok_s, record, metrics, workload, sweep, record_gpu) continue if best is None or tok_s > best["tok_s"]: - best = _trial_entry(tok_s, record, metrics, workload, record_gpu) + best = _trial_entry(tok_s, record, metrics, workload, sweep, record_gpu) return { "count": count, "best": best, @@ -644,11 +639,55 @@ def _summarize_session_trials(path: Path) -> Optional[dict[str, Any]]: } +def _select_trial_point( + record: dict[str, Any], max_ttft: Optional[float] +) -> Optional[tuple[float, dict[str, Any], dict[str, Any], list[dict[str, Any]]]]: + """The sweep point a trial is judged by, as `select_benchmark` picks it for a + preset: the highest throughput within `max_ttft`, else the closest TTFT. + A record from before the sweep holds a single `benchmark`. Returns + `(tok_s, metrics, workload, sweep)`, where `sweep` is the metrics of every + point with usable numbers, or `None` when there is none.""" + benchmarks = record.get("benchmarks") + if not isinstance(benchmarks, list): + benchmarks = [record.get("benchmark")] + points: list[tuple[bool, float, Any, dict[str, Any], dict[str, Any]]] = [] + for benchmark in benchmarks: + if not isinstance(benchmark, dict): + continue + metrics = benchmark.get("metrics") or {} + workload = benchmark.get("workload") or {} + duration = metrics.get("duration_seconds") + tokens = metrics.get("total_output_tokens") + if not isinstance(duration, (int, float)) or duration <= 0: + continue + if not isinstance(tokens, (int, float)): + continue + ttft = (metrics.get("ttft_ms") or {}).get("p50") + within = not isinstance(max_ttft, (int, float)) or ( + isinstance(ttft, (int, float)) and ttft <= max_ttft + ) + points.append((within, tokens / duration, ttft, metrics, workload)) + if not points: + return None + sweep = [p[3] for p in points] + passing = [p for p in points if p[0]] + if passing: + _, tok_s, _, metrics, workload = max(passing, key=lambda p: p[1]) + else: + # `ttft` may be missing in a raw record; sort those last. + _, tok_s, _, metrics, workload = min( + points, + key=lambda p: (p[2] if isinstance(p[2], (int, float)) else float("inf"), -p[1]), + ) + return tok_s, metrics, workload, sweep + + def _trial_entry( tok_s: float, record: dict[str, Any], metrics: dict[str, Any], workload: dict[str, Any], + sweep: list[dict[str, Any]], gpu: Optional[str], ) -> dict[str, Any]: ttft = (metrics.get("ttft_ms") or {}).get("p50") @@ -659,12 +698,30 @@ def _trial_entry( "tok_s": tok_s, "tpot_ms": tpot if isinstance(tpot, (int, float)) and tpot > 0 else None, "ttft_ms": ttft if isinstance(ttft, (int, float)) else None, + # The trial's spread across its sweep, as `[low, high]`, None when no + # point reports the value. One point gives `low == high`. + "tok_s_range": _range([m["total_output_tokens"] / m["duration_seconds"] for m in sweep]), + "per_user_tok_s_range": _range( + [1000 / v for v in (_latency(m, "tpot_ms", "mean") for m in sweep) if v] + ), + "ttft_ms_range": _range( + [v for v in (_latency(m, "ttft_ms", "p50") for m in sweep) if v is not None] + ), "context_length": context_length if isinstance(context_length, int) else None, "concurrency": workload.get("concurrency"), "gpu": gpu, } +def _latency(metrics: dict[str, Any], name: str, stat: str) -> Optional[float]: + value = (metrics.get(name) or {}).get(stat) + return value if isinstance(value, (int, float)) and value >= 0 else None + + +def _range(values: list[float]) -> Optional[list[float]]: + return [min(values), max(values)] if values else None + + def _format_trial_gpu(record: dict[str, Any]) -> Optional[str]: """A trial records its hardware in one of two formats, as described in `system_prompt.md`: `{"resources": {...}}` without node groups, and diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 643522c503..c473a5b4b4 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -31,6 +31,7 @@ PresetBenchmark, PresetVerificationReplicaGroup, PresetWorkload, + select_benchmark, ) from dstack._internal.core.models.runs import JobStatus, Run, RunStatus @@ -99,6 +100,10 @@ def build_verified_preset( _check_report_answers_request(report, preset_configuration) if service.model is None or service.model.name != preset_configuration.model.api_model_name: raise CLIError("The agent's final service model name does not match the requested model") + benchmarks = sorted( + (_normalized_benchmark(b, preset_configuration) for b in report.benchmarks), + key=lambda b: b.workload.concurrency, + ) return build_preset( name=name, service=_portable_service( @@ -114,7 +119,8 @@ def build_verified_preset( # it cannot be confused with the service's client-facing model name. repo=report.model, context_length=report.context_length, - benchmark=_normalized_benchmark(report.benchmark, preset_configuration), + benchmarks=benchmarks, + benchmark=select_benchmark(benchmarks, preset_configuration.max_ttft), best_trial=report.trial, configuration=preset_configuration, preset_id=preset_id, @@ -139,9 +145,11 @@ def _check_report_answers_request( report: PresetAgentSuccess, configuration: PresetConfiguration ) -> None: """The report must answer what the configuration asked: the same benchmark - workload, and the requested model — exactly when it was exact, any variant of - the base otherwise.""" - _check_workload_answers_request(report.benchmark.workload, configuration) + workload at every concurrency of the sweep, and the requested model — exactly + when it was exact, any variant of the base otherwise.""" + for benchmark in report.benchmarks: + _check_workload_answers_request(benchmark.workload, configuration) + _check_sweep_answers_request(report.benchmarks, configuration) if configuration.model.allows_variant_selection: if report.base != configuration.model.api_model_name: raise CLIError("The agent's final report base does not match the requested model") @@ -172,10 +180,18 @@ def _check_workload_answers_request( f"The agent's final benchmark shared prefix of {workload.shared_prefix_tokens}" f" tokens does not match the requested {shared_prefix_tokens}" ) - if configuration.concurrency is not None and workload.concurrency != configuration.concurrency: + + +def _check_sweep_answers_request( + benchmarks: Sequence[PresetBenchmark], configuration: PresetConfiguration +) -> None: + """One benchmark per requested concurrency, no more and no fewer: a missing + point would hide where the service stops meeting the constraints.""" + reported = sorted(b.workload.concurrency for b in benchmarks) + if configuration.concurrency is not None and reported != configuration.concurrency: raise CLIError( - f"The agent's final benchmark concurrency of {workload.concurrency} does not match the" - f" requested concurrency of {configuration.concurrency}" + f"The agent's final benchmark concurrencies {reported} do not match the" + f" requested {configuration.concurrency}" ) diff --git a/src/dstack/_internal/cli/services/presets/workspace.py b/src/dstack/_internal/cli/services/presets/workspace.py index 97ade98b38..7d5b609cf6 100644 --- a/src/dstack/_internal/cli/services/presets/workspace.py +++ b/src/dstack/_internal/cli/services/presets/workspace.py @@ -10,7 +10,7 @@ from contextlib import suppress from dataclasses import dataclass from pathlib import Path -from typing import Optional, Sequence +from typing import Any, Optional, Sequence from dstack._internal.cli.models.preset_agent import PresetSessionWorkspace from dstack._internal.cli.services.presets.session import ( @@ -300,7 +300,7 @@ def _copy_session_records(source_root: Path, target_root: Path) -> bool: for name in (_CONSTRAINTS_FILENAME, _FINAL_REPORT_FILENAME): if (source_root / name).is_file(): target_root.mkdir(parents=True, exist_ok=True) - shutil.copyfile(source_root / name, target_root / name) + _copy_record(source_root / name, target_root / name) copied = True for group, filenames in ( (TRIALS_DIRNAME, ("trial.json", "task.dstack.yml")), @@ -316,7 +316,7 @@ def _copy_session_records(source_root: Path, target_root: Path) -> bool: for name in filenames: if (record_dir / name).is_file(): target_dir.mkdir(parents=True, exist_ok=True) - shutil.copyfile(record_dir / name, target_dir / name) + _copy_record(record_dir / name, target_dir / name) copied = True patches = record_dir / "patches" if group == TRIALS_DIRNAME and patches.is_dir(): @@ -325,6 +325,51 @@ def _copy_session_records(source_root: Path, target_root: Path) -> bool: return copied +# The records an agent reads through `previous/`, upgraded on copy so a +# session recorded before concurrency sweeps reads in the format the agent +# writes. The originals stay as they were written. +_UPGRADED_RECORDS = (_CONSTRAINTS_FILENAME, _FINAL_REPORT_FILENAME, "trial.json") + + +def _copy_record(source: Path, target: Path) -> None: + if source.name not in _UPGRADED_RECORDS: + shutil.copyfile(source, target) + return + try: + data = json.loads(source.read_text(encoding="utf-8")) + except (OSError, ValueError): + # A torn or foreign file is still the agent's to read as written. + shutil.copyfile(source, target) + return + upgraded = _upgrade_pre_sweep_record(source.name, data) + if upgraded is data: + shutil.copyfile(source, target) + return + target.write_text(json.dumps(upgraded, indent=2) + "\n", encoding="utf-8") + + +def _upgrade_pre_sweep_record(name: str, data: Any) -> Any: + # Before the sweep, `constraints.json` held one `concurrency` and trial and + # final records held one `benchmark`: that one point is the whole sweep. + if not isinstance(data, dict): + return data + if name == _CONSTRAINTS_FILENAME: + concurrency = data.get("concurrency") + if isinstance(concurrency, int): + return {**data, "concurrency": [concurrency]} + return data + if "benchmark" not in data or "benchmarks" in data: + return data + upgraded = {} + for key, value in data.items(): + if key == "benchmark": + # `null` means no benchmark could be done; it stays `null`. + upgraded["benchmarks"] = None if value is None else [value] + else: + upgraded[key] = value + return upgraded + + def _install_skills(target_dir: Path) -> None: source_dir = _get_skills_dir() for skill_name in _SKILL_NAMES: diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 6cf8e423ff..bb2caaafa3 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -1774,12 +1774,14 @@ class PresetConfiguration( ), ] = None concurrency: Annotated[ - Optional[PositiveInt], + Optional[list[PositiveInt]], Field( + min_length=1, description=( - "The number of simultaneous requests used for benchmarks during preset" - " creation. Required for creation" - ) + "The numbers of simultaneous requests every benchmark is run at during" + " preset creation: a single value, a list, `2,3`, or a doubling range" + " `2..16` (2, 4, 8, 16). Required for creation" + ), ), ] = None input_tokens: Annotated[ @@ -1866,6 +1868,22 @@ def effective_output_tokens(self) -> int: def effective_baseline(self) -> bool: return self.baseline if self.baseline is not None else DEFAULT_BASELINE + @field_validator( + "concurrency", + mode="before", + json_schema_input_type=Union[PositiveInt, str, list[PositiveInt]], + ) + @classmethod + def parse_concurrency(cls, value: Any) -> Any: + return _parse_concurrency_sweep(value) + + @field_validator("concurrency") + @classmethod + def sort_concurrency(cls, value: Optional[list[int]]) -> Optional[list[int]]: + # Ascending and unique, whichever way it was written: the sweep is compared + # to what the agent reports, and the report is ordered the same way. + return sorted(set(value)) if value is not None else None + @field_validator("dataset") @classmethod def validate_dataset_name(cls, value: Optional[str]) -> Optional[str]: @@ -1946,6 +1964,33 @@ def validate_prompt(cls, value: Any) -> Any: return value +def _parse_concurrency_sweep(value: Any) -> Any: + # `8`, `2,3`, `[2, 3]`, or `2..16`. A range doubles from its start and must + # land exactly on its end, so `3..16` is rejected rather than stopping at 12 + # or overshooting to 24. Element validation is left to the field type. + if value is None or isinstance(value, (bool, list)): + return value + if isinstance(value, int): + return [value] + if not isinstance(value, str): + return value + text = value.replace(" ", "") + if ".." not in text: + return text.split(",") + start_text, _, end_text = text.partition("..") + if not start_text.isdigit() or not end_text.isdigit(): + raise ValueError("a concurrency range needs both ends, e.g. `2..16`") + start, end = int(start_text), int(end_text) + if start < 1 or end < start: + raise ValueError("a concurrency range must go from a positive start up to its end") + values = [start] + while values[-1] < end: + values.append(values[-1] * 2) + if values[-1] != end: + raise ValueError(f"concurrency range end {end} is not reachable by doubling from {start}") + return values + + def _validate_model(value: Any, *, field: str) -> str: if not isinstance(value, str) or not value.strip(): raise ValueError(f"Preset model {field} must be a non-empty string") diff --git a/src/dstack/_internal/core/models/presets.py b/src/dstack/_internal/core/models/presets.py index 95e87fb29f..1258dcb1b1 100644 --- a/src/dstack/_internal/core/models/presets.py +++ b/src/dstack/_internal/core/models/presets.py @@ -128,6 +128,22 @@ def validate_metrics(self) -> Self: return self +def select_benchmark( + benchmarks: Sequence[PresetBenchmark], max_ttft: Optional[int] +) -> PresetBenchmark: + """Picks the sweep point a preset is judged by. + + Rule 1: Keep the points whose p50 TTFT is within `max_ttft`. + Rule 2: If any are kept, pick the one with the highest output throughput. + Rule 3: If none are kept, pick the one with the lowest p50 TTFT, so the + preset records how close it got to the cap rather than how fast it ran. + Rule 4: If two points tie on TTFT in rule 3, pick the higher throughput.""" + passing = [b for b in benchmarks if max_ttft is None or b.metrics.ttft_ms.p50 <= max_ttft] + if passing: + return max(passing, key=lambda b: b.effective_output_tok_per_s) + return min(benchmarks, key=lambda b: (b.metrics.ttft_ms.p50, -b.effective_output_tok_per_s)) + + class PresetVerificationReplicaGroup(CoreModel): # The service replica group this was measured for. name: str @@ -153,16 +169,35 @@ class PortablePreset(CoreModel): # stored as-is. `files` paths are stored relative to the preset directory, # absolute after load. service: ServiceConfiguration + benchmarks: Annotated[List[PresetBenchmark], Field(min_length=1)] + """The final service benchmarked at every concurrency of the sweep, ascending.""" benchmark: PresetBenchmark + """The one of `benchmarks` the preset is judged and listed by, chosen by + `select_benchmark`. Kept as its own field so clients that predate the sweep + keep reading it.""" # The hardware it was verified on: the actual resources of every running # replica, by service replica group. verified_on: List[PresetVerificationReplicaGroup] + @model_validator(mode="before") + @classmethod + def wrap_legacy_single_benchmark(cls, data: Any) -> Any: + # Before the sweep, a preset was benchmarked at one concurrency and + # stored it as `benchmark` alone. That one point is its whole sweep. + if isinstance(data, dict) and "benchmark" in data and "benchmarks" not in data: + data = {**data, "benchmarks": [data["benchmark"]]} + return data + @model_validator(mode="after") def validate_artifact(self) -> Self: service = self.service if service.model is None: raise ValueError("preset service must specify model") + concurrencies = [b.workload.concurrency for b in self.benchmarks] + if concurrencies != sorted(set(concurrencies)): + raise ValueError("preset benchmarks must be at unique, ascending concurrencies") + if self.benchmark not in self.benchmarks: + raise ValueError("preset benchmark must be one of its benchmarks") if any(group.resources is None for group in service.replica_groups): raise ValueError("preset service must specify resources") for field in PRESET_EXCLUDED_FIELDS: @@ -304,7 +339,7 @@ class PresetConstraints(CoreModel): min_context_length: PositiveInt max_ttft: PositiveInt trials_num: PositiveInt - concurrency: PositiveInt + concurrency: Annotated[list[PositiveInt], Field(min_length=1)] baseline: bool fleets: Annotated[list[str], Field(min_length=1)] env: list[str] diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index add3afc533..144d2e1a2b 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -171,6 +171,7 @@ def get_preset( ), context_length=context_length, best_trial=1, + benchmarks=[get_preset_benchmark()], benchmark=get_preset_benchmark(), verified_on=[ PresetVerificationReplicaGroup(name=DEFAULT_REPLICA_GROUP_NAME, replicas=[resources]) @@ -278,7 +279,7 @@ def get_successful_preset_report(run: Run) -> PresetAgentSuccess: "base": "Qwen/Qwen3.5-27B", "model": "community/Qwen3.5-27B-GPTQ-Int4", "context_length": 32768, - "benchmark": get_preset_benchmark(), + "benchmarks": [get_preset_benchmark()], } ) diff --git a/src/tests/_internal/cli/services/presets/test_agent.py b/src/tests/_internal/cli/services/presets/test_agent.py index e55c3fd013..4167331685 100644 --- a/src/tests/_internal/cli/services/presets/test_agent.py +++ b/src/tests/_internal/cli/services/presets/test_agent.py @@ -1642,6 +1642,9 @@ def test_counts_records_even_when_trials_share_a_task(self, tmp_path): "tok_s": 2300.0, "tpot_ms": None, "ttft_ms": None, + "tok_s_range": [2300.0, 2300.0], + "per_user_tok_s_range": None, + "ttft_ms_range": None, "context_length": None, "concurrency": 8, "gpu": "A40:48GB:1", @@ -1684,6 +1687,132 @@ def test_a_trial_still_in_flight_is_not_counted(self, tmp_path): assert summary["count"] == 1 assert summary["series"] == [2000.0] + def test_judges_a_sweep_by_its_fastest_point_within_max_ttft(self, tmp_path): + def point(concurrency, tok_s, ttft_p50): + return { + "workload": {"concurrency": concurrency}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": tok_s, + "ttft_ms": {"p50": ttft_p50}, + }, + } + + record = { + "resources": {}, + # Throughput rises with concurrency, but the top point breaches the cap. + "benchmarks": [point(2, 500.0, 100.0), point(4, 800.0, 300.0), point(8, 900.0, 900.0)], + } + trials_dir = _write_trials(tmp_path, [record]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + assert summary["series"] == [800.0] + best = summary["best"] + assert best is not None + assert best["concurrency"] == 4 + assert best["ttft_ms"] == 300.0 + # The selected point is c=4; the ranges still cover every point. + assert best["ttft_ms_range"] == [100.0, 900.0] + + def test_ranges_come_from_the_selected_trial_only(self, tmp_path): + def point(concurrency, tokens, ttft, tpot): + return { + "workload": {"concurrency": concurrency}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": tokens, + "ttft_ms": {"p50": ttft}, + "tpot_ms": {"mean": tpot}, + }, + } + + slow = {"benchmarks": [point(4, 100.0, 50.0, 10.0), point(8, 200.0, 80.0, 20.0)]} + fast = {"benchmarks": [point(4, 300.0, 400.0, 30.0), point(8, 600.0, 450.0, 40.0)]} + trials_dir = _write_trials(tmp_path, [slow, fast]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + best = summary["best"] + assert best is not None + assert best["concurrency"] == 8 + assert best["ttft_ms_range"] == [400.0, 450.0] + assert best["per_user_tok_s_range"] == [25.0, 1000 / 30.0] + + def test_a_partial_sweep_ranges_over_its_usable_points(self, tmp_path): + usable = { + "workload": {"concurrency": 4}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": 100.0, + "ttft_ms": {"p50": 200.0}, + "tpot_ms": {"mean": 20.0}, + }, + } + broken = {"workload": {"concurrency": 8}, "metrics": {"duration_seconds": 0}} + trials_dir = _write_trials(tmp_path, [{"benchmarks": [usable, broken]}]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + best = summary["best"] + assert best is not None + assert best["ttft_ms_range"] == [200.0, 200.0] + assert best["per_user_tok_s_range"] == [50.0, 50.0] + + def test_a_pre_sweep_record_has_a_single_point_range(self, tmp_path): + record = { + "benchmark": { + "workload": {"concurrency": 8}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": 100.0, + "ttft_ms": {"p50": 240.0}, + "tpot_ms": {"mean": 25.0}, + }, + } + } + trials_dir = _write_trials(tmp_path, [record]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + best = summary["best"] + assert best is not None + assert best["ttft_ms_range"] == [240.0, 240.0] + + def test_judges_a_sweep_that_breached_everywhere_by_its_closest_point(self, tmp_path): + record = { + "benchmarks": [ + { + "workload": {"concurrency": 4}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": 800.0, + "ttft_ms": {"p50": 700.0}, + }, + }, + { + "workload": {"concurrency": 8}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": 900.0, + "ttft_ms": {"p50": 900.0}, + }, + }, + ] + } + trials_dir = _write_trials(tmp_path, [record]) + + summary = _summarize_session_trials(trials_dir, max_ttft=500) + assert summary is not None + + best = summary["best"] + assert best is not None + assert best["concurrency"] == 4 + def test_a_torn_result_copy_reads_as_still_in_flight(self, tmp_path): trials_dir = _write_trials(tmp_path, []) (trials_dir / "1").mkdir(parents=True) diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index c6203023f4..c4c4dcf09e 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -863,7 +863,7 @@ def test_renders_defaults_for_the_optional_fields(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": 8, + "concurrency": [8], "input_tokens": 1024, "output_tokens": 1024, "shared_prefix_tokens": 0, @@ -897,7 +897,7 @@ def test_renders_custom_dataset_without_request_shape(self): "min_context_length": 32768, "max_ttft": 5000, "trials_num": 3, - "concurrency": 8, + "concurrency": [8], "dataset": "spec_bench", "baseline": True, "fleets": ["gpu-fleet"], @@ -927,7 +927,7 @@ def test_renders_configured_values(self): assert data["model"] == {"repo": "Qwen/Qwen3-32B-AWQ", "name": "qwen3"} assert data["min_context_length"] == 32768 assert data["trials_num"] == 10 - assert data["concurrency"] == 16 + assert data["concurrency"] == [16] assert data["fleets"] == ["a", "b"] diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index e5e963388b..749b971b5c 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -32,6 +32,7 @@ def test_formats_second_scale_ttft_without_scientific_notation(self): ttft.mean = 8148.3 ttft.p50 = 8151.4 ttft.p99 = 8334.2 + preset.benchmarks = [preset.benchmark] output = output_module.format_preset_benchmark(preset, verbose=True) @@ -42,6 +43,28 @@ def test_formats_second_scale_ttft_without_scientific_notation(self): assert "ttft=8.15s" in output assert "e+03" not in output + def test_shows_a_sweep_as_ranges_and_names_the_selected_point(self): + preset = get_preset() + points = [] + for concurrency, tpot, ttft, tokens in ((4, 30.0, 425.0, 1024), (16, 52.0, 861.0, 4096)): + point = preset.benchmark.model_copy(deep=True) + point.workload.concurrency = concurrency + point.metrics.tpot_ms.mean = tpot + point.metrics.ttft_ms.p50 = ttft + point.metrics.total_output_tokens = tokens + points.append(point) + preset.benchmarks = points + preset.benchmark = points[1] + + output = output_module.format_preset_benchmark(preset, verbose=True) + + assert output == "tps/user=19.2..33.3 c=16 tps=21.1..84.2 ttft=425ms..861ms ctx=32K" + + def test_shows_single_values_without_a_sweep(self): + output = output_module.format_preset_benchmark(get_preset()) + + assert output == "tps/user=133 ttft=108ms ctx=32K" + class TestFormatPresetObjective: def test_shows_the_requested_shared_prefix(self): @@ -140,7 +163,7 @@ def test_prints_submitted_column(self, monkeypatch): assert "2 months ago" in output.getvalue() -def _session_row(session: dict) -> dict: +def _session_row(session: dict, *, verbose: bool = False) -> dict: table = Table(box=None) for column in ( "BASE", @@ -154,7 +177,7 @@ def _session_row(session: dict) -> dict: "SUBMITTED", ): table.add_column(column) - _add_session(table, session) + _add_session(table, session, verbose=verbose) return { column.header: "".join(str(cell) for cell in column._cells) for column in table.columns } @@ -171,7 +194,7 @@ def test_shows_progress_after_status_and_best_benchmark(self): "count": 2, "best": { "tok_s": 2339.0, - "tpot_ms": 3.42, + "per_user_tok_s_range": [292.0, 292.0], "concurrency": 8, "gpu": "A40:48GB:1", }, @@ -202,6 +225,84 @@ def test_shows_the_shared_prefix_when_the_workload_has_one(self): assert row["CONSTRAINTS"] == ("[secondary]io=8K/1K prefix=90% c=162[/]") + def test_shows_a_sweep_and_the_point_the_best_trial_is_judged_by(self): + row = _session_row( + { + "id": "c7e18d52", + "status": "running", + "constraints": {"dataset": "spec_bench", "concurrency": [2, 4, 8, 16]}, + "trials": { + "count": 1, + "best": { + "tok_s": 2339.0, + "per_user_tok_s_range": [292.0, 292.0], + "concurrency": 8, + "gpu": None, + }, + }, + } + ) + + assert row["CONSTRAINTS"] == "[secondary]data=spec_bench c=2..16[/]" + assert row["BENCHMARK"].startswith("tps/user=292 c=8") + + def test_shows_a_sweep_session_as_ranges(self): + row = _session_row( + { + "id": "6ac0bec5", + "status": "interrupted", + "constraints": {"concurrency": [4, 8, 16]}, + "trials": { + "count": 1, + "best": { + "tok_s": 380.0, + "tok_s_range": [132.0, 380.0], + "per_user_tok_s_range": [24.3, 33.3], + "ttft_ms_range": [425.0, 861.0], + "context_length": 98304, + "concurrency": 16, + }, + }, + }, + verbose=True, + ) + + assert row["BENCHMARK"] == ( + "tps/user=24.3..33.3 c=16 tps=132..380 ttft=425ms..861ms ctx=96K" + ) + + def test_a_failed_sweep_keeps_its_ranges_and_its_mark(self): + row = _session_row( + { + "id": "6ac0bec5", + "status": "failed", + "constraints": {"concurrency": [4, 8]}, + "trials": { + "count": 1, + "best": None, + "best_failed": { + "tok_s": 200.0, + "per_user_tok_s_range": [25.0, 33.3], + "ttft_ms_range": [900.0, 1100.0], + "concurrency": 4, + }, + }, + } + ) + + assert row["BENCHMARK"] == "[secondary]*tps/user=25..33.3 c=4 ttft=900ms..1.1s[/]" + + def test_shows_a_sweep_that_does_not_double_as_a_list(self): + row = _session_row( + { + "id": "c7e18d52", + "status": "running", + "constraints": {"dataset": "spec_bench", "concurrency": [2, 3]}, + } + ) + + assert row["CONSTRAINTS"] == "[secondary]data=spec_bench c=2,3[/]" + def test_shows_the_dataset_for_a_session_with_a_custom_dataset(self): row = _session_row( { @@ -511,8 +612,8 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): "best": None, "best_failed": { "tok_s": 100.0, - "tpot_ms": 34.4, - "ttft_ms": 4300.0, + "per_user_tok_s_range": [29.1, 29.1], + "ttft_ms_range": [4300.0, 4300.0], "context_length": 1048576, "concurrency": 4, "gpu": "MI300X:192GB:1", @@ -527,6 +628,50 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): # styling is absent or invisible to the reader. assert "*tps/user" in row["BENCHMARK"] + def test_a_sweep_trial_record_reaches_the_row_as_ranges(self, tmp_path): + from dstack._internal.cli.services.presets.session import _summarize_session_trials + + # The summary writes the keys the row reads; only both together prove it. + def point(concurrency, tokens, ttft, tpot): + return { + "workload": {"concurrency": concurrency}, + "metrics": { + "duration_seconds": 1.0, + "total_output_tokens": tokens, + "ttft_ms": {"p50": ttft}, + "tpot_ms": {"mean": tpot}, + }, + } + + trials_dir = _write_trials( + tmp_path, + [ + { + "context_length": 98304, + "benchmarks": [ + point(4, 132.0, 425.0, 30.0), + point(8, 237.0, 691.0, 34.0), + point(16, 380.0, 861.0, 41.2), + ], + } + ], + ) + + trials = _summarize_session_trials(trials_dir, max_ttft=2000) + row = _session_row( + { + "id": "6ac0bec5", + "status": "interrupted", + "constraints": {"concurrency": [4, 8, 16], "max_ttft": 2000}, + "trials": trials, + }, + verbose=True, + ) + + assert row["BENCHMARK"] == ( + "tps/user=24.3..33.3 c=16 tps=132..380 ttft=425ms..861ms ctx=96K" + ) + class TestFailedTrialSpark: def test_a_trial_that_broke_a_constraint_is_charted_but_not_the_best(self): diff --git a/src/tests/_internal/cli/services/presets/test_report_schema.py b/src/tests/_internal/cli/services/presets/test_report_schema.py index 9298e06d2a..0e9eb437e7 100644 --- a/src/tests/_internal/cli/services/presets/test_report_schema.py +++ b/src/tests/_internal/cli/services/presets/test_report_schema.py @@ -19,7 +19,7 @@ "base": None, "model": None, "context_length": None, - "benchmark": None, + "benchmarks": None, "failure_summary": None, } # Keywords the OpenAI strict structured-output mode rejects. @@ -50,6 +50,15 @@ def _walk(node): yield from _walk(item) +class TestGetReportJsonSchema: + def test_offers_only_the_sweep_shape(self): + # The legacy `benchmark` is accepted at parse only; agents are never offered it. + properties = get_report_json_schema()["properties"] + + assert "benchmarks" in properties + assert "benchmark" not in properties + + class TestGetStrictReportJsonSchema: def test_every_object_requires_all_of_its_properties(self): objects = [ @@ -106,31 +115,33 @@ def test_keeps_the_agent_facing_shape(self): "base": "Qwen/Qwen2.5-0.5B-Instruct", "model": "Qwen/Qwen2.5-0.5B-Instruct", "context_length": 2048, - "benchmark": { - "tool": "vllm.bench serve", - "tool_version": "0.27.0", - "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", - "workload": { - "api": "chat_completions", - "dataset": None, - "num_requests": 32, - "input_tokens": 128, - "output_tokens": 64, - "concurrency": 1, - "shared_prefix_tokens": 0, - }, - "metrics": { - "successful_requests": 32, - "failed_requests": 0, - "duration_seconds": 40.5, - "total_input_tokens": 4096, - "total_output_tokens": 2048, - "output_tok_per_s": 50.6, - "per_user_tok_per_s": 50.6, - "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, - "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, - }, - }, + "benchmarks": [ + { + "tool": "vllm.bench serve", + "tool_version": "0.27.0", + "command": "vllm bench serve --model Qwen/Qwen2.5-0.5B-Instruct", + "workload": { + "api": "chat_completions", + "dataset": None, + "num_requests": 32, + "input_tokens": 128, + "output_tokens": 64, + "concurrency": 1, + "shared_prefix_tokens": 0, + }, + "metrics": { + "successful_requests": 32, + "failed_requests": 0, + "duration_seconds": 40.5, + "total_input_tokens": 4096, + "total_output_tokens": 2048, + "output_tok_per_s": 50.6, + "per_user_tok_per_s": 50.6, + "ttft_ms": {"mean": 210.0, "p50": 200.0, "p99": 300.0}, + "tpot_ms": {"mean": 19.8, "p50": 19.5, "p99": 25.0}, + }, + } + ], }, PresetAgentSuccess, ), @@ -146,5 +157,5 @@ def test_reports_codex_produced_against_the_schema_parse(self, data, expected): assert isinstance(report, expected) if isinstance(report, PresetAgentSuccess): assert isinstance(report.service_yaml, ServiceConfiguration) - assert report.benchmark.workload.dataset is None - assert report.benchmark.workload.shared_prefix_tokens == 0 + assert report.benchmarks[0].workload.dataset is None + assert report.benchmarks[0].workload.shared_prefix_tokens == 0 diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index fb774a68ff..0dbcc66929 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -383,7 +383,7 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): assert preset.configuration.input_tokens == 10000 assert preset.configuration.output_tokens == 1500 assert preset.configuration.shared_prefix_tokens == 0 - assert preset.configuration.concurrency == 4 + assert preset.configuration.concurrency == [4] # `priority` is not an excluded field, so the regrouping keeps it. assert preset.service.priority == 0 assert [group.name for group in preset.verified_on] == [ @@ -392,6 +392,8 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): assert get_or_error(preset.verified_on[0].replicas[0].gpu).name == ["MI300X"] # The old format never recorded a dataset name for a synthetic workload. assert preset.benchmark.workload.dataset is None + # One concurrency, so the one benchmark is the whole sweep. + assert preset.benchmarks == [preset.benchmark] assert store.list() == [preset] def test_upgrade_maps_validation_replicas_to_replica_groups(self, tmp_path: Path): diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index c7a4d9ea80..97e2ab1321 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -4,7 +4,11 @@ import pytest from pydantic import ValidationError -from dstack._internal.cli.models.preset_agent import AnyPresetAgentResult, PresetAgentSuccess +from dstack._internal.cli.models.preset_agent import ( + AnyPresetAgentResult, + PresetAgentFailure, + PresetAgentSuccess, +) from dstack._internal.cli.services.presets.agent import ( PresetAgentProcessOutput, ) @@ -61,6 +65,7 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): base_model="Qwen/Qwen3.5-27B", repo="community/Qwen3.5-27B-GPTQ-Int4", context_length=32768, + benchmarks=[get_preset_benchmark()], benchmark=get_preset_benchmark(), configuration=configuration, best_trial=1, @@ -76,12 +81,12 @@ def test_stores_only_the_creation_contract_not_this_machine_s_deployment(self): assert preset.configuration.spot_policy is None assert "secret" not in preset.model_dump_json() - def test_successful_report_requires_benchmark(self): + def test_successful_report_requires_benchmarks(self): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() - data.pop("benchmark") + data.pop("benchmarks") - with pytest.raises(ValidationError, match="benchmark"): + with pytest.raises(ValidationError, match="benchmarks"): validate_extra_ignore(AnyPresetAgentResult, data) def test_builds_portable_self_contained_preset(self, tmp_path): @@ -222,11 +227,11 @@ def test_rejects_a_benchmark_without_the_requested_shared_prefix(self, tmp_path) created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): + def test_rejects_benchmarks_at_other_concurrencies(self, tmp_path): run = get_running_service_run() with pytest.raises( - CLIError, match="concurrency of 8 does not match the requested concurrency of 4" + CLIError, match=r"concurrencies \[8\] do not match the requested \[4\]" ): build_verified_preset( run=run, @@ -239,6 +244,44 @@ def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) + def test_rejects_a_partial_sweep(self, tmp_path): + run = get_running_service_run() + + with pytest.raises( + CLIError, match=r"concurrencies \[2, 4\] do not match the requested \[2, 4, 8\]" + ): + build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(concurrency="2..8"), + report=_sweep_report(run, [2, 4]), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_stores_every_point_and_selects_the_fastest_within_max_ttft(self, tmp_path): + run = get_running_service_run() + + preset = build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(concurrency="2..8", max_ttft=500), + # Reported out of order; throughput rises with concurrency, but the top + # point breaches the TTFT cap. + report=_sweep_report( + run, [8, 2, 4], duration_seconds={2: 8.0, 4: 4.0, 8: 2.0}, ttft_p50={8: 900.0} + ), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + assert [b.workload.concurrency for b in preset.benchmarks] == [2, 4, 8] + assert preset.benchmark.workload.concurrency == 4 + def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): run = get_running_service_run() report = _dataset_report(run, dataset="spec_bench") @@ -346,22 +389,22 @@ def test_redacts_known_secret_in_benchmark_command_instead_of_failing(self, tmp_ run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["benchmarks"][0]["command"] = ( "python bench.py --header 'Authorization: Bearer sk-live-0123456789abcdef'" ) report = self._load(tmp_path, data, redacted_values=("sk-live-0123456789abcdef",)) assert isinstance(report, PresetAgentSuccess) - assert report.benchmark is not None - assert report.benchmark.command.endswith("Bearer [redacted]'") - assert "sk-live" not in report.benchmark.command + assert report.benchmarks + assert report.benchmarks[0].command.endswith("Bearer [redacted]'") + assert "sk-live" not in report.benchmarks[0].command def test_still_rejects_unknown_bearer_token(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["benchmarks"][0]["command"] = ( "curl -H 'Authorization: Bearer sk-unknown-9876543210fedcba'" ) @@ -374,7 +417,7 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): run = get_running_service_run() data = get_successful_preset_report(run).model_dump() data["run_id"] = str(data["run_id"]) - data["benchmark"]["command"] = ( + data["benchmarks"][0]["command"] = ( "./benchenv/bin/python bench_service.py --base $DSTACK_SERVER_URL/x" " (auth via DSTACK_TOKEN bearer header from env)" ) @@ -382,11 +425,46 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): report = self._load(tmp_path, data, redacted_values=()) assert isinstance(report, PresetAgentSuccess) - assert report.benchmark is not None - assert "bearer header" in report.benchmark.command + assert report.benchmarks + assert "bearer header" in report.benchmarks[0].command + + def test_reads_a_pre_sweep_success_as_a_one_point_sweep(self, tmp_path): + # A session detached under a pre-sweep CLI finishes with that CLI's schema. + run = get_running_service_run() + data = get_successful_preset_report(run).model_dump() + data["run_id"] = str(data["run_id"]) + data["benchmark"] = data.pop("benchmarks")[0] + + report = self._load(tmp_path, data, redacted_values=()) + + assert isinstance(report, PresetAgentSuccess) + assert [b.workload.concurrency for b in report.benchmarks] == [1] + + def test_reads_a_sweep_success_unchanged(self, tmp_path): + run = get_running_service_run() + data = get_successful_preset_report(run).model_dump() + data["run_id"] = str(data["run_id"]) + second = {**data["benchmarks"][0], "workload": {**data["benchmarks"][0]["workload"]}} + second["workload"]["concurrency"] = 2 + data["benchmarks"].append(second) + + report = self._load(tmp_path, data, redacted_values=()) + + assert isinstance(report, PresetAgentSuccess) + assert [b.workload.concurrency for b in report.benchmarks] == [1, 2] + + def test_leaves_a_failure_report_alone(self, tmp_path): + report = self._load( + tmp_path, + {"success": False, "failure_summary": "no configuration met max_ttft"}, + redacted_values=(), + ) + + assert isinstance(report, PresetAgentFailure) + assert report.failure_summary == "no configuration met max_ttft" -def _shared_prefix_configuration() -> PresetConfiguration: +def _shared_prefix_configuration(**overrides: Any) -> PresetConfiguration: """The configuration from dstackai/dstack#4198: a shared prefix and no dataset.""" return PresetConfiguration.model_validate( { @@ -399,6 +477,7 @@ def _shared_prefix_configuration() -> PresetConfiguration: "input_tokens": 131072, "output_tokens": 512, "shared_prefix_tokens": 130048, + **overrides, } ) @@ -407,6 +486,29 @@ def _shared_prefix_report(run: Run, **workload: Any) -> PresetAgentSuccess: return _report_with_workload(run, {**SHARED_PREFIX_WORKLOAD, **workload}) +def _sweep_report( + run: Run, + concurrencies: list[int], + *, + duration_seconds: dict[int, float] | None = None, + ttft_p50: dict[int, float] | None = None, +) -> PresetAgentSuccess: + """One shared-prefix benchmark per concurrency, in the given order. The + metrics are the fixture's unless overridden per concurrency.""" + benchmarks = [] + for concurrency in concurrencies: + benchmark = get_preset_benchmark() + benchmark.workload = PresetWorkload.model_validate( + {**SHARED_PREFIX_WORKLOAD, "concurrency": concurrency} + ) + if duration_seconds and concurrency in duration_seconds: + benchmark.metrics.duration_seconds = duration_seconds[concurrency] + if ttft_p50 and concurrency in ttft_p50: + benchmark.metrics.ttft_ms.p50 = ttft_p50[concurrency] + benchmarks.append(benchmark) + return get_successful_preset_report(run).model_copy(update={"benchmarks": benchmarks}) + + def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: # A dataset defines the request shape, so the workload records what it measured. return _report_with_workload( @@ -425,4 +527,4 @@ def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: def _report_with_workload(run: Run, workload: dict[str, Any]) -> PresetAgentSuccess: benchmark = get_preset_benchmark() benchmark.workload = PresetWorkload.model_validate(workload) - return get_successful_preset_report(run).model_copy(update={"benchmark": benchmark}) + return get_successful_preset_report(run).model_copy(update={"benchmarks": [benchmark]}) diff --git a/src/tests/_internal/cli/services/presets/test_workspace.py b/src/tests/_internal/cli/services/presets/test_workspace.py index a10832688e..3c60ab227a 100644 --- a/src/tests/_internal/cli/services/presets/test_workspace.py +++ b/src/tests/_internal/cli/services/presets/test_workspace.py @@ -1,3 +1,5 @@ +import json + import pytest from dstack._internal.cli.services.presets.session import PresetSession @@ -81,3 +83,60 @@ def test_a_session_without_records_warns(self, tmp_path, capsys): assert "empty000 has no records" in capsys.readouterr().err assert not (workspace.path / "previous" / "empty000").exists() + + def test_upgrades_pre_sweep_records_to_the_sweep_format(self, tmp_path): + session = _previous_session(tmp_path) + root = session.path + benchmark = {"workload": {"concurrency": 8}, "metrics": {}} + (root / "constraints.json").write_text(json.dumps({"max_ttft": 2000, "concurrency": 8})) + (root / "trials" / "1" / "trial.json").write_text( + json.dumps({"context_length": 8192, "benchmark": benchmark, "learned": "x"}) + ) + (root / "trials" / "2").mkdir() + # A trial that never produced a benchmark. + (root / "trials" / "2" / "trial.json").write_text( + json.dumps({"benchmark": None, "learned": "y", "failed": True}) + ) + (root / "final_report.json").write_text( + json.dumps({"success": True, "benchmark": benchmark}) + ) + workspace = _workspace(tmp_path) + + install_previous_records(workspace, [session]) + + target = workspace.path / "previous" / "8d3b01aa" + assert json.loads((target / "constraints.json").read_text()) == { + "max_ttft": 2000, + "concurrency": [8], + } + trial = json.loads((target / "trials" / "1" / "trial.json").read_text()) + # Same position as the field it replaces. + assert list(trial) == ["context_length", "benchmarks", "learned"] + assert trial["benchmarks"] == [benchmark] + failed = json.loads((target / "trials" / "2" / "trial.json").read_text()) + assert failed["benchmarks"] is None + report = json.loads((target / "final_report.json").read_text()) + assert report == {"success": True, "benchmarks": [benchmark]} + # The session's own records are not rewritten. + assert json.loads((root / "constraints.json").read_text())["concurrency"] == 8 + + def test_copies_sweep_records_unchanged(self, tmp_path): + session = _previous_session(tmp_path) + text = '{"benchmarks": [{"workload": {"concurrency": 4}}], "learned": "x"}' + (session.path / "trials" / "1" / "trial.json").write_text(text) + workspace = _workspace(tmp_path) + + install_previous_records(workspace, [session]) + + copied = workspace.path / "previous" / "8d3b01aa" / "trials" / "1" / "trial.json" + assert copied.read_text() == text + + def test_copies_an_unreadable_record_as_written(self, tmp_path): + session = _previous_session(tmp_path) + (session.path / "trials" / "1" / "trial.json").write_text('{"benchmark": {"met') + workspace = _workspace(tmp_path) + + install_previous_records(workspace, [session]) + + copied = workspace.path / "previous" / "8d3b01aa" / "trials" / "1" / "trial.json" + assert copied.read_text() == '{"benchmark": {"met' diff --git a/src/tests/_internal/core/models/test_configurations.py b/src/tests/_internal/core/models/test_configurations.py index a77656c43d..ec9f35a65e 100644 --- a/src/tests/_internal/core/models/test_configurations.py +++ b/src/tests/_internal/core/models/test_configurations.py @@ -1344,6 +1344,49 @@ def test_defaults_to_a_synthetic_workload(self): assert configuration.dataset is None +class TestPresetConfigurationConcurrency: + @pytest.mark.parametrize( + ("value", "expected"), + [ + pytest.param(10, [10], id="int"), + pytest.param("2,3", [2, 3], id="comma"), + pytest.param([3, 2, 2], [2, 3], id="list-sorted-deduped"), + pytest.param("2..16", [2, 4, 8, 16], id="range"), + pytest.param("8..8", [8], id="range-single"), + pytest.param(" 2 .. 8 ", [2, 4, 8], id="range-spaces"), + ], + ) + def test_normalizes_to_list(self, value, expected): + conf = PresetConfiguration.model_validate( + {"type": "preset", "base": "m", "concurrency": value} + ) + + assert conf.concurrency == expected + + @pytest.mark.parametrize( + ("value", "error"), + [ + pytest.param("3..16", "not reachable by doubling from 3", id="range-off-chain"), + pytest.param("..16", "needs both ends", id="range-no-start"), + pytest.param("2..", "needs both ends", id="range-no-end"), + pytest.param("16..2", "up to its end", id="range-descending"), + pytest.param([], "at least 1", id="empty-list"), + pytest.param(0, "greater than 0", id="zero"), + pytest.param("2,x", "valid integer", id="comma-garbage"), + ], + ) + def test_rejects(self, value, error): + with pytest.raises(ValidationError, match=error): + PresetConfiguration.model_validate( + {"type": "preset", "base": "m", "concurrency": value} + ) + + def test_stays_unset_when_omitted(self): + conf = PresetConfiguration.model_validate({"type": "preset", "base": "m"}) + + assert conf.concurrency is None + + class TestPresetConfigurationSchema: def test_schema_does_not_require_model(self): # `model` is filled from the `base`/`repo` shorthand by a before-validator, diff --git a/src/tests/_internal/core/models/test_presets.py b/src/tests/_internal/core/models/test_presets.py index e12a881663..bd1f8133d3 100644 --- a/src/tests/_internal/core/models/test_presets.py +++ b/src/tests/_internal/core/models/test_presets.py @@ -5,6 +5,7 @@ from dstack._internal.core.models.presets import ( PresetBenchmark, + select_benchmark, validate_preset_file_path, validate_preset_file_paths, ) @@ -152,3 +153,32 @@ def test_keeps_a_dataset_named_random_without_a_prefix(self): benchmark = PresetBenchmark.model_validate(data) assert benchmark.workload.dataset == "random" + + +def _benchmark(*, concurrency: int, tok_s: float, ttft_p50: float) -> PresetBenchmark: + data = get_benchmark_data(get_workload_data(concurrency=concurrency)) + # The fixture's 2048 output tokens over `duration` give `tok_s`. + data["metrics"]["duration_seconds"] = 2048 / tok_s + data["metrics"]["ttft_ms"]["p50"] = ttft_p50 + return PresetBenchmark.model_validate(data) + + +class TestSelectBenchmark: + def test_picks_the_fastest_point_within_max_ttft(self): + over = _benchmark(concurrency=16, tok_s=600, ttft_p50=900) + ok = _benchmark(concurrency=8, tok_s=500, ttft_p50=300) + + assert select_benchmark([ok, over], max_ttft=500) is ok + + def test_falls_back_to_the_closest_ttft_when_none_passes(self): + # Everything breached: the record is how near it got, not how fast it ran. + closer = _benchmark(concurrency=8, tok_s=500, ttft_p50=900) + faster = _benchmark(concurrency=16, tok_s=600, ttft_p50=950) + + assert select_benchmark([closer, faster], max_ttft=500) is closer + + def test_breaks_a_ttft_tie_by_throughput(self): + slow = _benchmark(concurrency=8, tok_s=500, ttft_p50=900) + fast = _benchmark(concurrency=16, tok_s=600, ttft_p50=900) + + assert select_benchmark([slow, fast], max_ttft=500) is fast From 8e1efac477409a29761c5ddef091e732d46fb3ba Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Thu, 5 Nov 2026 15:44:51 +0545 Subject: [PATCH 2/5] Minor Update --- .../_internal/cli/models/preset_agent.py | 6 ++---- .../_internal/cli/services/presets/output.py | 8 +++---- .../presets/resources/system_prompt.md | 3 +-- .../_internal/cli/services/presets/session.py | 21 ++++++++++++------- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/dstack/_internal/cli/models/preset_agent.py b/src/dstack/_internal/cli/models/preset_agent.py index 918e622737..c6574aa7bb 100644 --- a/src/dstack/_internal/cli/models/preset_agent.py +++ b/src/dstack/_internal/cli/models/preset_agent.py @@ -59,10 +59,8 @@ class PresetAgentSuccess(CoreModel): @model_validator(mode="before") @classmethod def wrap_legacy_single_benchmark(cls, data: Any) -> Any: - # An agent process started by a pre-sweep CLI reports one `benchmark`, and - # a session detached before the upgrade is attached and verified by the - # new CLI. The one point is its whole sweep; the sweep check in `verify` - # still requires it to be the requested concurrency. + # Older agents report a single `benchmark`. Wrap it in a list so + # the new CLI can read reports from sessions started before the upgrade. if isinstance(data, dict) and "benchmark" in data and "benchmarks" not in data: data = dict(data) data["benchmarks"] = [data.pop("benchmark")] diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index f5cf9fde07..3e75d07e50 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -214,7 +214,7 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False per_user = best.get("per_user_tok_s_range") if per_user: parts.append(f"tps/user={_format_range(per_user, _format_number)}") - # Which point of a sweep the numbers are from; redundant for one value. + # Show the selected concurrency when the sweep has multiple values. if isinstance(sweep, list) and len(sweep) > 1 and best.get("concurrency"): parts.append(f"c={best['concurrency']}") if verbose: @@ -349,8 +349,7 @@ def _breaches_constraints(preset: AnyStoredPreset) -> bool: def format_preset_benchmark(preset: AnyStoredPreset, *, verbose: bool = False) -> str: - # A sweep shows each metric as its range across concurrencies, and `c=` names - # the selected point. + # Show benchmark ranges across the sweep and identify the selected concurrency. benchmarks = preset.benchmarks parts = [ "tps/user=" @@ -379,8 +378,7 @@ def _format_range(values: list[float], fmt: Callable[[float], str]) -> str: def _format_concurrency(value: Any) -> str: - """A sweep in the shape the user wrote it: `8`, `2,3`, or `2..16` when every - step doubles. A pre-sweep record carries a bare int.""" + """Format concurrency values for display, for example: 8, 2,3, or 2..16.""" if not isinstance(value, list): return str(value) if len(value) > 1 and all(b == a * 2 for a, b in zip(value, value[1:])): diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 79605eeec3..80a282c185 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -333,8 +333,7 @@ During trials, run benchmarks via SSH inside the task, directly against the serving engine: run one benchmark per value of `concurrency`, using `dataset``input_tokens`, `output_tokens`, and `shared_prefix_tokens` from `constraints.json`, and measure all trials the same -way so that their results are comparable with each other. Use a different -seed for each point of the sweep. +way so that their results are comparable with each other. In case the task is using a router (see `## Cluster Placement`), run benchmarks via SSH inside the router node, directly against the router engine, never against a diff --git a/src/dstack/_internal/cli/services/presets/session.py b/src/dstack/_internal/cli/services/presets/session.py index dc09cc2b5d..e56eeadd28 100644 --- a/src/dstack/_internal/cli/services/presets/session.py +++ b/src/dstack/_internal/cli/services/presets/session.py @@ -642,11 +642,16 @@ def _summarize_session_trials( def _select_trial_point( record: dict[str, Any], max_ttft: Optional[float] ) -> Optional[tuple[float, dict[str, Any], dict[str, Any], list[dict[str, Any]]]]: - """The sweep point a trial is judged by, as `select_benchmark` picks it for a - preset: the highest throughput within `max_ttft`, else the closest TTFT. - A record from before the sweep holds a single `benchmark`. Returns - `(tok_s, metrics, workload, sweep)`, where `sweep` is the metrics of every - point with usable numbers, or `None` when there is none.""" + """Select the benchmark point used to judge a trial. + + Choose the highest-throughput point whose p50 TTFT meets `max_ttft`. + If no point meets the limit, choose the point with the lowest p50 TTFT. + Older trial records may contain a single `benchmark` instead of `benchmarks`. + + Return `(tok_s, metrics, workload, sweep)`, where `metrics` and `workload` + describe the selected point and `sweep` contains metrics for all usable + points. Return `None` if no point has usable throughput data. + """ benchmarks = record.get("benchmarks") if not isinstance(benchmarks, list): benchmarks = [record.get("benchmark")] @@ -698,8 +703,10 @@ def _trial_entry( "tok_s": tok_s, "tpot_ms": tpot if isinstance(tpot, (int, float)) and tpot > 0 else None, "ttft_ms": ttft if isinstance(ttft, (int, float)) else None, - # The trial's spread across its sweep, as `[low, high]`, None when no - # point reports the value. One point gives `low == high`. + # Store the lowest and highest throughput and TTFT values across the trial's + # concurrency points. For example, throughput values 132, 237, and 380 + # tokens/sec become [132, 380]. One value becomes [value, value]; no values + # becomes None. "tok_s_range": _range([m["total_output_tokens"] / m["duration_seconds"] for m in sweep]), "per_user_tok_s_range": _range( [1000 / v for v in (_latency(m, "tpot_ms", "mean") for m in sweep) if v] From 7533695ee44387f0551e91fec41a67e44eec26fb Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Sun, 8 Nov 2026 08:34:22 +0545 Subject: [PATCH 3/5] Update system_prompt.md Update system_prompt.md Update system_prompt.md Update system_prompt.md Update system_prompt.md Update system_prompt.md --- .../presets/resources/system_prompt.md | 116 ++++++++++-------- 1 file changed, 68 insertions(+), 48 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 80a282c185..0c5276a0b3 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -5,21 +5,9 @@ hardware, through sequential experimental trials. ## Performance -Performance is the highest `output_tok_per_s` a configuration reaches across -the `concurrency` sweep from `constraints.json` (see `## Benchmark`), counting -only sweep points that meet `max_ttft`, the maximum p50 time to first token, in -milliseconds (see `# Constraints`). Points that breach `max_ttft` are ignored. -For example, with `max_ttft` of 2000: - -| `concurrency` | `output_tok_per_s` | `ttft_ms.p50` | -|---|---|---| -| 4 | 130 | 400 | -| 8 | 240 | 1500 | -| 16 | 380 | 2400 | - -The point at 16 breaches `max_ttft`, so it is ignored. Performance is 240, the -highest throughput among the remaining points. Benchmark tools report several -throughput numbers; performance is the output one. +Performance is `output_tok_per_s`, the output token throughput of a benchmark +run at the `concurrency` from `constraints.json` (see `## Benchmark`). Benchmark +tools report several throughput numbers; performance is the output one. Significantly better performance goes beyond what conventional configurations and published benchmarks reach; they are not the ceiling. Estimate the @@ -47,9 +35,10 @@ Field semantics: - `max_ttft`: the maximum p50 time to first token, in milliseconds, that any benchmark in this session may report. - `trials_num`: the number of trials in this session. -- `concurrency`: the numbers of simultaneous requests every benchmark in - this session is run at, one benchmark per value. The list is fixed so - that benchmark results are comparable across trials. +- `concurrency`: a list of simultaneous request counts used for every + benchmark in this session. For example, `[2, 4, 8]` means running the + benchmark once each at 2, 4, and 8 simultaneous requests. The list is + fixed so that benchmark results are comparable across trials. - `dataset`: the benchmark dataset for every benchmark in this session; see `## Benchmark`. @@ -257,7 +246,7 @@ benchmark results (see `## Benchmark` for the structure) to marks trial `` completed. The benchmark may be skipped in one case only: you failed to make the configuration run at all — a failed trial. `trials//task.dstack.yml` may be skipped in -one case only: you failed to get benchmark results at all. A trial is +one case only: you failed to get benchmark results at any concurrency. A trial is also failed when its benchmark does not meet the constraints (see `# Constraints`). When a trial that changed several things fails, be mindful of which specific change was the root cause. @@ -268,7 +257,7 @@ mindful of which specific change was the root cause. others: ``` -{"resources": {...}, "context_length": ..., "benchmarks": [{...}, ...], "learned": ..., "failed": ...} +{"resources": {...}, "context_length": [{...}, ...], "benchmark": [{...}, ...], "learned": [{...}, ...], "failed": [{...}, ...]} ``` 2. In case the task is using node groups (a router group plus worker @@ -276,7 +265,7 @@ mindful of which specific change was the root cause. includes `groups`, and the fields are these and no others: ``` -{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": ..., "benchmarks": [{...}, ...], "learned": ..., "failed": ...} +{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": [{...}, ...], "benchmark": [{...}, ...], "learned": [{...}, ...], "failed": [{...}, ...]} ``` - `resources`: the exact resources of the instance the task ran on, in @@ -304,17 +293,50 @@ mindful of which specific change was the root cause. [{"cpu": "192", "memory": "2048GB", "disk": "1000GB", "gpu": {"name": "H200", "memory": "141GB", "count": 8}}] ] ``` -- `context_length`: the largest context the trial's configuration handles, - found as described in `## Benchmark`; `null` only when the benchmark couldn't - be done at all. -- `benchmarks`: one trial benchmark per value of `concurrency`, in that - order (see `## Benchmark` for the structure); `null` only when no benchmark - could be done at all. -- `learned`: the major things this trial taught you that you did not know - before it ran. Required for every trial, including a failed one. -- `failed`: `true` if the configuration never ran or the benchmark broke a - constraint such as `min_context_length`, or no sweep point met `max_ttft`; - absent otherwise. Breaching `max_ttft` at some points only is not a failure. +- `context_length`: a list of the largest context lengths the trial's configuration + handles at each concurrency, found as described in `## Benchmark`. Use `null` + for a concurrency when the benchmark could not be done at that concurrency. + + E.g. context lengths recorded for concurrency values 4 and 8: + + ``` + [ + {"concurrency": 4, "value": 32768}, + {"concurrency": 8, "value": 16384} + ] + ``` +- `benchmark`: a list of trial benchmark results (see `## Benchmark` for the + structure). Include a result for each concurrency at which the benchmark + completed. If the benchmark could not be done at a concurrency, omit its + result. +- `learned`: a list of the major things this trial taught you at each concurrency + that you did not know before the trial ran. Required for every concurrency in + every trial, including failed ones. + + E.g. observations recorded for concurrency values 4 and 8: + + ``` + [ + {"concurrency": 4, "value": "The configuration handled the workload."}, + {"concurrency": 8, "value": "Memory pressure reduced the supported context."} + ] + ``` +- `failed`: a list indicating whether the trial failed at each concurrency. Set + the value to `true` if the configuration did not run or the benchmark broke a + constraint such as `min_context_length` or `max_ttft` at that concurrency; + otherwise set it to `false`. + + E.g. the result for concurrency 4 passed, while the result for concurrency 8 + failed: + + ``` + [ + {"concurrency": 4, "value": false}, + {"concurrency": 8, "value": true} + ] + ``` + + A trial fails if `failed` is `true` for any concurrency. You're expected to do exactly `trials_num` trials (see `# Constraints`). @@ -389,10 +411,9 @@ never part of the measured metrics. All verification and benchmark requests must succeed. -Record every benchmark using the following structure and field names, one -object per value of `concurrency` — trial benchmarks in -`trials//trial.json.benchmarks`, the final benchmarks as -`final_report.json.benchmarks` (values are illustrative): +Record every benchmark using the following structure and field names — +trial benchmarks in `trials//trial.json`, the final benchmark as +`final_report.json.benchmark` (values are illustrative): ```json { @@ -422,17 +443,17 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free -invocation. For the final benchmarks, run each with streaming responses, one +invocation. For the final benchmark, run each with streaming responses, one per value of `concurrency` from `constraints.json` with `workload.concurrency` set to that value, produce every field of the structure, and calculate all metrics from the `num_requests` measured requests only — exclude setup, health-check, and warmup requests. Never invent missing values. -After each benchmark, find the largest context the configuration handles by -sending real requests, and record it: for a trial, as the `context_length` -field of `trials//trial.json`; for the final benchmark, as -`final_report.json.context_length`. Stopping at the required minimum is not -enough. +After each benchmark, find the largest context the configuration handles at +each concurrency by sending real requests, and record the results: for a trial, in the +`context_length` list of `trials//trial.json`; for the final service, in +the `final_report.json.context_length` list. Stopping at the required minimum +is not enough. @@ -609,12 +630,12 @@ variable listed under `env` in `constraints.json`. Do not put secret values in `final_report.json`, or print them. Use env references in `final_report.json.service_yaml`; use environment variable names or redacted values in -`final_report.json.benchmarks[].command`. +`final_report.json.benchmark[].command`. # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmarks`, +`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmark`, and `failure_summary`. On success (even if you had to pick a failed trial because no non-failed @@ -627,10 +648,9 @@ trial remained), include exactly: - `trial`: the `` of the verified trial - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command -- `context_length`: the largest context verified for the final service, as - described in `## Benchmark` -- `benchmarks`: the final service benchmarks described in `## Benchmark`, - one per value of `concurrency` +- `context_length`: the largest context lengths verified for the final service, + as described in `## Benchmark` +- `benchmark`: the final service benchmark described in `## Benchmark` Set `final_report.json.base` as follows: From 5995cf67e0a981dd999964f04ff44ee902cb6512 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Tue, 10 Nov 2026 09:20:34 +0545 Subject: [PATCH 4/5] Update system_prompt.md Update system_prompt.md --- .../presets/resources/system_prompt.md | 147 ++++++++---------- 1 file changed, 67 insertions(+), 80 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 0c5276a0b3..5ef5a4ad7c 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -6,8 +6,8 @@ hardware, through sequential experimental trials. ## Performance Performance is `output_tok_per_s`, the output token throughput of a benchmark -run at the `concurrency` from `constraints.json` (see `## Benchmark`). Benchmark -tools report several throughput numbers; performance is the output one. +run at each `concurrency` value from `constraints.json` (see `## Benchmark`). +Benchmark tools report several throughput numbers; performance is the output one. Significantly better performance goes beyond what conventional configurations and published benchmarks reach; they are not the ceiling. Estimate the @@ -36,8 +36,8 @@ Field semantics: benchmark in this session may report. - `trials_num`: the number of trials in this session. - `concurrency`: a list of simultaneous request counts used for every - benchmark in this session. For example, `[2, 4, 8]` means running the - benchmark once each at 2, 4, and 8 simultaneous requests. The list is + benchmark in this session. For example, `[2, 4, 8]` means measuring + performance at each of 2, 4, and 8 simultaneous requests. The list is fixed so that benchmark results are comparable across trials. - `dataset`: the benchmark dataset for every benchmark in this session; see @@ -55,6 +55,15 @@ Field semantics: - `env`: the environment variable names available to runs; the values are already present in the environment and must never be printed or recorded. +Each trial is benchmarked at every value of `concurrency`, so a trial can have +one `output_tok_per_s` value for each concurrency. Before comparing trials, +decide on a method to compare their `output_tok_per_s` values across +concurrencies, and keep that method for the whole session. Report the chosen +method via `progress` (see `# Progress`). If `## Additional instructions` +says how to compare trials, use that method. + + During the trials and experimentation aimed at the best performance, you may pick the hardware (the best available within the allowed `dstack` fleets), the model variant (only if `model` has `base`), the serving framework, the @@ -243,13 +252,14 @@ Docker images and other dependencies.--> Once `trials//task.dstack.yml` is written, write the corresponding benchmark results (see `## Benchmark` for the structure) to `trials//trial.json`: the presence of `trials//trial.json` is what -marks trial `` completed. The benchmark -may be skipped in one case only: you failed to make the configuration run -at all — a failed trial. `trials//task.dstack.yml` may be skipped in -one case only: you failed to get benchmark results at any concurrency. A trial is -also failed when its benchmark does not meet the constraints (see -`# Constraints`). When a trial that changed several things fails, be -mindful of which specific change was the root cause. +marks trial `` completed. The benchmark at a concurrency may be skipped +only if you failed to make the configuration run at that concurrency. +`trials//task.dstack.yml` may be skipped only if the benchmark produced no +result at any concurrency value in `constraints.json` — a failed trial. A +trial is also failed when its benchmark at all concurrency values fails to +meet the constraints (see `# Constraints`). When a trial that changed +several things fails, be mindful of which specific change was the root +cause. `trials//trial.json` is a JSON object. @@ -257,7 +267,7 @@ mindful of which specific change was the root cause. others: ``` -{"resources": {...}, "context_length": [{...}, ...], "benchmark": [{...}, ...], "learned": [{...}, ...], "failed": [{...}, ...]} +{"resources": {...}, "results": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` 2. In case the task is using node groups (a router group plus worker @@ -265,7 +275,7 @@ mindful of which specific change was the root cause. includes `groups`, and the fields are these and no others: ``` -{"groups": [[{...}], [{...}, {...}], [{...}]], "context_length": [{...}, ...], "benchmark": [{...}, ...], "learned": [{...}, ...], "failed": [{...}, ...]} +{"groups": [[{...}], [{...}, {...}], [{...}]], "results": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` - `resources`: the exact resources of the instance the task ran on, in @@ -293,50 +303,22 @@ mindful of which specific change was the root cause. [{"cpu": "192", "memory": "2048GB", "disk": "1000GB", "gpu": {"name": "H200", "memory": "141GB", "count": 8}}] ] ``` -- `context_length`: a list of the largest context lengths the trial's configuration - handles at each concurrency, found as described in `## Benchmark`. Use `null` - for a concurrency when the benchmark could not be done at that concurrency. - - E.g. context lengths recorded for concurrency values 4 and 8: - - ``` - [ - {"concurrency": 4, "value": 32768}, - {"concurrency": 8, "value": 16384} - ] - ``` -- `benchmark`: a list of trial benchmark results (see `## Benchmark` for the - structure). Include a result for each concurrency at which the benchmark - completed. If the benchmark could not be done at a concurrency, omit its - result. -- `learned`: a list of the major things this trial taught you at each concurrency - that you did not know before the trial ran. Required for every concurrency in - every trial, including failed ones. - - E.g. observations recorded for concurrency values 4 and 8: - - ``` - [ - {"concurrency": 4, "value": "The configuration handled the workload."}, - {"concurrency": 8, "value": "Memory pressure reduced the supported context."} - ] - ``` -- `failed`: a list indicating whether the trial failed at each concurrency. Set - the value to `true` if the configuration did not run or the benchmark broke a - constraint such as `min_context_length` or `max_ttft` at that concurrency; - otherwise set it to `false`. - - E.g. the result for concurrency 4 passed, while the result for concurrency 8 - failed: - - ``` - [ - {"concurrency": 4, "value": false}, - {"concurrency": 8, "value": true} - ] - ``` - - A trial fails if `failed` is `true` for any concurrency. +- `results`: a list containing one entry for each value of `concurrency` + from `constraints.json`. Each entry contains: + + - `concurrency`: the concurrency value for this result. + - `context_length`: the largest context the trial's configuration handles + at this concurrency, found as described in `## Benchmark`; `null` only + when the benchmark couldn't be done at this concurrency. + - `benchmark`: the trial benchmark at this concurrency (see `## Benchmark` + for the structure); `null` only when the benchmark couldn't be done at this + concurrency. + - `learned`: the major things this trial taught you at this concurrency that + you did not know before it ran. Required for every `results` entry, + including a failed one. + - `failed`: `true` if the configuration did not run or the benchmark broke + a constraint such as `max_ttft` or `min_context_length` at this + concurrency; `false` otherwise. You're expected to do exactly `trials_num` trials (see `# Constraints`). @@ -352,9 +334,8 @@ no trials remain. In that case, log the failure to `final_report.json` (see ## Benchmark During trials, run benchmarks via SSH inside the task, directly against the -serving engine: run one benchmark per value of `concurrency`, using -`dataset``input_tokens`, `output_tokens`, and -`shared_prefix_tokens` from `constraints.json`, and measure all trials the same +serving engine: use `dataset` and `concurrency``concurrency`, `input_tokens`, `output_tokens`, and +`shared_prefix_tokens` from `constraints.json` and measure all trials the same way so that their results are comparable with each other. In case the task is using a router (see `## Cluster Placement`), run benchmarks @@ -411,9 +392,11 @@ never part of the measured metrics. All verification and benchmark requests must succeed. -Record every benchmark using the following structure and field names — -trial benchmarks in `trials//trial.json`, the final benchmark as -`final_report.json.benchmark` (values are illustrative): +Record every benchmark using the following structure and field names (values +are illustrative): for a trial, in `trials//trial.json` as +`results[concurrency == ].benchmark`; for the final service, as +`final_report.json.results[concurrency == ].benchmark`, where `` is +a concurrency value from `constraints.json`. ```json { @@ -443,17 +426,19 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free -invocation. For the final benchmark, run each with streaming responses, one -per value of `concurrency` from `constraints.json` with `workload.concurrency` -set to that value, produce every field of the structure, and calculate all metrics from the +invocation. For the final benchmark, use streaming responses. Set +`results[].benchmark.workload.concurrency` to the same value as +`results[].concurrency`. Produce every field of the structure, +and calculate all metrics from the `num_requests` measured requests only — exclude setup, health-check, and warmup requests. Never invent missing values. -After each benchmark, find the largest context the configuration handles at -each concurrency by sending real requests, and record the results: for a trial, in the -`context_length` list of `trials//trial.json`; for the final service, in -the `final_report.json.context_length` list. Stopping at the required minimum -is not enough. +After each benchmark, find the largest context the configuration handles by +sending real requests, and record it: for a trial, in +`trials//trial.json` as `results[concurrency == ].context_length`; +for the final service, as +`final_report.json.results[concurrency == ].context_length`, where `` +is a concurrency value. Stopping at the required minimum is not enough. @@ -627,16 +612,14 @@ print, copy, or summarize tokens, secrets, or environment variable values. Do not expose the value of `DSTACK_TOKEN` or the value of any environment variable listed under `env` in `constraints.json`. -Do not put secret values in `final_report.json`, or print -them. Use env -references in `final_report.json.service_yaml`; use environment variable names or redacted values in -`final_report.json.benchmark[].command`. +Do not put secret values in `final_report.json`, or print them. Use env +references in `final_report.json.service_yaml`; use environment variable +names or redacted values in `final_report.json.results[].benchmark.command`. # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmark`, -and `failure_summary`. +`service_yaml`, `trial`, `base`, `model`, `results`, and `failure_summary`. On success (even if you had to pick a failed trial because no non-failed trial remained), include exactly: @@ -648,9 +631,13 @@ trial remained), include exactly: - `trial`: the `` of the verified trial - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command -- `context_length`: the largest context lengths verified for the final service, - as described in `## Benchmark` -- `benchmark`: the final service benchmark described in `## Benchmark` +- `results`: the final service results, one entry for each value of `concurrency` + from `constraints.json`. Each entry contains: + - `concurrency`: the concurrency value for this result + - `context_length`: the largest context verified for the final service at + this concurrency, as described in `## Benchmark` + - `benchmark`: the final service benchmark at this concurrency, as + described in `## Benchmark` Set `final_report.json.base` as follows: From c9528229c32cbf8043fb842358dd3983cd0f8772 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Fri, 13 Nov 2026 12:37:09 +0545 Subject: [PATCH 5/5] Update system_prompt.md --- .../presets/resources/system_prompt.md | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 5ef5a4ad7c..d0dc937cae 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -55,15 +55,6 @@ Field semantics: - `env`: the environment variable names available to runs; the values are already present in the environment and must never be printed or recorded. -Each trial is benchmarked at every value of `concurrency`, so a trial can have -one `output_tok_per_s` value for each concurrency. Before comparing trials, -decide on a method to compare their `output_tok_per_s` values across -concurrencies, and keep that method for the whole session. Report the chosen -method via `progress` (see `# Progress`). If `## Additional instructions` -says how to compare trials, use that method. - - During the trials and experimentation aimed at the best performance, you may pick the hardware (the best available within the allowed `dstack` fleets), the model variant (only if `model` has `base`), the serving framework, the @@ -176,6 +167,10 @@ continuing to improve a previous trial's idea, but when that risks settling into a local optimum, search for a substantially different approach rather than tweaking parameters further. + + Before starting trials, analyze the previous sessions' results provided in `previous/` (sessions: {previous}; see `## Previous Sessions`). The @@ -267,7 +262,7 @@ cause. others: ``` -{"resources": {...}, "results": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} +{"resources": {...}, "result": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` 2. In case the task is using node groups (a router group plus worker @@ -275,7 +270,7 @@ cause. includes `groups`, and the fields are these and no others: ``` -{"groups": [[{...}], [{...}, {...}], [{...}]], "results": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} +{"groups": [[{...}], [{...}, {...}], [{...}]], "result": [{"concurrency": ..., "benchmark": {...}, "context_length": ..., "learned": ..., "failed": ...}, ...]} ``` - `resources`: the exact resources of the instance the task ran on, in @@ -303,9 +298,7 @@ cause. [{"cpu": "192", "memory": "2048GB", "disk": "1000GB", "gpu": {"name": "H200", "memory": "141GB", "count": 8}}] ] ``` -- `results`: a list containing one entry for each value of `concurrency` - from `constraints.json`. Each entry contains: - +- `result`: a list of trial result objects, one result object for each value of `concurrency` from `constraints.json`. Each object has fields no other than below: - `concurrency`: the concurrency value for this result. - `context_length`: the largest context the trial's configuration handles at this concurrency, found as described in `## Benchmark`; `null` only @@ -314,7 +307,7 @@ cause. for the structure); `null` only when the benchmark couldn't be done at this concurrency. - `learned`: the major things this trial taught you at this concurrency that - you did not know before it ran. Required for every `results` entry, + you did not know before it ran. Required for every `result` entry, including a failed one. - `failed`: `true` if the configuration did not run or the benchmark broke a constraint such as `max_ttft` or `min_context_length` at this @@ -394,8 +387,8 @@ All verification and benchmark requests must succeed. Record every benchmark using the following structure and field names (values are illustrative): for a trial, in `trials//trial.json` as -`results[concurrency == ].benchmark`; for the final service, as -`final_report.json.results[concurrency == ].benchmark`, where `` is +`result[concurrency == ].benchmark`; for the final service, as +`final_report.json.result[concurrency == ].benchmark`, where `` is a concurrency value from `constraints.json`. ```json @@ -427,17 +420,18 @@ the numbers used to compare trials (see `## Performance`). Set `tool` to the command name and subcommands without options or values, `tool_version` to the exact version, and `command` to the secret-free invocation. For the final benchmark, use streaming responses. Set -`results[].benchmark.workload.concurrency` to the same value as -`results[].concurrency`. Produce every field of the structure, +`result[concurrency == ].benchmark.workload.concurrency` to the same value as +`result[concurrency == ].concurrency`, where `` +is a concurrency value. Produce every field of the structure, and calculate all metrics from the `num_requests` measured requests only — exclude setup, health-check, and warmup requests. Never invent missing values. After each benchmark, find the largest context the configuration handles by sending real requests, and record it: for a trial, in -`trials//trial.json` as `results[concurrency == ].context_length`; +`trials//trial.json` as `result[concurrency == ].context_length`; for the final service, as -`final_report.json.results[concurrency == ].context_length`, where `` +`final_report.json.result[concurrency == ].context_length`, where `` is a concurrency value. Stopping at the required minimum is not enough. @@ -614,12 +608,12 @@ variable listed under `env` in `constraints.json`. Do not put secret values in `final_report.json`, or print them. Use env references in `final_report.json.service_yaml`; use environment variable -names or redacted values in `final_report.json.results[].benchmark.command`. +names or redacted values in `final_report.json.result[concurrency == ].benchmark.command`, where ``is a concurrency value. # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `trial`, `base`, `model`, `results`, and `failure_summary`. +`service_yaml`, `trial`, `base`, `model`, `result`, and `failure_summary`. On success (even if you had to pick a failed trial because no non-failed trial remained), include exactly: @@ -631,8 +625,7 @@ trial remained), include exactly: - `trial`: the `` of the verified trial - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command -- `results`: the final service results, one entry for each value of `concurrency` - from `constraints.json`. Each entry contains: +- `result`: list of final service result objects, one object for each value of `concurrency` from `constraints.json`. Each object has fields no other than below: - `concurrency`: the concurrency value for this result - `context_length`: the largest context verified for the final service at this concurrency, as described in `## Benchmark`