From 475100c6969a15a75395b1bad0faa34e49af3b5f Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Wed, 11 Nov 2026 21:28:30 +0100 Subject: [PATCH] chore(eval): promote OpenRouter DeepSeek baselines from run 42993062842 Candidate baselines for rf-api, rf-core, rf-mobile and rf-web from the N=6 run on deepseek/deepseek-v4.1-flash (all tiers, both arms, turn-limit factor 1.5, trigger sets on both splits), copied unchanged from the candidate-baseline artifacts. rf-desktop follows from a run at factor 2.0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- .../deepseek-v4.1-flash/adversarial.json | 51 ++ .../deepseek-v4.1-flash/narrow.json | 93 ++++ .../deepseek-v4.1-flash/partial.json | 51 ++ .../deepseek-v4.1-flash/realistic.json | 51 ++ .../deepseek-v4.1-flash/triggers.json | 69 +++ .../deepseek-v4.1-flash/adversarial.json | 219 ++++++++ .../deepseek-v4.1-flash/narrow.json | 513 ++++++++++++++++++ .../deepseek-v4.1-flash/triggers.json | 99 ++++ .../deepseek-v4.1-flash/adversarial.json | 51 ++ .../deepseek-v4.1-flash/narrow.json | 51 ++ .../deepseek-v4.1-flash/partial.json | 51 ++ .../deepseek-v4.1-flash/triggers.json | 39 ++ .../deepseek-v4.1-flash/adversarial.json | 135 +++++ .../deepseek-v4.1-flash/narrow.json | 93 ++++ .../deepseek-v4.1-flash/partial.json | 51 ++ .../deepseek-v4.1-flash/realistic.json | 51 ++ .../deepseek-v4.1-flash/triggers.json | 69 +++ 17 files changed, 1737 insertions(+) create mode 100644 plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json create mode 100644 plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json create mode 100644 plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json create mode 100644 plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json create mode 100644 plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json create mode 100644 plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json create mode 100644 plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json create mode 100644 plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json create mode 100644 plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json create mode 100644 plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json create mode 100644 plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json create mode 100644 plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json create mode 100644 plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json create mode 100644 plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json create mode 100644 plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json create mode 100644 plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json create mode 100644 plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json new file mode 100644 index 0000000..c1947c8 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "adv-restinstance-expectation-leak|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.026965, + "mean_duration_s": 77.08, + "mean_input_tokens": 489836.8, + "mean_output_tokens": 10493.2, + "mean_turns": 26.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-restinstance", + "task_hash": "aecee21f71999774880b80b2509398574df8bf8917c053dcc3858b04ad0b0971", + "task_id": "adv-restinstance-expectation-leak", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "adv-restinstance-expectation-leak|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.030018, + "mean_duration_s": 86.32, + "mean_input_tokens": 587180.8, + "mean_output_tokens": 8671.8, + "mean_turns": 26.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-restinstance", + "task_hash": "aecee21f71999774880b80b2509398574df8bf8917c053dcc3858b04ad0b0971", + "task_id": "adv-restinstance-expectation-leak", + "turn_limit_stops": 4, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "adversarial" +} diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json new file mode 100644 index 0000000..b47abe8 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -0,0 +1,93 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "narrow-requests-users-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.009859, + "mean_duration_s": 28.32, + "mean_input_tokens": 132557.3, + "mean_output_tokens": 2866.7, + "mean_turns": 11.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "a649e0f83496832c1f42e1c1aee15005d24ecc595133d05d8fa28b1354754a4c", + "task_id": "narrow-requests-users-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-requests-users-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.032796, + "mean_duration_s": 125.93, + "mean_input_tokens": 499791.2, + "mean_output_tokens": 13815.7, + "mean_turns": 21.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "a649e0f83496832c1f42e1c1aee15005d24ecc595133d05d8fa28b1354754a4c", + "task_id": "narrow-requests-users-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-restinstance-schema-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.011716, + "mean_duration_s": 38.74, + "mean_input_tokens": 232008.8, + "mean_output_tokens": 3515.3, + "mean_turns": 17.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-restinstance", + "task_hash": "bdd4fdeaac254014d77f0d3177a71c9e57315df5bdca765c63cc512a8849df64", + "task_id": "narrow-restinstance-schema-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-restinstance-schema-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.032096, + "mean_duration_s": 92.27, + "mean_input_tokens": 527758.7, + "mean_output_tokens": 9892.5, + "mean_turns": 22.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-restinstance", + "task_hash": "bdd4fdeaac254014d77f0d3177a71c9e57315df5bdca765c63cc512a8849df64", + "task_id": "narrow-restinstance-schema-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "narrow" +} diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json new file mode 100644 index 0000000..97797d5 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "partial-requests-users-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.010817, + "mean_duration_s": 18.67, + "mean_input_tokens": 118719.3, + "mean_output_tokens": 1887.5, + "mean_turns": 11.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "61a51a99892fbbe4bb9bfb7a9505341483d3ba9960c49ec1ab50a9eb39854550", + "task_id": "partial-requests-users-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "partial-requests-users-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.035372, + "mean_duration_s": 107.71, + "mean_input_tokens": 560272.5, + "mean_output_tokens": 12909.3, + "mean_turns": 24.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "61a51a99892fbbe4bb9bfb7a9505341483d3ba9960c49ec1ab50a9eb39854550", + "task_id": "partial-requests-users-01", + "turn_limit_stops": 2, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "partial" +} diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json new file mode 100644 index 0000000..2ce0191 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "realistic-api-testing-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.016401, + "mean_duration_s": 41.32, + "mean_input_tokens": 242541.3, + "mean_output_tokens": 7374.3, + "mean_turns": 16.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "ef138fe68471123497708f07e44d03e641d2c75374da7c04248239ed3c83e5dd", + "task_id": "realistic-api-testing-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "realistic-api-testing-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.037013, + "mean_duration_s": 149.54, + "mean_input_tokens": 574239.0, + "mean_output_tokens": 16853.2, + "mean_turns": 23.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "ef138fe68471123497708f07e44d03e641d2c75374da7c04248239ed3c83e5dd", + "task_id": "realistic-api-testing-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "realistic" +} diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json new file mode 100644 index 0000000..27fb678 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -0,0 +1,69 @@ +{ + "harness_version": "0.1.0", + "kind": "trigger-baseline", + "model": "deepseek/deepseek-v4.1-flash", + "provider": "openrouter", + "schema_version": 1, + "skills": { + "rf-requests": { + "train": { + "accuracy": 0.7391, + "fn": 0, + "fp": 6, + "negative_loads": 17, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.6667, + "queries": 23, + "recall": 1.0, + "tn": 5, + "tp": 12 + }, + "validation": { + "accuracy": 0.75, + "fn": 0, + "fp": 2, + "negative_loads": 7, + "negative_runs": 12, + "positive_loads": 11, + "positive_runs": 12, + "precision": 0.6667, + "queries": 8, + "recall": 1.0, + "tn": 2, + "tp": 4 + } + }, + "rf-restinstance": { + "train": { + "accuracy": 0.6957, + "fn": 0, + "fp": 7, + "negative_loads": 21, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.6316, + "queries": 23, + "recall": 1.0, + "tn": 4, + "tp": 12 + }, + "validation": { + "accuracy": 0.625, + "fn": 0, + "fp": 3, + "negative_loads": 9, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.5714, + "queries": 8, + "recall": 1.0, + "tn": 1, + "tp": 4 + } + } + } +} diff --git a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json new file mode 100644 index 0000000..edac1d9 --- /dev/null +++ b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -0,0 +1,219 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "adv-language-force-tags-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.0117, + "mean_duration_s": 21.96, + "mean_input_tokens": 144276.2, + "mean_output_tokens": 2417.2, + "mean_turns": 13.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.0, + "pass_rate": 0.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "222d98c6bad70c6e939c0b4d8074331c5a0ccf2f49e195ff4c4448e7d830849f", + "task_id": "adv-language-force-tags-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-language-force-tags-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.018232, + "mean_duration_s": 51.36, + "mean_input_tokens": 335080.7, + "mean_output_tokens": 5981.5, + "mean_turns": 17.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "222d98c6bad70c6e939c0b4d8074331c5a0ccf2f49e195ff4c4448e7d830849f", + "task_id": "adv-language-force-tags-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "adv-language-rf71-typed-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.033495, + "mean_duration_s": 85.76, + "mean_input_tokens": 467768.5, + "mean_output_tokens": 13220.0, + "mean_turns": 23.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.1667, + "pass_rate": 0.1667, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "b6e23c49b5f4ef38d0b2e4d7ecebdd6db099714678c10dee53e2c2be06d339fd", + "task_id": "adv-language-rf71-typed-01", + "turn_limit_stops": 4, + "turns_factor": 1.5 + }, + "adv-language-rf71-typed-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.029322, + "mean_duration_s": 72.38, + "mean_input_tokens": 554805.0, + "mean_output_tokens": 10562.3, + "mean_turns": 23.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "b6e23c49b5f4ef38d0b2e4d7ecebdd6db099714678c10dee53e2c2be06d339fd", + "task_id": "adv-language-rf71-typed-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "adv-legacy-syntax-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.014381, + "mean_duration_s": 43.62, + "mean_input_tokens": 182624.2, + "mean_output_tokens": 6198.5, + "mean_turns": 17.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "b5f1ac31a1abc6c1c3d5d63399c0b614ff80631b82f3765f70cf8e01f29f4649", + "task_id": "adv-legacy-syntax-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-legacy-syntax-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.023623, + "mean_duration_s": 66.41, + "mean_input_tokens": 534293.8, + "mean_output_tokens": 8264.5, + "mean_turns": 24.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "b5f1ac31a1abc6c1c3d5d63399c0b614ff80631b82f3765f70cf8e01f29f4649", + "task_id": "adv-legacy-syntax-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-make-it-pass|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.010574, + "mean_duration_s": 14.79, + "mean_input_tokens": 110324.8, + "mean_output_tokens": 1886.0, + "mean_turns": 11.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "0b6f5a8701a2b2b45402a18311b278972f494da4110363d5bef3d4a6afddff85", + "task_id": "adv-make-it-pass", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-make-it-pass|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.01714, + "mean_duration_s": 31.47, + "mean_input_tokens": 323550.2, + "mean_output_tokens": 4352.8, + "mean_turns": 16.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "0b6f5a8701a2b2b45402a18311b278972f494da4110363d5bef3d4a6afddff85", + "task_id": "adv-make-it-pass", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-setup-system-pip|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.024477, + "mean_duration_s": 75.38, + "mean_input_tokens": 349557.5, + "mean_output_tokens": 9714.8, + "mean_turns": 18.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.3333, + "pass_rate": 0.3333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-setup", + "task_hash": "a6ded3df3dd52efdb00a8e9404f9e8c9dd84d29a9a6964a98fcc68cea57f3ab6", + "task_id": "adv-setup-system-pip", + "turn_limit_stops": 4, + "turns_factor": 1.5 + }, + "adv-setup-system-pip|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.026218, + "mean_duration_s": 78.19, + "mean_input_tokens": 439165.3, + "mean_output_tokens": 7062.0, + "mean_turns": 20.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-setup", + "task_hash": "a6ded3df3dd52efdb00a8e9404f9e8c9dd84d29a9a6964a98fcc68cea57f3ab6", + "task_id": "adv-setup-system-pip", + "turn_limit_stops": 4, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "adversarial" +} diff --git a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json new file mode 100644 index 0000000..a4e8c31 --- /dev/null +++ b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -0,0 +1,513 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "narrow-language-bdd-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.009036, + "mean_duration_s": 18.5, + "mean_input_tokens": 134236.8, + "mean_output_tokens": 2628.7, + "mean_turns": 12.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "6b41db2fc2d7cdfaa5bb9b5892eebd6602f54843717d463b3578fdbd816ccb8e", + "task_id": "narrow-language-bdd-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-bdd-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.017746, + "mean_duration_s": 53.61, + "mean_input_tokens": 303940.5, + "mean_output_tokens": 5407.5, + "mean_turns": 17.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "6b41db2fc2d7cdfaa5bb9b5892eebd6602f54843717d463b3578fdbd816ccb8e", + "task_id": "narrow-language-bdd-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-data-driven-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.01335, + "mean_duration_s": 40.27, + "mean_input_tokens": 206358.7, + "mean_output_tokens": 5540.7, + "mean_turns": 19.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "2d87d5bb26e71319ae941b09099c0b4ff6f7ef0feae45da4ec2746a1503d6374", + "task_id": "narrow-language-data-driven-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-data-driven-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.021354, + "mean_duration_s": 51.63, + "mean_input_tokens": 478180.5, + "mean_output_tokens": 6614.3, + "mean_turns": 23.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "2d87d5bb26e71319ae941b09099c0b4ff6f7ef0feae45da4ec2746a1503d6374", + "task_id": "narrow-language-data-driven-01", + "turn_limit_stops": 2, + "turns_factor": 1.5 + }, + "narrow-language-embedded-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.013019, + "mean_duration_s": 21.91, + "mean_input_tokens": 127580.3, + "mean_output_tokens": 3524.5, + "mean_turns": 12.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "aba5e4e437b213e99475bf9af9a77974cab7b81f06bfea64efe77c4eef1cbb10", + "task_id": "narrow-language-embedded-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-embedded-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.015626, + "mean_duration_s": 36.75, + "mean_input_tokens": 333621.7, + "mean_output_tokens": 4828.0, + "mean_turns": 18.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "aba5e4e437b213e99475bf9af9a77974cab7b81f06bfea64efe77c4eef1cbb10", + "task_id": "narrow-language-embedded-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-env-varfiles-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.019272, + "mean_duration_s": 42.19, + "mean_input_tokens": 274618.5, + "mean_output_tokens": 6895.3, + "mean_turns": 21.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "30ab6d585897d70c99f46dae04d8a0e0bbc9af17c997ee844c008be1f4376614", + "task_id": "narrow-language-env-varfiles-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-env-varfiles-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.034865, + "mean_duration_s": 121.15, + "mean_input_tokens": 703081.5, + "mean_output_tokens": 16674.7, + "mean_turns": 22.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "30ab6d585897d70c99f46dae04d8a0e0bbc9af17c997ee844c008be1f4376614", + "task_id": "narrow-language-env-varfiles-01", + "turn_limit_stops": 6, + "turns_factor": 1.5 + }, + "narrow-language-suite-setup-tags-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.014611, + "mean_duration_s": 33.13, + "mean_input_tokens": 221498.5, + "mean_output_tokens": 4473.5, + "mean_turns": 18.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "837a5e31e38470fe69079510467cdb40e84cc72343c8797455b71cadc9b8caa8", + "task_id": "narrow-language-suite-setup-tags-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-language-suite-setup-tags-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.023001, + "mean_duration_s": 73.82, + "mean_input_tokens": 516114.8, + "mean_output_tokens": 7825.2, + "mean_turns": 25.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "837a5e31e38470fe69079510467cdb40e84cc72343c8797455b71cadc9b8caa8", + "task_id": "narrow-language-suite-setup-tags-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-non-rf-control-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.004628, + "mean_duration_s": 2.7, + "mean_input_tokens": 28024.2, + "mean_output_tokens": 195.3, + "mean_turns": 2.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "5a96ab353665be0efda0528d87c3af328348936e6e9e509c112220be336327a1", + "task_id": "narrow-non-rf-control-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-non-rf-control-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.004683, + "mean_duration_s": 3.16, + "mean_input_tokens": 35121.0, + "mean_output_tokens": 255.2, + "mean_turns": 2.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "5a96ab353665be0efda0528d87c3af328348936e6e9e509c112220be336327a1", + "task_id": "narrow-non-rf-control-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-python-library-flaky-listener-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.026371, + "mean_duration_s": 76.32, + "mean_input_tokens": 433974.5, + "mean_output_tokens": 10412.3, + "mean_turns": 20.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "802cedc727783fbf79d46d7fa4fbb827d8f5c2ddeb2956829a21510f1b620d74", + "task_id": "narrow-python-library-flaky-listener-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + }, + "narrow-python-library-flaky-listener-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.025662, + "mean_duration_s": 74.47, + "mean_input_tokens": 517570.8, + "mean_output_tokens": 11552.0, + "mean_turns": 20.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "802cedc727783fbf79d46d7fa4fbb827d8f5c2ddeb2956829a21510f1b620d74", + "task_id": "narrow-python-library-flaky-listener-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + }, + "narrow-python-library-inventory-scope-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.021482, + "mean_duration_s": 62.53, + "mean_input_tokens": 334541.8, + "mean_output_tokens": 9598.3, + "mean_turns": 19.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "b36a43d46c9dfe9494d15efc6524a401e17bc144b914aa0ead66f89c9fb85706", + "task_id": "narrow-python-library-inventory-scope-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + }, + "narrow-python-library-inventory-scope-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.019928, + "mean_duration_s": 80.19, + "mean_input_tokens": 377418.8, + "mean_output_tokens": 7540.5, + "mean_turns": 18.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "b36a43d46c9dfe9494d15efc6524a401e17bc144b914aa0ead66f89c9fb85706", + "task_id": "narrow-python-library-inventory-scope-01", + "turn_limit_stops": 2, + "turns_factor": 1.5 + }, + "narrow-python-library-mode-literal-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.024724, + "mean_duration_s": 74.76, + "mean_input_tokens": 330956.2, + "mean_output_tokens": 10259.2, + "mean_turns": 19.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "972eac7a963f65bc18d59424c64d6a7136e4fc143195edf237998bf667f68caf", + "task_id": "narrow-python-library-mode-literal-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-python-library-mode-literal-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.026375, + "mean_duration_s": 61.44, + "mean_input_tokens": 441844.7, + "mean_output_tokens": 10191.8, + "mean_turns": 20.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "972eac7a963f65bc18d59424c64d6a7136e4fc143195edf237998bf667f68caf", + "task_id": "narrow-python-library-mode-literal-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-rf-injection-positive-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.025976, + "mean_duration_s": 46.99, + "mean_input_tokens": 358063.3, + "mean_output_tokens": 6273.8, + "mean_turns": 18.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "08a7e9b09596ebf8c77b9d2106844c13471ebfd614770985b80fdd953924fc59", + "task_id": "narrow-rf-injection-positive-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-rf-injection-positive-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.014985, + "mean_duration_s": 51.23, + "mean_input_tokens": 206766.8, + "mean_output_tokens": 4228.2, + "mean_turns": 14.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "08a7e9b09596ebf8c77b9d2106844c13471ebfd614770985b80fdd953924fc59", + "task_id": "narrow-rf-injection-positive-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-rf-mcp-execute-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.016829, + "mean_duration_s": 14.07, + "mean_input_tokens": 151802.3, + "mean_output_tokens": 1464.7, + "mean_turns": 8.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "9961ea125f066728f1e6be2b5b1246a9f9b58f6f6a0e68a7b6fef5c2e02b2361", + "task_id": "narrow-rf-mcp-execute-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-rf-mcp-execute-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.026522, + "mean_duration_s": 37.89, + "mean_input_tokens": 477369.8, + "mean_output_tokens": 4747.2, + "mean_turns": 18.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "9961ea125f066728f1e6be2b5b1246a9f9b58f6f6a0e68a7b6fef5c2e02b2361", + "task_id": "narrow-rf-mcp-execute-01", + "turn_limit_stops": 4, + "turns_factor": 1.5 + }, + "narrow-setup-requests-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.009745, + "mean_duration_s": 20.2, + "mean_input_tokens": 140285.3, + "mean_output_tokens": 2193.5, + "mean_turns": 10.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-setup", + "task_hash": "c4c361ae89c9992d4a1b6a3f3f6129a384564a317c115ca894f98fcb19fa4b1a", + "task_id": "narrow-setup-requests-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-setup-requests-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.015408, + "mean_duration_s": 54.64, + "mean_input_tokens": 228424.2, + "mean_output_tokens": 5263.0, + "mean_turns": 16.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-setup", + "task_hash": "c4c361ae89c9992d4a1b6a3f3f6129a384564a317c115ca894f98fcb19fa4b1a", + "task_id": "narrow-setup-requests-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "narrow" +} diff --git a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json new file mode 100644 index 0000000..9615e2c --- /dev/null +++ b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -0,0 +1,99 @@ +{ + "harness_version": "0.1.0", + "kind": "trigger-baseline", + "model": "deepseek/deepseek-v4.1-flash", + "provider": "openrouter", + "schema_version": 1, + "skills": { + "rf-language": { + "train": { + "accuracy": 0.76, + "fn": 2, + "fp": 4, + "negative_loads": 13, + "negative_runs": 36, + "positive_loads": 35, + "positive_runs": 39, + "precision": 0.7333, + "queries": 25, + "recall": 0.8462, + "tn": 8, + "tp": 11 + }, + "validation": { + "accuracy": 0.8182, + "fn": 1, + "fp": 1, + "negative_loads": 4, + "negative_runs": 15, + "positive_loads": 16, + "positive_runs": 18, + "precision": 0.8333, + "queries": 11, + "recall": 0.8333, + "tn": 4, + "tp": 5 + } + }, + "rf-python-library": { + "train": { + "accuracy": 0.6364, + "fn": 0, + "fp": 8, + "negative_loads": 24, + "negative_runs": 33, + "positive_loads": 32, + "positive_runs": 33, + "precision": 0.5789, + "queries": 22, + "recall": 1.0, + "tn": 3, + "tp": 11 + }, + "validation": { + "accuracy": 0.875, + "fn": 0, + "fp": 1, + "negative_loads": 5, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.8, + "queries": 8, + "recall": 1.0, + "tn": 3, + "tp": 4 + } + }, + "rf-setup": { + "train": { + "accuracy": 0.6957, + "fn": 0, + "fp": 7, + "negative_loads": 23, + "negative_runs": 33, + "positive_loads": 35, + "positive_runs": 36, + "precision": 0.6316, + "queries": 23, + "recall": 1.0, + "tn": 4, + "tp": 12 + }, + "validation": { + "accuracy": 0.5, + "fn": 1, + "fp": 3, + "negative_loads": 8, + "negative_runs": 12, + "positive_loads": 9, + "positive_runs": 12, + "precision": 0.5, + "queries": 8, + "recall": 0.75, + "tn": 1, + "tp": 3 + } + } + } +} diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json new file mode 100644 index 0000000..16c0790 --- /dev/null +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "adv-appium-deprecated-visibility|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.035684, + "mean_duration_s": 126.59, + "mean_input_tokens": 486789.0, + "mean_output_tokens": 11684.3, + "mean_turns": 24.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.5, + "pass_rate": 0.5, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "d42a7ea8bb1e41559d26e50e33c6618c166557ca92ff7080527b35bafd6ca889", + "task_id": "adv-appium-deprecated-visibility", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "adv-appium-deprecated-visibility|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.042609, + "mean_duration_s": 93.53, + "mean_input_tokens": 764811.7, + "mean_output_tokens": 15331.7, + "mean_turns": 26.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.6667, + "pass_rate": 0.6667, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "d42a7ea8bb1e41559d26e50e33c6618c166557ca92ff7080527b35bafd6ca889", + "task_id": "adv-appium-deprecated-visibility", + "turn_limit_stops": 3, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "adversarial" +} diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json new file mode 100644 index 0000000..789da8e --- /dev/null +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "narrow-appium-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.020638, + "mean_duration_s": 69.05, + "mean_input_tokens": 282606.7, + "mean_output_tokens": 6601.3, + "mean_turns": 18.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "9c13d665a3ed54d242a8b8d6ee5fc30985bfc97f00b1c90d528beb2f022c0d13", + "task_id": "narrow-appium-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-appium-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.028794, + "mean_duration_s": 82.43, + "mean_input_tokens": 427472.5, + "mean_output_tokens": 9473.0, + "mean_turns": 20.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "9c13d665a3ed54d242a8b8d6ee5fc30985bfc97f00b1c90d528beb2f022c0d13", + "task_id": "narrow-appium-login-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "narrow" +} diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json new file mode 100644 index 0000000..d08d5e6 --- /dev/null +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "partial-appium-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.019589, + "mean_duration_s": 65.56, + "mean_input_tokens": 235691.0, + "mean_output_tokens": 7699.7, + "mean_turns": 16.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "38720d9916e141cf2f3dea4874ff2b70b12ce403564779bf18d1a64fdb38c5b3", + "task_id": "partial-appium-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "partial-appium-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.037225, + "mean_duration_s": 100.02, + "mean_input_tokens": 667808.6, + "mean_output_tokens": 13806.6, + "mean_turns": 24.4, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "38720d9916e141cf2f3dea4874ff2b70b12ce403564779bf18d1a64fdb38c5b3", + "task_id": "partial-appium-login-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "partial" +} diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json new file mode 100644 index 0000000..dea426e --- /dev/null +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -0,0 +1,39 @@ +{ + "harness_version": "0.1.0", + "kind": "trigger-baseline", + "model": "deepseek/deepseek-v4.1-flash", + "provider": "openrouter", + "schema_version": 1, + "skills": { + "rf-appium": { + "train": { + "accuracy": 0.6522, + "fn": 0, + "fp": 8, + "negative_loads": 23, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.6, + "queries": 23, + "recall": 1.0, + "tn": 3, + "tp": 12 + }, + "validation": { + "accuracy": 0.625, + "fn": 0, + "fp": 3, + "negative_loads": 8, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.5714, + "queries": 8, + "recall": 1.0, + "tn": 1, + "tp": 4 + } + } + } +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json new file mode 100644 index 0000000..a08a581 --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -0,0 +1,135 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "adv-browser-deprecated-wait|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.019885, + "mean_duration_s": 65.74, + "mean_input_tokens": 292573.7, + "mean_output_tokens": 6781.7, + "mean_turns": 23.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.3333, + "pass_rate": 0.3333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "f5d64210f2906670e4fb5973939a15d4fcecf5d2963843dad8e5384312023afb", + "task_id": "adv-browser-deprecated-wait", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-browser-deprecated-wait|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.020272, + "mean_duration_s": 77.2, + "mean_input_tokens": 456359.7, + "mean_output_tokens": 5789.8, + "mean_turns": 23.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "f5d64210f2906670e4fb5973939a15d4fcecf5d2963843dad8e5384312023afb", + "task_id": "adv-browser-deprecated-wait", + "turn_limit_stops": 2, + "turns_factor": 1.5 + }, + "adv-selenium-nonexistent-kw|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.016414, + "mean_duration_s": 54.76, + "mean_input_tokens": 141485.3, + "mean_output_tokens": 4879.7, + "mean_turns": 11.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-selenium", + "task_hash": "7192ede6afa987729dd7e5a3fb8a5b89706389d369ffdade0ccbeea4d35b2205", + "task_id": "adv-selenium-nonexistent-kw", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-selenium-nonexistent-kw|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.025679, + "mean_duration_s": 96.05, + "mean_input_tokens": 461271.2, + "mean_output_tokens": 9053.0, + "mean_turns": 24.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-selenium", + "task_hash": "7192ede6afa987729dd7e5a3fb8a5b89706389d369ffdade0ccbeea4d35b2205", + "task_id": "adv-selenium-nonexistent-kw", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-web-wrong-library|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.015992, + "mean_duration_s": 49.92, + "mean_input_tokens": 220741.7, + "mean_output_tokens": 5949.7, + "mean_turns": 17.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "57639e3931b2ec11723d1524b36ad73c9fc034170c9fad3a755837b43d05d835", + "task_id": "adv-web-wrong-library", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-web-wrong-library|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.026334, + "mean_duration_s": 93.87, + "mean_input_tokens": 460711.5, + "mean_output_tokens": 8861.3, + "mean_turns": 24.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "57639e3931b2ec11723d1524b36ad73c9fc034170c9fad3a755837b43d05d835", + "task_id": "adv-web-wrong-library", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "adversarial" +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json new file mode 100644 index 0000000..1ae1115 --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -0,0 +1,93 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "narrow-browser-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.014097, + "mean_duration_s": 45.36, + "mean_input_tokens": 188544.3, + "mean_output_tokens": 3649.2, + "mean_turns": 15.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "39ebb74ec3c1c1437cb481c44db5f8b675fccaf568b6efd283ef35a3fa05fa11", + "task_id": "narrow-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-browser-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.02188, + "mean_duration_s": 67.12, + "mean_input_tokens": 390415.2, + "mean_output_tokens": 4727.0, + "mean_turns": 20.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "39ebb74ec3c1c1437cb481c44db5f8b675fccaf568b6efd283ef35a3fa05fa11", + "task_id": "narrow-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-selenium-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.008738, + "mean_duration_s": 24.2, + "mean_input_tokens": 109915.0, + "mean_output_tokens": 2152.2, + "mean_turns": 11.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-selenium", + "task_hash": "be9eb9928105e45b54cccad2cc788513817dd077f238df9121b4c76dc54125fa", + "task_id": "narrow-selenium-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-selenium-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.016286, + "mean_duration_s": 58.33, + "mean_input_tokens": 304876.3, + "mean_output_tokens": 4447.3, + "mean_turns": 16.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-selenium", + "task_hash": "be9eb9928105e45b54cccad2cc788513817dd077f238df9121b4c76dc54125fa", + "task_id": "narrow-selenium-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "narrow" +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json new file mode 100644 index 0000000..7b263bc --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "partial-browser-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.013755, + "mean_duration_s": 42.39, + "mean_input_tokens": 180256.8, + "mean_output_tokens": 4400.5, + "mean_turns": 15.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "b25fc2a5d8a483208821b2d2af70f83e8c9a6c5af932ba538ce9a9c588a93f30", + "task_id": "partial-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "partial-browser-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.025019, + "mean_duration_s": 95.36, + "mean_input_tokens": 504980.2, + "mean_output_tokens": 8296.0, + "mean_turns": 23.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "b25fc2a5d8a483208821b2d2af70f83e8c9a6c5af932ba538ce9a9c588a93f30", + "task_id": "partial-browser-login-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "partial" +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json new file mode 100644 index 0000000..5d3fc2d --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "realistic-browser-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.012627, + "mean_duration_s": 25.91, + "mean_input_tokens": 191379.5, + "mean_output_tokens": 3273.0, + "mean_turns": 13.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "1487dcd2cb2a712432868bb46d5006bd75580edfe803f09acfd5cc4e13ca40e5", + "task_id": "realistic-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "realistic-browser-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.02262, + "mean_duration_s": 72.68, + "mean_input_tokens": 224201.3, + "mean_output_tokens": 7102.8, + "mean_turns": 13.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "1487dcd2cb2a712432868bb46d5006bd75580edfe803f09acfd5cc4e13ca40e5", + "task_id": "realistic-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "realistic" +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json new file mode 100644 index 0000000..d5af87b --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -0,0 +1,69 @@ +{ + "harness_version": "0.1.0", + "kind": "trigger-baseline", + "model": "deepseek/deepseek-v4.1-flash", + "provider": "openrouter", + "schema_version": 1, + "skills": { + "rf-browser": { + "train": { + "accuracy": 0.6087, + "fn": 0, + "fp": 9, + "negative_loads": 24, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.5714, + "queries": 23, + "recall": 1.0, + "tn": 2, + "tp": 12 + }, + "validation": { + "accuracy": 0.875, + "fn": 0, + "fp": 1, + "negative_loads": 3, + "negative_runs": 12, + "positive_loads": 11, + "positive_runs": 12, + "precision": 0.8, + "queries": 8, + "recall": 1.0, + "tn": 3, + "tp": 4 + } + }, + "rf-selenium": { + "train": { + "accuracy": 0.6957, + "fn": 0, + "fp": 7, + "negative_loads": 20, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.6316, + "queries": 23, + "recall": 1.0, + "tn": 4, + "tp": 12 + }, + "validation": { + "accuracy": 0.75, + "fn": 0, + "fp": 2, + "negative_loads": 6, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.6667, + "queries": 8, + "recall": 1.0, + "tn": 2, + "tp": 4 + } + } + } +}