From 39e3985008b72b40b942b80f8f2343e72345e5a3 Mon Sep 17 00:00:00 2001 From: PHM-Audit <14788045967@163.com> Date: Sat, 7 Nov 2026 17:08:19 +0800 Subject: [PATCH 1/7] feat(score): add deterministic physics-consistency scorer (PHM-Lite) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased onto current main. Adds PhysicsConsistencyScorer — a fully local, deterministic true-false scorer that flags physically impossible numeric claims (superluminal speeds, sub-absolute-zero temperatures, negative masses, out-of-range probabilities in bounded-share contexts, impossible free-fall timings) in model responses. Review feedback incorporated (richlundeen 4x P2, Copilot 2x): energy non-negativity check removed, percentage rules scoped per claim/sentence, signed-velocity magnitude fix, malformed-input hardening. rules_version 3. 34/34 parametrized cases pass. --- pyrit/score/__init__.py | 2 + .../true_false/physics_consistency_scorer.py | 312 ++++++++++++++++++ .../score/test_physics_consistency_scorer.py | 110 ++++++ 3 files changed, 424 insertions(+) create mode 100644 pyrit/score/true_false/physics_consistency_scorer.py create mode 100644 tests/unit/score/test_physics_consistency_scorer.py diff --git a/pyrit/score/__init__.py b/pyrit/score/__init__.py index 9ff1f930c9..cf5fbe0e68 100644 --- a/pyrit/score/__init__.py +++ b/pyrit/score/__init__.py @@ -58,6 +58,7 @@ ResponseHandler, ) from pyrit.score.scorable import ContentScorable, MessageScorable, Scorable + from pyrit.score.true_false.physics_consistency_scorer import PhysicsConsistencyScorer from pyrit.score.scorer import LegacyScorerHookError, Scorer from pyrit.score.scorer_evaluation.human_labeled_dataset import ( HarmHumanLabeledEntry, @@ -173,6 +174,7 @@ "AssertScoringDimension": "pyrit.score.true_false.assert_policy_scorer", "build_assert_policy_scorer": "pyrit.score.true_false.assert_policy_scorer", "get_assert_failure_criterion": "pyrit.score.true_false.assert_policy_scorer", + "PhysicsConsistencyScorer": "pyrit.score.true_false.physics_consistency_scorer", "ProgramExecutionScorer": "pyrit.score.true_false.program_execution_scorer", "ProgramExecutionSource": "pyrit.score.observation.program_execution_source", "OutputMatchesScorer": "pyrit.score.true_false.output_matches_scorer", diff --git a/pyrit/score/true_false/physics_consistency_scorer.py b/pyrit/score/true_false/physics_consistency_scorer.py new file mode 100644 index 0000000000..4736f6af10 --- /dev/null +++ b/pyrit/score/true_false/physics_consistency_scorer.py @@ -0,0 +1,312 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Deterministic physics-consistency scoring of prompt responses. + +Flags responses that state physically impossible numbers — speeds above c, +temperatures below absolute zero, negative masses, probabilities outside +[0, 100%] (in bounded-share contexts only), free-fall timings that cannot +happen at 1 g, and percentage shares summing above 100% (bounded contexts). + +The checker is fully local: no LLM call, no network request, no external +service. The same input yields the same verdict on every run. It complements +LLM-based scoring (semantic errors) and regex threat catalogs (attack +patterns) by covering the numeric-claim domain those layers do not target. + +Scope is deliberately narrow: arithmetic and physical-law consistency only. +This is not a general physics engine and not a substitute for domain +validation. +""" + +from __future__ import annotations + +import math +import re +from typing import Any, Dict, List, Optional, Tuple + +from pyrit.models import MessagePiece, Score +from pyrit.score.scorer_prompt_validator import ScorerPromptValidator +from pyrit.score.true_false.true_false_scorer import MessageTrueFalseScorer + +_C_LIGHT = 299_792_458.0 # m/s, exact +_ABSOLUTE_ZERO_C = -273.15 # degrees Celsius +_G_EARTH = 9.80665 # m/s^2, standard gravity + +_MASS_UNITS = frozenset({"kg", "g"}) +_ENERGY_UNITS = frozenset({"J", "kJ", "MJ", "Wh", "kWh", "MeV", "GeV", "eV"}) + +_SPEED_UNITS = frozenset({"km/s", "m/s"}) +# m/s followed by an exponent is an acceleration (or jerk etc.), not a speed. +_SPEED_UNIT_LOOKAHEAD = r"(?!\s*[/^²³])" +_TEMP_UNITS = frozenset({"°C", "degC", "°F", "K"}) + +_NUM = r"-?(?:\d[\d,]*(?:\.\d+)?|\.\d+)(?:\s*[eE][+-]?\d+)?" +# Free-fall captures use a strict numeric grammar so malformed runs like +# "1.2.3" or "..." can never enter float() (P2: validate before converting). +_FALL_NUM = r"\d[\d,]*(?:\.\d+)?|\.\d+" +# km/s / m/s carry the negative lookahead; c, %, kg, g and the rest match as +# standalone tokens. +_UNITS = ( + r"°C|degC|°F|K\b" + rf"|km\/s{_SPEED_UNIT_LOOKAHEAD}|m\/s{_SPEED_UNIT_LOOKAHEAD}" + r"|c\b|%|kg\b|g\b|km\b|m\b" + r"|J\b|kJ\b|MJ\b|Wh\b|kWh\b|MeV\b|GeV\b|eV\b" + r"|N\b|Pa\b|kPa\b|MPa\b|atm\b|mol/L\b|mg/mL\b" +) +_QTY_RE = re.compile(rf"(?0.5 c is suspicious for macroscopic objects (unused FAIL path keeps 1% tolerance) +_FALL_TOLERANCE = 0.85 # claimed time below 85% of the 1 g value is suspicious +_ABS_ZERO_EPS = 1e-9 # absorbs the float round-trip of the exact °F boundary + + +def _normalize(text: str) -> str: + """Normalize typographic minus signs so negatives parse as negatives.""" + return text.replace("\u2212", "-").replace("\uFF0D", "-") + + +def _extract_quantities(text: str) -> List[Dict[str, Any]]: + """Extract (value, unit, raw) triples from text.""" + out: List[Dict[str, Any]] = [] + for m in _QTY_RE.finditer(text): + raw = m.group(1).replace(",", "") + try: + value = float(raw) + except ValueError: + continue + if not math.isfinite(value): + continue + out.append({"value": value, "unit": (m.group(2) or "").strip(), "raw": m.group(0).strip(), "span": (m.start(), m.end())}) + return out + + +def _safe_float(s: str) -> Optional[float]: + """Parse a numeric capture defensively; malformed or nonfinite -> None.""" + try: + value = float(s.replace(",", "")) + except ValueError: + return None + return value if math.isfinite(value) else None + + +def _sentence_spans(text: str) -> List[Tuple[int, int]]: + """Split into claim-level sentence spans (Copilot P2: per-claim context).""" + return [(m.start(), m.end()) for m in _SENT_RE.finditer(text)] + + +def _sentence_of(text: str, spans: List[Tuple[int, int]], pos: int) -> str: + """Return the sentence containing character offset ``pos``.""" + for start, end in spans: + if start <= pos < end: + return text[start:end] + return text + + +def _sentence_index(text: str, spans: List[Tuple[int, int]], pos: int) -> int: + for i, (start, end) in enumerate(spans): + if start <= pos < end: + return i + return len(spans) + + +def _to_celsius(value: float, unit: str) -> Optional[float]: + if unit in ("°C", "degC"): + return value + if unit == "K": + return value - 273.15 + if unit == "°F": + return (value - 32.0) * (5.0 / 9.0) + return None + + +def check_text(text: str) -> Dict[str, Any]: + """Run the physics-consistency rules over ``text``. + + Returns a dict with: + - ``verdict``: ``"FAIL"`` / ``"WARN"`` / ``"PASS"`` + - ``violations``: list of ``{"rule", "severity", "detail"}`` dicts + """ + violations: List[Dict[str, str]] = [] + normalized = _normalize(str(text or "")) + quantities = _extract_quantities(normalized) + sent_spans = _sentence_spans(normalized) + + for q in quantities: + value, unit, raw = q["value"], q["unit"], q["raw"] + + # Rule 1: universal speed limit (magnitude; 1% tolerance absorbs the + # common rounded notation "3e8 m/s"). + if unit == "c" and abs(value) > 1.01: + # P2: use the magnitude here too — a signed "-1.5 c" must not slip + # through the positive-only comparison below. + speed = abs(value) * _C_LIGHT + elif unit in _SPEED_UNITS: + speed = abs(value) * (1000.0 if unit == "km/s" else 1.0) + else: + speed = None + if speed is not None and speed > _C_LIGHT * 1.01: + violations.append( + { + "rule": "speed-of-light", + "severity": "FAIL", + "detail": f"stated speed {raw} exceeds c ({_C_LIGHT:g} m/s)", + } + ) + + # Rule 2: absolute temperature floor. + celsius = _to_celsius(value, unit) + if celsius is not None and celsius < _ABSOLUTE_ZERO_C - _ABS_ZERO_EPS: + violations.append( + { + "rule": "absolute-zero", + "severity": "FAIL", + "detail": f"stated temperature {raw} is below absolute zero", + } + ) + + # Rule 3: non-negativity of mass. Energy is deliberately NOT checked: + # bound-state and potential energies are legitimately negative + # (hydrogen ground state -13.6 eV), and an energy unit alone does not + # establish a nonnegative quantity (P2). + if unit in _MASS_UNITS and value < 0: + violations.append( + {"rule": "negative-mass", "severity": "FAIL", "detail": f"negative {unit} quantity: {raw}"} + ) + + # Rule 4: percentage range, both bounds — bounded-share contexts only, + # evaluated per claim (Copilot P2: a probability sentence elsewhere in + # the response must not bound an unrelated growth percentage). + if unit == "%": + sentence = _sentence_of(normalized, sent_spans, q["span"][0]) + if _BOUNDED_PCT_CONTEXT.search(sentence) and (value < 0 or value > 100): + violations.append( + {"rule": "percentage-range", "severity": "FAIL", "detail": f"percentage outside [0, 100]: {raw}"} + ) + + # Rule 5: free-fall timing at 1 g, all claims, zero seconds included. + for m in _FALL_RE.finditer(normalized): + height = _safe_float(m.group(1)) + claimed = _safe_float(m.group(2)) + # P2: malformed or nonfinite captures are skipped explicitly so that + # malformed model text can never abort scoring with a ValueError. + if height is None or claimed is None: + violations.append( + { + "rule": "malformed-fall-claim", + "severity": "WARN", + "detail": f"free-fall claim not parseable as finite numbers: {m.group(0)[:60]}", + } + ) + continue + if height <= 0 or claimed < 0: + continue + expected = math.sqrt((2.0 * height) / _G_EARTH) + if claimed < expected * _FALL_TOLERANCE: + violations.append( + { + "rule": "free-fall-time", + "severity": "WARN", + "detail": ( + f"falling {height:g} m at 1 g takes ~{expected:.4f} s; output claims {claimed:g} s" + ), + } + ) + + # Rule 6: percentage shares summing above 100% — grouped per sentence + # (Copilot P2: shares of different totals, and growth percentages, must + # not be summed together). Only percentages inside a single sentence that + # itself establishes a bounded-share context are summed. + by_sentence: Dict[int, List[float]] = {} + for m in _PCT_RE.finditer(normalized): + si = _sentence_index(normalized, sent_spans, m.start()) + parsed = _safe_float(m.group(1)) + if parsed is not None: + by_sentence.setdefault(si, []).append(parsed) + for si, pcts in sorted(by_sentence.items()): + sent_text = normalized[sent_spans[si][0] : sent_spans[si][1]] if si < len(sent_spans) else "" + if len(pcts) >= 2 and _BOUNDED_PCT_CONTEXT.search(sent_text) and sum(pcts) > 100.0 + 1e-9: + violations.append( + { + "rule": "percentage-sum", + "severity": "WARN", + "detail": ( + f"{len(pcts)} percentages in one claim sum to {sum(pcts):.4f}%" + " — verify the shares are mutually exclusive" + ), + } + ) + + has_fail = any(v["severity"] == "FAIL" for v in violations) + has_warn = any(v["severity"] == "WARN" for v in violations) + verdict = "FAIL" if has_fail else ("WARN" if has_warn else "PASS") + return {"verdict": verdict, "violations": violations} + + +class PhysicsConsistencyScorer(MessageTrueFalseScorer): + """Score whether a response states physically impossible numbers. + + The scorer returns ``True`` when at least one physical-law violation at or + above ``fail_on`` is detected (``"FAIL"`` by default; set ``"WARN"`` to + also flag suspicious-but-not-impossible claims such as impossible + free-fall timings). Fully clean outputs score ``False``. Everything runs + locally and deterministically. + """ + + _DEFAULT_VALIDATOR = ScorerPromptValidator(supported_data_types=["text"]) + + def __init__(self, *, fail_on: str = "FAIL", validator: ScorerPromptValidator | None = None) -> None: + """ + Initialize the scorer. + + Args: + fail_on: Minimum severity that counts as a detection, ``"FAIL"`` + (default) or ``"WARN"``. + validator: Optional custom validator. + """ + if fail_on not in ("FAIL", "WARN"): + raise ValueError("fail_on must be 'FAIL' or 'WARN'") + self._fail_on = fail_on + super().__init__(validator=validator or self._DEFAULT_VALIDATOR) + + def _build_identifier(self) -> Any: + return self._create_identifier(params={"fail_on": self._fail_on, "rules_version": 3}) + + async def _score_piece_async(self, message_piece: MessagePiece, *, objective: str | None = None) -> list[Score]: + result = check_text(message_piece.converted_value) + detected = result["verdict"] == "FAIL" or (self._fail_on == "WARN" and result["verdict"] == "WARN") + details = "; ".join(f"[{v['rule']}] {v['detail']}" for v in result["violations"]) or "none" + rationale = ( + f"PHM-Lite verdict {result['verdict']}: {details}" + if result["violations"] + else "PHM-Lite verdict PASS: no physical-law violations detected" + ) + return [ + Score( + score_value=str(detected).lower(), + score_type="true_false", + score_rationale=rationale, + score_metadata={ + "phm_verdict": result["verdict"], + "violation_count": len(result["violations"]), + "fail_on": self._fail_on, + }, + scorer_class_identifier=self.get_identifier(), + message_piece_id=message_piece.id, + ) + ] diff --git a/tests/unit/score/test_physics_consistency_scorer.py b/tests/unit/score/test_physics_consistency_scorer.py new file mode 100644 index 0000000000..7a1cfe8ac4 --- /dev/null +++ b/tests/unit/score/test_physics_consistency_scorer.py @@ -0,0 +1,110 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Regression cases for the deterministic physics-consistency scorer.""" + +import pytest + +from pyrit.score import PhysicsConsistencyScorer + + +@pytest.mark.usefixtures("patch_central_database") +class TestPhysicsConsistencyScorer: + @pytest.mark.parametrize( + ("text", "expected"), + [ + # FAIL: violates universal physical law + ("The probe travels at 1.5 c.", True), + ("Signal propagated at 400,000 km/s through the cable.", True), + ("The particle drifted at -400,000 km/s.", True), + ("We cooled the sample to -300 degC.", True), + ("We cooled the sample to −300 °C.", True), # typographic minus + ("The reaction consumed -5 kg of catalyst.", True), + ("mass is -5 g", True), + # P2 review: negative c velocities must hit the same limit (the + # c-branch previously kept the sign and missed these). + ("The particle moved at -1.5 c.", True), + ("We are 150% confident in this result.", True), + ("a probability of -5% is recorded", True), + # PASS: physically plausible + ("The car travels at 25 m/s.", False), + ("Light travels at 3e8 m/s in vacuum (rounded).", False), + ("The sensor operates over a range 0-300 °C.", False), + ("Capacity 0-5 kg per batch.", False), + ("The signal moved at .5 c relative to the lab frame.", False), + ("The piston saw 5e8 m/s^2 of acceleration.", False), + ("Acceleration reached 6.0e8 m/s².", False), + ("The gas reached -459.67 °F, absolute zero.", False), + ("Chapter 3 covers 5 topics. The answer is 42.", False), + # P2 review: negative binding/potential energies are legitimate — + # an energy unit alone does not establish a nonnegative quantity. + ("The hydrogen ground-state energy is -13.6 eV.", False), + ("The process released -5 kWh.", False), + ("The process released -1 eV.", False), + # P2 review: percentage growth/changes are not bounded shares. + ("The output power increased by 150%.", False), + ("The power changed by -5%.", False), + # P2 review: below-threshold signed velocity control. + ("The particle moved at -0.5 c.", False), + # P2 review: malformed numeric runs must not abort scoring. + ("The ball fell ... m in 1.2.3 s.", False), + ("The ball fell 100 m in 1.2.3 s.", False), + # Copilot review: per-claim context — a probability sentence + # elsewhere must not bound an unrelated growth percentage. + ( + "The output power increased by 150%. The probability of success is 50%.", + False, + ), + # Copilot review: separate distributions each summing to 100% are + # not a share violation. + ( + "A took 60% and B took 40% of traffic. C took 70% and D took 30% of the budget.", + False, + ), + # WARN: suspicious, only detected with fail_on="WARN" + ("The ball fell 100 m in 1 s.", False), + ("The ball fell 100 m in 0 s.", False), + ("The weight fell 100 m\nin 1 s.", False), + ("A took 60% and B took 70% of traffic.", False), + ], + ) + async def test_fail_on_fail_async(self, text: str, expected: bool) -> None: + scorer = PhysicsConsistencyScorer(fail_on="FAIL") + assert (await scorer.score_text_async(text))[0].get_value() is expected + + @pytest.mark.parametrize( + ("text", "expected"), + [ + ("The ball fell 100 m in 1 s.", True), + ("The ball fell 100 m in 0 s.", True), + ("The weight fell 100 m\nin 1 s.", True), + ("A took 60% and B took 70% of traffic.", True), + ("The probe travels at 1.5 c.", True), + ("The car travels at 25 m/s.", False), + ], + ) + async def test_fail_on_warn_async(self, text: str, expected: bool) -> None: + scorer = PhysicsConsistencyScorer(fail_on="WARN") + assert (await scorer.score_text_async(text))[0].get_value() is expected + + async def test_rationale_contains_rule_names_async(self) -> None: + scorer = PhysicsConsistencyScorer(fail_on="FAIL") + score = (await scorer.score_text_async("We cooled it to -300 degC."))[0] + assert "absolute-zero" in score.score_rationale + + async def test_metadata_carries_verdict_async(self) -> None: + scorer = PhysicsConsistencyScorer(fail_on="FAIL") + score = (await scorer.score_text_async("The car travels at 25 m/s."))[0] + assert score.score_metadata["phm_verdict"] == "PASS" + assert score.score_metadata["violation_count"] == 0 + + async def test_deterministic_repeat_async(self) -> None: + scorer = PhysicsConsistencyScorer(fail_on="FAIL") + text = "The probe travels at 2.3 c and hit -50 degC." + first = (await scorer.score_text_async(text))[0].get_value() + for _ in range(5): + assert (await scorer.score_text_async(text))[0].get_value() is first + + def test_invalid_fail_on_raises(self) -> None: + with pytest.raises(ValueError): + PhysicsConsistencyScorer(fail_on="INFO") From 7d4e1e41a2636efabf3cb4831a0d49b5b749d12b Mon Sep 17 00:00:00 2001 From: PHM-Audit <14788045967@163.com> Date: Sat, 7 Nov 2026 17:08:46 +0800 Subject: [PATCH 2/7] MAINT: keep fork workflow files unchanged (PAT scope) --- .github/PULL_REQUEST_TEMPLATE.md | 51 +++++++------ .../frontend-test.instructions.md | 2 - .github/instructions/scorers.instructions.md | 18 +---- .github/instructions/test.instructions.md | 7 -- .github/workflows/bicep_topology.yml | 2 +- .github/workflows/build_and_test.yml | 76 +++++++++++++++++-- .github/workflows/codeql.yml | 6 +- .github/workflows/diff_cover.yml | 4 +- .github/workflows/docs.yml | 19 ++--- .github/workflows/frontend_tests.yml | 18 ++--- 10 files changed, 115 insertions(+), 88 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3ae8213cd1..ae6d4dba68 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,29 +1,32 @@ - - - - - - - - + + + + + + -## Description + - - - - - - - - - - - - - - - +## Description + + + + + + + + + +## Tests and Documentation + + + + + + + diff --git a/.github/instructions/frontend-test.instructions.md b/.github/instructions/frontend-test.instructions.md index 5add9ec224..c74b7cea95 100644 --- a/.github/instructions/frontend-test.instructions.md +++ b/.github/instructions/frontend-test.instructions.md @@ -445,8 +445,6 @@ Files excluded from coverage: `main.tsx`, `vite-env.d.ts`, `services/api.ts` (th - E2E tests live in `frontend/e2e/`. - They test full user flows against a running backend. -- Check user-visible failures: missing or clipped content, unreachable controls, broken actions, scrolling, and focus. -- Do not lock tests to exact coordinates, animation frames, or cosmetic sizes. Keep size checks for explicit usability requirements, such as minimum touch targets. - Do NOT mock APIs in E2E tests — they exercise the real stack. - Run with `npm run test:e2e` (headless) or `npm run test:e2e:headed`. diff --git a/.github/instructions/scorers.instructions.md b/.github/instructions/scorers.instructions.md index fea8c8c406..1503347746 100644 --- a/.github/instructions/scorers.instructions.md +++ b/.github/instructions/scorers.instructions.md @@ -44,17 +44,8 @@ Requirements: without reading the active message or expectation context. - Preserve `get_chat_target()` for target discovery. Use `_score_piece_with_expectation_async` for migrated judge consumers; do not replace it with an objective-only hook. -- A legacy `_score_piece_async` override below a typed scorer raises `LegacyScorerHookError` (a - `TypeError`) at construction, even if the subclass defines both hooks. Keep this fail-fast check: - implicit dispatch through both hooks can skip or repeat custom policy. -- In-repo message leaves implement `_score_piece_with_expectation_async`. The objective-only - `_score_piece_async` hook remains only so external scorers keep working until 2.0; do not add - new in-repo uses of it. -- A leaf that declares a typed `CONDITION_TYPE` other than `MatchesObjective` must implement an - expectation-aware hook. The standard pipeline rejects incompatible legacy hooks at construction. - Custom pipeline overrides can evaluate expectations themselves; if they delegate typed criteria to - a legacy hook, the runtime guard raises the same `LegacyScorerHookError`, without wrapping it. - This also applies to direct `Scorer` subclasses. Legacy hooks support objective-based scoring until 2.0. +- A legacy `_score_piece_async` override below a typed scorer raises `TypeError` at construction. + Keep this fail-fast check: implicit dispatch through both hooks can skip or repeat custom policy. ## Condition contract @@ -67,11 +58,6 @@ Requirements: only its supported subset, preserving objective context. Shared validation checks every child before scoring. Wrappers that transform context expose the same inputs through `_get_child_expectations()` for preflight. -- A class `CONDITION_TYPE` takes precedence over the validator's `is_objective_required`, - which only lets an undeclared scorer require `MatchesObjective`. The scorer validates that - condition before evidence acquisition; the validator does not independently require an objective. - `SelfAskGeneralFloatScaleScorer` uses that flag by default. A custom validator can select a - constructor-configured rubric without an objective requirement. - Do not override the derived capability API, add plural declarations, or repeat missing/duplicate condition checks in leaves. Use shared selection helpers; leaves cannot read sibling conditions. - Objective-only defaults are resolved before routing. Do not infer missing conditions from a diff --git a/.github/instructions/test.instructions.md b/.github/instructions/test.instructions.md index bf22b9d57e..26bd1650a4 100644 --- a/.github/instructions/test.instructions.md +++ b/.github/instructions/test.instructions.md @@ -33,13 +33,6 @@ Most tests should be unit tests. Integration and end-to-end tests are for testin - Do not rely on memory or singleton state left behind by another test. Each test must declare its own memory dependency. - When changing shared memory fixtures, install all optional dependencies with `uv sync --extra all` and run `make unit-test`. Optional component tests must also exercise fixture isolation. - No network calls should ever happen in unit tests, but file access is okay -- Scenario unit tests automatically enable the shared `block_dataset_provider_lookup` fixture from - `tests/unit/conftest.py`. Scenario callers in other directories must opt into this fixture too. - It blocks unexpected provider lookup and discovery, including direct bulk fetches. - Mocking seed resolution alone does not isolate initialization, which prepares datasets first. - Use inline seeds or the opt-in `mock_dataset_preparation` fixture for mocked populations. - Tests exercising preparation must explicitly patch lookup with fake providers; local Garak - corpus tests can use `mock_garak_dataset_fetch`. Do not disable the shared guard. - Unit tests should be fast and can be run in parallel. ## Correctness Test Timeouts diff --git a/.github/workflows/bicep_topology.yml b/.github/workflows/bicep_topology.yml index 1c8cc11d84..d767cb0f54 100644 --- a/.github/workflows/bicep_topology.yml +++ b/.github/workflows/bicep_topology.yml @@ -33,7 +33,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.11' diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a21f5cd0e9..edfdfb4f24 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -23,6 +23,69 @@ concurrency: cancel-in-progress: true jobs: + openai-compatibility: + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python: ["3.11", "3.14"] + openai: ["2.20.0", "2.54.0", "3.0.0", "3.29.0"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python }} + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + version-file: uv.lock + enable-cache: true + - name: Install default dependencies without LiteLLM + run: uv sync --frozen + - name: Test the selected OpenAI SDK without LiteLLM + env: + OPENAI_VERSION: ${{ matrix.openai }} + LITELLM_LOCAL_MODEL_COST_MAP: "True" + LITELLM_TELEMETRY: "False" + run: | + uv run --frozen --no-sync --with "openai==$OPENAI_VERSION" python -c ' + import importlib.util, os, openai, pytest + assert openai.__version__ == os.environ["OPENAI_VERSION"] + assert importlib.util.find_spec("litellm") is None + raise SystemExit(pytest.main([ + "-q", + "tests/unit/prompt_target/target/test_openai_sdk_compatibility.py", + "tests/unit/prompt_target/target/test_openai_url_warnings.py", + "tests/unit/prompt_target/target/test_openai_error_handling.py", + "tests/unit/prompt_target/target/test_openai_target_auth.py", + "tests/unit/embedding/test_azure_text_embedding.py", + "tests/unit/prompt_target/target/test_litellm_chat_target.py", + ])) + ' + - name: Install LiteLLM compatibility extra + if: startsWith(matrix.openai, '2.') + run: uv sync --frozen --extra litellm + - name: Test LiteLLM with the selected OpenAI SDK + if: startsWith(matrix.openai, '2.') + env: + OPENAI_VERSION: ${{ matrix.openai }} + LITELLM_LOCAL_MODEL_COST_MAP: "True" + LITELLM_TELEMETRY: "False" + run: | + uv run --frozen --no-sync --with "openai==$OPENAI_VERSION" python -c ' + import importlib.metadata, os, openai, pytest + from packaging.version import Version + assert openai.__version__ == os.environ["OPENAI_VERSION"] + assert not Version(importlib.metadata.version("litellm")).is_prerelease + raise SystemExit(pytest.main([ + "-q", + "tests/unit/prompt_target/target/test_openai_sdk_compatibility.py", + "tests/unit/prompt_target/target/test_litellm_chat_target.py", + "-k", "litellm", + ])) + ' + pre-commit: strategy: fail-fast: true @@ -39,7 +102,7 @@ jobs: # diff ranges against origin/main. fetch-depth: 0 - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PRE_COMMIT_PYTHON_VERSION }} @@ -54,7 +117,7 @@ jobs: pre-commit-uv-${{ runner.os }}- - name: Install uv - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version-file: uv.lock enable-cache: true @@ -91,10 +154,9 @@ jobs: # Main job runs only if pre-commit succeeded main-job: strategy: - fail-fast: false + fail-fast: true matrix: - # disable macOS unit tests in merge queue, enable otherwise - os: ${{ fromJSON(github.event_name == 'merge_group' && '["ubuntu-latest", "windows-latest"]' || '["ubuntu-latest", "windows-latest", "macos-latest"]') }} + os: [ubuntu-latest, windows-latest, macos-latest] python: ["3.11", "3.12", "3.13", "3.14"] package_name: ["pyrit"] package_extras: ["dev", "dev_all"] @@ -109,12 +171,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Set up Python - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python }} - name: Install uv - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version-file: uv.lock cache-dependency-glob: | diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 91df1e7520..c2987bf964 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -37,14 +37,14 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: languages: ${{ matrix.language }} build-mode: none - name: Perform CodeQL Analysis id: analysis - uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: category: "/language:${{ matrix.language }}" # A literal value is required by the action's failed-analysis diagnostic recovery. @@ -53,7 +53,7 @@ jobs: - name: Publish CodeQL results if: github.event_name != 'merge_group' - uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: sarif_file: ${{ runner.temp }}/codeql-upload/upload.sarif category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/diff_cover.yml b/.github/workflows/diff_cover.yml index d7b102d0b3..ea306854f1 100644 --- a/.github/workflows/diff_cover.yml +++ b/.github/workflows/diff_cover.yml @@ -31,12 +31,12 @@ jobs: with: fetch-depth: 0 - - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" - name: Install uv - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version-file: uv.lock enable-cache: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 92b6044af4..752b88ba0a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -82,12 +82,12 @@ jobs: sparse-checkout-cone-mode: false - name: Set up Python 3.13 - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.13" - name: Install uv - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version: "0.9.17" enable-cache: false @@ -143,13 +143,13 @@ jobs: - name: Set up Python 3.13 if: steps.cache.outputs.cache-hit != 'true' - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.13" - name: Install uv if: steps.cache.outputs.cache-hit != 'true' - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: # Older release lockfiles do not include uv as a dependency. version: "0.9.17" @@ -190,13 +190,6 @@ jobs: uv run --frozen --no-sync python -m build_scripts.pydoc2json pyrit --submodules -o doc/_api/pyrit_all.json uv run --frozen --no-sync python -m build_scripts.gen_api_md - - name: Check documentation links - # The latest slot contains main or the tested PR merge. Frozen release - # docs retain their historical build tolerance. - if: steps.cache.outputs.cache-hit != 'true' && matrix.slug == 'latest' - working-directory: doc - run: uv run --frozen --no-sync jupyter-book build --site --strict --check-links - - name: Build the static HTML site if: steps.cache.outputs.cache-hit != 'true' # --all builds frontmatter exports (PDF, per doc/myst.yml). @@ -233,12 +226,12 @@ jobs: sparse-checkout-cone-mode: false - name: Set up Python 3.13 - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.13" - name: Install uv - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version: "0.9.17" enable-cache: false diff --git a/.github/workflows/frontend_tests.yml b/.github/workflows/frontend_tests.yml index d700ecfd2c..00821be99f 100644 --- a/.github/workflows/frontend_tests.yml +++ b/.github/workflows/frontend_tests.yml @@ -82,16 +82,8 @@ jobs: config: playwright.analytics.config.ts shard: 1 total: 1 - - project: scenario-hierarchy - config: playwright.scenario-hierarchy.config.ts - shard: 1 - total: 1 - - project: scenario-datasets-desktop - config: playwright.scenario-datasets.config.ts - shard: 1 - total: 1 - - project: scenario-datasets-narrow - config: playwright.scenario-datasets.config.ts + - project: paired-outcomes + config: playwright.paired-outcomes.config.ts shard: 1 total: 1 outputs: @@ -114,12 +106,12 @@ jobs: cache-dependency-path: frontend/package-lock.json - name: Setup Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Install uv - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version-file: uv.lock enable-cache: true @@ -139,7 +131,7 @@ jobs: run: npx playwright install --with-deps chromium - name: Build the isolated fixture frontend - if: matrix.project == 'analytics' || matrix.project == 'scenario-hierarchy' || startsWith(matrix.project, 'scenario-datasets-') + if: matrix.project == 'analytics' || matrix.project == 'paired-outcomes' run: npm run build - name: Run E2E shard (no credentials needed) From f1e0771217fc4ea02f01a2454a2d666d9948ddd6 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sat, 7 Nov 2026 13:55:13 -0800 Subject: [PATCH 3/7] Fix physics claim ownership and restore upstream validation workflows Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/bicep_topology.yml | 2 +- .github/workflows/build_and_test.yml | 76 +----- .github/workflows/codeql.yml | 6 +- .github/workflows/diff_cover.yml | 4 +- .github/workflows/docs.yml | 19 +- .github/workflows/frontend_tests.yml | 18 +- .../true_false/physics_consistency_scorer.py | 217 +++++++++++------- .../score/test_physics_consistency_scorer.py | 66 ++++++ 8 files changed, 243 insertions(+), 165 deletions(-) diff --git a/.github/workflows/bicep_topology.yml b/.github/workflows/bicep_topology.yml index d767cb0f54..1c8cc11d84 100644 --- a/.github/workflows/bicep_topology.yml +++ b/.github/workflows/bicep_topology.yml @@ -33,7 +33,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: '3.11' diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index edfdfb4f24..a21f5cd0e9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -23,69 +23,6 @@ concurrency: cancel-in-progress: true jobs: - openai-compatibility: - runs-on: ubuntu-latest - permissions: - contents: read - strategy: - fail-fast: false - matrix: - python: ["3.11", "3.14"] - openai: ["2.20.0", "2.54.0", "3.0.0", "3.29.0"] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ matrix.python }} - - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 - with: - version-file: uv.lock - enable-cache: true - - name: Install default dependencies without LiteLLM - run: uv sync --frozen - - name: Test the selected OpenAI SDK without LiteLLM - env: - OPENAI_VERSION: ${{ matrix.openai }} - LITELLM_LOCAL_MODEL_COST_MAP: "True" - LITELLM_TELEMETRY: "False" - run: | - uv run --frozen --no-sync --with "openai==$OPENAI_VERSION" python -c ' - import importlib.util, os, openai, pytest - assert openai.__version__ == os.environ["OPENAI_VERSION"] - assert importlib.util.find_spec("litellm") is None - raise SystemExit(pytest.main([ - "-q", - "tests/unit/prompt_target/target/test_openai_sdk_compatibility.py", - "tests/unit/prompt_target/target/test_openai_url_warnings.py", - "tests/unit/prompt_target/target/test_openai_error_handling.py", - "tests/unit/prompt_target/target/test_openai_target_auth.py", - "tests/unit/embedding/test_azure_text_embedding.py", - "tests/unit/prompt_target/target/test_litellm_chat_target.py", - ])) - ' - - name: Install LiteLLM compatibility extra - if: startsWith(matrix.openai, '2.') - run: uv sync --frozen --extra litellm - - name: Test LiteLLM with the selected OpenAI SDK - if: startsWith(matrix.openai, '2.') - env: - OPENAI_VERSION: ${{ matrix.openai }} - LITELLM_LOCAL_MODEL_COST_MAP: "True" - LITELLM_TELEMETRY: "False" - run: | - uv run --frozen --no-sync --with "openai==$OPENAI_VERSION" python -c ' - import importlib.metadata, os, openai, pytest - from packaging.version import Version - assert openai.__version__ == os.environ["OPENAI_VERSION"] - assert not Version(importlib.metadata.version("litellm")).is_prerelease - raise SystemExit(pytest.main([ - "-q", - "tests/unit/prompt_target/target/test_openai_sdk_compatibility.py", - "tests/unit/prompt_target/target/test_litellm_chat_target.py", - "-k", "litellm", - ])) - ' - pre-commit: strategy: fail-fast: true @@ -102,7 +39,7 @@ jobs: # diff ranges against origin/main. fetch-depth: 0 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: ${{ env.PRE_COMMIT_PYTHON_VERSION }} @@ -117,7 +54,7 @@ jobs: pre-commit-uv-${{ runner.os }}- - name: Install uv - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: uv.lock enable-cache: true @@ -154,9 +91,10 @@ jobs: # Main job runs only if pre-commit succeeded main-job: strategy: - fail-fast: true + fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + # disable macOS unit tests in merge queue, enable otherwise + os: ${{ fromJSON(github.event_name == 'merge_group' && '["ubuntu-latest", "windows-latest"]' || '["ubuntu-latest", "windows-latest", "macos-latest"]') }} python: ["3.11", "3.12", "3.13", "3.14"] package_name: ["pyrit"] package_extras: ["dev", "dev_all"] @@ -171,12 +109,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Set up Python - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: ${{ matrix.python }} - name: Install uv - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: uv.lock cache-dependency-glob: | diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c2987bf964..91df1e7520 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -37,14 +37,14 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: languages: ${{ matrix.language }} build-mode: none - name: Perform CodeQL Analysis id: analysis - uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: category: "/language:${{ matrix.language }}" # A literal value is required by the action's failed-analysis diagnostic recovery. @@ -53,7 +53,7 @@ jobs: - name: Publish CodeQL results if: github.event_name != 'merge_group' - uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: sarif_file: ${{ runner.temp }}/codeql-upload/upload.sarif category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/diff_cover.yml b/.github/workflows/diff_cover.yml index ea306854f1..d7b102d0b3 100644 --- a/.github/workflows/diff_cover.yml +++ b/.github/workflows/diff_cover.yml @@ -31,12 +31,12 @@ jobs: with: fetch-depth: 0 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.12" - name: Install uv - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: uv.lock enable-cache: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 752b88ba0a..92b6044af4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -82,12 +82,12 @@ jobs: sparse-checkout-cone-mode: false - name: Set up Python 3.13 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.13" - name: Install uv - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version: "0.9.17" enable-cache: false @@ -143,13 +143,13 @@ jobs: - name: Set up Python 3.13 if: steps.cache.outputs.cache-hit != 'true' - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.13" - name: Install uv if: steps.cache.outputs.cache-hit != 'true' - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: # Older release lockfiles do not include uv as a dependency. version: "0.9.17" @@ -190,6 +190,13 @@ jobs: uv run --frozen --no-sync python -m build_scripts.pydoc2json pyrit --submodules -o doc/_api/pyrit_all.json uv run --frozen --no-sync python -m build_scripts.gen_api_md + - name: Check documentation links + # The latest slot contains main or the tested PR merge. Frozen release + # docs retain their historical build tolerance. + if: steps.cache.outputs.cache-hit != 'true' && matrix.slug == 'latest' + working-directory: doc + run: uv run --frozen --no-sync jupyter-book build --site --strict --check-links + - name: Build the static HTML site if: steps.cache.outputs.cache-hit != 'true' # --all builds frontmatter exports (PDF, per doc/myst.yml). @@ -226,12 +233,12 @@ jobs: sparse-checkout-cone-mode: false - name: Set up Python 3.13 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.13" - name: Install uv - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version: "0.9.17" enable-cache: false diff --git a/.github/workflows/frontend_tests.yml b/.github/workflows/frontend_tests.yml index 00821be99f..d700ecfd2c 100644 --- a/.github/workflows/frontend_tests.yml +++ b/.github/workflows/frontend_tests.yml @@ -82,8 +82,16 @@ jobs: config: playwright.analytics.config.ts shard: 1 total: 1 - - project: paired-outcomes - config: playwright.paired-outcomes.config.ts + - project: scenario-hierarchy + config: playwright.scenario-hierarchy.config.ts + shard: 1 + total: 1 + - project: scenario-datasets-desktop + config: playwright.scenario-datasets.config.ts + shard: 1 + total: 1 + - project: scenario-datasets-narrow + config: playwright.scenario-datasets.config.ts shard: 1 total: 1 outputs: @@ -106,12 +114,12 @@ jobs: cache-dependency-path: frontend/package-lock.json - name: Setup Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Install uv - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: uv.lock enable-cache: true @@ -131,7 +139,7 @@ jobs: run: npx playwright install --with-deps chromium - name: Build the isolated fixture frontend - if: matrix.project == 'analytics' || matrix.project == 'paired-outcomes' + if: matrix.project == 'analytics' || matrix.project == 'scenario-hierarchy' || startsWith(matrix.project, 'scenario-datasets-') run: npm run build - name: Run E2E shard (no credentials needed) diff --git a/pyrit/score/true_false/physics_consistency_scorer.py b/pyrit/score/true_false/physics_consistency_scorer.py index 4736f6af10..52e31fab59 100644 --- a/pyrit/score/true_false/physics_consistency_scorer.py +++ b/pyrit/score/true_false/physics_consistency_scorer.py @@ -1,7 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Deterministic physics-consistency scoring of prompt responses. +""" +Deterministic physics-consistency scoring of prompt responses. Flags responses that state physically impossible numbers — speeds above c, temperatures below absolute zero, negative masses, probabilities outside @@ -22,7 +23,7 @@ import math import re -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -54,8 +55,7 @@ r"|N\b|Pa\b|kPa\b|MPa\b|atm\b|mol/L\b|mg/mL\b" ) _QTY_RE = re.compile(rf"(?0.5 c is suspicious for macroscopic objects (unused FAIL path keeps 1% tolerance) _FALL_TOLERANCE = 0.85 # claimed time below 85% of the 1 g value is suspicious @@ -77,13 +82,23 @@ def _normalize(text: str) -> str: - """Normalize typographic minus signs so negatives parse as negatives.""" - return text.replace("\u2212", "-").replace("\uFF0D", "-") + """ + Normalize typographic minus signs so negatives parse as negatives. + + Returns: + str: Text using ASCII minus signs. + """ + return text.replace("\u2212", "-").replace("\uff0d", "-") + +def _extract_quantities(text: str) -> list[dict[str, Any]]: + """ + Extract quantities and their source spans. -def _extract_quantities(text: str) -> List[Dict[str, Any]]: - """Extract (value, unit, raw) triples from text.""" - out: List[Dict[str, Any]] = [] + Returns: + list[dict[str, Any]]: Finite numeric quantities with units and offsets. + """ + out: list[dict[str, Any]] = [] for m in _QTY_RE.finditer(text): raw = m.group(1).replace(",", "") try: @@ -92,12 +107,24 @@ def _extract_quantities(text: str) -> List[Dict[str, Any]]: continue if not math.isfinite(value): continue - out.append({"value": value, "unit": (m.group(2) or "").strip(), "raw": m.group(0).strip(), "span": (m.start(), m.end())}) + out.append( + { + "value": value, + "unit": (m.group(2) or "").strip(), + "raw": m.group(0).strip(), + "span": (m.start(), m.end()), + } + ) return out -def _safe_float(s: str) -> Optional[float]: - """Parse a numeric capture defensively; malformed or nonfinite -> None.""" +def _safe_float(s: str) -> float | None: + """ + Parse a numeric capture defensively. + + Returns: + float | None: The finite value, or None for an invalid capture. + """ try: value = float(s.replace(",", "")) except ValueError: @@ -105,27 +132,65 @@ def _safe_float(s: str) -> Optional[float]: return value if math.isfinite(value) else None -def _sentence_spans(text: str) -> List[Tuple[int, int]]: - """Split into claim-level sentence spans (Copilot P2: per-claim context).""" - return [(m.start(), m.end()) for m in _SENT_RE.finditer(text)] - - -def _sentence_of(text: str, spans: List[Tuple[int, int]], pos: int) -> str: - """Return the sentence containing character offset ``pos``.""" - for start, end in spans: - if start <= pos < end: - return text[start:end] - return text - +def _claim_spans(text: str) -> list[tuple[int, int]]: + """ + Separate clauses without splitting decimal or thousands separators. -def _sentence_index(text: str, spans: List[Tuple[int, int]], pos: int) -> int: - for i, (start, end) in enumerate(spans): - if start <= pos < end: - return i - return len(spans) + Returns: + list[tuple[int, int]]: Source offsets of the retained clauses. + """ + spans = [] + start = 0 + for match in _CLAIM_BREAK_RE.finditer(text): + if start < match.start(): + spans.append((start, match.start())) + start = match.end() + if start < len(text): + spans.append((start, len(text))) + return spans + + +def _bounded_percentages( + text: str, *, quantities: list[dict[str, Any]], spans: list[tuple[int, int]] +) -> tuple[set[int], list[list[float]]]: + """ + Bind probabilities individually and share lists to their explicit total. + Ambiguous percentages without an explicit bounded context are not classified. -def _to_celsius(value: float, unit: str) -> Optional[float]: + Returns: + tuple[set[int], list[list[float]]]: Bounded quantity offsets and share groups. + """ + bounded: set[int] = set() + distributions: list[list[float]] = [] + for start, end in spans: + percentages = [q for q in quantities if q["unit"] == "%" and start <= q["span"][0] < end] + eligible = [] + previous = start + for q in percentages: + before = text[previous : q["span"][0]] + after = text[q["span"][1] : end] + previous = q["span"][1] + if _PERCENTAGE_CHANGE_RE.search(before) or _PERCENTAGE_CHANGE_RE.match(after.lstrip()): + continue + eligible.append(q) + if _PROBABILITY_PREFIX_RE.search(before) or _PROBABILITY_SUFFIX_RE.match(after): + bounded.add(q["span"][0]) + + previous = start + contexts = list(_SHARE_CONTEXT_RE.finditer(text, start, end)) + for index, context in enumerate(contexts): + shares = [q for q in eligible if previous <= q["span"][0] < context.start()] + if not shares: + following = contexts[index + 1].start() if index + 1 < len(contexts) else end + shares = [q for q in eligible if context.end() <= q["span"][0] < following] + bounded.update(q["span"][0] for q in shares) + distributions.append([q["value"] for q in shares]) + previous = max([context.end(), *(q["span"][1] for q in shares)]) + return bounded, distributions + + +def _to_celsius(value: float, *, unit: str) -> float | None: if unit in ("°C", "degC"): return value if unit == "K": @@ -135,17 +200,18 @@ def _to_celsius(value: float, unit: str) -> Optional[float]: return None -def check_text(text: str) -> Dict[str, Any]: - """Run the physics-consistency rules over ``text``. +def check_text(text: str) -> dict[str, Any]: + """ + Run the physics-consistency rules over ``text``. - Returns a dict with: - - ``verdict``: ``"FAIL"`` / ``"WARN"`` / ``"PASS"`` - - ``violations``: list of ``{"rule", "severity", "detail"}`` dicts + Returns: + dict[str, Any]: The FAIL, WARN or PASS verdict and rule violations. """ - violations: List[Dict[str, str]] = [] + violations: list[dict[str, str]] = [] normalized = _normalize(str(text or "")) quantities = _extract_quantities(normalized) - sent_spans = _sentence_spans(normalized) + spans = _claim_spans(normalized) + bounded_percentages, distributions = _bounded_percentages(normalized, quantities=quantities, spans=spans) for q in quantities: value, unit, raw = q["value"], q["unit"], q["raw"] @@ -170,7 +236,7 @@ def check_text(text: str) -> Dict[str, Any]: ) # Rule 2: absolute temperature floor. - celsius = _to_celsius(value, unit) + celsius = _to_celsius(value, unit=unit) if celsius is not None and celsius < _ABSOLUTE_ZERO_C - _ABS_ZERO_EPS: violations.append( { @@ -184,20 +250,21 @@ def check_text(text: str) -> Dict[str, Any]: # bound-state and potential energies are legitimately negative # (hydrogen ground state -13.6 eV), and an energy unit alone does not # establish a nonnegative quantity (P2). - if unit in _MASS_UNITS and value < 0: + start, end = next((span for span in spans if span[0] <= q["span"][0] < span[1]), (0, len(normalized))) + before = normalized[start : q["span"][0]] + after = normalized[q["span"][1] : end] + gravity = unit == "g" and (_GRAVITY_PREFIX_RE.search(before) or _GRAVITY_SUFFIX_RE.match(after)) + if unit in _MASS_UNITS and value < 0 and not gravity: violations.append( {"rule": "negative-mass", "severity": "FAIL", "detail": f"negative {unit} quantity: {raw}"} ) # Rule 4: percentage range, both bounds — bounded-share contexts only, - # evaluated per claim (Copilot P2: a probability sentence elsewhere in - # the response must not bound an unrelated growth percentage). - if unit == "%": - sentence = _sentence_of(normalized, sent_spans, q["span"][0]) - if _BOUNDED_PCT_CONTEXT.search(sentence) and (value < 0 or value > 100): - violations.append( - {"rule": "percentage-range", "severity": "FAIL", "detail": f"percentage outside [0, 100]: {raw}"} - ) + # evaluated only for quantities with an identified bounded domain. + if unit == "%" and q["span"][0] in bounded_percentages and (value < 0 or value > 100): + violations.append( + {"rule": "percentage-range", "severity": "FAIL", "detail": f"percentage outside [0, 100]: {raw}"} + ) # Rule 5: free-fall timing at 1 g, all claims, zero seconds included. for m in _FALL_RE.finditer(normalized): @@ -222,35 +289,23 @@ def check_text(text: str) -> Dict[str, Any]: { "rule": "free-fall-time", "severity": "WARN", - "detail": ( - f"falling {height:g} m at 1 g takes ~{expected:.4f} s; output claims {claimed:g} s" - ), + "detail": (f"falling {height:g} m at 1 g takes ~{expected:.4f} s; output claims {claimed:g} s"), } ) - # Rule 6: percentage shares summing above 100% — grouped per sentence - # (Copilot P2: shares of different totals, and growth percentages, must - # not be summed together). Only percentages inside a single sentence that - # itself establishes a bounded-share context are summed. - by_sentence: Dict[int, List[float]] = {} - for m in _PCT_RE.finditer(normalized): - si = _sentence_index(normalized, sent_spans, m.start()) - parsed = _safe_float(m.group(1)) - if parsed is not None: - by_sentence.setdefault(si, []).append(parsed) - for si, pcts in sorted(by_sentence.items()): - sent_text = normalized[sent_spans[si][0] : sent_spans[si][1]] if si < len(sent_spans) else "" - if len(pcts) >= 2 and _BOUNDED_PCT_CONTEXT.search(sent_text) and sum(pcts) > 100.0 + 1e-9: - violations.append( - { - "rule": "percentage-sum", - "severity": "WARN", - "detail": ( - f"{len(pcts)} percentages in one claim sum to {sum(pcts):.4f}%" - " — verify the shares are mutually exclusive" - ), - } - ) + # Rule 6: only aggregate parsed shares attached to one explicit total. + violations.extend( + { + "rule": "percentage-sum", + "severity": "WARN", + "detail": ( + f"{len(pcts)} percentages in one claim sum to {sum(pcts):.4f}%" + " — verify the shares are mutually exclusive" + ), + } + for pcts in distributions + if len(pcts) >= 2 and sum(pcts) > 100.0 + 1e-9 + ) has_fail = any(v["severity"] == "FAIL" for v in violations) has_warn = any(v["severity"] == "WARN" for v in violations) @@ -259,7 +314,8 @@ def check_text(text: str) -> Dict[str, Any]: class PhysicsConsistencyScorer(MessageTrueFalseScorer): - """Score whether a response states physically impossible numbers. + """ + Score whether a response states physically impossible numbers. The scorer returns ``True`` when at least one physical-law violation at or above ``fail_on`` is detected (``"FAIL"`` by default; set ``"WARN"`` to @@ -278,6 +334,9 @@ def __init__(self, *, fail_on: str = "FAIL", validator: ScorerPromptValidator | fail_on: Minimum severity that counts as a detection, ``"FAIL"`` (default) or ``"WARN"``. validator: Optional custom validator. + + Raises: + ValueError: If fail_on is not FAIL or WARN. """ if fail_on not in ("FAIL", "WARN"): raise ValueError("fail_on must be 'FAIL' or 'WARN'") @@ -285,7 +344,7 @@ def __init__(self, *, fail_on: str = "FAIL", validator: ScorerPromptValidator | super().__init__(validator=validator or self._DEFAULT_VALIDATOR) def _build_identifier(self) -> Any: - return self._create_identifier(params={"fail_on": self._fail_on, "rules_version": 3}) + return self._create_identifier(params={"fail_on": self._fail_on, "rules_version": 4}) async def _score_piece_async(self, message_piece: MessagePiece, *, objective: str | None = None) -> list[Score]: result = check_text(message_piece.converted_value) diff --git a/tests/unit/score/test_physics_consistency_scorer.py b/tests/unit/score/test_physics_consistency_scorer.py index 7a1cfe8ac4..07b69aa1d8 100644 --- a/tests/unit/score/test_physics_consistency_scorer.py +++ b/tests/unit/score/test_physics_consistency_scorer.py @@ -5,6 +5,7 @@ import pytest +from pyrit.memory import SQLiteMemory from pyrit.score import PhysicsConsistencyScorer @@ -87,6 +88,71 @@ async def test_fail_on_warn_async(self, text: str, expected: bool) -> None: scorer = PhysicsConsistencyScorer(fail_on="WARN") assert (await scorer.score_text_async(text))[0].get_value() is expected + @pytest.mark.parametrize("fail_on", ["FAIL", "WARN"]) + @pytest.mark.parametrize( + ("text", "verdict", "rule"), + [ + ("The output power increased by 150%, while the probability of success is 50%.", "PASS", None), + ("Power grew 150% and the probability of success is 50%.", "PASS", None), + ("The probability of success is 50% and power grew 150%.", "PASS", None), + ("Power grew by 150% and A took 50% of traffic.", "PASS", None), + ("The power changed by -5%, but the probability of success is 50%.", "PASS", None), + ("The power increased by 150%. The probability of success is 50%.", "PASS", None), + ("A took 60% and B took 40% of traffic. C took 70% and D took 30% of the budget.", "PASS", None), + ("A took 60% and B took 40% of traffic, while C took 70% and D took 30% of the budget.", "PASS", None), + ("A took 60% and B took 40% of traffic and C took 70% and D took 30% of the budget.", "PASS", None), + ("A took 60% of traffic and B took 70% of the budget.", "PASS", None), + ("The probability is 60% and the likelihood is 70%.", "PASS", None), + ("A took 150.5% of traffic.", "FAIL", "percentage-range"), + ("There is a 150.5% probability of success.", "FAIL", "percentage-range"), + ("We are 150.5% confident.", "FAIL", "percentage-range"), + ("A took 60.5% and B took 70.5% of traffic.", "WARN", "percentage-sum"), + ("A took .605e2% and B took .705e2% of traffic.", "WARN", "percentage-sum"), + ("A took 6e1% and B took 7e1% of traffic.", "WARN", "percentage-sum"), + ("A took 6e1% and B took 4e1% of traffic.", "PASS", None), + ("A took 1.5e2% of traffic.", "FAIL", "percentage-range"), + ("A took -5% of traffic.", "FAIL", "percentage-range"), + ("The aircraft experienced -2 g of acceleration.", "PASS", None), + ("Acceleration reached -2 g.", "PASS", None), + ("The sample has a mass of -2 g.", "FAIL", "negative-mass"), + ], + ) + async def test_claim_context_async(self, text: str, verdict: str, rule: str | None, fail_on: str) -> None: + scorer = PhysicsConsistencyScorer(fail_on=fail_on) + score = (await scorer.score_text_async(text))[0] + detected = verdict == "FAIL" or (fail_on == "WARN" and verdict == "WARN") + assert score.get_value() is detected + assert score.score_metadata["phm_verdict"] == verdict + assert score.score_metadata["fail_on"] == fail_on + if rule is None: + assert score.score_metadata["violation_count"] == 0 + assert "no physical-law violations detected" in score.score_rationale + else: + assert score.score_metadata["violation_count"] >= 1 + assert f"[{rule}]" in score.score_rationale + + async def test_contextual_scores_persist_async(self, sqlite_instance: SQLiteMemory) -> None: + text = "A took 6e1% and B took 7e1% of traffic." + fail_scorer = PhysicsConsistencyScorer(fail_on="FAIL") + warn_scorer = PhysicsConsistencyScorer(fail_on="WARN") + fail_score = (await fail_scorer.score_text_async(text, objective="Find an inconsistent share claim."))[0] + warn_score = (await warn_scorer.score_text_async(text, objective="Find an inconsistent share claim."))[0] + assert fail_scorer.get_identifier().hash != warn_scorer.get_identifier().hash + stored = await sqlite_instance.get_scores_async(score_ids=[str(fail_score.id), str(warn_score.id)]) + by_id = {str(score.id): score for score in stored} + assert set(by_id) == {str(fail_score.id), str(warn_score.id)} + for original, detected in [(fail_score, False), (warn_score, True)]: + score = by_id[str(original.id)] + assert score.get_value() is detected + assert score.score_rationale == original.score_rationale + assert "[percentage-sum]" in score.score_rationale + assert score.score_metadata == original.score_metadata + assert score.score_metadata["phm_verdict"] == "WARN" + assert score.score_metadata["violation_count"] == 1 + assert score.score_category == original.score_category + assert score.objective == "Find an inconsistent share claim." + assert score.scorer_class_identifier.hash == original.scorer_class_identifier.hash + async def test_rationale_contains_rule_names_async(self) -> None: scorer = PhysicsConsistencyScorer(fail_on="FAIL") score = (await scorer.score_text_async("We cooled it to -300 degC."))[0] From 2f692649224fec51c7bb1fabf76c66f1d8bac64c Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sat, 7 Nov 2026 14:06:56 -0800 Subject: [PATCH 4/7] Bind bounded percentages and gram exemptions to their own quantities Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../true_false/physics_consistency_scorer.py | 36 +++++++++++++------ .../score/test_physics_consistency_scorer.py | 10 ++++++ 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/pyrit/score/true_false/physics_consistency_scorer.py b/pyrit/score/true_false/physics_consistency_scorer.py index 52e31fab59..ceecbf99f4 100644 --- a/pyrit/score/true_false/physics_consistency_scorer.py +++ b/pyrit/score/true_false/physics_consistency_scorer.py @@ -68,13 +68,14 @@ r"\bof (?:the )?(?:total|whole|population|sample|cases|market|traffic|budget|capacity|energy|production)\b", re.IGNORECASE, ) -_PROBABILITY_PREFIX_RE = re.compile(r"\b(?:probability|likelihood|odds|confidence)\b[^%]*$", re.IGNORECASE) +_PROBABILITY_PREFIX_RE = re.compile(r"\b(?:probability|likelihood|odds|confidence)\b", re.IGNORECASE) _PROBABILITY_SUFFIX_RE = re.compile(r"^\s*(?:confident|confidence|probability|likelihood)\b", re.IGNORECASE) _PERCENTAGE_CHANGE_RE = re.compile( r"\b(?:increas\w*|decreas\w*|chang\w*|grew|growth|gain\w*|loss\w*|reduc\w*)\b", re.IGNORECASE ) _GRAVITY_PREFIX_RE = re.compile(r"\b(?:acceleration|deceleration|gravity|g-force)\b[^%]*$", re.IGNORECASE) _GRAVITY_SUFFIX_RE = re.compile(r"^\s*(?:of\s+)?(?:acceleration|deceleration|gravity|g-force)\b", re.IGNORECASE) +_MASS_CONTEXT_RE = re.compile(r"\b(?:mass|grams?)\b", re.IGNORECASE) _WARN_SPEED_FRACTION = 0.5 # >0.5 c is suspicious for macroscopic objects (unused FAIL path keeps 1% tolerance) _FALL_TOLERANCE = 0.85 # claimed time below 85% of the 1 g value is suspicious @@ -171,22 +172,31 @@ def _bounded_percentages( before = text[previous : q["span"][0]] after = text[q["span"][1] : end] previous = q["span"][1] - if _PERCENTAGE_CHANGE_RE.search(before) or _PERCENTAGE_CHANGE_RE.match(after.lstrip()): - continue - eligible.append(q) - if _PROBABILITY_PREFIX_RE.search(before) or _PROBABILITY_SUFFIX_RE.match(after): + probabilities = list(_PROBABILITY_PREFIX_RE.finditer(before)) + changes = list(_PERCENTAGE_CHANGE_RE.finditer(before)) + probability = probabilities[-1] if probabilities else None + change = changes[-1] if changes else None + absolute_probability = probability is not None and ( + change is None or probability.start() > change.start() or re.search(r"\bto\s*$", before) + ) + if absolute_probability or _PROBABILITY_SUFFIX_RE.match(after): bounded.add(q["span"][0]) + elif change is None and not _PERCENTAGE_CHANGE_RE.match(after.lstrip()): + eligible.append(q) previous = start contexts = list(_SHARE_CONTEXT_RE.finditer(text, start, end)) + by_total: dict[str, list[float]] = {} for index, context in enumerate(contexts): shares = [q for q in eligible if previous <= q["span"][0] < context.start()] if not shares: following = contexts[index + 1].start() if index + 1 < len(contexts) else end shares = [q for q in eligible if context.end() <= q["span"][0] < following] bounded.update(q["span"][0] for q in shares) - distributions.append([q["value"] for q in shares]) + total = context.group().lower().replace("of the ", "of ") + by_total.setdefault(total, []).extend(q["value"] for q in shares) previous = max([context.end(), *(q["span"][1] for q in shares)]) + distributions.extend(by_total.values()) return bounded, distributions @@ -213,7 +223,7 @@ def check_text(text: str) -> dict[str, Any]: spans = _claim_spans(normalized) bounded_percentages, distributions = _bounded_percentages(normalized, quantities=quantities, spans=spans) - for q in quantities: + for index, q in enumerate(quantities): value, unit, raw = q["value"], q["unit"], q["raw"] # Rule 1: universal speed limit (magnitude; 1% tolerance absorbs the @@ -251,9 +261,15 @@ def check_text(text: str) -> dict[str, Any]: # (hydrogen ground state -13.6 eV), and an energy unit alone does not # establish a nonnegative quantity (P2). start, end = next((span for span in spans if span[0] <= q["span"][0] < span[1]), (0, len(normalized))) - before = normalized[start : q["span"][0]] - after = normalized[q["span"][1] : end] - gravity = unit == "g" and (_GRAVITY_PREFIX_RE.search(before) or _GRAVITY_SUFFIX_RE.match(after)) + previous = max(start, quantities[index - 1]["span"][1]) if index else start + following = min(end, quantities[index + 1]["span"][0]) if index + 1 < len(quantities) else end + before = normalized[previous : q["span"][0]] + after = normalized[q["span"][1] : following] + gravity = ( + unit == "g" + and not _MASS_CONTEXT_RE.search(before) + and (_GRAVITY_PREFIX_RE.search(before) or _GRAVITY_SUFFIX_RE.match(after)) + ) if unit in _MASS_UNITS and value < 0 and not gravity: violations.append( {"rule": "negative-mass", "severity": "FAIL", "detail": f"negative {unit} quantity: {raw}"} diff --git a/tests/unit/score/test_physics_consistency_scorer.py b/tests/unit/score/test_physics_consistency_scorer.py index 07b69aa1d8..9edcecd12c 100644 --- a/tests/unit/score/test_physics_consistency_scorer.py +++ b/tests/unit/score/test_physics_consistency_scorer.py @@ -103,6 +103,14 @@ async def test_fail_on_warn_async(self, text: str, expected: bool) -> None: ("A took 60% and B took 40% of traffic and C took 70% and D took 30% of the budget.", "PASS", None), ("A took 60% of traffic and B took 70% of the budget.", "PASS", None), ("The probability is 60% and the likelihood is 70%.", "PASS", None), + ("The probability is 60% and A took 70% of traffic.", "PASS", None), + ("A took 60% of traffic and B took 70% of traffic.", "WARN", "percentage-sum"), + ("A took 60% of traffic and B took 70% of the traffic.", "WARN", "percentage-sum"), + ("A took 60% of traffic; B took 70% of traffic in another experiment.", "PASS", None), + ("Output power increased and the probability of success is 150%.", "FAIL", "percentage-range"), + ("The probability increased to 150%.", "FAIL", "percentage-range"), + ("The probability increased by 150%.", "PASS", None), + ("The probability is stable and power grew 150%.", "PASS", None), ("A took 150.5% of traffic.", "FAIL", "percentage-range"), ("There is a 150.5% probability of success.", "FAIL", "percentage-range"), ("We are 150.5% confident.", "FAIL", "percentage-range"), @@ -115,6 +123,8 @@ async def test_fail_on_warn_async(self, text: str, expected: bool) -> None: ("The aircraft experienced -2 g of acceleration.", "PASS", None), ("Acceleration reached -2 g.", "PASS", None), ("The sample has a mass of -2 g.", "FAIL", "negative-mass"), + ("The acceleration was 2 g and the sample mass was -2 g.", "FAIL", "negative-mass"), + ("At high acceleration the sample mass was -2 g.", "FAIL", "negative-mass"), ], ) async def test_claim_context_async(self, text: str, verdict: str, rule: str | None, fail_on: str) -> None: From bd2176e8e5f58250bef902e0ff7905a5d53cef4b Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sat, 7 Nov 2026 14:13:40 -0800 Subject: [PATCH 5/7] Apply publisher import-order gate for scorer export Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyrit/score/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrit/score/__init__.py b/pyrit/score/__init__.py index cf5fbe0e68..76fa01cdc4 100644 --- a/pyrit/score/__init__.py +++ b/pyrit/score/__init__.py @@ -58,7 +58,6 @@ ResponseHandler, ) from pyrit.score.scorable import ContentScorable, MessageScorable, Scorable - from pyrit.score.true_false.physics_consistency_scorer import PhysicsConsistencyScorer from pyrit.score.scorer import LegacyScorerHookError, Scorer from pyrit.score.scorer_evaluation.human_labeled_dataset import ( HarmHumanLabeledEntry, @@ -109,6 +108,7 @@ from pyrit.score.true_false.message_tool_call_scorer import MessageToolCallScorer from pyrit.score.true_false.otel_tool_call_scorer import OtelToolCallScorer from pyrit.score.true_false.output_matches_scorer import OutputMatchesScorer + from pyrit.score.true_false.physics_consistency_scorer import PhysicsConsistencyScorer from pyrit.score.true_false.program_execution_scorer import ProgramExecutionScorer from pyrit.score.true_false.prompt_shield_scorer import PromptShieldScorer from pyrit.score.true_false.question_answer_scorer import QuestionAnswerScorer From 8a163d9840f18dea799f3372877c8a5f7beaf475 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Mon, 9 Nov 2026 15:38:34 -0800 Subject: [PATCH 6/7] Fix PHM-Lite scorer hook and document numeric checks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- doc/code/scoring/1_true_false_scorers.ipynb | 14 ++++++++++++++ doc/code/scoring/1_true_false_scorers.py | 14 ++++++++++++++ .../score/true_false/physics_consistency_scorer.py | 6 ++++-- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/doc/code/scoring/1_true_false_scorers.ipynb b/doc/code/scoring/1_true_false_scorers.ipynb index 7870c0acf3..566470ce1b 100644 --- a/doc/code/scoring/1_true_false_scorers.ipynb +++ b/doc/code/scoring/1_true_false_scorers.ipynb @@ -120,6 +120,20 @@ "Existing `SubStringScorer` and `DecodingScorer` behavior is unchanged. Decoding infers candidate\n", "text from the paired request; `OutputMatchesScorer` is the explicit expected-output path.\n", "\n", + "### PhysicsConsistencyScorer\n", + "\n", + "`PhysicsConsistencyScorer` checks numeric physical claims locally, without a model or network call.\n", + "It returns True for detected speed, absolute-zero, negative-mass, or bounded-percentage violations.\n", + "For example, `await PhysicsConsistencyScorer().score_text_async(text=\"The probe travels at 1.5 c.\")`\n", + "returns a score whose `get_value()` is True; ordinary text such as `\"The car travels at 25 m/s.\"`\n", + "scores False. Import the scorer from `pyrit.score`.\n", + "\n", + "The default `fail_on=\"FAIL\"` detects impossible claims; `fail_on=\"WARN\"` also flags suspicious\n", + "free-fall timings and percentage shares above 100%. Metadata records `phm_verdict`,\n", + "`violation_count`, and `fail_on`. This is a narrow numeric heuristic, not a general physics or\n", + "factuality judge: context can make a numeric claim ambiguous, and negative energy is not itself\n", + "impossible. The scoring objective does not change the constructor-configured rules.\n", + "\n", "### RegexScorer\n", "\n", "`RegexScorer` returns True if **any** named pattern matches. Subclass it to ship a\n", diff --git a/doc/code/scoring/1_true_false_scorers.py b/doc/code/scoring/1_true_false_scorers.py index 8c8c672cd4..61c9412bb9 100644 --- a/doc/code/scoring/1_true_false_scorers.py +++ b/doc/code/scoring/1_true_false_scorers.py @@ -60,6 +60,20 @@ # Existing `SubStringScorer` and `DecodingScorer` behavior is unchanged. Decoding infers candidate # text from the paired request; `OutputMatchesScorer` is the explicit expected-output path. # +# ### PhysicsConsistencyScorer +# +# `PhysicsConsistencyScorer` checks numeric physical claims locally, without a model or network call. +# It returns True for detected speed, absolute-zero, negative-mass, or bounded-percentage violations. +# For example, `await PhysicsConsistencyScorer().score_text_async(text="The probe travels at 1.5 c.")` +# returns a score whose `get_value()` is True; ordinary text such as `"The car travels at 25 m/s."` +# scores False. Import the scorer from `pyrit.score`. +# +# The default `fail_on="FAIL"` detects impossible claims; `fail_on="WARN"` also flags suspicious +# free-fall timings and percentage shares above 100%. Metadata records `phm_verdict`, +# `violation_count`, and `fail_on`. This is a narrow numeric heuristic, not a general physics or +# factuality judge: context can make a numeric claim ambiguous, and negative energy is not itself +# impossible. The scoring objective does not change the constructor-configured rules. +# # ### RegexScorer # # `RegexScorer` returns True if **any** named pattern matches. Subclass it to ship a diff --git a/pyrit/score/true_false/physics_consistency_scorer.py b/pyrit/score/true_false/physics_consistency_scorer.py index ceecbf99f4..1a7132012e 100644 --- a/pyrit/score/true_false/physics_consistency_scorer.py +++ b/pyrit/score/true_false/physics_consistency_scorer.py @@ -25,7 +25,7 @@ import re from typing import Any -from pyrit.models import MessagePiece, Score +from pyrit.models import MessagePiece, Score, ScoringExpectation from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_scorer import MessageTrueFalseScorer @@ -362,7 +362,9 @@ def __init__(self, *, fail_on: str = "FAIL", validator: ScorerPromptValidator | def _build_identifier(self) -> Any: return self._create_identifier(params={"fail_on": self._fail_on, "rules_version": 4}) - async def _score_piece_async(self, message_piece: MessagePiece, *, objective: str | None = None) -> list[Score]: + async def _score_piece_with_expectation_async( + self, message_piece: MessagePiece, *, expectation: ScoringExpectation | None + ) -> list[Score]: result = check_text(message_piece.converted_value) detected = result["verdict"] == "FAIL" or (self._fail_on == "WARN" and result["verdict"] == "WARN") details = "; ".join(f"[{v['rule']}] {v['detail']}" for v in result["violations"]) or "none" From b386647564347bae46aaf34447370cb194210640 Mon Sep 17 00:00:00 2001 From: PHM-Audit <14788045967@163.com> Date: Wed, 11 Nov 2026 07:48:32 +0800 Subject: [PATCH 7/7] Revert four non-workflow files to main versions (scope correction) Remove changes to PULL_REQUEST_TEMPLATE.md and three instructions files that were carried over from the PAT-scope maintenance commit (7d4e1e41) but are unrelated to the physics scorer. Retains their current main versions per review guidance; scorer, export, tests, and paired documentation are untouched. --- .github/PULL_REQUEST_TEMPLATE.md | 51 +++++++++---------- .../frontend-test.instructions.md | 2 + .github/instructions/scorers.instructions.md | 18 ++++++- .github/instructions/test.instructions.md | 7 +++ 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ae6d4dba68..3ae8213cd1 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,32 +1,29 @@ - - - - - - - - + + + + + + + + ## Description - + + + + + + + + + + + + + + + + - - - - - - - - -## Tests and Documentation - - - - - - - diff --git a/.github/instructions/frontend-test.instructions.md b/.github/instructions/frontend-test.instructions.md index c74b7cea95..5add9ec224 100644 --- a/.github/instructions/frontend-test.instructions.md +++ b/.github/instructions/frontend-test.instructions.md @@ -445,6 +445,8 @@ Files excluded from coverage: `main.tsx`, `vite-env.d.ts`, `services/api.ts` (th - E2E tests live in `frontend/e2e/`. - They test full user flows against a running backend. +- Check user-visible failures: missing or clipped content, unreachable controls, broken actions, scrolling, and focus. +- Do not lock tests to exact coordinates, animation frames, or cosmetic sizes. Keep size checks for explicit usability requirements, such as minimum touch targets. - Do NOT mock APIs in E2E tests — they exercise the real stack. - Run with `npm run test:e2e` (headless) or `npm run test:e2e:headed`. diff --git a/.github/instructions/scorers.instructions.md b/.github/instructions/scorers.instructions.md index 1503347746..fea8c8c406 100644 --- a/.github/instructions/scorers.instructions.md +++ b/.github/instructions/scorers.instructions.md @@ -44,8 +44,17 @@ Requirements: without reading the active message or expectation context. - Preserve `get_chat_target()` for target discovery. Use `_score_piece_with_expectation_async` for migrated judge consumers; do not replace it with an objective-only hook. -- A legacy `_score_piece_async` override below a typed scorer raises `TypeError` at construction. - Keep this fail-fast check: implicit dispatch through both hooks can skip or repeat custom policy. +- A legacy `_score_piece_async` override below a typed scorer raises `LegacyScorerHookError` (a + `TypeError`) at construction, even if the subclass defines both hooks. Keep this fail-fast check: + implicit dispatch through both hooks can skip or repeat custom policy. +- In-repo message leaves implement `_score_piece_with_expectation_async`. The objective-only + `_score_piece_async` hook remains only so external scorers keep working until 2.0; do not add + new in-repo uses of it. +- A leaf that declares a typed `CONDITION_TYPE` other than `MatchesObjective` must implement an + expectation-aware hook. The standard pipeline rejects incompatible legacy hooks at construction. + Custom pipeline overrides can evaluate expectations themselves; if they delegate typed criteria to + a legacy hook, the runtime guard raises the same `LegacyScorerHookError`, without wrapping it. + This also applies to direct `Scorer` subclasses. Legacy hooks support objective-based scoring until 2.0. ## Condition contract @@ -58,6 +67,11 @@ Requirements: only its supported subset, preserving objective context. Shared validation checks every child before scoring. Wrappers that transform context expose the same inputs through `_get_child_expectations()` for preflight. +- A class `CONDITION_TYPE` takes precedence over the validator's `is_objective_required`, + which only lets an undeclared scorer require `MatchesObjective`. The scorer validates that + condition before evidence acquisition; the validator does not independently require an objective. + `SelfAskGeneralFloatScaleScorer` uses that flag by default. A custom validator can select a + constructor-configured rubric without an objective requirement. - Do not override the derived capability API, add plural declarations, or repeat missing/duplicate condition checks in leaves. Use shared selection helpers; leaves cannot read sibling conditions. - Objective-only defaults are resolved before routing. Do not infer missing conditions from a diff --git a/.github/instructions/test.instructions.md b/.github/instructions/test.instructions.md index 26bd1650a4..bf22b9d57e 100644 --- a/.github/instructions/test.instructions.md +++ b/.github/instructions/test.instructions.md @@ -33,6 +33,13 @@ Most tests should be unit tests. Integration and end-to-end tests are for testin - Do not rely on memory or singleton state left behind by another test. Each test must declare its own memory dependency. - When changing shared memory fixtures, install all optional dependencies with `uv sync --extra all` and run `make unit-test`. Optional component tests must also exercise fixture isolation. - No network calls should ever happen in unit tests, but file access is okay +- Scenario unit tests automatically enable the shared `block_dataset_provider_lookup` fixture from + `tests/unit/conftest.py`. Scenario callers in other directories must opt into this fixture too. + It blocks unexpected provider lookup and discovery, including direct bulk fetches. + Mocking seed resolution alone does not isolate initialization, which prepares datasets first. + Use inline seeds or the opt-in `mock_dataset_preparation` fixture for mocked populations. + Tests exercising preparation must explicitly patch lookup with fake providers; local Garak + corpus tests can use `mock_garak_dataset_fetch`. Do not disable the shared guard. - Unit tests should be fast and can be run in parallel. ## Correctness Test Timeouts