From 0de07def10759b4ce72968b9afef86bad726bfe9 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 10 Nov 2026 17:05:01 -0800 Subject: [PATCH] FIX: Apply Cyber's malware scoring question in its default scorer Cyber defined get_override_composite_scorer_questions_path(), but the Scenario base only calls _get_additional_scoring_questions(). The names diverged in b0e91f884 (#1695), so Cyber's default objective scorer never used malware.yaml and fell back to the registry default scorer or the inverted refusal scorer. Rename the override to the hook the base calls, bump VERSION to 4 since default scoring changes, and test that the default scorer is the malware question AND not-refused composite. Co-Authored-By: Claude Opus 5.5 --- pyrit/scenario/scenarios/airt/cyber.py | 12 ++++++------ tests/unit/scenario/airt/test_cyber.py | 24 +++++++++++++++++++++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pyrit/scenario/scenarios/airt/cyber.py b/pyrit/scenario/scenarios/airt/cyber.py index 6e82a567e0..26cd505fb3 100644 --- a/pyrit/scenario/scenarios/airt/cyber.py +++ b/pyrit/scenario/scenarios/airt/cyber.py @@ -65,18 +65,18 @@ class Cyber(Scenario): techniques. """ - #: Bumped from 2 → 3 by dropping the ``core`` pool gate so the selectable - #: technique pool (and the ``all`` aggregate) reflects whatever the initializer - #: registered. ``use_cached`` only matches prior runs at the current ``VERSION``. - VERSION: int = 3 + #: Bumped from 3 → 4 because the default objective scorer now applies the malware + #: question; earlier versions never did. ``use_cached`` only matches prior runs at + #: the current ``VERSION``. + VERSION: int = 4 @classmethod - def get_override_composite_scorer_questions_path(cls) -> list[Path]: + def _get_additional_scoring_questions(cls) -> list[Path]: """ Override true/false question paths for cyber objective scoring. Returns: - Sequence[Path]: Paths to true/false question paths for cyber objective scoring. + list[Path]: Paths to true/false question paths for cyber objective scoring. """ return [SCORER_SEED_PROMPT_PATH / "true_false_question" / "malware.yaml"] diff --git a/tests/unit/scenario/airt/test_cyber.py b/tests/unit/scenario/airt/test_cyber.py index 3d7bc19624..268d1f44f4 100644 --- a/tests/unit/scenario/airt/test_cyber.py +++ b/tests/unit/scenario/airt/test_cyber.py @@ -13,7 +13,7 @@ from pyrit.registry import AttackTechniqueRegistry from pyrit.scenario import DatasetAttackConfiguration from pyrit.scenario.scenarios.airt import Cyber -from pyrit.score import TrueFalseScorer +from pyrit.score import TrueFalseCompositeScorer, TrueFalseInverterScorer, TrueFalseScorer from pyrit.setup.initializers.techniques import ( build_technique_factories, ) @@ -131,8 +131,8 @@ def _make_seed_groups(name: str) -> list[AttackSeedGroup]: class TestCyberBasic: """Tests for Cyber initialization and class properties.""" - def test_version_is_3(self): - assert Cyber.VERSION == 3 + def test_version_is_4(self): + assert Cyber.VERSION == 4 def test_get_technique_class(self): strat = _technique_class() @@ -187,6 +187,24 @@ def test_initialization_with_default_scorer(self): scenario = Cyber() assert scenario._objective_scorer_identifier is None + def test_default_objective_scorer_uses_malware_question(self): + scorer_target = MagicMock(spec=PromptTarget) + scorer_target.capabilities.includes.return_value = True + scorer_target.get_identifier.return_value = _mock_target_id("MockScorerTarget") + registry = MagicMock() + registry.instances.get_by_tag.return_value = [] + + with ( + patch("pyrit.scenario.core.scenario.ScorerRegistry.get_registry_singleton", return_value=registry), + patch("pyrit.scenario.core.scenario.get_default_scorer_target", return_value=scorer_target), + ): + scorer = Cyber()._get_default_objective_scorer() + + assert isinstance(scorer, TrueFalseCompositeScorer) + malware_scorer, inverted_refusal_scorer = scorer._scorers + assert malware_scorer._score_category == ["malware"] + assert isinstance(inverted_refusal_scorer, TrueFalseInverterScorer) + def test_scenario_name_is_cyber(self, mock_objective_scorer): scenario = Cyber(objective_scorer=mock_objective_scorer) assert scenario.name == "Cyber"