diff --git a/pyrit/scenario/scenarios/airt/cyber.py b/pyrit/scenario/scenarios/airt/cyber.py index 6e82a567e0..26cd505fb3 100644 --- a/pyrit/scenario/scenarios/airt/cyber.py +++ b/pyrit/scenario/scenarios/airt/cyber.py @@ -65,18 +65,18 @@ class Cyber(Scenario): techniques. """ - #: Bumped from 2 → 3 by dropping the ``core`` pool gate so the selectable - #: technique pool (and the ``all`` aggregate) reflects whatever the initializer - #: registered. ``use_cached`` only matches prior runs at the current ``VERSION``. - VERSION: int = 3 + #: Bumped from 3 → 4 because the default objective scorer now applies the malware + #: question; earlier versions never did. ``use_cached`` only matches prior runs at + #: the current ``VERSION``. + VERSION: int = 4 @classmethod - def get_override_composite_scorer_questions_path(cls) -> list[Path]: + def _get_additional_scoring_questions(cls) -> list[Path]: """ Override true/false question paths for cyber objective scoring. Returns: - Sequence[Path]: Paths to true/false question paths for cyber objective scoring. + list[Path]: Paths to true/false question paths for cyber objective scoring. """ return [SCORER_SEED_PROMPT_PATH / "true_false_question" / "malware.yaml"] diff --git a/tests/unit/scenario/airt/test_cyber.py b/tests/unit/scenario/airt/test_cyber.py index 3d7bc19624..268d1f44f4 100644 --- a/tests/unit/scenario/airt/test_cyber.py +++ b/tests/unit/scenario/airt/test_cyber.py @@ -13,7 +13,7 @@ from pyrit.registry import AttackTechniqueRegistry from pyrit.scenario import DatasetAttackConfiguration from pyrit.scenario.scenarios.airt import Cyber -from pyrit.score import TrueFalseScorer +from pyrit.score import TrueFalseCompositeScorer, TrueFalseInverterScorer, TrueFalseScorer from pyrit.setup.initializers.techniques import ( build_technique_factories, ) @@ -131,8 +131,8 @@ def _make_seed_groups(name: str) -> list[AttackSeedGroup]: class TestCyberBasic: """Tests for Cyber initialization and class properties.""" - def test_version_is_3(self): - assert Cyber.VERSION == 3 + def test_version_is_4(self): + assert Cyber.VERSION == 4 def test_get_technique_class(self): strat = _technique_class() @@ -187,6 +187,24 @@ def test_initialization_with_default_scorer(self): scenario = Cyber() assert scenario._objective_scorer_identifier is None + def test_default_objective_scorer_uses_malware_question(self): + scorer_target = MagicMock(spec=PromptTarget) + scorer_target.capabilities.includes.return_value = True + scorer_target.get_identifier.return_value = _mock_target_id("MockScorerTarget") + registry = MagicMock() + registry.instances.get_by_tag.return_value = [] + + with ( + patch("pyrit.scenario.core.scenario.ScorerRegistry.get_registry_singleton", return_value=registry), + patch("pyrit.scenario.core.scenario.get_default_scorer_target", return_value=scorer_target), + ): + scorer = Cyber()._get_default_objective_scorer() + + assert isinstance(scorer, TrueFalseCompositeScorer) + malware_scorer, inverted_refusal_scorer = scorer._scorers + assert malware_scorer._score_category == ["malware"] + assert isinstance(inverted_refusal_scorer, TrueFalseInverterScorer) + def test_scenario_name_is_cyber(self, mock_objective_scorer): scenario = Cyber(objective_scorer=mock_objective_scorer) assert scenario.name == "Cyber"