Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions pyrit/scenario/scenarios/airt/cyber.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,18 @@ class Cyber(Scenario):
techniques.
"""

#: Bumped from 2 → 3 by dropping the ``core`` pool gate so the selectable
#: technique pool (and the ``all`` aggregate) reflects whatever the initializer
#: registered. ``use_cached`` only matches prior runs at the current ``VERSION``.
VERSION: int = 3
#: Bumped from 3 → 4 because the default objective scorer now applies the malware
#: question; earlier versions never did. ``use_cached`` only matches prior runs at
#: the current ``VERSION``.
VERSION: int = 4

@classmethod
def get_override_composite_scorer_questions_path(cls) -> list[Path]:
def _get_additional_scoring_questions(cls) -> list[Path]:
"""
Override true/false question paths for cyber objective scoring.

Returns:
Sequence[Path]: Paths to true/false question paths for cyber objective scoring.
list[Path]: Paths to true/false question paths for cyber objective scoring.
"""
return [SCORER_SEED_PROMPT_PATH / "true_false_question" / "malware.yaml"]

Expand Down
24 changes: 21 additions & 3 deletions tests/unit/scenario/airt/test_cyber.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from pyrit.registry import AttackTechniqueRegistry
from pyrit.scenario import DatasetAttackConfiguration
from pyrit.scenario.scenarios.airt import Cyber
from pyrit.score import TrueFalseScorer
from pyrit.score import TrueFalseCompositeScorer, TrueFalseInverterScorer, TrueFalseScorer
from pyrit.setup.initializers.techniques import (
build_technique_factories,
)
Expand Down Expand Up @@ -131,8 +131,8 @@ def _make_seed_groups(name: str) -> list[AttackSeedGroup]:
class TestCyberBasic:
"""Tests for Cyber initialization and class properties."""

def test_version_is_3(self):
assert Cyber.VERSION == 3
def test_version_is_4(self):
assert Cyber.VERSION == 4

def test_get_technique_class(self):
strat = _technique_class()
Expand Down Expand Up @@ -187,6 +187,24 @@ def test_initialization_with_default_scorer(self):
scenario = Cyber()
assert scenario._objective_scorer_identifier is None

def test_default_objective_scorer_uses_malware_question(self):
scorer_target = MagicMock(spec=PromptTarget)
scorer_target.capabilities.includes.return_value = True
scorer_target.get_identifier.return_value = _mock_target_id("MockScorerTarget")
registry = MagicMock()
registry.instances.get_by_tag.return_value = []

with (
patch("pyrit.scenario.core.scenario.ScorerRegistry.get_registry_singleton", return_value=registry),
patch("pyrit.scenario.core.scenario.get_default_scorer_target", return_value=scorer_target),
):
scorer = Cyber()._get_default_objective_scorer()

assert isinstance(scorer, TrueFalseCompositeScorer)
malware_scorer, inverted_refusal_scorer = scorer._scorers
assert malware_scorer._score_category == ["malware"]
assert isinstance(inverted_refusal_scorer, TrueFalseInverterScorer)

def test_scenario_name_is_cyber(self, mock_objective_scorer):
scenario = Cyber(objective_scorer=mock_objective_scorer)
assert scenario.name == "Cyber"
Expand Down
Loading