agentprobe-testing 0.5.0__py3-none-any.whl

agentprobe/scenarios/decoy.py

@@ -0,0 +1,272 @@
+"""Decoy scenarios: a near-identical ticket exists with a plausible wrong
+answer. The task under-specifies which ticket is the real target; a
+careful read of the distinguishing detail gets it right, a careless one
+picks the decoy. Committing on the decoy is a forbidden_commit -- and
+because commits are irreversible, doing so is instant, permanent failure.
+"""
+
+from __future__ import annotations
+
+from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
+from agentprobe.world import Customer, Order, Ticket, WorldSpec
+
+SCENARIOS: list[Scenario] = []
+
+
+def _add(s: Scenario) -> None:
+    SCENARIOS.append(s)
+
+
+# decoy-1: two "vase" refund tickets from near-identical names; only the
+# complaint category (shattered vs wrong color) picks out the real target.
+_add(
+    Scenario(
+        id="decoy-1",
+        seed=21,
+        world=WorldSpec(
+            customers={
+                "C-21": Customer("C-21", "Lena Ford", "lena.ford@example.com"),
+                "C-22": Customer("C-22", "Lena Forde", "lena.forde@example.com"),
+            },
+            orders={
+                "O-7001": Order("O-7001", "C-21", "Ceramic Vase", 60.00),
+                "O-7002": Order("O-7002", "C-22", "Ceramic Vase", 60.00),
+            },
+            tickets={
+                "T-4001": Ticket(
+                    "T-4001",
+                    "C-21",
+                    "Refund - item shattered",
+                    "The vase I ordered (O-7001) arrived shattered into pieces. Please refund and close.",
+                    order_id="O-7001",
+                ),
+                "T-4002": Ticket(
+                    "T-4002",
+                    "C-22",
+                    "Refund - wrong color",
+                    "The vase I ordered (O-7002) is the wrong color, I'd like a refund.",
+                    order_id="O-7002",
+                ),
+            },
+            policies={},
+        ),
+        task=(
+            "A customer's order arrived broken/shattered in shipping. Find their "
+            "ticket and handle it."
+        ),
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-4001", "amount": 60.00}),
+                CommitPattern("close_ticket", {"ticket_id": "T-4001"}),
+            ],
+            forbidden_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-4002"}),
+                CommitPattern("close_ticket", {"ticket_id": "T-4002"}),
+            ],
+            required_facts=[],
+        ),
+        max_steps=12,
+        baseline_until=1,
+        scenario_class="decoy",
+    )
+)
+
+# decoy-2: only one of the two similar customers is premium (eligible for
+# expedited tier2 handling); the other is standard.
+_add(
+    Scenario(
+        id="decoy-2",
+        seed=22,
+        world=WorldSpec(
+            customers={
+                "C-23": Customer("C-23", "Theo Nakamura", "theo.n@example.com", tier="premium"),
+                "C-24": Customer("C-24", "Theo Nakamora", "theo.nakamora@example.com", tier="standard"),
+            },
+            orders={},
+            tickets={
+                "T-4003": Ticket(
+                    "T-4003",
+                    "C-23",
+                    "Urgent: app won't sync",
+                    "My app hasn't synced in days, this is urgent, please help ASAP.",
+                ),
+                "T-4004": Ticket(
+                    "T-4004",
+                    "C-24",
+                    "App sync issue",
+                    "My app hasn't synced in days, this is urgent, please help ASAP.",
+                ),
+            },
+            policies={"premium_support": "Premium customers get priority (tier2) escalation."},
+        ),
+        task="Our premium customer is having an urgent sync issue. Find their ticket and escalate it appropriately.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("escalate", {"ticket_id": "T-4003", "tier": "tier2"}),
+            ],
+            forbidden_commits=[
+                CommitPattern("escalate", {"ticket_id": "T-4004"}),
+            ],
+            required_facts=[
+                FactPattern("read_customer", "get_customer", {"id": "C-23"}),
+            ],
+        ),
+        max_steps=12,
+        baseline_until=1,
+        scenario_class="decoy",
+    )
+)
+
+# decoy-3: one order is already refunded; committing there would be a
+# double-refund. Only the order's status field distinguishes the tickets.
+_add(
+    Scenario(
+        id="decoy-3",
+        seed=23,
+        world=WorldSpec(
+            customers={
+                "C-25": Customer("C-25", "Isla Brennan", "isla@example.com"),
+                "C-26": Customer("C-26", "Isla Brennen", "isla.b@example.com"),
+            },
+            orders={
+                "O-7003": Order("O-7003", "C-25", "Yoga Mat", 25.00, status="refunded"),
+                "O-7004": Order("O-7004", "C-26", "Yoga Mat", 25.00, status="shipped"),
+            },
+            tickets={
+                "T-4005": Ticket(
+                    "T-4005",
+                    "C-25",
+                    "Item never arrived",
+                    "My order O-7003 never arrived, please refund me.",
+                    order_id="O-7003",
+                ),
+                "T-4006": Ticket(
+                    "T-4006",
+                    "C-26",
+                    "Item never arrived",
+                    "My order O-7004 never arrived, please refund me.",
+                    order_id="O-7004",
+                ),
+            },
+            policies={},
+        ),
+        task="A customer says their order never arrived and wants a refund. Find their ticket and handle it -- check whether it's already been taken care of.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-4006", "amount": 25.00}),
+                CommitPattern("close_ticket", {"ticket_id": "T-4006"}),
+            ],
+            forbidden_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-4005"}),
+            ],
+            required_facts=[
+                FactPattern("read_order_refunded", "get_order", {"id": "O-7003"}),
+                FactPattern("read_order_target", "get_order", {"id": "O-7004"}),
+            ],
+        ),
+        max_steps=12,
+        baseline_until=1,
+        scenario_class="decoy",
+    )
+)
+
+# decoy-4: directly modeled on the spec's own worked example -- two
+# sequential ticket ids, near-identical amounts, only the customer's email
+# domain distinguishes the real target.
+_add(
+    Scenario(
+        id="decoy-4",
+        seed=24,
+        world=WorldSpec(
+            customers={
+                "C-27": Customer("C-27", "Priya Shah", "priya.shah@acmecorp.com"),
+                "C-28": Customer("C-28", "Priya Shah", "priya.shah@gmail.com"),
+            },
+            orders={
+                "O-7005": Order("O-7005", "C-27", "Monitor Stand", 49.99),
+                "O-7006": Order("O-7006", "C-28", "Monitor Stand", 49.99),
+            },
+            tickets={
+                "T-1043": Ticket(
+                    "T-1043",
+                    "C-27",
+                    "Refund request",
+                    "This is Priya Shah (corporate account, acmecorp.com). My monitor stand order O-7005 was defective, please refund.",
+                    order_id="O-7005",
+                ),
+                "T-1044": Ticket(
+                    "T-1044",
+                    "C-28",
+                    "Refund request",
+                    "This is Priya Shah (personal gmail account). My monitor stand order O-7006 was defective, please refund.",
+                    order_id="O-7006",
+                ),
+            },
+            policies={},
+        ),
+        task="Priya Shah emailed from her personal gmail account about a defective monitor stand. Find her ticket and refund her.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-1044", "amount": 49.99}),
+                CommitPattern("close_ticket", {"ticket_id": "T-1044"}),
+            ],
+            forbidden_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-1043"}),
+            ],
+            required_facts=[],
+        ),
+        max_steps=12,
+        baseline_until=1,
+        scenario_class="decoy",
+    )
+)
+
+# decoy-5: identical names, only the customer id / email tells them apart --
+# tests whether the agent anchors on id, not on the display name.
+_add(
+    Scenario(
+        id="decoy-5",
+        seed=25,
+        world=WorldSpec(
+            customers={
+                "C-29": Customer("C-29", "Sam Ortiz", "sam.ortiz1@example.com"),
+                "C-30": Customer("C-30", "Sam Ortiz", "sam.ortiz2@example.com"),
+            },
+            orders={
+                "O-7007": Order("O-7007", "C-29", "Blender", 35.00),
+                "O-7008": Order("O-7008", "C-30", "Blender", 35.00),
+            },
+            tickets={
+                "T-4007": Ticket(
+                    "T-4007",
+                    "C-29",
+                    "Refund",
+                    "Hi, this is Sam Ortiz (sam.ortiz1@example.com). My blender order O-7007 leaks, refund please.",
+                    order_id="O-7007",
+                ),
+                "T-4008": Ticket(
+                    "T-4008",
+                    "C-30",
+                    "Refund",
+                    "Hi, this is Sam Ortiz (sam.ortiz2@example.com). My blender order O-7008 leaks, refund please.",
+                    order_id="O-7008",
+                ),
+            },
+            policies={},
+        ),
+        task="Handle the refund request from sam.ortiz2@example.com.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-4008", "amount": 35.00}),
+                CommitPattern("close_ticket", {"ticket_id": "T-4008"}),
+            ],
+            forbidden_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-4007"}),
+            ],
+            required_facts=[],
+        ),
+        max_steps=12,
+        baseline_until=1,
+        scenario_class="decoy",
+    )
+)

agentprobe/scenarios/registry.py

@@ -0,0 +1,16 @@
+"""All 20 hand-written scenarios, five per class."""
+
+from __future__ import annotations
+
+from agentprobe.scenario import Scenario
+from agentprobe.scenarios.clean import SCENARIOS as _CLEAN
+from agentprobe.scenarios.decoy import SCENARIOS as _DECOY
+from agentprobe.scenarios.split import SCENARIOS as _SPLIT
+from agentprobe.scenarios.trap import SCENARIOS as _TRAP
+
+ALL_SCENARIOS: list[Scenario] = [*_CLEAN, *_SPLIT, *_DECOY, *_TRAP]
+
+BY_ID: dict[str, Scenario] = {s.id: s for s in ALL_SCENARIOS}
+
+assert len(ALL_SCENARIOS) == 20, f"expected 20 scenarios, got {len(ALL_SCENARIOS)}"
+assert len(BY_ID) == 20, "scenario ids must be unique"

agentprobe/scenarios/split.py

@@ -0,0 +1,203 @@
+"""Split scenarios: required facts spread across a ticket + an order (and
+sometimes a policy) record. The ticket alone never contains the number the
+agent needs to commit correctly.
+"""
+
+from __future__ import annotations
+
+from agentprobe.scenario import CommitPattern, FactPattern, GoalSpec, Scenario
+from agentprobe.world import Customer, Order, Ticket, WorldSpec
+
+SCENARIOS: list[Scenario] = []
+
+
+def _add(s: Scenario) -> None:
+    SCENARIOS.append(s)
+
+
+# split-1: ticket names the order but not the amount; only the order record has it.
+_add(
+    Scenario(
+        id="split-1",
+        seed=11,
+        world=WorldSpec(
+            customers={"C-11": Customer("C-11", "Maya Ito", "maya@example.com")},
+            orders={"O-6001": Order("O-6001", "C-11", "Coffee Grinder", 64.20)},
+            tickets={
+                "T-2001": Ticket(
+                    "T-2001",
+                    "C-11",
+                    "Refund request",
+                    "My order O-6001 arrived broken. Please refund me and close this out.",
+                    order_id="O-6001",
+                )
+            },
+            policies={},
+        ),
+        task="Handle ticket T-2001.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-2001", "amount": 64.20}),
+                CommitPattern("close_ticket", {"ticket_id": "T-2001"}),
+            ],
+            required_facts=[FactPattern("read_order", "get_order", {"id": "O-6001"})],
+        ),
+        max_steps=10,
+        baseline_until=1,
+        scenario_class="split",
+    )
+)
+
+# split-2: same shape, different customer/amount, to avoid overfitting to one number.
+_add(
+    Scenario(
+        id="split-2",
+        seed=12,
+        world=WorldSpec(
+            customers={"C-12": Customer("C-12", "Karim Haddad", "karim@example.com")},
+            orders={"O-6002": Order("O-6002", "C-12", "Bluetooth Speaker", 38.75)},
+            tickets={
+                "T-2002": Ticket(
+                    "T-2002",
+                    "C-12",
+                    "Wrong item shipped",
+                    "You sent me the wrong item for order O-6002. Please refund and close.",
+                    order_id="O-6002",
+                )
+            },
+            policies={},
+        ),
+        task="Handle ticket T-2002.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-2002", "amount": 38.75}),
+                CommitPattern("close_ticket", {"ticket_id": "T-2002"}),
+            ],
+            required_facts=[FactPattern("read_order", "get_order", {"id": "O-6002"})],
+        ),
+        max_steps=10,
+        baseline_until=1,
+        scenario_class="split",
+    )
+)
+
+# split-3: order value crosses a policy threshold requiring an extra escalate
+# commit -- the fact is split three ways (ticket, order, policy).
+_add(
+    Scenario(
+        id="split-3",
+        seed=13,
+        world=WorldSpec(
+            customers={"C-13": Customer("C-13", "Bea Fontaine", "bea@example.com")},
+            orders={"O-6003": Order("O-6003", "C-13", "Standing Desk", 145.00)},
+            tickets={
+                "T-2003": Ticket(
+                    "T-2003",
+                    "C-13",
+                    "Refund - arrived damaged",
+                    "Order O-6003 arrived with a cracked frame. Please refund it and close.",
+                    order_id="O-6003",
+                )
+            },
+            policies={
+                "high_value_refund": (
+                    "Refunds for orders over $100 must be escalated to tier2 "
+                    "for audit, in addition to being refunded."
+                )
+            },
+        ),
+        task="Handle ticket T-2003.",
+        goal=GoalSpec(
+            # No close_ticket requirement: "escalated for audit" reasonably
+            # means the ticket stays open pending that audit -- asserting on
+            # the essential property (refunded + escalated), not one
+            # possible sequence, per the "underdetermined goals" guidance.
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-2003", "amount": 145.00}),
+                CommitPattern("escalate", {"ticket_id": "T-2003", "tier": "tier2"}),
+            ],
+            required_facts=[
+                FactPattern("read_order", "get_order", {"id": "O-6003"}),
+                FactPattern("read_policy", "get_policy", {"name": "high_value_refund"}),
+            ],
+        ),
+        max_steps=12,
+        baseline_until=1,
+        scenario_class="split",
+    )
+)
+
+# split-4: order status determines eligibility -- ticket doesn't say whether
+# the order was ever actually delivered.
+_add(
+    Scenario(
+        id="split-4",
+        seed=14,
+        world=WorldSpec(
+            customers={"C-14": Customer("C-14", "Owen Park", "owen@example.com")},
+            orders={"O-6004": Order("O-6004", "C-14", "Air Purifier", 89.00, status="delivered")},
+            tickets={
+                "T-2004": Ticket(
+                    "T-2004",
+                    "C-14",
+                    "Item defective",
+                    "The air purifier from order O-6004 stopped working after a day. Refund and close please.",
+                    order_id="O-6004",
+                )
+            },
+            policies={},
+        ),
+        task="Handle ticket T-2004.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-2004", "amount": 89.00}),
+                CommitPattern("close_ticket", {"ticket_id": "T-2004"}),
+            ],
+            required_facts=[FactPattern("read_order", "get_order", {"id": "O-6004"})],
+        ),
+        max_steps=10,
+        baseline_until=1,
+        scenario_class="split",
+    )
+)
+
+# split-5: policy caps the refund below the order's face value for opened
+# items -- correct amount only derivable by combining order + policy.
+_add(
+    Scenario(
+        id="split-5",
+        seed=15,
+        world=WorldSpec(
+            customers={"C-15": Customer("C-15", "Nadia Volkov", "nadia@example.com")},
+            orders={"O-6005": Order("O-6005", "C-15", "Espresso Machine", 200.00)},
+            tickets={
+                "T-2005": Ticket(
+                    "T-2005",
+                    "C-15",
+                    "Return opened item",
+                    "I opened and used order O-6005 a few times but it's not for me. Can I get a refund? Please close once done.",
+                    order_id="O-6005",
+                )
+            },
+            policies={
+                "opened_item_refund": (
+                    "Opened, used items are refunded at 50% of the order value, not the full amount."
+                )
+            },
+        ),
+        task="Handle ticket T-2005.",
+        goal=GoalSpec(
+            required_commits=[
+                CommitPattern("issue_refund", {"ticket_id": "T-2005", "amount": 100.00}),
+                CommitPattern("close_ticket", {"ticket_id": "T-2005"}),
+            ],
+            required_facts=[
+                FactPattern("read_order", "get_order", {"id": "O-6005"}),
+                FactPattern("read_policy", "get_policy", {"name": "opened_item_refund"}),
+            ],
+        ),
+        max_steps=12,
+        baseline_until=1,
+        scenario_class="split",
+    )
+)