sys1bench 0.3.0__py3-none-any.whl

sys1bench/generators/log_triage.py

@@ -0,0 +1,100 @@
+"""Log triage generator: JSON log records with injected distractor keys.
+
+  severity   (score 0..3)   -> from the log level and error class, per a stated policy
+  owning_team (choice)      -> from the service name (a mapping is given in the state as `team_directory`)
+  page_oncall (noul)        -> policy: severity >= 2 AND the environment is production
+The team directory lives in the state, so the model must read a lookup table rather than guess; the paging rule
+combines two fields. Distractors are irrelevant keys with plausible values.
+"""
+
+from __future__ import annotations
+
+import json
+
+from ..schemas import Controls, LabelProvenance, Level, Option, Question, TaskItem
+from .base import BaseGenerator, canary_for, estimate_tokens, pad_to_tokens, register_generator
+
+SERVICES = {"checkout-api": "payments", "ledger": "payments", "search-indexer": "discovery", "ranker": "discovery", "auth-gateway": "identity",
+            "session-store": "identity", "notify-worker": "messaging", "mailer": "messaging", "cdn-edge": "platform", "k8s-autoscaler": "platform"}
+TEAMS = [("payments", "Checkout, ledger, refunds"), ("discovery", "Search and ranking"), ("identity", "Auth, sessions, SSO"),
+         ("messaging", "Email, push, in-app notifications"), ("platform", "Infra, CDN, orchestration")]
+# (level, error class, base severity, message)
+EVENTS = [
+    ("INFO", "deploy", 0, "deployment {ver} rolled out to {pct}% of pods"),
+    ("INFO", "cache", 0, "cache warmup complete in {ms} ms"),
+    ("WARN", "latency", 1, "p99 latency {ms} ms exceeds SLO of 800 ms for {n} minutes"),
+    ("WARN", "retry", 1, "upstream {dep} returned 503; retry {n}/5 succeeded"),
+    ("ERROR", "exception", 2, "NullPointerException in {fn} handling request {rid}; {n} requests failed"),
+    ("ERROR", "db", 2, "connection pool exhausted; {n} queries queued > 5 s"),
+    ("CRITICAL", "outage", 3, "health check failing on all replicas for {n} minutes; traffic returning 5xx"),
+    ("CRITICAL", "data", 3, "write path returned inconsistent results; {n} records may be lost"),
+    ("ERROR", "auth", 2, "token validation failing for {pct}% of requests since {ver}"),
+    ("WARN", "disk", 1, "disk usage at {pct}% on {n} nodes"),
+]
+ENVS = ["production", "production", "staging", "canary"]
+DISTRACTORS = {"trace_sampling": "0.1", "build_sha": "a3f9c21", "region": "eu-central-1", "feature_flags": ["new-ranker", "dark-mode"],
+               "previous_incident": "INC-2041 (resolved)", "runbook_url": "https://runbooks.internal/generic", "request_headers": {"x-ab": "B"}}
+SEV = [Level(level=0, description="Informational; no action."), Level(level=1, description="Degraded; investigate within the day."),
+       Level(level=2, description="Errors affecting users; fix within hours."), Level(level=3, description="Outage or data loss; all hands now.")]
+SEV_INSTR = "Severity 0-3 from the log record. Policy: INFO=0, WARN=1, ERROR=2, CRITICAL=3."
+
+
+@register_generator("log_triage")
+class LogTriageGenerator(BaseGenerator):
+    version = "1.0.0"
+
+    def generate(self) -> list[TaskItem]:
+        kb = self.knobs
+        rng, ctl = self.rng("items"), self.rng("controls")
+        items = []
+        for i in range(kb.n):
+            distract = ctl.random() < kb.distractor_density
+            noise_draw = ctl.random() < kb.label_noise
+            service = rng.choice(list(SERVICES))
+            level, cls, sev, msg = rng.choice(EVENTS)
+            env = rng.choice(ENVS)
+            msg = msg.format(ver=f"v{rng.randrange(2, 9)}.{rng.randrange(0, 30)}", pct=rng.randrange(5, 100), ms=rng.randrange(120, 4000),
+                             n=rng.randrange(1, 60), dep=rng.choice(list(SERVICES)), fn=rng.choice(["chargeCard", "buildIndex", "verifyToken", "renderEmail"]),
+                             rid=f"req-{rng.randrange(10**5, 10**6)}")
+            record = {"ts": f"2026-09-{rng.randrange(1, 28):02d}T{rng.randrange(0, 24):02d}:{rng.randrange(0, 60):02d}:00Z", "level": level, "service": service,
+                      "env": env, "message": msg, "error_class": cls}
+            if distract:
+                for k in ctl.sample(list(DISTRACTORS), 3):
+                    record[k] = DISTRACTORS[k]
+            state = {"log": record, "team_directory": SERVICES,
+                     "paging_policy": "Page the on-call engineer only when severity is 2 or higher AND env is production."}
+            if kb.target_tokens:
+                state["context"] = pad_to_tokens(ctl, "", kb.target_tokens)
+            team = SERVICES[service]
+            page = sev >= 2 and env == "production"
+            noise = False
+            if kb.label_noise and noise_draw:
+                noise = True
+                team = ctl.choice([t for t, _ in TEAMS if t != team])
+                sev = ctl.choice([x for x in range(4) if x != sev])
+                page = not page
+            tid = f"logs_{kb.seed}_{i:05d}"
+            items.append(TaskItem(
+                task_id=tid, tier="G", domain="log_triage", language=kb.language, state=state,
+                state_tokens=estimate_tokens(json.dumps(state)),
+                questions={
+                    "severity": Question(type="score", instructions=SEV_INSTR, criteria=SEV, ground_truth=sev, framing_group="logs.severity"),
+                    "owning_team": Question(type="choice", instructions="Which team owns the service in `log.service`? Use `team_directory`.",
+                                            criteria=[Option(key=k, description=d) for k, d in TEAMS], ground_truth=team, framing_group="logs.owning_team"),
+                    "page_oncall": Question(type="noul", instructions="Per `paging_policy`, should the on-call engineer be paged for this record?",
+                                            ground_truth=page, framing_group="logs.page_oncall"),
+                },
+                label_provenance=LabelProvenance(source="generator", generator=self.generator_id, seed=kb.seed, noise_injected=noise),
+                controls=Controls(distractor_density=kb.distractor_density), cost_matrix=self.cost_matrices(), canary=canary_for(tid),
+                metadata={"level": level, "env": env, "service": service},
+            ))
+        return items
+
+    def regex_rules(self) -> dict:
+        return {"logs.severity": {"0": [r'"level": "INFO"'], "1": [r'"level": "WARN"'], "2": [r'"level": "ERROR"'], "3": [r'"level": "CRITICAL"']},
+                "logs.owning_team": {t: [rf'"service": "{s}"' for s, tt in SERVICES.items() if tt == t] for t, _ in TEAMS},
+                "logs.page_oncall": [r'"level": "(ERROR|CRITICAL)"[^}]*"env": "production"|"env": "production"[^}]*"level": "(ERROR|CRITICAL)"']}
+
+    def cost_matrices(self):
+        return {"page_oncall": {"true": {"true": 0.0, "false": 50.0}, "false": {"true": 5.0, "false": 0.0}},
+                "severity": {str(t): {str(p): float(abs(t - p) * (3 if p < t else 1)) for p in range(4)} for t in range(4)}}

sys1bench/generators/multilingual_tickets.py

@@ -0,0 +1,84 @@
+"""Multilingual support tickets: the support_tickets task templated in German, Spanish, French and Hindi (Devanagari),
+with English questions and criteria. Labels follow the same policy as the English generator. Used for Laya's language
+router and for Jev's language behaviour; scripts other than Latin are the collapse case reported by Convai.
+"""
+
+from __future__ import annotations
+
+from ..schemas import Controls, LabelProvenance, Option, Question, TaskItem
+from .base import BaseGenerator, canary_for, estimate_tokens, register_generator
+from .support_tickets import PRIORITY_INSTRUCTIONS, PRIORITY_LEVELS, QUEUES
+
+# queue, base urgency, {lang: template}
+T = [
+    ("billing", 1, {"de": "Mir wurde am {day} zweimal {amt} € für Bestellung #{order} abgebucht. Es darf nur eine Abbuchung geben.",
+                    "es": "Me cobraron dos veces {amt} € el {day} por el pedido #{order}. Solo debería haber un cargo.",
+                    "fr": "J'ai été débité deux fois de {amt} € le {day} pour la commande #{order}. Il ne devrait y avoir qu'un seul débit.",
+                    "hi": "ऑर्डर #{order} के लिए {day} को मुझसे दो बार ₹{amt} लिए गए। केवल एक ही शुल्क होना चाहिए।"}),
+    ("shipping", 1, {"de": "Bestellung #{order} gilt seit {day} als zugestellt, aber nichts ist angekommen.",
+                     "es": "El pedido #{order} figura como entregado el {day}, pero no ha llegado nada.",
+                     "fr": "La commande #{order} est indiquée livrée le {day}, mais rien n'est arrivé.",
+                     "hi": "ऑर्डर #{order} {day} को डिलीवर दिखाया गया है, लेकिन कुछ भी नहीं आया।"}),
+    ("account", 2, {"de": "Ich bin ausgesperrt. Die Zwei-Faktor-Codes gehen an eine alte Nummer.",
+                    "es": "No puedo entrar. Los códigos de dos factores van a un número antiguo.",
+                    "fr": "Je suis bloqué. Les codes à deux facteurs partent vers un ancien numéro.",
+                    "hi": "मैं लॉक आउट हो गया हूँ। टू-फैक्टर कोड पुराने नंबर पर जा रहे हैं।"}),
+    ("technical", 2, {"de": "Die App stürzt seit dem Update am {day} beim Start ab. Fehlercode {code}.",
+                      "es": "La aplicación se cierra al abrirla desde la actualización del {day}. Código de error {code}.",
+                      "fr": "L'application plante au démarrage depuis la mise à jour du {day}. Code d'erreur {code}.",
+                      "hi": "{day} के अपडेट के बाद से ऐप खोलते ही क्रैश हो जाता है। एरर कोड {code}।"}),
+    ("cancellation", 1, {"de": "Bitte kündigen Sie mein Abo vor der Verlängerung am {day}.",
+                         "es": "Por favor cancelen mi suscripción antes de la renovación del {day}.",
+                         "fr": "Veuillez annuler mon abonnement avant le renouvellement du {day}.",
+                         "hi": "कृपया {day} के नवीनीकरण से पहले मेरी सदस्यता रद्द करें।"}),
+    ("sales", 0, {"de": "Gibt es einen Rabatt bei jährlicher Zahlung für das Team-Paket?",
+                  "es": "¿Hay descuento por pago anual en el plan de equipo?",
+                  "fr": "Y a-t-il une remise pour un paiement annuel sur l'offre équipe ?",
+                  "hi": "टीम प्लान पर वार्षिक भुगतान के लिए कोई छूट है?"}),
+]
+ANGRY = {"de": "Das ist völlig inakzeptabel, ich bin wütend.", "es": "Esto es completamente inaceptable, estoy furioso.",
+         "fr": "C'est totalement inacceptable, je suis furieux.", "hi": "यह पूरी तरह अस्वीकार्य है, मैं बहुत गुस्से में हूँ।"}
+CALM = {"de": "Vielen Dank im Voraus.", "es": "Gracias de antemano.", "fr": "Merci d'avance.", "hi": "अग्रिम धन्यवाद।"}
+CTX = {"de": "Kontext: Kundenstufe {tier}.", "es": "Contexto: nivel de cliente {tier}.", "fr": "Contexte : niveau client {tier}.", "hi": "संदर्भ: ग्राहक स्तर {tier}।"}
+TIERS = ["free", "standard", "gold", "enterprise"]
+
+
+@register_generator("multilingual_tickets")
+class MultilingualTicketsGenerator(BaseGenerator):
+    version = "1.0.0"
+
+    def generate(self) -> list[TaskItem]:
+        kb = self.knobs
+        rng = self.rng("items")
+        langs = [kb.language] if kb.language in ("de", "es", "fr", "hi") else ["de", "es", "fr", "hi"]
+        queues = [q for q in QUEUES if q[0] in {t[0] for t in T}]
+        items = []
+        for i in range(kb.n):
+            lang = langs[i % len(langs)]
+            queue, base, tmpl = rng.choice(T)
+            angry = rng.random() < 0.3
+            tier = rng.choice(TIERS)
+            body = tmpl[lang].format(day=rng.choice(["Montag", "3.", "gestern"]) if lang == "de" else rng.choice(["lunes", "3", "ayer"]) if lang == "es"
+                                     else rng.choice(["lundi", "3", "hier"]) if lang == "fr" else rng.choice(["सोमवार", "3 तारीख", "कल"]),
+                                     amt=rng.choice([19, 45, 89, 249]), order=rng.randrange(10000, 99999), code=f"E{rng.randrange(100, 999)}")
+            state = f"{body} {(ANGRY if angry else CALM)[lang]}\n{CTX[lang].format(tier=tier)}"
+            priority = min(4, base + (1 if angry else 0) + (1 if tier in ("gold", "enterprise") else 0))
+            tid = f"mltickets_{kb.seed}_{lang}_{i:05d}"
+            items.append(TaskItem(
+                task_id=tid, tier="G", domain="support_triage", language=lang, state=state, state_tokens=estimate_tokens(state),
+                questions={
+                    "queue": Question(type="choice", instructions="Which queue should handle this ticket?", criteria=[Option(key=k, description=d) for k, d in queues],
+                                      ground_truth=queue, framing_group="tickets.queue"),
+                    "is_angry": Question(type="noul", instructions="Is the customer expressing anger or hostility?", ground_truth=angry, framing_group="tickets.is_angry"),
+                    "priority": Question(type="score", instructions=PRIORITY_INSTRUCTIONS, criteria=PRIORITY_LEVELS, ground_truth=priority, framing_group="tickets.priority"),
+                },
+                label_provenance=LabelProvenance(source="generator", generator=self.generator_id, seed=kb.seed), controls=Controls(),
+                canary=canary_for(tid), metadata={"lang": lang, "tier": tier, "angry": angry, "base_urgency": base},
+            ))
+        return items
+
+    def regex_rules(self) -> dict:
+        return {"tickets.is_angry": [r"inakzeptabel|inaceptable|furieux|अस्वीकार्य"]}
+
+    def cost_matrices(self):
+        return {}

sys1bench/generators/phishing_email.py

@@ -0,0 +1,127 @@
+"""Phishing email generator (is_phishing: noul, attack_class: choice, urgency: score) with a built-in
+5-way decomposition for Suite B (sender mismatch, credential request, urgency language, link/domain
+mismatch, unexpected attachment) combined by a fixed rule.
+"""
+
+from __future__ import annotations
+
+import json
+
+from ..schemas import Controls, LabelProvenance, Level, Option, Question, TaskItem
+from .base import BaseGenerator, canary_for, estimate_tokens, pad_to_tokens, register_generator
+
+BRANDS = ["Northwind Bank", "Contoso Cloud", "Fabrikam Shipping", "Tailspin Payroll", "Woodgrove HR"]
+LEGIT_DOMAINS = {"Northwind Bank": "northwindbank.com", "Contoso Cloud": "contoso.com", "Fabrikam Shipping": "fabrikam.com",
+                 "Tailspin Payroll": "tailspin.com", "Woodgrove HR": "woodgrove.com"}
+LOOKALIKE = {"northwindbank.com": "northwind-bank-secure.com", "contoso.com": "contoso-cloud-login.net",
+             "fabrikam.com": "fabrikam-delivery.co", "tailspin.com": "tailspin-payroll.info", "woodgrove.com": "woodgrove-hr.org"}
+
+CLASSES = [
+    ("credential_harvest", "Asks the recipient to log in or verify credentials via a link"),
+    ("invoice_fraud", "Fake invoice or payment redirection"),
+    ("malware_attachment", "Unexpected attachment the recipient is urged to open"),
+    ("ceo_fraud", "Impersonates an executive requesting an urgent transfer or gift cards"),
+    ("benign", "Legitimate message with no attack"),
+]
+URGENCY = [Level(level=0, description="No time pressure"), Level(level=1, description="Mild: 'when you get a chance'"),
+           Level(level=2, description="Moderate: action requested within days"), Level(level=3, description="High: within 24 hours or account consequences")]
+
+LEGIT_TEMPLATES = [
+    ("Your {brand} statement is ready", "Hi {name}, your monthly statement is available in the app under Documents. No action is required.", 0, False),
+    ("Delivery update for order {order}", "Hi {name}, your parcel is out for delivery today. Track it in your account.", 0, False),
+    ("Team lunch Thursday", "Hi {name}, we're doing lunch Thursday at noon. Reply if you have dietary needs.", 1, False),
+    ("Q3 timesheet reminder", "Hi {name}, timesheets are due Friday. Submit through the usual portal when you get a chance.", 1, False),
+    ("Invoice {order} attached", "Hi {name}, attached is the invoice we discussed on the call. Let me know if anything looks off.", 1, True),
+]
+PHISH_TEMPLATES = [
+    ("credential_harvest", "Urgent: verify your {brand} account", "Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://{fake}/verify or your account will be suspended.", 3, False),
+    ("credential_harvest", "Password expires today", "Your {brand} password expires today. Reset now: https://{fake}/reset", 3, False),
+    ("invoice_fraud", "Updated banking details for invoice {order}", "Hi {name}, please note our bank details have changed. Remit invoice {order} to the new account in the attached PDF by end of week.", 2, True),
+    ("malware_attachment", "Scanned document", "Please see the attached scan (Document_{order}.zip). Open to review.", 1, True),
+    ("ceo_fraud", "Quick favour", "{name}, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone", 3, False),
+]
+NAMES = ["Alex", "Priya", "Jordan", "Mei", "Sam", "Fatima", "Luca", "Noor"]
+
+
+@register_generator("phishing_email")
+class PhishingEmailGenerator(BaseGenerator):
+    version = "1.0.1"
+
+    def generate(self) -> list[TaskItem]:
+        kb = self.knobs
+        rng = self.rng("items")
+        ctl = self.rng("controls")
+        items: list[TaskItem] = []
+        for i in range(kb.n):
+            distract = ctl.random() < kb.distractor_density
+            noise_draw = ctl.random() < kb.label_noise
+            brand = rng.choice(BRANDS)
+            legit_dom = LEGIT_DOMAINS[brand]
+            name = rng.choice(NAMES)
+            order = rng.randrange(10000, 99999)
+            is_phish = rng.random() < 0.5
+            if is_phish:
+                cls, subj, body, urg, attach = rng.choice(PHISH_TEMPLATES)
+                sender_dom = LOOKALIKE[legit_dom] if rng.random() < 0.8 else legit_dom  # 20% spoof the real domain
+                fake = LOOKALIKE[legit_dom]
+            else:
+                subj, body, urg, attach = rng.choice(LEGIT_TEMPLATES)
+                cls, sender_dom, fake = "benign", legit_dom, legit_dom
+            body = body.format(brand=brand, name=name, order=order, fake=fake)
+            subj = subj.format(brand=brand, order=order)
+            state = {"from": f"{brand} <no-reply@{sender_dom}>", "to": f"{name.lower()}@example-corp.com", "subject": subj,
+                     "body": body, "attachments": ([f"file_{order}.{'zip' if cls == 'malware_attachment' else 'pdf'}"] if attach else []),
+                     "known_brand_domain": legit_dom}
+            if distract:
+                state["thread_history"] = "Earlier: 'Thanks for the quick turnaround last month!'"
+            if kb.target_tokens:
+                state["footer"] = pad_to_tokens(ctl, "", kb.target_tokens - estimate_tokens(body))
+            noise = False
+            truth_phish, truth_cls, truth_urg = is_phish, cls, urg
+            if kb.label_noise and noise_draw:
+                noise = True
+                truth_phish = not truth_phish
+                truth_cls = ctl.choice([c[0] for c in CLASSES if c[0] != cls])
+            tid = f"phish_{kb.seed}_{i:05d}"
+            # decomposition sub-questions (Suite B): each is a noul with rule-derived truth
+            sub = {
+                "sender_mismatch": ("Does the sender domain differ from the known brand domain?", sender_dom != legit_dom),
+                "asks_credentials": ("Does the message ask the recipient to log in, verify, or reset credentials via a link?", cls == "credential_harvest"),
+                "urgent_language": ("Does the message pressure the recipient to act within 24 hours or threaten consequences?", urg >= 3),
+                "payment_redirect": ("Does the message ask to change payment details or send money or gift cards?", cls in ("invoice_fraud", "ceo_fraud")),
+                # v1.0.1: invoice-fraud mails also push an unrequested attachment ("new account in the attached PDF"); the legit
+                # "Invoice attached" template was discussed on a call, so it stays false.
+                "unexpected_attachment": ("Does the message urge opening an attachment the recipient did not request?", cls in ("malware_attachment", "invoice_fraud")),
+            }
+            questions = {
+                "is_phishing": Question(type="noul", instructions="Is this email a phishing or social-engineering attack?",
+                                        ground_truth=truth_phish, framing_group="phish.is_phishing"),
+                "attack_class": Question(type="choice", instructions="Classify the email.", criteria=[Option(key=k, description=d) for k, d in CLASSES],
+                                         ground_truth=truth_cls, framing_group="phish.attack_class"),
+                "urgency": Question(type="score", instructions="Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).",
+                                    criteria=URGENCY, ground_truth=truth_urg, framing_group="phish.urgency"),
+            }
+            for k, (instr, truth) in sub.items():
+                questions[f"sub_{k}"] = Question(type="noul", instructions=instr, ground_truth=truth,
+                                                 framing_group=f"phish.sub.{k}", decomposition_of="is_phishing")
+            items.append(TaskItem(
+                task_id=tid, tier="G", domain="email_security", language=kb.language, state=state,
+                state_tokens=estimate_tokens(json.dumps(state)), questions=questions,
+                label_provenance=LabelProvenance(source="generator", generator=self.generator_id, seed=kb.seed, noise_injected=noise),
+                controls=Controls(distractor_density=kb.distractor_density), cost_matrix=self.cost_matrices(), canary=canary_for(tid),
+                metadata={"class": cls, "sender_domain": sender_dom, "decomposition_rule": "phishing if sender_mismatch or any of the other four"},
+            ))
+        return items
+
+    def regex_rules(self) -> dict:
+        return {
+            "phish.is_phishing": [r"verify your", r"suspended", r"expires today", r"gift cards", r"bank details have changed", r"\.zip", r"-secure\.com|-login\.net|-delivery\.co|-payroll\.info|-hr\.org"],
+            "phish.attack_class": {"credential_harvest": [r"verify", r"reset now", r"password"], "invoice_fraud": [r"bank details", r"remit"],
+                                   "malware_attachment": [r"\.zip", r"scanned document"], "ceo_fraud": [r"gift cards", r"confidential"],
+                                   "benign": [r"no action is required", r"team lunch", r"timesheet", r"out for delivery"]},
+            "phish.urgency": {"3": [r"24 hours|today|suspended"], "2": [r"end of week"], "1": [r"when you get a chance|let me know"], "0": [r"no action"]},
+        }
+
+    def cost_matrices(self) -> dict[str, dict[str, dict[str, float]]]:
+        # missing a phish costs 20; false alarm costs 1
+        return {"is_phishing": {"true": {"true": 0.0, "false": 20.0}, "false": {"true": 1.0, "false": 0.0}}}

sys1bench/generators/policy_compliance.py

@@ -0,0 +1,90 @@
+"""Policy compliance generator: a short policy (3 to 5 numbered clauses) plus a request.
+
+  compliant       (noul)  -> no clause is violated
+  violated_clause (choice over clause ids plus a required "none")  -> the single violated clause, or none
+  severity        (score 0..2) -> 0 compliant, 1 minor clause, 2 major clause (clause severities stated in the policy)
+Requests are built from the same clause set, so the label is exact and the "none" option is correct for a stated
+fraction of items. Nothing in the request text names the clause.
+"""
+
+from __future__ import annotations
+
+import json
+
+from ..schemas import Controls, LabelProvenance, Level, Option, Question, TaskItem
+from .base import BaseGenerator, canary_for, estimate_tokens, pad_to_tokens, register_generator
+
+# clause: (id, text, severity 1|2, violating request templates, compliant request templates)
+CLAUSES = [
+    ("C1", "Refunds are only issued within 30 days of purchase.", 2,
+     ["I bought this {days} days ago and want a full refund.", "Please refund order #{order}; it was placed {days} days back."],
+     ["I bought this {sdays} days ago and want a full refund.", "Order #{order} from {sdays} days ago, please refund."]),
+    ("C2", "Account details may only be shared with the verified account holder.", 2,
+     ["I'm calling for my husband; can you read me his last four transactions?", "My roommate's account: what email is on file? He asked me to check."],
+     ["Can you read me my own last four transactions? I've verified via the security question.", "What email is on my account? I'm the holder and verified."]),
+    ("C3", "Discount codes cannot be combined with other promotions.", 1,
+     ["Apply code SAVE20 on top of the current summer sale, please.", "Stack my loyalty coupon with the Black Friday price."],
+     ["Apply code SAVE20 to my order; no other promotions are on it.", "Use my loyalty coupon on this full-price item."]),
+    ("C4", "Shipping address changes are not possible after dispatch.", 1,
+     ["My order shipped this morning; change the address to my office.", "Tracking says dispatched; please redirect it to my parents' house."],
+     ["My order hasn't shipped yet; change the address to my office.", "Order still processing, please update the delivery address."]),
+    ("C5", "Support agents must not provide legal or tax advice.", 2,
+     ["Should I declare this refund as income on my tax return?", "Is it legal for my employer to deduct this from my pay?"],
+     ["Can you send me the invoice so my accountant can handle the taxes?", "Where do I download my annual statement?"]),
+]
+SEV = [Level(level=0, description="Compliant: no clause is violated."), Level(level=1, description="Minor violation (a clause marked minor)."),
+       Level(level=2, description="Major violation (a clause marked major).")]
+
+
+@register_generator("policy_compliance")
+class PolicyComplianceGenerator(BaseGenerator):
+    version = "1.0.0"
+
+    def generate(self) -> list[TaskItem]:
+        kb = self.knobs
+        rng, ctl = self.rng("items"), self.rng("controls")
+        none_frac = kb.none_correct_frac if kb.none_correct_frac else 0.3
+        items = []
+        for i in range(kb.n):
+            noise_draw = ctl.random() < kb.label_noise
+            k = rng.randrange(3, 6)
+            clauses = rng.sample(CLAUSES, k)
+            policy = "\n".join(f"{cid}. {txt} [{'major' if sev == 2 else 'minor'}]" for cid, txt, sev, _, _ in clauses)
+            compliant = rng.random() < none_frac
+            target = rng.choice(clauses)
+            tmpl = rng.choice(target[4] if compliant else target[3])
+            request = tmpl.format(days=rng.randrange(31, 120), sdays=rng.randrange(1, 29), order=rng.randrange(10000, 99999))
+            state = {"policy": policy, "customer_request": request}
+            if kb.target_tokens:
+                state["history"] = pad_to_tokens(ctl, "", kb.target_tokens)
+            violated = "none" if compliant else target[0]
+            sev = 0 if compliant else target[2]
+            noise = False
+            if kb.label_noise and noise_draw:
+                noise = True
+                compliant = not compliant
+                violated = "none" if compliant else target[0]
+                sev = 0 if compliant else target[2]
+            options = [Option(key=cid, description=txt) for cid, txt, _, _, _ in clauses] + [Option(key="none", description="No clause is violated.")]
+            tid = f"policy_{kb.seed}_{i:05d}"
+            items.append(TaskItem(
+                task_id=tid, tier="G", domain="policy_compliance", language=kb.language, state=state, state_tokens=estimate_tokens(json.dumps(state)),
+                questions={
+                    "compliant": Question(type="noul", instructions="Can the request be fulfilled without violating any clause of `policy`?",
+                                          ground_truth=compliant, framing_group="policy.compliant"),
+                    "violated_clause": Question(type="choice", instructions="Which clause of `policy` would fulfilling `customer_request` violate? Choose none if it complies.",
+                                                criteria=options, ground_truth=violated, framing_group="policy.violated_clause"),
+                    "severity": Question(type="score", instructions="Violation severity per the [minor]/[major] tags in `policy`; 0 if compliant.",
+                                         criteria=SEV, ground_truth=sev, framing_group="policy.severity"),
+                },
+                label_provenance=LabelProvenance(source="generator", generator=self.generator_id, seed=kb.seed, noise_injected=noise),
+                controls=Controls(none_correct=compliant), cost_matrix=self.cost_matrices(), canary=canary_for(tid),
+                metadata={"target_clause": target[0], "k": k},
+            ))
+        return items
+
+    def regex_rules(self) -> dict:
+        return {"policy.compliant": [r"(?!)"]}
+
+    def cost_matrices(self):
+        return {"compliant": {"true": {"true": 0.0, "false": 2.0}, "false": {"true": 10.0, "false": 0.0}}}

sys1bench/generators/rag_relevance.py

@@ -0,0 +1,122 @@
+"""RAG relevance generator: query plus K candidate passages.
+
+Questions:
+  best_passage  (choice over K passage ids, K up to 255)  -> the one passage that answers the query
+  is_relevant   (noul)  -> does passage `focus` answer the query (focus is a random candidate; 50% positives)
+  relevance     (score 0..3) -> grade of `focus`: 0 unrelated, 1 same topic, 2 partial, 3 answers it
+
+Passages are built from a fact table: each fact has a topic, an entity, an attribute and a value. Distractors are
+(a) same topic different entity (hard), (b) same entity different attribute (hard), (c) unrelated (easy). Labels are
+exact by construction, so this is the cardinality sweep generator for Suite C (K in 2..255).
+"""
+
+from __future__ import annotations
+
+import json
+import random
+
+from ..schemas import Controls, LabelProvenance, Level, Option, Question, TaskItem
+from .base import BaseGenerator, canary_for, estimate_tokens, pad_to_tokens, register_generator
+
+TOPICS = {
+    "planets": (["Mercury", "Venus", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"],
+                {"moons": lambda r: str(r.randrange(0, 90)), "day_length_hours": lambda r: str(r.randrange(10, 5900)),
+                 "discovery_year": lambda r: str(r.randrange(1600, 1950)), "mean_temp_c": lambda r: str(r.randrange(-220, 470))}),
+    "companies": (["Northwind", "Contoso", "Fabrikam", "Tailspin", "Woodgrove", "Litware", "Adatum", "Proseware"],
+                  {"headquarters": lambda r: r.choice(["Oslo", "Lisbon", "Austin", "Kyoto", "Nairobi", "Toronto"]),
+                   "founded": lambda r: str(r.randrange(1950, 2022)), "employees": lambda r: str(r.randrange(20, 90000)),
+                   "ceo": lambda r: r.choice(["A. Okafor", "M. Lindqvist", "R. Tanaka", "S. Mehta", "J. Alvarez"])}),
+    "recipes": (["ratatouille", "bibimbap", "moussaka", "feijoada", "pho", "tagine", "paella", "goulash"],
+                {"origin_country": lambda r: r.choice(["France", "Korea", "Greece", "Brazil", "Vietnam", "Morocco", "Spain", "Hungary"]),
+                 "cook_time_minutes": lambda r: str(r.randrange(15, 240)), "main_protein": lambda r: r.choice(["beef", "pork", "chicken", "lamb", "none", "fish"]),
+                 "servings": lambda r: str(r.randrange(2, 12))}),
+    "software": (["Kestrel", "Nimbus", "Quartz", "Vellum", "Marlin", "Orchid", "Tundra", "Zephyr"],
+                 {"latest_version": lambda r: f"{r.randrange(1, 9)}.{r.randrange(0, 20)}", "license": lambda r: r.choice(["MIT", "Apache-2.0", "GPL-3.0", "BSD-3", "proprietary"]),
+                  "language": lambda r: r.choice(["Rust", "Go", "Python", "TypeScript", "C++"]), "first_release": lambda r: str(r.randrange(2005, 2026))}),
+}
+TEMPLATES = ["{entity}'s {attr_h} is {value}.", "According to the {topic} handbook, the {attr_h} of {entity} is {value}.",
+             "Note: {entity} — {attr_h}: {value}.", "For {entity}, records list the {attr_h} as {value}."]
+QUERY_TEMPLATES = ["What is the {attr_h} of {entity}?", "{entity}: {attr_h}?", "Tell me {entity}'s {attr_h}.", "Look up the {attr_h} for {entity}."]
+REL_LEVELS = [Level(level=0, description="Unrelated to the query's topic or entity."),
+              Level(level=1, description="Same topic but about a different entity or attribute; does not answer the query."),
+              Level(level=2, description="About the right entity but a different attribute, or the right attribute for a different entity."),
+              Level(level=3, description="Directly states the value the query asks for.")]
+
+
+def _fact(rng: random.Random, topic: str, entity: str, attr: str) -> tuple[str, str]:
+    ents, attrs = TOPICS[topic]
+    value = attrs[attr](rng)
+    text = rng.choice(TEMPLATES).format(entity=entity, attr_h=attr.replace("_", " "), value=value, topic=topic)
+    return text, value
+
+
+@register_generator("rag_relevance")
+class RagRelevanceGenerator(BaseGenerator):
+    version = "1.0.0"
+
+    def generate(self) -> list[TaskItem]:
+        kb = self.knobs
+        rng = self.rng("items")
+        ctl = self.rng("controls")
+        K = max(2, min(kb.cardinality or 5, 255))
+        items: list[TaskItem] = []
+        for i in range(kb.n):
+            topic = rng.choice(list(TOPICS))
+            ents, attrs = TOPICS[topic]
+            entity, attr = rng.choice(ents), rng.choice(list(attrs))
+            query = rng.choice(QUERY_TEMPLATES).format(entity=entity, attr_h=attr.replace("_", " "))
+            gold_text, _ = _fact(rng, topic, entity, attr)
+            passages: list[tuple[str, int]] = [(gold_text, 3)]  # (text, relevance grade)
+            # hard distractors
+            while len(passages) < K:
+                kind = rng.random()
+                if kind < 0.35:  # same entity, other attribute
+                    other_attr = rng.choice([a for a in attrs if a != attr])
+                    passages.append((_fact(rng, topic, entity, other_attr)[0], 2))
+                elif kind < 0.7:  # same attribute, other entity
+                    other_ent = rng.choice([e for e in ents if e != entity])
+                    passages.append((_fact(rng, topic, other_ent, attr)[0], 2))
+                elif kind < 0.85:  # same topic, other entity and attribute
+                    other_ent = rng.choice([e for e in ents if e != entity])
+                    other_attr = rng.choice([a for a in attrs if a != attr])
+                    passages.append((_fact(rng, topic, other_ent, other_attr)[0], 1))
+                else:  # unrelated topic
+                    ot = rng.choice([t for t in TOPICS if t != topic])
+                    oe, oa = rng.choice(TOPICS[ot][0]), rng.choice(list(TOPICS[ot][1]))
+                    passages.append((_fact(rng, ot, oe, oa)[0], 0))
+            rng.shuffle(passages)
+            ids = [f"p{j:03d}" for j in range(K)]
+            gold_id = ids[[g for _, g in passages].index(3)]
+            focus_idx = rng.randrange(K) if rng.random() < 0.5 else ids.index(gold_id)
+            focus_id, focus_grade = ids[focus_idx], passages[focus_idx][1]
+            state = {"query": query, "passages": {pid: txt for pid, (txt, _) in zip(ids, passages)}, "focus": focus_id}
+            noise_draw = ctl.random() < kb.label_noise
+            if kb.target_tokens:
+                state["notes"] = pad_to_tokens(ctl, "", kb.target_tokens)
+            noise = False
+            if kb.label_noise and noise_draw:
+                noise = True
+                gold_id = ctl.choice([p for p in ids if p != gold_id])
+            tid = f"rag_{kb.seed}_K{K}_{i:05d}"
+            items.append(TaskItem(
+                task_id=tid, tier="G", domain="rag_relevance", language=kb.language, state=state,
+                state_tokens=estimate_tokens(json.dumps(state)),
+                questions={
+                    "best_passage": Question(type="choice", instructions="Which passage in `passages` directly answers `query`?",
+                                             criteria=[Option(key=pid, description="") for pid in ids], ground_truth=gold_id, framing_group="rag.best_passage"),
+                    "is_relevant": Question(type="noul", instructions="Does the passage with id `focus` directly answer `query`?",
+                                            ground_truth=(focus_grade == 3), framing_group="rag.is_relevant"),
+                    "relevance": Question(type="score", instructions="Grade how well the passage with id `focus` answers `query`.",
+                                          criteria=REL_LEVELS, ground_truth=focus_grade, framing_group="rag.relevance"),
+                },
+                label_provenance=LabelProvenance(source="generator", generator=self.generator_id, seed=kb.seed, noise_injected=noise),
+                controls=Controls(distractor_density=kb.distractor_density), cost_matrix=self.cost_matrices(), canary=canary_for(tid),
+                metadata={"topic": topic, "entity": entity, "attribute": attr, "K": K, "focus_grade": focus_grade},
+            ))
+        return items
+
+    def regex_rules(self) -> dict:
+        return {"rag.is_relevant": [r"(?!)"]}  # no sensible regex baseline: passages are ids; kept so the rule file exists
+
+    def cost_matrices(self) -> dict[str, dict[str, dict[str, float]]]:
+        return {"is_relevant": {"true": {"true": 0.0, "false": 5.0}, "false": {"true": 1.0, "false": 0.0}}}