nomosguard 0.1.0__py3-none-any.whl

nomosguard/__init__.py

@@ -0,0 +1,9 @@
+"""NomosGuard: deterministic security reasoning core.
+
+An LLM may read security events and write structured, evidence-citing
+claims; this package turns those claims into tamper-evident decisions.
+Language models never cross the decision boundary: the ledger, the rule
+engine, and the policy gate are fully deterministic.
+"""
+
+__version__ = "0.1.0"

nomosguard/benchmark/__init__.py

@@ -0,0 +1,12 @@
+"""Attack-scenario benchmark: measure what the core catches and misses.
+
+The corpus is deterministic and committed. run_suite.py produces a results
+file from an actual run — recall, false-positive rate, and fail-closed
+correctness. The honesty section in RESULTS.md states what the corpus does
+not cover.
+"""
+
+from .scenarios import Scenario, build_all_scenarios
+from .run_suite import run_suite
+
+__all__ = ["Scenario", "build_all_scenarios", "run_suite"]

nomosguard/benchmark/results/RESULTS.md

@@ -0,0 +1,21 @@
+# NomosGuard benchmark results
+
+Run at: 2026-10-08T23:54:36.236216+00:00
+Scenarios: 6
+
+## Metrics (measured, not asserted)
+
+- recall: 2/2 = 100.0%
+- false positives: 0
+- fail-closed on incomplete evidence: OK
+
+## Per-scenario
+
+| scenario | expected | derived | passed |
+|---|---|---|---|
+| positive_vulnerable_exposure | [['researcher', 'orders_db']] | [['researcher', 'orders_db']] | PASS |
+| negative_benign_workflow | [] | [] | PASS |
+| multi_agent_only_one_exposed | [['agent_a', 'orders_db']] | [['agent_a', 'orders_db']] | PASS |
+| incomplete_evidence | [] | [] | PASS |
+| policy_violation | [] | [] | PASS |
+| no_vulnerability_no_exposure | [] | [] | PASS |

nomosguard/benchmark/results/results.json

@@ -0,0 +1,81 @@
+{
+  "scenarios": 6,
+  "recall": 1.0,
+  "caught": 2,
+  "expected_total": 2,
+  "derived_total": 2,
+  "false_positives": 0,
+  "fail_closed_ok": true,
+  "all_passed": true,
+  "run_at": "2026-10-08T23:54:36.236216+00:00",
+  "results": [
+    {
+      "name": "positive_vulnerable_exposure",
+      "expected": [
+        [
+          "researcher",
+          "orders_db"
+        ]
+      ],
+      "derived": [
+        [
+          "researcher",
+          "orders_db"
+        ]
+      ],
+      "passed": true,
+      "fail_closed_ok": true,
+      "gate_decision": "BLOCK"
+    },
+    {
+      "name": "negative_benign_workflow",
+      "expected": [],
+      "derived": [],
+      "passed": true,
+      "fail_closed_ok": true,
+      "gate_decision": "BLOCK"
+    },
+    {
+      "name": "multi_agent_only_one_exposed",
+      "expected": [
+        [
+          "agent_a",
+          "orders_db"
+        ]
+      ],
+      "derived": [
+        [
+          "agent_a",
+          "orders_db"
+        ]
+      ],
+      "passed": true,
+      "fail_closed_ok": true,
+      "gate_decision": "BLOCK"
+    },
+    {
+      "name": "incomplete_evidence",
+      "expected": [],
+      "derived": [],
+      "passed": true,
+      "fail_closed_ok": true,
+      "gate_decision": "BLOCK"
+    },
+    {
+      "name": "policy_violation",
+      "expected": [],
+      "derived": [],
+      "passed": true,
+      "fail_closed_ok": true,
+      "gate_decision": "ALERT"
+    },
+    {
+      "name": "no_vulnerability_no_exposure",
+      "expected": [],
+      "derived": [],
+      "passed": true,
+      "fail_closed_ok": true,
+      "gate_decision": "BLOCK"
+    }
+  ]
+}

nomosguard/benchmark/run_suite.py

@@ -0,0 +1,185 @@
+"""Run the benchmark corpus and measure recall / false-positive / fail-closed.
+
+Every number in the results file comes from actually running the scenarios
+through the real engine and gate. Nothing is hand-computed.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from ..ledger import Claim, EvidenceLedger
+from ..rules import Pattern, Rule, RuleEngine
+from ..gate import Decision, PolicyGate, PolicyRule
+from .scenarios import Scenario, build_all_scenarios
+
+
+def default_rules() -> list[Rule]:
+    return [
+        Rule(
+            name="tool_on_vulnerable_component",
+            body=(
+                Pattern("?agent", "calls", "?tool"),
+                Pattern("?agent", "operates_on", "?component"),
+                Pattern("?component", "has_vulnerability", "?cve"),
+            ),
+            head=Pattern("?agent", "exposes", "?component"),
+            description="An agent operating a tool on a vulnerable component exposes it.",
+        ),
+        Rule(
+            name="policy_denied_action",
+            body=(
+                Pattern("?agent", "calls", "?tool"),
+                Pattern("?agent", "policy_denies", "?action"),
+            ),
+            head=Pattern("?agent", "violates", "policy"),
+            description="An agent calling a tool whose action is policy-denied violates policy.",
+        ),
+    ]
+
+
+def default_policy_rules() -> list[PolicyRule]:
+    return [
+        PolicyRule(
+            name="block_vulnerable_exposure",
+            match_relation="exposes",
+            decision=Decision.BLOCK,
+            description="Exposure of a vulnerable component is blocked.",
+        ),
+        PolicyRule(
+            name="alert_policy_violation",
+            match_relation="violates",
+            decision=Decision.ALERT,
+            description="Policy violations raise an alert.",
+        ),
+    ]
+
+
+@dataclass
+class ScenarioResult:
+    name: str
+    expected: set
+    derived: set
+    passed: bool
+    fail_closed_ok: bool
+    gate_decision: str
+
+
+def run_scenario(scenario: Scenario) -> ScenarioResult:
+    ledger = EvidenceLedger()
+    for spec in scenario.claims:
+        ledger.append(Claim(kind=spec.kind, payload=spec.payload, evidence=spec.evidence))
+
+    engine = RuleEngine(rules=default_rules())
+    engine.add_facts_from_ledger(ledger)
+    engine.derive()
+
+    derived = {
+        (f.subject, f.object) for f in engine.facts if f.relation == "exposes"
+    }
+    gate = PolicyGate(policy_rules=default_policy_rules(), fallback=Decision.BLOCK)
+    decision = gate.evaluate_with_fallback(engine)
+
+    expected = set(scenario.expected_exposed)
+    passed = derived == expected
+
+    # fail-closed correctness: with no derived path the gate must BLOCK
+    fail_closed_ok = True
+    if scenario.incomplete:
+        fail_closed_ok = (
+            derived == set()
+            and decision["decision"] == "BLOCK"
+            and decision["policy_rule"] == "FALLBACK"
+        )
+
+    return ScenarioResult(
+        name=scenario.name,
+        expected=expected,
+        derived=derived,
+        passed=passed,
+        fail_closed_ok=fail_closed_ok,
+        gate_decision=decision["decision"],
+    )
+
+
+def run_suite(output_dir: str | Path | None = None) -> dict[str, Any]:
+    """Run every scenario, compute metrics, optionally write results files.
+
+    Metrics (all measured, never asserted):
+    - recall: caught positives / total expected positives
+    - false_positive: derived on scenarios expecting nothing
+    - fail_closed: incomplete-evidence scenarios that fell back to BLOCK
+    """
+    scenarios = build_all_scenarios()
+    results = [run_scenario(s) for s in scenarios]
+
+    total_expected = sum(len(r.expected) for r in results)
+    total_derived = sum(len(r.derived) for r in results)
+    caught = sum(len(r.expected & r.derived) for r in results)
+    false_positives = sum(
+        1 for r in results if r.expected == set() and r.derived != set()
+    )
+    incomplete = [r for r in results if any(s.incomplete for s in scenarios if s.name == r.name)]
+    fail_closed_ok = all(r.fail_closed_ok for r in incomplete) if incomplete else True
+
+    recall = caught / total_expected if total_expected else 1.0
+
+    summary = {
+        "scenarios": len(results),
+        "recall": recall,
+        "caught": caught,
+        "expected_total": total_expected,
+        "derived_total": total_derived,
+        "false_positives": false_positives,
+        "fail_closed_ok": fail_closed_ok,
+        "all_passed": all(r.passed for r in results),
+        "run_at": datetime.now(timezone.utc).isoformat(),
+        "results": [
+            {
+                "name": r.name,
+                "expected": sorted(list(p) for p in r.expected),
+                "derived": sorted(list(p) for p in r.derived),
+                "passed": r.passed,
+                "fail_closed_ok": r.fail_closed_ok,
+                "gate_decision": r.gate_decision,
+            }
+            for r in results
+        ],
+    }
+
+    if output_dir is not None:
+        out = Path(output_dir)
+        out.mkdir(parents=True, exist_ok=True)
+        (out / "results.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+        (out / "RESULTS.md").write_text(_render_markdown(summary), encoding="utf-8")
+
+    return summary
+
+
+def _render_markdown(summary: dict[str, Any]) -> str:
+    lines = [
+        "# NomosGuard benchmark results",
+        "",
+        f"Run at: {summary['run_at']}",
+        f"Scenarios: {summary['scenarios']}",
+        "",
+        "## Metrics (measured, not asserted)",
+        "",
+        f"- recall: {summary['caught']}/{summary['expected_total']} = {summary['recall']:.1%}",
+        f"- false positives: {summary['false_positives']}",
+        f"- fail-closed on incomplete evidence: {'OK' if summary['fail_closed_ok'] else 'FAILED'}",
+        "",
+        "## Per-scenario",
+        "",
+        "| scenario | expected | derived | passed |",
+        "|---|---|---|---|",
+    ]
+    for r in summary["results"]:
+        lines.append(
+            f"| {r['name']} | {r['expected']} | {r['derived']} | {'PASS' if r['passed'] else 'FAIL'} |"
+        )
+    return "\n".join(lines) + "\n"

nomosguard/benchmark/run_suite_main.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""Entry point: python -m benchmark.run_suite_main [--output DIR]"""
+
+import argparse
+
+from .run_suite import run_suite
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run the NomosGuard benchmark corpus")
+    parser.add_argument(
+        "--output",
+        default=None,
+        help="Directory to write results.json and RESULTS.md (omit to print only)",
+    )
+    args = parser.parse_args()
+
+    summary = run_suite(output_dir=args.output)
+    import json
+
+    print(json.dumps(
+        {
+            "scenarios": summary["scenarios"],
+            "recall": summary["recall"],
+            "caught": summary["caught"],
+            "expected_total": summary["expected_total"],
+            "false_positives": summary["false_positives"],
+            "fail_closed_ok": summary["fail_closed_ok"],
+            "all_passed": summary["all_passed"],
+        },
+        indent=2,
+    ))
+    return 0 if summary["all_passed"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

nomosguard/benchmark/scenarios.py

@@ -0,0 +1,129 @@
+"""The benchmark corpus: scenarios with known ground truth.
+
+Each scenario is a small evidence set (claims the way a caller would assert
+them) plus the expected outcome — the set of (agent, component) pairs the
+rules SHOULD derive as "exposes", computed by hand from the facts, not by
+running the engine.
+
+Fact model (the extractor emits):
+    (agent, calls, tool)             — the agent invoked the tool
+    (agent, operates_on, target)     — the AGENT operates on the target
+
+The agent-scoped operates_on is deliberate: a tool invoked by two agents
+on different targets does NOT make both agents operate on both targets.
+The evidence stays honest — an agent operates only on what it called.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ClaimSpec:
+    kind: str
+    payload: dict
+    evidence: str
+
+
+@dataclass(frozen=True)
+class Scenario:
+    name: str
+    description: str
+    claims: tuple[ClaimSpec, ...]
+    expected_exposed: frozenset  # (agent, component) pairs the rules MUST derive
+    incomplete: bool = False      # True: gate must apply its fail-closed default
+
+
+def _tool_call(agent: str, tool: str, target: str, line: int) -> ClaimSpec:
+    return ClaimSpec(
+        kind="tool_call",
+        payload={"agent": agent, "tool": tool, "target": target},
+        evidence=f"log line {line}: {agent} invoked {tool} on {target}",
+    )
+
+
+def _vuln(component: str, cve: str) -> ClaimSpec:
+    return ClaimSpec(
+        kind="vulnerability",
+        payload={"component": component, "cve": cve, "severity": "high"},
+        evidence=f"NVD record {cve} affects {component}",
+    )
+
+
+def _policy_deny(agent: str, action: str) -> ClaimSpec:
+    return ClaimSpec(
+        kind="policy_rule",
+        payload={"subject": agent, "action": action, "resource": "*", "effect": "deny"},
+        evidence=f"policy doc: {agent} is denied {action}",
+    )
+
+
+def build_all_scenarios() -> list[Scenario]:
+    """The full corpus. Deterministic: no randomness, no files, no network.
+
+    Expected sets are hand-derived from the facts:
+      exposed(a, c) <=> a calls some tool, a operates_on c, c has_vulnerability
+    """
+    return [
+        Scenario(
+            name="positive_vulnerable_exposure",
+            description="An agent calls a tool on a component with a known CVE.",
+            claims=(
+                _tool_call("researcher", "sql_query", "orders_db", 42),
+                _vuln("orders_db", "CVE-2026-1234"),
+            ),
+            expected_exposed=frozenset({("researcher", "orders_db")}),
+        ),
+        Scenario(
+            name="negative_benign_workflow",
+            description="A benign multi-tool workflow on non-vulnerable components must not fire.",
+            claims=(
+                _tool_call("researcher", "sql_query", "analytics_db", 1),
+                _tool_call("researcher", "file_reader", "/data/clean.csv", 2),
+                _tool_call("researcher", "chart_maker", "/data/charts", 3),
+            ),
+            expected_exposed=frozenset(),
+        ),
+        Scenario(
+            name="multi_agent_only_one_exposed",
+            description=(
+                "Two agents call the same tool on different targets; only one "
+                "target is vulnerable. The fact model must NOT make agent_b "
+                "exposed to agent_a's target — operates_on is agent-scoped."
+            ),
+            claims=(
+                _tool_call("agent_a", "sql_query", "orders_db", 1),
+                _tool_call("agent_b", "sql_query", "customers_db", 2),
+                _vuln("orders_db", "CVE-2026-1234"),
+            ),
+            expected_exposed=frozenset({("agent_a", "orders_db")}),
+        ),
+        Scenario(
+            name="incomplete_evidence",
+            description="A tool call with no vulnerability record — fail-closed default applies.",
+            claims=(
+                _tool_call("researcher", "sql_query", "orders_db", 1),
+            ),
+            expected_exposed=frozenset(),
+            incomplete=True,
+        ),
+        Scenario(
+            name="policy_violation",
+            description="An agent calls a tool whose action is policy-denied.",
+            claims=(
+                _tool_call("researcher", "sql_query", "orders_db", 1),
+                _policy_deny("researcher", "sql_query"),
+            ),
+            expected_exposed=frozenset(),
+        ),
+        Scenario(
+            name="no_vulnerability_no_exposure",
+            description="A tool call on a component with NO vulnerability record derives nothing.",
+            claims=(
+                _tool_call("researcher", "sql_query", "orders_db", 1),
+                _vuln("customers_db", "CVE-2026-9999"),
+            ),
+            expected_exposed=frozenset(),
+        ),
+    ]

nomosguard/demo.py

@@ -0,0 +1,88 @@
+"""End-to-end demo: tool-call evidence -> derivation -> fail-closed gate.
+
+Runs a committed scenario through the full deterministic chain and prints
+the real output. Run with:
+
+    python -m nomosguard.demo
+"""
+
+from __future__ import annotations
+
+import json
+
+from .ledger import Claim, EvidenceLedger
+from .rules import Pattern, Rule, RuleEngine
+from .gate import Decision, PolicyGate, PolicyRule
+
+
+def build_scenario() -> EvidenceLedger:
+    """A committed, deterministic tool-call scenario.
+
+    Scenario: agent 'researcher' calls a data-analysis tool that operates
+    on a database component which has a known CVE. Policy denies access
+    to components with vulnerabilities.
+    """
+    ledger = EvidenceLedger()
+    ledger.append(
+        Claim(
+            kind="tool_call",
+            payload={"agent": "researcher", "tool": "sql_query", "target": "orders_db"},
+            evidence="agent log line 42: researcher invoked sql_query on orders_db",
+        )
+    )
+    ledger.append(
+        Claim(
+            kind="vulnerability",
+            payload={"component": "orders_db", "cve": "CVE-2026-1234", "severity": "high"},
+            evidence="NVD record CVE-2026-1234 affects orders_db (SQL injection)",
+        )
+    )
+    return ledger
+
+
+def run() -> dict:
+    ledger = build_scenario()
+    ok, detail = ledger.verify()
+
+    engine = RuleEngine(
+        rules=[
+            Rule(
+                name="tool_on_vulnerable_component",
+                body=(
+                    Pattern("?agent", "calls", "?tool"),
+                    Pattern("?agent", "operates_on", "?component"),
+                    Pattern("?component", "has_vulnerability", "?cve"),
+                ),
+                head=Pattern("?agent", "exposes", "?component"),
+                description="An agent operating a tool on a vulnerable component exposes it.",
+            )
+        ]
+    )
+    engine.add_facts_from_ledger(ledger)
+    derivation = engine.derive()
+
+    gate = PolicyGate(
+        policy_rules=[
+            PolicyRule(
+                name="block_vulnerable_exposure",
+                match_relation="exposes",
+                decision=Decision.BLOCK,
+                description="Exposure of a vulnerable component is blocked.",
+            )
+        ],
+        fallback=Decision.BLOCK,
+    )
+    result = gate.evaluate_with_fallback(engine)
+
+    return {
+        "ledger_verified": ok,
+        "ledger_detail": detail,
+        "entries": len(ledger.entries),
+        "derived_facts": [str(f) for f in engine.facts],
+        "derivation": derivation,
+        "gate_decision": result,
+    }
+
+
+if __name__ == "__main__":
+    print(json.dumps(run(), indent=2))