sqlguard-ninja 0.1.0__py3-none-any.whl

sqlguard_ninja/_data/benchmarks/datasets/v1/manifest.yaml

@@ -0,0 +1,15 @@
+name: sqlguard-ninja-core
+version: "1.0.0"
+description: >
+  Labelled SQL policy-enforcement cases for a synthetic multi-tenant retail schema.
+  Expected outcomes follow the policy semantics in docs/guides/BENCHMARK_METHODOLOGY.md and
+  are independent of any particular validator.
+policy: ../../policies/retail_saas.yaml
+default_context:
+  tenant_id: 1
+case_files:
+  - cases/*.yaml
+  - generated/*.yaml
+generator:
+  script: scripts/generate_corpus.py
+  seed: 20261024

sqlguard_ninja/_data/benchmarks/policies/retail_saas.yaml

@@ -0,0 +1,259 @@
+# Benchmark policy for the synthetic multi-tenant retail SaaS schema.
+# All data described here is synthetic. Classification labels are assigned explicitly;
+# SQLGuard Ninja never infers sensitivity from column names when enforcing.
+version: 1
+name: retail_saas
+description: Multi-tenant retail platform with support, warehouse, HR, finance and reporting roles.
+dialect: postgres
+
+labels:
+  personal_identifier:
+    description: Directly identifies a natural person (name, date of birth).
+    sensitivity: high
+    masking: pseudonymise or aggregate
+  contact:
+    description: Email, phone or postal address.
+    sensitivity: high
+    masking: return the email domain or a masked value
+  location:
+    description: City-level location.
+    sensitivity: medium
+  financial:
+    description: Bank details and individual financial amounts.
+    sensitivity: high
+    masking: aggregate
+  payment_card:
+    description: Primary account number (PCI DSS scope).
+    sensitivity: critical
+    masking: use card_last4 and card_brand
+  hr:
+    description: Employment terms and performance.
+    sensitivity: high
+    masking: aggregate by department
+  government_id:
+    description: National identification numbers.
+    sensitivity: critical
+  special_category:
+    description: Health data (GDPR Art. 9) — configured explicitly for this schema.
+    sensitivity: critical
+    special_category: true
+  free_text:
+    description: Unstructured text that may contain personal data.
+    sensitivity: medium
+
+catalog:
+  default_schema: public
+  tables:
+    tenants:
+      columns: {id: int, name: text, plan: text, created_at: timestamp}
+      estimated_rows: 500
+    customers:
+      tenant_column: tenant_id
+      estimated_rows: 2000000
+      columns:
+        id: {type: int, identifying: true}
+        tenant_id: int
+        full_name: {labels: [personal_identifier]}
+        email: {labels: [contact], alternatives: [email_domain]}
+        email_domain: text
+        phone: {labels: [contact]}
+        date_of_birth: {type: date, labels: [personal_identifier]}
+        country: text
+        city: {labels: [location]}
+        loyalty_tier: text
+        marketing_opt_in: boolean
+        created_at: timestamp
+        profile:
+          type: json
+          json_fields:
+            preferences.newsletter: []
+            preferences.language: []
+            health.allergies: [special_category]
+            payment.card_number: [payment_card]
+    orders:
+      tenant_column: tenant_id
+      estimated_rows: 10000000
+      columns:
+        id: {type: int, identifying: false}
+        tenant_id: int
+        customer_id: int
+        status: text
+        total_amount: numeric
+        currency: text
+        created_at: timestamp
+        shipping_address: {labels: [contact, location]}
+    order_items:
+      tenant_column: tenant_id
+      estimated_rows: 30000000
+      columns: {id: int, tenant_id: int, order_id: int, product_id: int, quantity: int, unit_price: numeric}
+    products:
+      tenant_column: tenant_id
+      estimated_rows: 50000
+      columns: {id: int, tenant_id: int, sku: text, name: text, category: text, price: numeric}
+    payments:
+      tenant_column: tenant_id
+      estimated_rows: 10000000
+      columns:
+        id: int
+        tenant_id: int
+        order_id: int
+        card_number: {labels: [payment_card], alternatives: [card_last4]}
+        card_last4: text
+        card_brand: text
+        iban: {labels: [financial]}
+        amount: numeric
+        status: text
+    shipments:
+      tenant_column: tenant_id
+      estimated_rows: 9000000
+      columns:
+        id: int
+        tenant_id: int
+        order_id: int
+        warehouse_id: int
+        carrier: text
+        tracking_number: text
+        status: text
+        shipped_at: timestamp
+        delivery_address: {labels: [contact, location]}
+    warehouses:
+      estimated_rows: 40
+      columns: {id: int, name: text, region: text}
+    employees:
+      estimated_rows: 5000
+      columns:
+        id: {type: int, identifying: true}
+        full_name: {labels: [personal_identifier]}
+        email: {labels: [contact]}
+        department: text
+        title: text
+        salary: {type: numeric, labels: [hr, financial]}
+        bank_account: {labels: [financial]}
+        national_id: {labels: [government_id]}
+        health_notes: {labels: [special_category]}
+        hire_date: date
+        manager_id: int
+        warehouse_id: int
+    support_tickets:
+      tenant_column: tenant_id
+      estimated_rows: 3000000
+      columns:
+        id: int
+        tenant_id: int
+        customer_id: int
+        subject: text
+        body: {labels: [free_text]}
+        status: text
+        priority: text
+        created_at: timestamp
+
+roles:
+  support_analyst:
+    description: Customer support. Order status yes; full payment details no.
+    tables:
+      customers: [id, tenant_id, full_name, email, email_domain, country, city, loyalty_tier, created_at, profile]
+      orders: "*"
+      order_items: "*"
+      products: "*"
+      shipments: "*"
+      support_tickets: "*"
+      payments: [id, tenant_id, order_id, card_last4, card_brand, amount, status]
+    labels:
+      personal_identifier: allow
+      contact: allow
+      location: allow
+      free_text: allow
+      payment_card: deny
+      financial: deny
+      special_category: deny
+
+  warehouse_analyst:
+    description: Shipments and stock. No HR or customer contact data.
+    tables:
+      shipments: "*"
+      warehouses: "*"
+      products: "*"
+      order_items: "*"
+      orders: [id, tenant_id, status, created_at]
+    labels:
+      location: aggregate_only
+    unlisted_label_access: deny
+
+  hr_analyst:
+    description: HR operations. Salary only in aggregate; never national ids or health notes.
+    tenant_scoped: false
+    tables:
+      employees: "*"
+      warehouses: "*"
+    labels:
+      personal_identifier: allow
+      contact: allow
+      hr: allow
+      financial: aggregate_only
+      government_id: deny
+      special_category: deny
+
+  finance_analyst:
+    description: Revenue and payments, without card numbers or customer contact details.
+    tables:
+      orders: [id, tenant_id, customer_id, status, total_amount, currency, created_at]
+      order_items: "*"
+      products: "*"
+      payments: "*"
+    labels:
+      financial: allow
+      payment_card: deny
+
+  reporting_agent:
+    description: Automated reporting. Aggregates only for anything personal.
+    tables:
+      orders: [id, tenant_id, customer_id, status, total_amount, currency, created_at]
+      order_items: "*"
+      products: "*"
+      customers: [id, tenant_id, country, city, loyalty_tier, created_at, email_domain, email]
+    labels:
+      location: aggregate_only
+      contact: deny
+    limits:
+      max_limit: 1000
+      max_joins: 4
+      max_subquery_depth: 3
+
+  platform_admin:
+    description: Internal cross-tenant operations. Sensitive classes need review.
+    tenant_scoped: false
+    tables:
+      tenants: "*"
+      customers: "*"
+      orders: "*"
+      order_items: "*"
+      products: "*"
+      payments: "*"
+      shipments: "*"
+      warehouses: "*"
+      support_tickets: "*"
+    labels:
+      personal_identifier: allow
+      contact: allow
+      location: allow
+      free_text: allow
+      financial: review
+      payment_card: deny
+      special_category: deny
+
+tenant:
+  context_key: tenant_id
+  parameter_names: [tenant_id]
+
+functions:
+  mode: denylist
+
+limits:
+  max_joins: 6
+  max_subquery_depth: 4
+  max_limit: 10000
+
+cost:
+  review_score: 50
+  deny_score: 80
+  large_table_rows: 1000000

sqlguard_ninja/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

sqlguard_ninja/benchmarking/__init__.py

@@ -0,0 +1 @@
+"""Benchmark engine: corpus, adapters, runner, metrics, comparison and reports."""

sqlguard_ninja/benchmarking/adapters.py

@@ -0,0 +1,303 @@
+"""Benchmark adapters: a uniform interface over different SQL validators.
+
+* :class:`NativeAdapter`      — SQLGuard Ninja's own :class:`~sqlguard_ninja.Guard`.
+* :class:`CallableAdapter`    — any user function ``module:function``.
+* :class:`SubprocessAdapter`  — a worker script run by *another* Python interpreter
+  (one virtual environment per external tool), speaking JSON Lines over stdin/stdout.
+  This isolates tools whose dependencies or import names conflict.
+
+An adapter declares the case categories its policy model can express (``capabilities``).
+Reports show metrics on all cases and on the capability subset, so a tool is not
+penalised in a headline number for features it never claimed.
+"""
+
+from __future__ import annotations
+
+import importlib
+import json
+import subprocess
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Protocol
+
+from sqlguard_ninja._version import __version__
+from sqlguard_ninja.benchmarking.dataset import BenchmarkCase
+from sqlguard_ninja.benchmarking.translate import translate_for_role
+from sqlguard_ninja.core.findings import Decision
+from sqlguard_ninja.core.guard import Guard
+from sqlguard_ninja.policy.model import Policy
+
+WORKERS_DIR = Path(__file__).parent / "workers"
+
+ALL_CATEGORIES: frozenset[str] | None = None
+"""Sentinel: the adapter claims every category."""
+
+
+@dataclass(frozen=True)
+class AdapterVerdict:
+    decision: Decision
+    reasons: tuple[str, ...] = ()
+    latency_ms: float = 0.0
+    error: str | None = None
+    """Set when the validator crashed; the decision is then DENY (fail closed) and the
+    error is counted separately in reports."""
+
+
+class BenchmarkAdapter(Protocol):
+    name: str
+
+    @property
+    def version(self) -> str: ...
+
+    @property
+    def capabilities(self) -> frozenset[str] | None: ...
+
+    def evaluate(self, case: BenchmarkCase) -> AdapterVerdict: ...
+
+    def close(self) -> None: ...
+
+
+class NativeAdapter:
+    """Evaluate cases with SQLGuard Ninja's built-in guard."""
+
+    name = "native"
+
+    def __init__(self, policy: Policy, guard: Guard | None = None) -> None:
+        self.guard = guard or Guard(policy)
+
+    @property
+    def version(self) -> str:
+        return __version__
+
+    @property
+    def capabilities(self) -> frozenset[str] | None:
+        return ALL_CATEGORIES
+
+    def evaluate(self, case: BenchmarkCase) -> AdapterVerdict:
+        started = time.perf_counter()
+        result = self.guard.validate(
+            case.sql, role=case.role, dialect=case.dialect, context=case.context
+        )
+        elapsed = (time.perf_counter() - started) * 1000
+        reasons = tuple(f.rule_id for f in result.blocking_findings)
+        return AdapterVerdict(result.decision, reasons, elapsed)
+
+    def close(self) -> None:
+        return None
+
+
+def _coerce_decision(value: Any) -> tuple[Decision, tuple[str, ...]]:
+    """Normalise the many result shapes a user validator may return."""
+    reasons: tuple[str, ...] = ()
+    if isinstance(value, tuple) and len(value) == 2:
+        value, raw = value
+        reasons = tuple(str(r) for r in (raw or ()))
+    if isinstance(value, dict):
+        reasons = tuple(str(r) for r in value.get("reasons", ()))
+        value = value.get("decision", value.get("allowed"))
+    if isinstance(value, Decision):
+        return value, reasons
+    if isinstance(value, bool):
+        return (Decision.ALLOW if value else Decision.DENY), reasons
+    if isinstance(value, str) and value.lower() in {d.value for d in Decision}:
+        return Decision(value.lower()), reasons
+    raise TypeError(f"cannot interpret validator result {value!r}")
+
+
+class CallableAdapter:
+    """Wrap a Python callable ``fn(sql, dialect, case) -> bool | str | dict | Decision``."""
+
+    def __init__(
+        self,
+        fn: Callable[[str, str, BenchmarkCase], Any],
+        name: str = "callable",
+        version: str = "unknown",
+        capabilities: frozenset[str] | None = ALL_CATEGORIES,
+    ) -> None:
+        self.fn = fn
+        self.name = name
+        self._version = version
+        self._capabilities = capabilities
+
+    @classmethod
+    def from_spec(cls, spec: str) -> CallableAdapter:
+        """Load ``package.module:function``."""
+        module_name, _, attr = spec.partition(":")
+        if not module_name or not attr:
+            raise ValueError("callable adapter spec must look like 'module:function'")
+        fn = getattr(importlib.import_module(module_name), attr)
+        return cls(fn, name=f"callable:{spec}")
+
+    @property
+    def version(self) -> str:
+        return self._version
+
+    @property
+    def capabilities(self) -> frozenset[str] | None:
+        return self._capabilities
+
+    def evaluate(self, case: BenchmarkCase) -> AdapterVerdict:
+        started = time.perf_counter()
+        try:
+            raw = self.fn(case.sql, case.dialect, case)
+            decision, reasons = _coerce_decision(raw)
+            error = None
+        except Exception as exc:  # a crashing validator must not crash the benchmark
+            decision, reasons, error = Decision.DENY, (), f"{type(exc).__name__}: {exc}"
+        return AdapterVerdict(decision, reasons, (time.perf_counter() - started) * 1000, error)
+
+    def close(self) -> None:
+        return None
+
+
+@dataclass
+class ExternalToolSpec:
+    """Metadata for a bundled external worker."""
+
+    name: str
+    worker: str
+    package: str
+    pip_spec: str
+    capabilities: frozenset[str]
+    notes: str = ""
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+_BASE_CAPS = frozenset(
+    {
+        "destructive",
+        "multi_statement",
+        "comments_formatting",
+        "unauthorized_table",
+        "unauthorized_column",
+        "lookalike_safe",
+        "legitimate_complex",
+        "cte_bypass",
+        "nested_query",
+        "join_subquery_policy",
+        "unicode_identifiers",
+    }
+)
+
+EXTERNAL_TOOLS: dict[str, ExternalToolSpec] = {
+    "sql-guardrail": ExternalToolSpec(
+        name="sql-guardrail",
+        worker="sql_guardrail_worker.py",
+        package="sql-guardrail",
+        pip_spec="sql-guardrail==0.5.2",
+        capabilities=_BASE_CAPS | {"tenant_isolation", "cost"},
+        notes="Column allow/deny lists and require_predicate; no roles, labels or review.",
+    ),
+    "agent-sql-guard": ExternalToolSpec(
+        name="agent-sql-guard",
+        worker="agent_sql_guard_worker.py",
+        package="agent-sql-guard",
+        pip_spec="agent-sql-guard==0.2.0",
+        capabilities=_BASE_CAPS | {"sensitive_field"},
+        notes="Table allowlist and a name-based PII denylist; no tenant predicates.",
+    ),
+    "krish2105-sqlguard": ExternalToolSpec(
+        name="krish2105-sqlguard",
+        worker="krish_sqlguard_worker.py",
+        package="sqlguard",
+        pip_spec="sqlguard @ git+https://github.com/krish2105/sqlguard@62327f214ed8c9ea82ecf820bf04c4b55a6161a2",
+        capabilities=_BASE_CAPS,
+        notes="Read-only AST check with a schema allow-list; not published on PyPI.",
+    ),
+}
+
+
+class SubprocessAdapter:
+    """Run an external validator in its own interpreter via a JSON Lines worker.
+
+    The worker receives one request per line::
+
+        {"id": ..., "sql": ..., "dialect": ..., "context": {...}, "config": {...}}
+
+    and answers with ``{"decision": "allow|deny|review", "reasons": [...],
+    "latency_ms": float, "error": str|null}``. Its first line is a hello message carrying
+    ``{"library": ..., "version": ...}``.
+    """
+
+    def __init__(
+        self,
+        tool: ExternalToolSpec,
+        python: str | Path,
+        policy: Policy,
+        timeout_s: float = 30.0,
+    ) -> None:
+        self.tool = tool
+        self.name = tool.name
+        self.policy = policy
+        self.timeout_s = timeout_s
+        worker = WORKERS_DIR / tool.worker
+        # -I: isolated mode, so the worker cannot import from the current directory.
+        self._proc = subprocess.Popen(  # noqa: S603 - fixed argv, no shell
+            [str(python), "-I", str(worker)],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            encoding="utf-8",
+            bufsize=1,
+        )
+        hello = self._read()
+        if "error" in hello and hello.get("error"):
+            self.close()
+            raise RuntimeError(f"{tool.name} worker failed to start: {hello['error']}")
+        self._version = str(hello.get("version", "unknown"))
+        self._configs: dict[tuple[str | None, str], dict[str, Any]] = {}
+
+    def _read(self) -> dict[str, Any]:
+        assert self._proc.stdout is not None
+        line = self._proc.stdout.readline()
+        if not line:
+            stderr = self._proc.stderr.read() if self._proc.stderr else ""
+            raise RuntimeError(f"{self.tool.name} worker exited: {stderr[-500:]}")
+        data: dict[str, Any] = json.loads(line)
+        return data
+
+    @property
+    def version(self) -> str:
+        return self._version
+
+    @property
+    def capabilities(self) -> frozenset[str] | None:
+        return self.tool.capabilities
+
+    def evaluate(self, case: BenchmarkCase) -> AdapterVerdict:
+        key = (case.role, case.dialect)
+        if key not in self._configs:
+            self._configs[key] = translate_for_role(self.policy, case.role, case.dialect)
+        request = {
+            "id": case.id,
+            "sql": case.sql,
+            "dialect": case.dialect,
+            "context": case.context,
+            "config": self._configs[key],
+        }
+        assert self._proc.stdin is not None
+        self._proc.stdin.write(json.dumps(request) + "\n")
+        self._proc.stdin.flush()
+        reply = self._read()
+        decision = Decision(reply.get("decision", "deny"))
+        return AdapterVerdict(
+            decision=decision,
+            reasons=tuple(reply.get("reasons", ())),
+            latency_ms=float(reply.get("latency_ms", 0.0)),
+            error=reply.get("error"),
+        )
+
+    def close(self) -> None:
+        if self._proc.poll() is None:
+            if self._proc.stdin:
+                self._proc.stdin.close()
+            try:
+                self._proc.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                self._proc.kill()
+        for stream in (self._proc.stdout, self._proc.stderr):
+            if stream:
+                stream.close()

sqlguard_ninja/benchmarking/builtin.py

@@ -0,0 +1,39 @@
+"""Locate the reference benchmark corpus and policy shipped with the package.
+
+In a wheel, the files are packaged under ``sqlguard_ninja/_data/benchmarks``. In a source
+checkout (editable install), they are read from the repository's ``benchmarks/`` directory.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+_PACKAGE = Path(__file__).resolve().parents[1]
+_CANDIDATES = (_PACKAGE / "_data" / "benchmarks", _PACKAGE.parents[1] / "benchmarks")
+
+PREFIX = "builtin:"
+
+
+def benchmarks_root() -> Path:
+    for candidate in _CANDIDATES:
+        if (candidate / "datasets").is_dir() and (candidate / "policies").is_dir():
+            return candidate
+    raise FileNotFoundError("the bundled benchmark data is missing from this installation")
+
+
+def resolve(spec: str | Path, kind: str) -> Path:
+    """Resolve ``builtin:<name>`` for ``kind`` 'dataset' or 'policy'; pass other paths through."""
+    text = str(spec)
+    if not text.startswith(PREFIX):
+        return Path(text)
+    name = text.removeprefix(PREFIX) or ("v1" if kind == "dataset" else "retail_saas")
+    root = benchmarks_root()
+    path = root / "datasets" / name if kind == "dataset" else root / "policies" / f"{name}.yaml"
+    if not path.exists():
+        available = sorted(
+            p.name if kind == "dataset" else p.stem
+            for p in (root / ("datasets" if kind == "dataset" else "policies")).iterdir()
+            if not p.name.startswith(".")
+        )
+        raise FileNotFoundError(f"no built-in {kind} {name!r}; available: {available}")
+    return path