sqlguard-ninja 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. sqlguard_ninja/__init__.py +35 -0
  2. sqlguard_ninja/_data/benchmarks/datasets/v1/cases/01_destructive.yaml +169 -0
  3. sqlguard_ninja/_data/benchmarks/datasets/v1/cases/02_multi_statement_and_formatting.yaml +155 -0
  4. sqlguard_ninja/_data/benchmarks/datasets/v1/cases/03_table_and_column_access.yaml +205 -0
  5. sqlguard_ninja/_data/benchmarks/datasets/v1/cases/04_privacy.yaml +245 -0
  6. sqlguard_ninja/_data/benchmarks/datasets/v1/cases/05_tenant_isolation.yaml +167 -0
  7. sqlguard_ninja/_data/benchmarks/datasets/v1/cases/06_cte_nested_unicode.yaml +168 -0
  8. sqlguard_ninja/_data/benchmarks/datasets/v1/cases/07_legitimate_lookalike_cost_dialect.yaml +243 -0
  9. sqlguard_ninja/_data/benchmarks/datasets/v1/generated/grant_matrix.yaml +573 -0
  10. sqlguard_ninja/_data/benchmarks/datasets/v1/generated/label_matrix.yaml +403 -0
  11. sqlguard_ninja/_data/benchmarks/datasets/v1/generated/tenant_matrix.yaml +684 -0
  12. sqlguard_ninja/_data/benchmarks/datasets/v1/generated/transforms.yaml +927 -0
  13. sqlguard_ninja/_data/benchmarks/datasets/v1/manifest.yaml +15 -0
  14. sqlguard_ninja/_data/benchmarks/policies/retail_saas.yaml +259 -0
  15. sqlguard_ninja/_version.py +1 -0
  16. sqlguard_ninja/benchmarking/__init__.py +1 -0
  17. sqlguard_ninja/benchmarking/adapters.py +303 -0
  18. sqlguard_ninja/benchmarking/builtin.py +39 -0
  19. sqlguard_ninja/benchmarking/compare.py +168 -0
  20. sqlguard_ninja/benchmarking/dataset.py +265 -0
  21. sqlguard_ninja/benchmarking/fixtures.py +274 -0
  22. sqlguard_ninja/benchmarking/metrics.py +114 -0
  23. sqlguard_ninja/benchmarking/runner.py +126 -0
  24. sqlguard_ninja/benchmarking/translate.py +64 -0
  25. sqlguard_ninja/benchmarking/workers/_protocol.py +53 -0
  26. sqlguard_ninja/benchmarking/workers/agent_sql_guard_worker.py +40 -0
  27. sqlguard_ninja/benchmarking/workers/krish_sqlguard_worker.py +25 -0
  28. sqlguard_ninja/benchmarking/workers/sql_guardrail_worker.py +48 -0
  29. sqlguard_ninja/cli/__init__.py +1 -0
  30. sqlguard_ninja/cli/config.py +76 -0
  31. sqlguard_ninja/cli/main.py +433 -0
  32. sqlguard_ninja/cli/templates.py +91 -0
  33. sqlguard_ninja/core/__init__.py +1 -0
  34. sqlguard_ninja/core/analysis.py +581 -0
  35. sqlguard_ninja/core/findings.py +133 -0
  36. sqlguard_ninja/core/guard.py +256 -0
  37. sqlguard_ninja/core/rules/__init__.py +1 -0
  38. sqlguard_ninja/core/rules/access.py +127 -0
  39. sqlguard_ninja/core/rules/base.py +40 -0
  40. sqlguard_ninja/core/rules/catalog.py +61 -0
  41. sqlguard_ninja/core/rules/functions.py +197 -0
  42. sqlguard_ninja/core/rules/statements.py +99 -0
  43. sqlguard_ninja/core/rules/structure.py +72 -0
  44. sqlguard_ninja/core/rules/tenant.py +196 -0
  45. sqlguard_ninja/integrations/__init__.py +43 -0
  46. sqlguard_ninja/integrations/audit.py +109 -0
  47. sqlguard_ninja/integrations/checkpoint.py +342 -0
  48. sqlguard_ninja/integrations/executors.py +64 -0
  49. sqlguard_ninja/integrations/fastapi.py +89 -0
  50. sqlguard_ninja/integrations/langchain.py +67 -0
  51. sqlguard_ninja/integrations/langgraph.py +87 -0
  52. sqlguard_ninja/policy/__init__.py +39 -0
  53. sqlguard_ninja/policy/model.py +370 -0
  54. sqlguard_ninja/privacy/__init__.py +9 -0
  55. sqlguard_ninja/privacy/rules.py +286 -0
  56. sqlguard_ninja/privacy/suggest.py +46 -0
  57. sqlguard_ninja/py.typed +0 -0
  58. sqlguard_ninja/query_cost/__init__.py +1 -0
  59. sqlguard_ninja/query_cost/postgres.py +226 -0
  60. sqlguard_ninja/query_cost/static.py +241 -0
  61. sqlguard_ninja/reporting/__init__.py +1 -0
  62. sqlguard_ninja/reporting/markdown.py +137 -0
  63. sqlguard_ninja-0.1.0.dist-info/METADATA +370 -0
  64. sqlguard_ninja-0.1.0.dist-info/RECORD +68 -0
  65. sqlguard_ninja-0.1.0.dist-info/WHEEL +4 -0
  66. sqlguard_ninja-0.1.0.dist-info/entry_points.txt +2 -0
  67. sqlguard_ninja-0.1.0.dist-info/licenses/LICENSE +202 -0
  68. sqlguard_ninja-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +56 -0
@@ -0,0 +1,274 @@
1
+ """Synthetic fixture database for the benchmark schema (SQLite and PostgreSQL).
2
+
3
+ The data is generated deterministically from a seed. It contains no real personal data:
4
+ names, emails, card numbers and identifiers are fabricated and use reserved example
5
+ domains and test card ranges.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import random
12
+ import sqlite3
13
+ from collections.abc import Iterator
14
+ from datetime import date, datetime, timedelta
15
+ from pathlib import Path
16
+ from typing import Any, Protocol
17
+
18
+ from sqlguard_ninja.policy.model import Catalog, Policy
19
+
20
+ DEFAULT_SEED = 20261024
21
+
22
+ _TYPE_MAP = {
23
+ "sqlite": {
24
+ "int": "INTEGER",
25
+ "text": "TEXT",
26
+ "numeric": "NUMERIC",
27
+ "timestamp": "TEXT",
28
+ "date": "TEXT",
29
+ "boolean": "INTEGER",
30
+ "json": "TEXT",
31
+ },
32
+ "postgres": {
33
+ "int": "BIGINT",
34
+ "text": "TEXT",
35
+ "numeric": "NUMERIC(14,2)",
36
+ "timestamp": "TIMESTAMP",
37
+ "date": "DATE",
38
+ "boolean": "BOOLEAN",
39
+ "json": "JSONB",
40
+ },
41
+ }
42
+
43
+ ROW_COUNTS = {
44
+ "tenants": 3,
45
+ "warehouses": 4,
46
+ "customers": 60,
47
+ "products": 30,
48
+ "orders": 200,
49
+ "order_items": 400,
50
+ "payments": 200,
51
+ "shipments": 180,
52
+ "employees": 40,
53
+ "support_tickets": 80,
54
+ }
55
+
56
+ _FIRST = ["Alex", "Sam", "Robin", "Kim", "Jordan", "Taylor", "Morgan", "Casey", "Avery", "Riley"]
57
+ _LAST = ["Example", "Sample", "Testperson", "Placeholder", "Demo", "Fixture", "Synthetic"]
58
+ _CITIES = ["Berlin", "Lisbon", "Dubai", "Pune", "Oslo", "Austin", "Lyon", "Kraków"]
59
+ _COUNTRIES = ["DE", "PT", "AE", "IN", "NO", "US", "FR", "PL"]
60
+
61
+
62
+ class _DBConnection(Protocol):
63
+ def cursor(self) -> Any: ...
64
+ def commit(self) -> None: ...
65
+
66
+
67
+ def schema_ddl(catalog: Catalog, dialect: str) -> list[str]:
68
+ """CREATE TABLE statements derived from the policy catalog."""
69
+ types = _TYPE_MAP[dialect]
70
+ statements = []
71
+ for name, table in catalog.tables.items():
72
+ cols = []
73
+ for col, spec in table.columns.items():
74
+ sql_type = types.get(spec.type, "TEXT")
75
+ pk = " PRIMARY KEY" if col == "id" else ""
76
+ cols.append(f"{col} {sql_type}{pk}")
77
+ statements.append(f"CREATE TABLE {name} ({', '.join(cols)})")
78
+ return statements
79
+
80
+
81
+ def _rows(seed: int) -> Iterator[tuple[str, list[dict[str, Any]]]]:
82
+ rng = random.Random(seed)
83
+ base = datetime(2026, 1, 1)
84
+
85
+ def ts(days: int = 300) -> str:
86
+ return (base + timedelta(minutes=rng.randrange(days * 24 * 60))).isoformat(sep=" ")
87
+
88
+ def person() -> str:
89
+ return f"{rng.choice(_FIRST)} {rng.choice(_LAST)}"
90
+
91
+ tenants = [
92
+ {"id": i, "name": f"Tenant {i}", "plan": rng.choice(["free", "pro"]), "created_at": ts()}
93
+ for i in range(1, ROW_COUNTS["tenants"] + 1)
94
+ ]
95
+ yield "tenants", tenants
96
+ yield (
97
+ "warehouses",
98
+ [
99
+ {"id": i, "name": f"WH-{i}", "region": rng.choice(["EU", "US", "MEA", "APAC"])}
100
+ for i in range(1, ROW_COUNTS["warehouses"] + 1)
101
+ ],
102
+ )
103
+ customers = []
104
+ for i in range(1, ROW_COUNTS["customers"] + 1):
105
+ k = rng.randrange(len(_CITIES))
106
+ customers.append(
107
+ {
108
+ "id": i,
109
+ "tenant_id": (i % 3) + 1,
110
+ "full_name": person(),
111
+ "email": f"user{i}@example.com",
112
+ "email_domain": "example.com",
113
+ "phone": f"+00 555 {i:04d}",
114
+ "date_of_birth": date(
115
+ 1960 + rng.randrange(40), 1 + rng.randrange(12), 1
116
+ ).isoformat(),
117
+ "country": _COUNTRIES[k],
118
+ "city": _CITIES[k],
119
+ "loyalty_tier": rng.choice(["bronze", "silver", "gold"]),
120
+ "marketing_opt_in": rng.random() < 0.5,
121
+ "created_at": ts(),
122
+ "profile": json.dumps(
123
+ {
124
+ "preferences": {"newsletter": rng.random() < 0.5, "language": "en"},
125
+ "health": {"allergies": rng.choice(["none", "synthetic-allergen"])},
126
+ "payment": {"card_number": "4111111111111111"},
127
+ }
128
+ ),
129
+ }
130
+ )
131
+ yield "customers", customers
132
+ products = [
133
+ {
134
+ "id": i,
135
+ "tenant_id": (i % 3) + 1,
136
+ "sku": f"SKU-{i:04d}",
137
+ "name": f"Product {i}",
138
+ "category": rng.choice(["books", "garden", "toys", "tools"]),
139
+ "price": round(rng.uniform(2, 200), 2),
140
+ }
141
+ for i in range(1, ROW_COUNTS["products"] + 1)
142
+ ]
143
+ yield "products", products
144
+ orders = []
145
+ for i in range(1, ROW_COUNTS["orders"] + 1):
146
+ cust = rng.choice(customers)
147
+ orders.append(
148
+ {
149
+ "id": i,
150
+ "tenant_id": cust["tenant_id"],
151
+ "customer_id": cust["id"],
152
+ "status": rng.choice(["new", "paid", "shipped", "delivered", "cancelled"]),
153
+ "total_amount": round(rng.uniform(5, 900), 2),
154
+ "currency": "EUR",
155
+ "created_at": ts(),
156
+ "shipping_address": f"{i} Example Street, {cust['city']}",
157
+ }
158
+ )
159
+ yield "orders", orders
160
+ yield (
161
+ "order_items",
162
+ [
163
+ {
164
+ "id": i,
165
+ "tenant_id": (o := rng.choice(orders))["tenant_id"],
166
+ "order_id": o["id"],
167
+ "product_id": rng.choice(products)["id"],
168
+ "quantity": 1 + rng.randrange(5),
169
+ "unit_price": round(rng.uniform(2, 200), 2),
170
+ }
171
+ for i in range(1, ROW_COUNTS["order_items"] + 1)
172
+ ],
173
+ )
174
+ yield (
175
+ "payments",
176
+ [
177
+ {
178
+ "id": o["id"],
179
+ "tenant_id": o["tenant_id"],
180
+ "order_id": o["id"],
181
+ "card_number": "4111111111111111", # published test card number
182
+ "card_last4": "1111",
183
+ "card_brand": "visa",
184
+ "iban": "XX00TEST0000000000",
185
+ "amount": o["total_amount"],
186
+ "status": rng.choice(["captured", "refunded", "failed"]),
187
+ }
188
+ for o in orders[: ROW_COUNTS["payments"]]
189
+ ],
190
+ )
191
+ yield (
192
+ "shipments",
193
+ [
194
+ {
195
+ "id": o["id"],
196
+ "tenant_id": o["tenant_id"],
197
+ "order_id": o["id"],
198
+ "warehouse_id": 1 + rng.randrange(ROW_COUNTS["warehouses"]),
199
+ "carrier": rng.choice(["DHL", "UPS", "Aramex"]),
200
+ "tracking_number": f"TRK{o['id']:06d}",
201
+ "status": rng.choice(["in_transit", "delivered", "returned"]),
202
+ "shipped_at": ts(),
203
+ "delivery_address": o["shipping_address"],
204
+ }
205
+ for o in orders[: ROW_COUNTS["shipments"]]
206
+ ],
207
+ )
208
+ yield (
209
+ "employees",
210
+ [
211
+ {
212
+ "id": i,
213
+ "full_name": person(),
214
+ "email": f"employee{i}@example.org",
215
+ "department": rng.choice(["warehouse", "support", "finance", "engineering"]),
216
+ "title": rng.choice(["associate", "lead", "manager"]),
217
+ "salary": round(rng.uniform(30000, 120000), 2),
218
+ "bank_account": f"XX00TEST{i:010d}",
219
+ "national_id": f"TEST-{i:06d}",
220
+ "health_notes": "synthetic",
221
+ "hire_date": date(2015 + rng.randrange(10), 1 + rng.randrange(12), 1).isoformat(),
222
+ "manager_id": None if i <= 3 else 1 + rng.randrange(3),
223
+ "warehouse_id": 1 + rng.randrange(ROW_COUNTS["warehouses"]),
224
+ }
225
+ for i in range(1, ROW_COUNTS["employees"] + 1)
226
+ ],
227
+ )
228
+ yield (
229
+ "support_tickets",
230
+ [
231
+ {
232
+ "id": i,
233
+ "tenant_id": (c := rng.choice(customers))["tenant_id"],
234
+ "customer_id": c["id"],
235
+ "subject": rng.choice(["Late delivery", "Refund", "Login issue"]),
236
+ "body": "Synthetic ticket body.",
237
+ "status": rng.choice(["open", "closed"]),
238
+ "priority": rng.choice(["low", "high"]),
239
+ "created_at": ts(),
240
+ }
241
+ for i in range(1, ROW_COUNTS["support_tickets"] + 1)
242
+ ],
243
+ )
244
+
245
+
246
+ def populate(conn: _DBConnection, policy: Policy, dialect: str, seed: int = DEFAULT_SEED) -> None:
247
+ """Create the schema and load synthetic rows. Values are always bound, never interpolated."""
248
+ marker = "?" if dialect == "sqlite" else "%s"
249
+ cur = conn.cursor()
250
+ for stmt in schema_ddl(policy.catalog, dialect):
251
+ cur.execute(stmt)
252
+ for table, rows in _rows(seed):
253
+ if not rows:
254
+ continue
255
+ columns = list(policy.catalog.tables[table].columns)
256
+ placeholders = ", ".join([marker] * len(columns))
257
+ # Table and column names come from the validated policy catalog, not from input.
258
+ insert = f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({placeholders})" # noqa: S608
259
+ params = [tuple(row.get(c) for c in columns) for row in rows]
260
+ cur.executemany(insert, params)
261
+ conn.commit()
262
+
263
+
264
+ def create_sqlite(
265
+ policy: Policy, path: str | Path = ":memory:", seed: int = DEFAULT_SEED
266
+ ) -> sqlite3.Connection:
267
+ """Return a SQLite connection with the fixture schema and data loaded.
268
+
269
+ The connection may be shared across threads (web servers run handlers in a pool);
270
+ :func:`sqlguard_ninja.integrations.executors.sqlite_executor` serialises access.
271
+ """
272
+ conn = sqlite3.connect(str(path), check_same_thread=False)
273
+ populate(conn, policy, "sqlite", seed)
274
+ return conn
@@ -0,0 +1,114 @@
1
+ """Detection metrics with explicit definitions.
2
+
3
+ Positive class: a case whose expected outcome is *not* ALLOW (``deny`` or ``review``), i.e.
4
+ something that must not run unattended. A prediction is positive when the validator's decision
5
+ is not ALLOW. On top of the binary view, ``exact_match_rate`` measures how often the
6
+ three-way decision (allow/review/deny) matches exactly.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import math
12
+ from collections import defaultdict
13
+ from collections.abc import Iterable
14
+ from dataclasses import dataclass
15
+ from typing import Any
16
+
17
+ from sqlguard_ninja.core.findings import Decision
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class CaseOutcome:
22
+ id: str
23
+ category: str
24
+ dialect: str
25
+ provenance: str
26
+ expected: Decision
27
+ actual: Decision
28
+ latency_ms: float
29
+ error: str | None = None
30
+ in_capability: bool = True
31
+
32
+ @property
33
+ def unsafe(self) -> bool:
34
+ return self.expected is not Decision.ALLOW
35
+
36
+ @property
37
+ def flagged(self) -> bool:
38
+ return self.actual is not Decision.ALLOW
39
+
40
+ @property
41
+ def kind(self) -> str:
42
+ if self.unsafe and self.flagged:
43
+ return "tp"
44
+ if self.unsafe:
45
+ return "fn"
46
+ if self.flagged:
47
+ return "fp"
48
+ return "tn"
49
+
50
+
51
+ def _ratio(num: int, den: int) -> float | None:
52
+ return None if den == 0 else round(num / den, 4)
53
+
54
+
55
+ def percentile(values: list[float], pct: float) -> float | None:
56
+ """Nearest-rank percentile (deterministic, no interpolation)."""
57
+ if not values:
58
+ return None
59
+ ordered = sorted(values)
60
+ rank = max(1, math.ceil(pct / 100 * len(ordered)))
61
+ return round(ordered[rank - 1], 4)
62
+
63
+
64
+ def summarise(outcomes: Iterable[CaseOutcome]) -> dict[str, Any]:
65
+ items = list(outcomes)
66
+ counts = {"tp": 0, "fp": 0, "tn": 0, "fn": 0}
67
+ for o in items:
68
+ counts[o.kind] += 1
69
+ tp, fp, tn, fn = counts["tp"], counts["fp"], counts["tn"], counts["fn"]
70
+ latencies = [o.latency_ms for o in items]
71
+ return {
72
+ "cases": len(items),
73
+ "unsafe_cases": tp + fn,
74
+ "safe_cases": tn + fp,
75
+ **counts,
76
+ "recall": _ratio(tp, tp + fn),
77
+ "precision": _ratio(tp, tp + fp),
78
+ "false_positive_rate": _ratio(fp, fp + tn),
79
+ "false_negative_rate": _ratio(fn, fn + tp),
80
+ "exact_match_rate": _ratio(sum(o.expected is o.actual for o in items), len(items)),
81
+ "errors": sum(o.error is not None for o in items),
82
+ "latency_ms": {
83
+ "p50": percentile(latencies, 50),
84
+ "p95": percentile(latencies, 95),
85
+ "p99": percentile(latencies, 99),
86
+ "max": round(max(latencies), 4) if latencies else None,
87
+ },
88
+ }
89
+
90
+
91
+ def grouped(outcomes: list[CaseOutcome], attr: str) -> dict[str, dict[str, Any]]:
92
+ groups: dict[str, list[CaseOutcome]] = defaultdict(list)
93
+ for o in outcomes:
94
+ key = getattr(o, attr)
95
+ if attr == "provenance":
96
+ key = key.split(":")[0] if not key.startswith("generated:") else key.split("@")[0]
97
+ groups[key].append(o)
98
+ return {k: summarise(v) for k, v in sorted(groups.items())}
99
+
100
+
101
+ def compute_metrics(outcomes: list[CaseOutcome], examples: int = 15) -> dict[str, Any]:
102
+ capable = [o for o in outcomes if o.in_capability]
103
+ return {
104
+ "overall": summarise(outcomes),
105
+ "within_capability": summarise(capable),
106
+ "by_category": grouped(outcomes, "category"),
107
+ "by_dialect": grouped(outcomes, "dialect"),
108
+ "by_provenance": grouped(outcomes, "provenance"),
109
+ "false_negative_examples": [o.id for o in outcomes if o.kind == "fn"][:examples],
110
+ "false_positive_examples": [o.id for o in outcomes if o.kind == "fp"][:examples],
111
+ "decision_mismatch_examples": [
112
+ o.id for o in outcomes if o.expected is not o.actual and o.kind in ("tp", "tn")
113
+ ][:examples],
114
+ }
@@ -0,0 +1,126 @@
1
+ """Run a dataset through an adapter and produce a versioned, machine-readable result."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import platform
7
+ import subprocess
8
+ import sys
9
+ from datetime import UTC, datetime
10
+ from importlib.metadata import PackageNotFoundError, version
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ from sqlguard_ninja.benchmarking.adapters import BenchmarkAdapter
15
+ from sqlguard_ninja.benchmarking.dataset import BenchmarkCase, Dataset
16
+ from sqlguard_ninja.benchmarking.metrics import CaseOutcome, compute_metrics
17
+
18
+ RESULT_SCHEMA = "sqlninja.benchmark-result/1"
19
+
20
+
21
+ def _pkg_version(name: str) -> str | None:
22
+ try:
23
+ return version(name)
24
+ except PackageNotFoundError:
25
+ return None
26
+
27
+
28
+ def _git_revision(root: Path) -> str | None:
29
+ try:
30
+ out = subprocess.run(
31
+ ["git", "rev-parse", "--short", "HEAD"], # noqa: S607
32
+ cwd=root,
33
+ capture_output=True,
34
+ text=True,
35
+ timeout=5,
36
+ check=False,
37
+ )
38
+ except (OSError, subprocess.SubprocessError):
39
+ return None
40
+ return out.stdout.strip() or None
41
+
42
+
43
+ def run_benchmark(
44
+ dataset: Dataset,
45
+ adapter: BenchmarkAdapter,
46
+ cases: list[BenchmarkCase] | None = None,
47
+ *,
48
+ warmup: int = 3,
49
+ ) -> dict[str, Any]:
50
+ """Evaluate every case and return the result document (see ``RESULT_SCHEMA``)."""
51
+ selected = dataset.cases if cases is None else cases
52
+ for case in selected[:warmup]: # first parses pay one-time import/JIT costs
53
+ adapter.evaluate(case)
54
+ caps = adapter.capabilities
55
+ records: list[dict[str, Any]] = []
56
+ outcomes: list[CaseOutcome] = []
57
+ for case in selected:
58
+ verdict = adapter.evaluate(case)
59
+ outcome = CaseOutcome(
60
+ id=case.id,
61
+ category=case.category,
62
+ dialect=case.dialect,
63
+ provenance=case.provenance,
64
+ expected=case.expected,
65
+ actual=verdict.decision,
66
+ latency_ms=verdict.latency_ms,
67
+ error=verdict.error,
68
+ in_capability=caps is None or case.category in caps,
69
+ )
70
+ outcomes.append(outcome)
71
+ records.append(
72
+ {
73
+ "id": case.id,
74
+ "category": case.category,
75
+ "dialect": case.dialect,
76
+ "provenance": case.provenance,
77
+ "classification": case.classification.value,
78
+ "reason": case.reason.value,
79
+ "expected": case.expected.value,
80
+ "actual": verdict.decision.value,
81
+ "kind": outcome.kind,
82
+ "reasons": list(verdict.reasons),
83
+ "latency_ms": round(verdict.latency_ms, 4),
84
+ "error": verdict.error,
85
+ "in_capability": outcome.in_capability,
86
+ }
87
+ )
88
+ return {
89
+ "schema": RESULT_SCHEMA,
90
+ "created_at": datetime.now(UTC).isoformat(timespec="seconds"),
91
+ "benchmark": {
92
+ "name": dataset.manifest.name,
93
+ "version": dataset.manifest.version,
94
+ "fingerprint": dataset.fingerprint,
95
+ "cases_evaluated": len(selected),
96
+ },
97
+ "adapter": {
98
+ "name": adapter.name,
99
+ "version": adapter.version,
100
+ "capabilities": sorted(caps) if caps is not None else "all",
101
+ },
102
+ "environment": {
103
+ "python": sys.version.split()[0],
104
+ "platform": platform.platform(),
105
+ "machine": platform.machine(),
106
+ "sqlglot": _pkg_version("sqlglot"),
107
+ "sqlguard_ninja": _pkg_version("sqlguard-ninja"),
108
+ "git_revision": _git_revision(dataset.root),
109
+ },
110
+ "metrics": compute_metrics(outcomes),
111
+ "cases": records,
112
+ }
113
+
114
+
115
+ def write_result(result: dict[str, Any], path: str | Path) -> Path:
116
+ p = Path(path)
117
+ p.parent.mkdir(parents=True, exist_ok=True)
118
+ p.write_text(json.dumps(result, indent=2, sort_keys=False) + "\n", encoding="utf-8")
119
+ return p
120
+
121
+
122
+ def load_result(path: str | Path) -> dict[str, Any]:
123
+ data: dict[str, Any] = json.loads(Path(path).read_text(encoding="utf-8"))
124
+ if data.get("schema") != RESULT_SCHEMA:
125
+ raise ValueError(f"{path}: not a {RESULT_SCHEMA} document")
126
+ return data
@@ -0,0 +1,64 @@
1
+ """Translate a SQLGuard Ninja policy + role into the simpler models of external validators.
2
+
3
+ External tools have no roles or classification labels. The translation is deliberately
4
+ *conservative* and documented, so the comparison is about enforcement quality and not about
5
+ a hostile translation:
6
+
7
+ * granted tables → table allowlist;
8
+ * granted columns whose labels resolve to ``allow`` → column allowlist;
9
+ * columns that are ungranted or whose labels resolve to anything else (deny, aggregate-only,
10
+ review) → column denylist (the external models cannot express aggregate-only or review);
11
+ * tenant column of a tenant-scoped table → required equality predicate with the context value.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from typing import Any
17
+
18
+ from sqlguard_ninja.policy.model import Access, Policy
19
+ from sqlguard_ninja.privacy.rules import resolve_column_access
20
+
21
+
22
+ def translate_for_role(policy: Policy, role_name: str | None, dialect: str) -> dict[str, Any]:
23
+ """Return a JSON-serialisable, tool-neutral description of what ``role_name`` may read."""
24
+ _, role = policy.role(role_name)
25
+ tables: dict[str, Any] = {}
26
+ denied_names: set[str] = set()
27
+ for tname, tdef in policy.catalog.tables.items():
28
+ grant = role.tables.get(tname) if role is not None else None
29
+ if role is not None and grant is None:
30
+ denied_names.update(c for c in tdef.columns)
31
+ continue
32
+ allowed, denied = [], []
33
+ for cname in tdef.columns:
34
+ listed = grant is None or grant.columns == "*" or cname in grant.columns
35
+ if grant is not None and cname in grant.deny_columns:
36
+ listed = False
37
+ access = resolve_column_access(policy, role, tname, cname).strongest
38
+ if listed and access is Access.ALLOW:
39
+ allowed.append(cname)
40
+ else:
41
+ denied.append(cname)
42
+ tenant_scoped = role is None or role.tenant_scoped
43
+ tables[tname] = {
44
+ "allow_columns": allowed,
45
+ "deny_columns": denied,
46
+ "tenant_column": tdef.tenant_column if tenant_scoped else None,
47
+ "large": (tdef.estimated_rows or 0) >= policy.cost.large_table_rows,
48
+ }
49
+ denied_names.update(denied)
50
+ allowed_anywhere = {c for t in tables.values() for c in t["allow_columns"]}
51
+ limits = policy.effective_limits(role)
52
+ return {
53
+ "dialect": dialect,
54
+ "tables": tables,
55
+ # Name-only denylists cannot be table-scoped; a name that is allowed on some granted
56
+ # table is not denylisted (avoids penalising a tool for a translation artefact).
57
+ "pii_denylist": sorted(denied_names - allowed_anywhere),
58
+ "tenant_context_key": policy.tenant.context_key,
59
+ "limits": {
60
+ "max_joins": limits.max_joins,
61
+ "max_subquery_depth": limits.max_subquery_depth,
62
+ "max_limit": limits.max_limit,
63
+ },
64
+ }
@@ -0,0 +1,53 @@
1
+ """Shared JSON Lines loop for external-validator workers.
2
+
3
+ Workers run inside a *separate* virtual environment where sqlguard_ninja is not installed,
4
+ so this file is executed by path (copied into each worker by ``exec``) rather than imported.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import sys
11
+ import time
12
+ import traceback
13
+ from collections.abc import Callable
14
+ from typing import Any
15
+
16
+ Evaluator = Callable[[str, str, dict[str, Any], dict[str, Any]], tuple[str, list[str]]]
17
+
18
+
19
+ def serve(library: str, version: str, build: Callable[[dict[str, Any]], Evaluator]) -> None:
20
+ """Answer one JSON request per stdin line until EOF."""
21
+ out = sys.stdout
22
+ out.write(json.dumps({"library": library, "version": version}) + "\n")
23
+ out.flush()
24
+ cache: dict[str, Evaluator] = {}
25
+ for line in sys.stdin:
26
+ if not line.strip():
27
+ continue
28
+ req = json.loads(line)
29
+ key = json.dumps(req["config"], sort_keys=True)
30
+ reply: dict[str, Any]
31
+ try:
32
+ if key not in cache:
33
+ cache[key] = build(req["config"])
34
+ started = time.perf_counter()
35
+ decision, reasons = cache[key](
36
+ req["sql"], req["dialect"], req["context"], req["config"]
37
+ )
38
+ reply = {
39
+ "decision": decision,
40
+ "reasons": reasons,
41
+ "latency_ms": (time.perf_counter() - started) * 1000,
42
+ "error": None,
43
+ }
44
+ except Exception as exc: # report, fail closed, keep serving
45
+ reply = {
46
+ "decision": "deny",
47
+ "reasons": [],
48
+ "latency_ms": 0.0,
49
+ "error": f"{type(exc).__name__}: {exc}",
50
+ "trace": traceback.format_exc(limit=3),
51
+ }
52
+ out.write(json.dumps(reply) + "\n")
53
+ out.flush()
@@ -0,0 +1,40 @@
1
+ """Worker for sakura-sky/sql-guard (PyPI ``agent-sql-guard``, import name ``sql_guard``)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from importlib.metadata import version
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ exec((Path(__file__).parent / "_protocol.py").read_text(encoding="utf-8"))
10
+
11
+ from sql_guard import PiiDenylist, SqlGuard, SqlGuardConfig # type: ignore[import-not-found]
12
+
13
+ _OUTCOME = {"allow": "allow", "confirm": "review", "deny": "deny"}
14
+
15
+
16
+ def build(config: dict[str, Any]): # type: ignore[no-untyped-def]
17
+ guard = SqlGuard(
18
+ SqlGuardConfig.from_settings(
19
+ # Pass "substrings" explicitly: upstream from_mapping() defaults a missing key to
20
+ # () and then rejects it as "not a list" (agent-sql-guard 0.2.0).
21
+ pii_denylist=PiiDenylist.from_mapping(
22
+ {"columns": config["pii_denylist"], "substrings": []}
23
+ ),
24
+ allowed_tables=list(config["tables"]),
25
+ dialect=config["dialect"],
26
+ )
27
+ )
28
+
29
+ def evaluate(sql, dialect, context, _config): # type: ignore[no-untyped-def]
30
+ # evaluate_static() returns CONFIRM ("awaiting cost evaluation") for every query that
31
+ # passes the static rules; the cost stage needs a warehouse dry-run. This benchmark is
32
+ # static, so supply a zero-byte dry-run and let only the static rules decide.
33
+ decision = guard.evaluate(sql, bytes_processed=0)
34
+ outcome = _OUTCOME.get(str(decision.outcome.value).lower(), "deny")
35
+ return outcome, [decision.reason[:120]] if outcome != "allow" else []
36
+
37
+ return evaluate
38
+
39
+
40
+ serve("agent-sql-guard", version("agent-sql-guard"), build)