sqlguard-ninja 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlguard_ninja/__init__.py +35 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/cases/01_destructive.yaml +169 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/cases/02_multi_statement_and_formatting.yaml +155 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/cases/03_table_and_column_access.yaml +205 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/cases/04_privacy.yaml +245 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/cases/05_tenant_isolation.yaml +167 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/cases/06_cte_nested_unicode.yaml +168 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/cases/07_legitimate_lookalike_cost_dialect.yaml +243 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/generated/grant_matrix.yaml +573 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/generated/label_matrix.yaml +403 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/generated/tenant_matrix.yaml +684 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/generated/transforms.yaml +927 -0
- sqlguard_ninja/_data/benchmarks/datasets/v1/manifest.yaml +15 -0
- sqlguard_ninja/_data/benchmarks/policies/retail_saas.yaml +259 -0
- sqlguard_ninja/_version.py +1 -0
- sqlguard_ninja/benchmarking/__init__.py +1 -0
- sqlguard_ninja/benchmarking/adapters.py +303 -0
- sqlguard_ninja/benchmarking/builtin.py +39 -0
- sqlguard_ninja/benchmarking/compare.py +168 -0
- sqlguard_ninja/benchmarking/dataset.py +265 -0
- sqlguard_ninja/benchmarking/fixtures.py +274 -0
- sqlguard_ninja/benchmarking/metrics.py +114 -0
- sqlguard_ninja/benchmarking/runner.py +126 -0
- sqlguard_ninja/benchmarking/translate.py +64 -0
- sqlguard_ninja/benchmarking/workers/_protocol.py +53 -0
- sqlguard_ninja/benchmarking/workers/agent_sql_guard_worker.py +40 -0
- sqlguard_ninja/benchmarking/workers/krish_sqlguard_worker.py +25 -0
- sqlguard_ninja/benchmarking/workers/sql_guardrail_worker.py +48 -0
- sqlguard_ninja/cli/__init__.py +1 -0
- sqlguard_ninja/cli/config.py +76 -0
- sqlguard_ninja/cli/main.py +433 -0
- sqlguard_ninja/cli/templates.py +91 -0
- sqlguard_ninja/core/__init__.py +1 -0
- sqlguard_ninja/core/analysis.py +581 -0
- sqlguard_ninja/core/findings.py +133 -0
- sqlguard_ninja/core/guard.py +256 -0
- sqlguard_ninja/core/rules/__init__.py +1 -0
- sqlguard_ninja/core/rules/access.py +127 -0
- sqlguard_ninja/core/rules/base.py +40 -0
- sqlguard_ninja/core/rules/catalog.py +61 -0
- sqlguard_ninja/core/rules/functions.py +197 -0
- sqlguard_ninja/core/rules/statements.py +99 -0
- sqlguard_ninja/core/rules/structure.py +72 -0
- sqlguard_ninja/core/rules/tenant.py +196 -0
- sqlguard_ninja/integrations/__init__.py +43 -0
- sqlguard_ninja/integrations/audit.py +109 -0
- sqlguard_ninja/integrations/checkpoint.py +342 -0
- sqlguard_ninja/integrations/executors.py +64 -0
- sqlguard_ninja/integrations/fastapi.py +89 -0
- sqlguard_ninja/integrations/langchain.py +67 -0
- sqlguard_ninja/integrations/langgraph.py +87 -0
- sqlguard_ninja/policy/__init__.py +39 -0
- sqlguard_ninja/policy/model.py +370 -0
- sqlguard_ninja/privacy/__init__.py +9 -0
- sqlguard_ninja/privacy/rules.py +286 -0
- sqlguard_ninja/privacy/suggest.py +46 -0
- sqlguard_ninja/py.typed +0 -0
- sqlguard_ninja/query_cost/__init__.py +1 -0
- sqlguard_ninja/query_cost/postgres.py +226 -0
- sqlguard_ninja/query_cost/static.py +241 -0
- sqlguard_ninja/reporting/__init__.py +1 -0
- sqlguard_ninja/reporting/markdown.py +137 -0
- sqlguard_ninja-0.1.0.dist-info/METADATA +370 -0
- sqlguard_ninja-0.1.0.dist-info/RECORD +68 -0
- sqlguard_ninja-0.1.0.dist-info/WHEEL +4 -0
- sqlguard_ninja-0.1.0.dist-info/entry_points.txt +2 -0
- sqlguard_ninja-0.1.0.dist-info/licenses/LICENSE +202 -0
- sqlguard_ninja-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +56 -0
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""Synthetic fixture database for the benchmark schema (SQLite and PostgreSQL).
|
|
2
|
+
|
|
3
|
+
The data is generated deterministically from a seed. It contains no real personal data:
|
|
4
|
+
names, emails, card numbers and identifiers are fabricated and use reserved example
|
|
5
|
+
domains and test card ranges.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import random
|
|
12
|
+
import sqlite3
|
|
13
|
+
from collections.abc import Iterator
|
|
14
|
+
from datetime import date, datetime, timedelta
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Protocol
|
|
17
|
+
|
|
18
|
+
from sqlguard_ninja.policy.model import Catalog, Policy
|
|
19
|
+
|
|
20
|
+
DEFAULT_SEED = 20261024
|
|
21
|
+
|
|
22
|
+
_TYPE_MAP = {
|
|
23
|
+
"sqlite": {
|
|
24
|
+
"int": "INTEGER",
|
|
25
|
+
"text": "TEXT",
|
|
26
|
+
"numeric": "NUMERIC",
|
|
27
|
+
"timestamp": "TEXT",
|
|
28
|
+
"date": "TEXT",
|
|
29
|
+
"boolean": "INTEGER",
|
|
30
|
+
"json": "TEXT",
|
|
31
|
+
},
|
|
32
|
+
"postgres": {
|
|
33
|
+
"int": "BIGINT",
|
|
34
|
+
"text": "TEXT",
|
|
35
|
+
"numeric": "NUMERIC(14,2)",
|
|
36
|
+
"timestamp": "TIMESTAMP",
|
|
37
|
+
"date": "DATE",
|
|
38
|
+
"boolean": "BOOLEAN",
|
|
39
|
+
"json": "JSONB",
|
|
40
|
+
},
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
ROW_COUNTS = {
|
|
44
|
+
"tenants": 3,
|
|
45
|
+
"warehouses": 4,
|
|
46
|
+
"customers": 60,
|
|
47
|
+
"products": 30,
|
|
48
|
+
"orders": 200,
|
|
49
|
+
"order_items": 400,
|
|
50
|
+
"payments": 200,
|
|
51
|
+
"shipments": 180,
|
|
52
|
+
"employees": 40,
|
|
53
|
+
"support_tickets": 80,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
_FIRST = ["Alex", "Sam", "Robin", "Kim", "Jordan", "Taylor", "Morgan", "Casey", "Avery", "Riley"]
|
|
57
|
+
_LAST = ["Example", "Sample", "Testperson", "Placeholder", "Demo", "Fixture", "Synthetic"]
|
|
58
|
+
_CITIES = ["Berlin", "Lisbon", "Dubai", "Pune", "Oslo", "Austin", "Lyon", "Kraków"]
|
|
59
|
+
_COUNTRIES = ["DE", "PT", "AE", "IN", "NO", "US", "FR", "PL"]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class _DBConnection(Protocol):
|
|
63
|
+
def cursor(self) -> Any: ...
|
|
64
|
+
def commit(self) -> None: ...
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def schema_ddl(catalog: Catalog, dialect: str) -> list[str]:
|
|
68
|
+
"""CREATE TABLE statements derived from the policy catalog."""
|
|
69
|
+
types = _TYPE_MAP[dialect]
|
|
70
|
+
statements = []
|
|
71
|
+
for name, table in catalog.tables.items():
|
|
72
|
+
cols = []
|
|
73
|
+
for col, spec in table.columns.items():
|
|
74
|
+
sql_type = types.get(spec.type, "TEXT")
|
|
75
|
+
pk = " PRIMARY KEY" if col == "id" else ""
|
|
76
|
+
cols.append(f"{col} {sql_type}{pk}")
|
|
77
|
+
statements.append(f"CREATE TABLE {name} ({', '.join(cols)})")
|
|
78
|
+
return statements
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _rows(seed: int) -> Iterator[tuple[str, list[dict[str, Any]]]]:
|
|
82
|
+
rng = random.Random(seed)
|
|
83
|
+
base = datetime(2026, 1, 1)
|
|
84
|
+
|
|
85
|
+
def ts(days: int = 300) -> str:
|
|
86
|
+
return (base + timedelta(minutes=rng.randrange(days * 24 * 60))).isoformat(sep=" ")
|
|
87
|
+
|
|
88
|
+
def person() -> str:
|
|
89
|
+
return f"{rng.choice(_FIRST)} {rng.choice(_LAST)}"
|
|
90
|
+
|
|
91
|
+
tenants = [
|
|
92
|
+
{"id": i, "name": f"Tenant {i}", "plan": rng.choice(["free", "pro"]), "created_at": ts()}
|
|
93
|
+
for i in range(1, ROW_COUNTS["tenants"] + 1)
|
|
94
|
+
]
|
|
95
|
+
yield "tenants", tenants
|
|
96
|
+
yield (
|
|
97
|
+
"warehouses",
|
|
98
|
+
[
|
|
99
|
+
{"id": i, "name": f"WH-{i}", "region": rng.choice(["EU", "US", "MEA", "APAC"])}
|
|
100
|
+
for i in range(1, ROW_COUNTS["warehouses"] + 1)
|
|
101
|
+
],
|
|
102
|
+
)
|
|
103
|
+
customers = []
|
|
104
|
+
for i in range(1, ROW_COUNTS["customers"] + 1):
|
|
105
|
+
k = rng.randrange(len(_CITIES))
|
|
106
|
+
customers.append(
|
|
107
|
+
{
|
|
108
|
+
"id": i,
|
|
109
|
+
"tenant_id": (i % 3) + 1,
|
|
110
|
+
"full_name": person(),
|
|
111
|
+
"email": f"user{i}@example.com",
|
|
112
|
+
"email_domain": "example.com",
|
|
113
|
+
"phone": f"+00 555 {i:04d}",
|
|
114
|
+
"date_of_birth": date(
|
|
115
|
+
1960 + rng.randrange(40), 1 + rng.randrange(12), 1
|
|
116
|
+
).isoformat(),
|
|
117
|
+
"country": _COUNTRIES[k],
|
|
118
|
+
"city": _CITIES[k],
|
|
119
|
+
"loyalty_tier": rng.choice(["bronze", "silver", "gold"]),
|
|
120
|
+
"marketing_opt_in": rng.random() < 0.5,
|
|
121
|
+
"created_at": ts(),
|
|
122
|
+
"profile": json.dumps(
|
|
123
|
+
{
|
|
124
|
+
"preferences": {"newsletter": rng.random() < 0.5, "language": "en"},
|
|
125
|
+
"health": {"allergies": rng.choice(["none", "synthetic-allergen"])},
|
|
126
|
+
"payment": {"card_number": "4111111111111111"},
|
|
127
|
+
}
|
|
128
|
+
),
|
|
129
|
+
}
|
|
130
|
+
)
|
|
131
|
+
yield "customers", customers
|
|
132
|
+
products = [
|
|
133
|
+
{
|
|
134
|
+
"id": i,
|
|
135
|
+
"tenant_id": (i % 3) + 1,
|
|
136
|
+
"sku": f"SKU-{i:04d}",
|
|
137
|
+
"name": f"Product {i}",
|
|
138
|
+
"category": rng.choice(["books", "garden", "toys", "tools"]),
|
|
139
|
+
"price": round(rng.uniform(2, 200), 2),
|
|
140
|
+
}
|
|
141
|
+
for i in range(1, ROW_COUNTS["products"] + 1)
|
|
142
|
+
]
|
|
143
|
+
yield "products", products
|
|
144
|
+
orders = []
|
|
145
|
+
for i in range(1, ROW_COUNTS["orders"] + 1):
|
|
146
|
+
cust = rng.choice(customers)
|
|
147
|
+
orders.append(
|
|
148
|
+
{
|
|
149
|
+
"id": i,
|
|
150
|
+
"tenant_id": cust["tenant_id"],
|
|
151
|
+
"customer_id": cust["id"],
|
|
152
|
+
"status": rng.choice(["new", "paid", "shipped", "delivered", "cancelled"]),
|
|
153
|
+
"total_amount": round(rng.uniform(5, 900), 2),
|
|
154
|
+
"currency": "EUR",
|
|
155
|
+
"created_at": ts(),
|
|
156
|
+
"shipping_address": f"{i} Example Street, {cust['city']}",
|
|
157
|
+
}
|
|
158
|
+
)
|
|
159
|
+
yield "orders", orders
|
|
160
|
+
yield (
|
|
161
|
+
"order_items",
|
|
162
|
+
[
|
|
163
|
+
{
|
|
164
|
+
"id": i,
|
|
165
|
+
"tenant_id": (o := rng.choice(orders))["tenant_id"],
|
|
166
|
+
"order_id": o["id"],
|
|
167
|
+
"product_id": rng.choice(products)["id"],
|
|
168
|
+
"quantity": 1 + rng.randrange(5),
|
|
169
|
+
"unit_price": round(rng.uniform(2, 200), 2),
|
|
170
|
+
}
|
|
171
|
+
for i in range(1, ROW_COUNTS["order_items"] + 1)
|
|
172
|
+
],
|
|
173
|
+
)
|
|
174
|
+
yield (
|
|
175
|
+
"payments",
|
|
176
|
+
[
|
|
177
|
+
{
|
|
178
|
+
"id": o["id"],
|
|
179
|
+
"tenant_id": o["tenant_id"],
|
|
180
|
+
"order_id": o["id"],
|
|
181
|
+
"card_number": "4111111111111111", # published test card number
|
|
182
|
+
"card_last4": "1111",
|
|
183
|
+
"card_brand": "visa",
|
|
184
|
+
"iban": "XX00TEST0000000000",
|
|
185
|
+
"amount": o["total_amount"],
|
|
186
|
+
"status": rng.choice(["captured", "refunded", "failed"]),
|
|
187
|
+
}
|
|
188
|
+
for o in orders[: ROW_COUNTS["payments"]]
|
|
189
|
+
],
|
|
190
|
+
)
|
|
191
|
+
yield (
|
|
192
|
+
"shipments",
|
|
193
|
+
[
|
|
194
|
+
{
|
|
195
|
+
"id": o["id"],
|
|
196
|
+
"tenant_id": o["tenant_id"],
|
|
197
|
+
"order_id": o["id"],
|
|
198
|
+
"warehouse_id": 1 + rng.randrange(ROW_COUNTS["warehouses"]),
|
|
199
|
+
"carrier": rng.choice(["DHL", "UPS", "Aramex"]),
|
|
200
|
+
"tracking_number": f"TRK{o['id']:06d}",
|
|
201
|
+
"status": rng.choice(["in_transit", "delivered", "returned"]),
|
|
202
|
+
"shipped_at": ts(),
|
|
203
|
+
"delivery_address": o["shipping_address"],
|
|
204
|
+
}
|
|
205
|
+
for o in orders[: ROW_COUNTS["shipments"]]
|
|
206
|
+
],
|
|
207
|
+
)
|
|
208
|
+
yield (
|
|
209
|
+
"employees",
|
|
210
|
+
[
|
|
211
|
+
{
|
|
212
|
+
"id": i,
|
|
213
|
+
"full_name": person(),
|
|
214
|
+
"email": f"employee{i}@example.org",
|
|
215
|
+
"department": rng.choice(["warehouse", "support", "finance", "engineering"]),
|
|
216
|
+
"title": rng.choice(["associate", "lead", "manager"]),
|
|
217
|
+
"salary": round(rng.uniform(30000, 120000), 2),
|
|
218
|
+
"bank_account": f"XX00TEST{i:010d}",
|
|
219
|
+
"national_id": f"TEST-{i:06d}",
|
|
220
|
+
"health_notes": "synthetic",
|
|
221
|
+
"hire_date": date(2015 + rng.randrange(10), 1 + rng.randrange(12), 1).isoformat(),
|
|
222
|
+
"manager_id": None if i <= 3 else 1 + rng.randrange(3),
|
|
223
|
+
"warehouse_id": 1 + rng.randrange(ROW_COUNTS["warehouses"]),
|
|
224
|
+
}
|
|
225
|
+
for i in range(1, ROW_COUNTS["employees"] + 1)
|
|
226
|
+
],
|
|
227
|
+
)
|
|
228
|
+
yield (
|
|
229
|
+
"support_tickets",
|
|
230
|
+
[
|
|
231
|
+
{
|
|
232
|
+
"id": i,
|
|
233
|
+
"tenant_id": (c := rng.choice(customers))["tenant_id"],
|
|
234
|
+
"customer_id": c["id"],
|
|
235
|
+
"subject": rng.choice(["Late delivery", "Refund", "Login issue"]),
|
|
236
|
+
"body": "Synthetic ticket body.",
|
|
237
|
+
"status": rng.choice(["open", "closed"]),
|
|
238
|
+
"priority": rng.choice(["low", "high"]),
|
|
239
|
+
"created_at": ts(),
|
|
240
|
+
}
|
|
241
|
+
for i in range(1, ROW_COUNTS["support_tickets"] + 1)
|
|
242
|
+
],
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def populate(conn: _DBConnection, policy: Policy, dialect: str, seed: int = DEFAULT_SEED) -> None:
|
|
247
|
+
"""Create the schema and load synthetic rows. Values are always bound, never interpolated."""
|
|
248
|
+
marker = "?" if dialect == "sqlite" else "%s"
|
|
249
|
+
cur = conn.cursor()
|
|
250
|
+
for stmt in schema_ddl(policy.catalog, dialect):
|
|
251
|
+
cur.execute(stmt)
|
|
252
|
+
for table, rows in _rows(seed):
|
|
253
|
+
if not rows:
|
|
254
|
+
continue
|
|
255
|
+
columns = list(policy.catalog.tables[table].columns)
|
|
256
|
+
placeholders = ", ".join([marker] * len(columns))
|
|
257
|
+
# Table and column names come from the validated policy catalog, not from input.
|
|
258
|
+
insert = f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({placeholders})" # noqa: S608
|
|
259
|
+
params = [tuple(row.get(c) for c in columns) for row in rows]
|
|
260
|
+
cur.executemany(insert, params)
|
|
261
|
+
conn.commit()
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def create_sqlite(
|
|
265
|
+
policy: Policy, path: str | Path = ":memory:", seed: int = DEFAULT_SEED
|
|
266
|
+
) -> sqlite3.Connection:
|
|
267
|
+
"""Return a SQLite connection with the fixture schema and data loaded.
|
|
268
|
+
|
|
269
|
+
The connection may be shared across threads (web servers run handlers in a pool);
|
|
270
|
+
:func:`sqlguard_ninja.integrations.executors.sqlite_executor` serialises access.
|
|
271
|
+
"""
|
|
272
|
+
conn = sqlite3.connect(str(path), check_same_thread=False)
|
|
273
|
+
populate(conn, policy, "sqlite", seed)
|
|
274
|
+
return conn
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Detection metrics with explicit definitions.
|
|
2
|
+
|
|
3
|
+
Positive class: a case whose expected outcome is *not* ALLOW (``deny`` or ``review``), i.e.
|
|
4
|
+
something that must not run unattended. A prediction is positive when the validator's decision
|
|
5
|
+
is not ALLOW. On top of the binary view, ``exact_match_rate`` measures how often the
|
|
6
|
+
three-way decision (allow/review/deny) matches exactly.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
from collections import defaultdict
|
|
13
|
+
from collections.abc import Iterable
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from sqlguard_ninja.core.findings import Decision
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class CaseOutcome:
|
|
22
|
+
id: str
|
|
23
|
+
category: str
|
|
24
|
+
dialect: str
|
|
25
|
+
provenance: str
|
|
26
|
+
expected: Decision
|
|
27
|
+
actual: Decision
|
|
28
|
+
latency_ms: float
|
|
29
|
+
error: str | None = None
|
|
30
|
+
in_capability: bool = True
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def unsafe(self) -> bool:
|
|
34
|
+
return self.expected is not Decision.ALLOW
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def flagged(self) -> bool:
|
|
38
|
+
return self.actual is not Decision.ALLOW
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def kind(self) -> str:
|
|
42
|
+
if self.unsafe and self.flagged:
|
|
43
|
+
return "tp"
|
|
44
|
+
if self.unsafe:
|
|
45
|
+
return "fn"
|
|
46
|
+
if self.flagged:
|
|
47
|
+
return "fp"
|
|
48
|
+
return "tn"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _ratio(num: int, den: int) -> float | None:
|
|
52
|
+
return None if den == 0 else round(num / den, 4)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def percentile(values: list[float], pct: float) -> float | None:
|
|
56
|
+
"""Nearest-rank percentile (deterministic, no interpolation)."""
|
|
57
|
+
if not values:
|
|
58
|
+
return None
|
|
59
|
+
ordered = sorted(values)
|
|
60
|
+
rank = max(1, math.ceil(pct / 100 * len(ordered)))
|
|
61
|
+
return round(ordered[rank - 1], 4)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def summarise(outcomes: Iterable[CaseOutcome]) -> dict[str, Any]:
|
|
65
|
+
items = list(outcomes)
|
|
66
|
+
counts = {"tp": 0, "fp": 0, "tn": 0, "fn": 0}
|
|
67
|
+
for o in items:
|
|
68
|
+
counts[o.kind] += 1
|
|
69
|
+
tp, fp, tn, fn = counts["tp"], counts["fp"], counts["tn"], counts["fn"]
|
|
70
|
+
latencies = [o.latency_ms for o in items]
|
|
71
|
+
return {
|
|
72
|
+
"cases": len(items),
|
|
73
|
+
"unsafe_cases": tp + fn,
|
|
74
|
+
"safe_cases": tn + fp,
|
|
75
|
+
**counts,
|
|
76
|
+
"recall": _ratio(tp, tp + fn),
|
|
77
|
+
"precision": _ratio(tp, tp + fp),
|
|
78
|
+
"false_positive_rate": _ratio(fp, fp + tn),
|
|
79
|
+
"false_negative_rate": _ratio(fn, fn + tp),
|
|
80
|
+
"exact_match_rate": _ratio(sum(o.expected is o.actual for o in items), len(items)),
|
|
81
|
+
"errors": sum(o.error is not None for o in items),
|
|
82
|
+
"latency_ms": {
|
|
83
|
+
"p50": percentile(latencies, 50),
|
|
84
|
+
"p95": percentile(latencies, 95),
|
|
85
|
+
"p99": percentile(latencies, 99),
|
|
86
|
+
"max": round(max(latencies), 4) if latencies else None,
|
|
87
|
+
},
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def grouped(outcomes: list[CaseOutcome], attr: str) -> dict[str, dict[str, Any]]:
|
|
92
|
+
groups: dict[str, list[CaseOutcome]] = defaultdict(list)
|
|
93
|
+
for o in outcomes:
|
|
94
|
+
key = getattr(o, attr)
|
|
95
|
+
if attr == "provenance":
|
|
96
|
+
key = key.split(":")[0] if not key.startswith("generated:") else key.split("@")[0]
|
|
97
|
+
groups[key].append(o)
|
|
98
|
+
return {k: summarise(v) for k, v in sorted(groups.items())}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def compute_metrics(outcomes: list[CaseOutcome], examples: int = 15) -> dict[str, Any]:
|
|
102
|
+
capable = [o for o in outcomes if o.in_capability]
|
|
103
|
+
return {
|
|
104
|
+
"overall": summarise(outcomes),
|
|
105
|
+
"within_capability": summarise(capable),
|
|
106
|
+
"by_category": grouped(outcomes, "category"),
|
|
107
|
+
"by_dialect": grouped(outcomes, "dialect"),
|
|
108
|
+
"by_provenance": grouped(outcomes, "provenance"),
|
|
109
|
+
"false_negative_examples": [o.id for o in outcomes if o.kind == "fn"][:examples],
|
|
110
|
+
"false_positive_examples": [o.id for o in outcomes if o.kind == "fp"][:examples],
|
|
111
|
+
"decision_mismatch_examples": [
|
|
112
|
+
o.id for o in outcomes if o.expected is not o.actual and o.kind in ("tp", "tn")
|
|
113
|
+
][:examples],
|
|
114
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Run a dataset through an adapter and produce a versioned, machine-readable result."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import platform
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
from datetime import UTC, datetime
|
|
10
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from sqlguard_ninja.benchmarking.adapters import BenchmarkAdapter
|
|
15
|
+
from sqlguard_ninja.benchmarking.dataset import BenchmarkCase, Dataset
|
|
16
|
+
from sqlguard_ninja.benchmarking.metrics import CaseOutcome, compute_metrics
|
|
17
|
+
|
|
18
|
+
RESULT_SCHEMA = "sqlninja.benchmark-result/1"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _pkg_version(name: str) -> str | None:
|
|
22
|
+
try:
|
|
23
|
+
return version(name)
|
|
24
|
+
except PackageNotFoundError:
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _git_revision(root: Path) -> str | None:
|
|
29
|
+
try:
|
|
30
|
+
out = subprocess.run(
|
|
31
|
+
["git", "rev-parse", "--short", "HEAD"], # noqa: S607
|
|
32
|
+
cwd=root,
|
|
33
|
+
capture_output=True,
|
|
34
|
+
text=True,
|
|
35
|
+
timeout=5,
|
|
36
|
+
check=False,
|
|
37
|
+
)
|
|
38
|
+
except (OSError, subprocess.SubprocessError):
|
|
39
|
+
return None
|
|
40
|
+
return out.stdout.strip() or None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def run_benchmark(
|
|
44
|
+
dataset: Dataset,
|
|
45
|
+
adapter: BenchmarkAdapter,
|
|
46
|
+
cases: list[BenchmarkCase] | None = None,
|
|
47
|
+
*,
|
|
48
|
+
warmup: int = 3,
|
|
49
|
+
) -> dict[str, Any]:
|
|
50
|
+
"""Evaluate every case and return the result document (see ``RESULT_SCHEMA``)."""
|
|
51
|
+
selected = dataset.cases if cases is None else cases
|
|
52
|
+
for case in selected[:warmup]: # first parses pay one-time import/JIT costs
|
|
53
|
+
adapter.evaluate(case)
|
|
54
|
+
caps = adapter.capabilities
|
|
55
|
+
records: list[dict[str, Any]] = []
|
|
56
|
+
outcomes: list[CaseOutcome] = []
|
|
57
|
+
for case in selected:
|
|
58
|
+
verdict = adapter.evaluate(case)
|
|
59
|
+
outcome = CaseOutcome(
|
|
60
|
+
id=case.id,
|
|
61
|
+
category=case.category,
|
|
62
|
+
dialect=case.dialect,
|
|
63
|
+
provenance=case.provenance,
|
|
64
|
+
expected=case.expected,
|
|
65
|
+
actual=verdict.decision,
|
|
66
|
+
latency_ms=verdict.latency_ms,
|
|
67
|
+
error=verdict.error,
|
|
68
|
+
in_capability=caps is None or case.category in caps,
|
|
69
|
+
)
|
|
70
|
+
outcomes.append(outcome)
|
|
71
|
+
records.append(
|
|
72
|
+
{
|
|
73
|
+
"id": case.id,
|
|
74
|
+
"category": case.category,
|
|
75
|
+
"dialect": case.dialect,
|
|
76
|
+
"provenance": case.provenance,
|
|
77
|
+
"classification": case.classification.value,
|
|
78
|
+
"reason": case.reason.value,
|
|
79
|
+
"expected": case.expected.value,
|
|
80
|
+
"actual": verdict.decision.value,
|
|
81
|
+
"kind": outcome.kind,
|
|
82
|
+
"reasons": list(verdict.reasons),
|
|
83
|
+
"latency_ms": round(verdict.latency_ms, 4),
|
|
84
|
+
"error": verdict.error,
|
|
85
|
+
"in_capability": outcome.in_capability,
|
|
86
|
+
}
|
|
87
|
+
)
|
|
88
|
+
return {
|
|
89
|
+
"schema": RESULT_SCHEMA,
|
|
90
|
+
"created_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
|
91
|
+
"benchmark": {
|
|
92
|
+
"name": dataset.manifest.name,
|
|
93
|
+
"version": dataset.manifest.version,
|
|
94
|
+
"fingerprint": dataset.fingerprint,
|
|
95
|
+
"cases_evaluated": len(selected),
|
|
96
|
+
},
|
|
97
|
+
"adapter": {
|
|
98
|
+
"name": adapter.name,
|
|
99
|
+
"version": adapter.version,
|
|
100
|
+
"capabilities": sorted(caps) if caps is not None else "all",
|
|
101
|
+
},
|
|
102
|
+
"environment": {
|
|
103
|
+
"python": sys.version.split()[0],
|
|
104
|
+
"platform": platform.platform(),
|
|
105
|
+
"machine": platform.machine(),
|
|
106
|
+
"sqlglot": _pkg_version("sqlglot"),
|
|
107
|
+
"sqlguard_ninja": _pkg_version("sqlguard-ninja"),
|
|
108
|
+
"git_revision": _git_revision(dataset.root),
|
|
109
|
+
},
|
|
110
|
+
"metrics": compute_metrics(outcomes),
|
|
111
|
+
"cases": records,
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def write_result(result: dict[str, Any], path: str | Path) -> Path:
|
|
116
|
+
p = Path(path)
|
|
117
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
p.write_text(json.dumps(result, indent=2, sort_keys=False) + "\n", encoding="utf-8")
|
|
119
|
+
return p
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def load_result(path: str | Path) -> dict[str, Any]:
|
|
123
|
+
data: dict[str, Any] = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
124
|
+
if data.get("schema") != RESULT_SCHEMA:
|
|
125
|
+
raise ValueError(f"{path}: not a {RESULT_SCHEMA} document")
|
|
126
|
+
return data
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Translate a SQLGuard Ninja policy + role into the simpler models of external validators.
|
|
2
|
+
|
|
3
|
+
External tools have no roles or classification labels. The translation is deliberately
|
|
4
|
+
*conservative* and documented, so the comparison is about enforcement quality and not about
|
|
5
|
+
a hostile translation:
|
|
6
|
+
|
|
7
|
+
* granted tables → table allowlist;
|
|
8
|
+
* granted columns whose labels resolve to ``allow`` → column allowlist;
|
|
9
|
+
* columns that are ungranted or whose labels resolve to anything else (deny, aggregate-only,
|
|
10
|
+
review) → column denylist (the external models cannot express aggregate-only or review);
|
|
11
|
+
* tenant column of a tenant-scoped table → required equality predicate with the context value.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from sqlguard_ninja.policy.model import Access, Policy
|
|
19
|
+
from sqlguard_ninja.privacy.rules import resolve_column_access
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def translate_for_role(policy: Policy, role_name: str | None, dialect: str) -> dict[str, Any]:
|
|
23
|
+
"""Return a JSON-serialisable, tool-neutral description of what ``role_name`` may read."""
|
|
24
|
+
_, role = policy.role(role_name)
|
|
25
|
+
tables: dict[str, Any] = {}
|
|
26
|
+
denied_names: set[str] = set()
|
|
27
|
+
for tname, tdef in policy.catalog.tables.items():
|
|
28
|
+
grant = role.tables.get(tname) if role is not None else None
|
|
29
|
+
if role is not None and grant is None:
|
|
30
|
+
denied_names.update(c for c in tdef.columns)
|
|
31
|
+
continue
|
|
32
|
+
allowed, denied = [], []
|
|
33
|
+
for cname in tdef.columns:
|
|
34
|
+
listed = grant is None or grant.columns == "*" or cname in grant.columns
|
|
35
|
+
if grant is not None and cname in grant.deny_columns:
|
|
36
|
+
listed = False
|
|
37
|
+
access = resolve_column_access(policy, role, tname, cname).strongest
|
|
38
|
+
if listed and access is Access.ALLOW:
|
|
39
|
+
allowed.append(cname)
|
|
40
|
+
else:
|
|
41
|
+
denied.append(cname)
|
|
42
|
+
tenant_scoped = role is None or role.tenant_scoped
|
|
43
|
+
tables[tname] = {
|
|
44
|
+
"allow_columns": allowed,
|
|
45
|
+
"deny_columns": denied,
|
|
46
|
+
"tenant_column": tdef.tenant_column if tenant_scoped else None,
|
|
47
|
+
"large": (tdef.estimated_rows or 0) >= policy.cost.large_table_rows,
|
|
48
|
+
}
|
|
49
|
+
denied_names.update(denied)
|
|
50
|
+
allowed_anywhere = {c for t in tables.values() for c in t["allow_columns"]}
|
|
51
|
+
limits = policy.effective_limits(role)
|
|
52
|
+
return {
|
|
53
|
+
"dialect": dialect,
|
|
54
|
+
"tables": tables,
|
|
55
|
+
# Name-only denylists cannot be table-scoped; a name that is allowed on some granted
|
|
56
|
+
# table is not denylisted (avoids penalising a tool for a translation artefact).
|
|
57
|
+
"pii_denylist": sorted(denied_names - allowed_anywhere),
|
|
58
|
+
"tenant_context_key": policy.tenant.context_key,
|
|
59
|
+
"limits": {
|
|
60
|
+
"max_joins": limits.max_joins,
|
|
61
|
+
"max_subquery_depth": limits.max_subquery_depth,
|
|
62
|
+
"max_limit": limits.max_limit,
|
|
63
|
+
},
|
|
64
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Shared JSON Lines loop for external-validator workers.
|
|
2
|
+
|
|
3
|
+
Workers run inside a *separate* virtual environment where sqlguard_ninja is not installed,
|
|
4
|
+
so this file is executed by path (copied into each worker by ``exec``) rather than imported.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
import traceback
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
Evaluator = Callable[[str, str, dict[str, Any], dict[str, Any]], tuple[str, list[str]]]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def serve(library: str, version: str, build: Callable[[dict[str, Any]], Evaluator]) -> None:
|
|
20
|
+
"""Answer one JSON request per stdin line until EOF."""
|
|
21
|
+
out = sys.stdout
|
|
22
|
+
out.write(json.dumps({"library": library, "version": version}) + "\n")
|
|
23
|
+
out.flush()
|
|
24
|
+
cache: dict[str, Evaluator] = {}
|
|
25
|
+
for line in sys.stdin:
|
|
26
|
+
if not line.strip():
|
|
27
|
+
continue
|
|
28
|
+
req = json.loads(line)
|
|
29
|
+
key = json.dumps(req["config"], sort_keys=True)
|
|
30
|
+
reply: dict[str, Any]
|
|
31
|
+
try:
|
|
32
|
+
if key not in cache:
|
|
33
|
+
cache[key] = build(req["config"])
|
|
34
|
+
started = time.perf_counter()
|
|
35
|
+
decision, reasons = cache[key](
|
|
36
|
+
req["sql"], req["dialect"], req["context"], req["config"]
|
|
37
|
+
)
|
|
38
|
+
reply = {
|
|
39
|
+
"decision": decision,
|
|
40
|
+
"reasons": reasons,
|
|
41
|
+
"latency_ms": (time.perf_counter() - started) * 1000,
|
|
42
|
+
"error": None,
|
|
43
|
+
}
|
|
44
|
+
except Exception as exc: # report, fail closed, keep serving
|
|
45
|
+
reply = {
|
|
46
|
+
"decision": "deny",
|
|
47
|
+
"reasons": [],
|
|
48
|
+
"latency_ms": 0.0,
|
|
49
|
+
"error": f"{type(exc).__name__}: {exc}",
|
|
50
|
+
"trace": traceback.format_exc(limit=3),
|
|
51
|
+
}
|
|
52
|
+
out.write(json.dumps(reply) + "\n")
|
|
53
|
+
out.flush()
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Worker for sakura-sky/sql-guard (PyPI ``agent-sql-guard``, import name ``sql_guard``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import version
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
exec((Path(__file__).parent / "_protocol.py").read_text(encoding="utf-8"))
|
|
10
|
+
|
|
11
|
+
from sql_guard import PiiDenylist, SqlGuard, SqlGuardConfig # type: ignore[import-not-found]
|
|
12
|
+
|
|
13
|
+
_OUTCOME = {"allow": "allow", "confirm": "review", "deny": "deny"}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def build(config: dict[str, Any]): # type: ignore[no-untyped-def]
|
|
17
|
+
guard = SqlGuard(
|
|
18
|
+
SqlGuardConfig.from_settings(
|
|
19
|
+
# Pass "substrings" explicitly: upstream from_mapping() defaults a missing key to
|
|
20
|
+
# () and then rejects it as "not a list" (agent-sql-guard 0.2.0).
|
|
21
|
+
pii_denylist=PiiDenylist.from_mapping(
|
|
22
|
+
{"columns": config["pii_denylist"], "substrings": []}
|
|
23
|
+
),
|
|
24
|
+
allowed_tables=list(config["tables"]),
|
|
25
|
+
dialect=config["dialect"],
|
|
26
|
+
)
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
def evaluate(sql, dialect, context, _config): # type: ignore[no-untyped-def]
|
|
30
|
+
# evaluate_static() returns CONFIRM ("awaiting cost evaluation") for every query that
|
|
31
|
+
# passes the static rules; the cost stage needs a warehouse dry-run. This benchmark is
|
|
32
|
+
# static, so supply a zero-byte dry-run and let only the static rules decide.
|
|
33
|
+
decision = guard.evaluate(sql, bytes_processed=0)
|
|
34
|
+
outcome = _OUTCOME.get(str(decision.outcome.value).lower(), "deny")
|
|
35
|
+
return outcome, [decision.reason[:120]] if outcome != "allow" else []
|
|
36
|
+
|
|
37
|
+
return evaluate
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
serve("agent-sql-guard", version("agent-sql-guard"), build)
|