toolgraph 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toolgraph/__init__.py +3 -0
- toolgraph/artifact_safety.py +97 -0
- toolgraph/artifacts.py +122 -0
- toolgraph/cli.py +951 -0
- toolgraph/config.py +106 -0
- toolgraph/control_plan.py +494 -0
- toolgraph/crawler/__init__.py +0 -0
- toolgraph/crawler/client.py +120 -0
- toolgraph/crawler/crawl.py +103 -0
- toolgraph/crawler/transports.py +60 -0
- toolgraph/graph/__init__.py +0 -0
- toolgraph/graph/driver.py +275 -0
- toolgraph/graph/loader.py +280 -0
- toolgraph/graph/queries.py +984 -0
- toolgraph/graph/schema.py +166 -0
- toolgraph/graph/selector.py +423 -0
- toolgraph/graph/store.py +57 -0
- toolgraph/manifest/__init__.py +0 -0
- toolgraph/manifest/ingest.py +542 -0
- toolgraph/manifest/parser.py +26 -0
- toolgraph/models.py +192 -0
- toolgraph/policy_bundle.py +212 -0
- toolgraph/policy_review.py +97 -0
- toolgraph/preflight.py +77 -0
- toolgraph/quickstart/governance.yaml +28 -0
- toolgraph/quickstart/policy_gateway_server.py +31 -0
- toolgraph/quickstart/servers.yaml +5 -0
- toolgraph/redaction.py +82 -0
- toolgraph/review_candidates.py +579 -0
- toolgraph/server/__init__.py +0 -0
- toolgraph/server/app.py +306 -0
- toolgraph/uris.py +64 -0
- toolgraph/yaml_loader.py +68 -0
- toolgraph-0.0.1.dist-info/METADATA +674 -0
- toolgraph-0.0.1.dist-info/RECORD +38 -0
- toolgraph-0.0.1.dist-info/WHEEL +4 -0
- toolgraph-0.0.1.dist-info/entry_points.txt +2 -0
- toolgraph-0.0.1.dist-info/licenses/LICENSE +191 -0
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
"""Ingest authored governance into the graph (declarative + idempotent).
|
|
2
|
+
|
|
3
|
+
The manifest is the single source of truth for authored edges and is applied
|
|
4
|
+
atomically: a fully-valid manifest clears all CAN_CALL/READS/WRITES/GOVERNED_BY
|
|
5
|
+
edges and re-applies (so revoking a grant in YAML removes it), while a manifest
|
|
6
|
+
with ANY unresolved reference is rejected whole and the graph is left untouched.
|
|
7
|
+
This matters because a partial apply could clear an existing DENY and skip its
|
|
8
|
+
replacement, leaving a false ALLOW. Crawl-owned EXPOSES/PROVIDES are untouched.
|
|
9
|
+
|
|
10
|
+
Tool references resolve to a unique key ("<server>::<tool>" exact, or a bare
|
|
11
|
+
name when it matches exactly one tool). Bare names matching multiple servers,
|
|
12
|
+
unknown tools, and undefined policy ids become warnings that abort the ingest.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
|
|
19
|
+
from neo4j import ManagedTransaction
|
|
20
|
+
|
|
21
|
+
from toolgraph.graph.driver import session
|
|
22
|
+
from toolgraph.graph.queries import resolve_tool_refs
|
|
23
|
+
from toolgraph.graph.schema import bump_generation, exception_key
|
|
24
|
+
from toolgraph.manifest.parser import governance_digest
|
|
25
|
+
from toolgraph.models import Governance, GovernedByBinding, edge_provenance_props
|
|
26
|
+
from toolgraph.uris import is_resource_ref, normalize_resource_uri
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class IngestReport:
|
|
31
|
+
"""Outcome of a manifest ingest (ADR-0002).
|
|
32
|
+
|
|
33
|
+
``warnings`` reject the whole manifest — graph untouched (the atomicity
|
|
34
|
+
contract). ``notices`` are non-fatal advisories: today, drift — a tool
|
|
35
|
+
ref that resolves to a node no server currently exposes. A clean apply
|
|
36
|
+
has empty ``warnings`` but may still carry ``notices``; with
|
|
37
|
+
``strict_drift`` drift notices are promoted into ``warnings``.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
warnings: list[str] = field(default_factory=list)
|
|
41
|
+
notices: list[str] = field(default_factory=list)
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def applied(self) -> bool:
|
|
45
|
+
return not self.warnings
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _normalize_bindings(items: list) -> list[GovernedByBinding]:
|
|
49
|
+
"""Lift heterogeneous list items to GovernedByBinding instances.
|
|
50
|
+
|
|
51
|
+
Three input shapes can land here:
|
|
52
|
+
- ``str`` — bare policy id (backward-compat YAML shape; or list[str]
|
|
53
|
+
injected via Pydantic ``model_copy(update=...)`` which BYPASSES
|
|
54
|
+
validators; test_review_fixes.py:223 hits exactly this path).
|
|
55
|
+
- ``dict`` — raw object from ``model_copy(update=...)`` that wasn't
|
|
56
|
+
re-validated; coerce via ``model_validate``.
|
|
57
|
+
- ``GovernedByBinding`` — already-validated object from the YAML parser.
|
|
58
|
+
"""
|
|
59
|
+
out: list[GovernedByBinding] = []
|
|
60
|
+
for item in items:
|
|
61
|
+
if isinstance(item, str):
|
|
62
|
+
out.append(GovernedByBinding(policy=item))
|
|
63
|
+
elif isinstance(item, GovernedByBinding):
|
|
64
|
+
out.append(item)
|
|
65
|
+
else:
|
|
66
|
+
out.append(GovernedByBinding.model_validate(item))
|
|
67
|
+
return out
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _resolve(resolved: dict[str, list[str]], ref: str) -> tuple[str | None, list[str]]:
|
|
71
|
+
"""Return (unique_key, all_matches). unique_key is set only on exactly one match.
|
|
72
|
+
|
|
73
|
+
Shares the exact resolution rule with check_access so write-side and
|
|
74
|
+
read-side tool identity can never drift.
|
|
75
|
+
"""
|
|
76
|
+
keys = resolved[ref]
|
|
77
|
+
return (keys[0] if len(keys) == 1 else None, keys)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _resolve_problem(ref: str, matches: list[str]) -> str:
|
|
81
|
+
return f"ambiguous {matches}" if matches else "not found"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _ingest(
|
|
85
|
+
tx: ManagedTransaction,
|
|
86
|
+
gov: Governance,
|
|
87
|
+
strict_drift: bool = False,
|
|
88
|
+
digest: str | None = None,
|
|
89
|
+
) -> tuple[list[str], list[str]]:
|
|
90
|
+
"""Two phases in one transaction; returns (warnings, notices).
|
|
91
|
+
|
|
92
|
+
Phase 1 validates every reference read-only and collects warnings. If ANY
|
|
93
|
+
warning is found the manifest is rejected whole — nothing is mutated — so a
|
|
94
|
+
typo can never wipe an existing DENY and leave a false ALLOW behind. Phase 2
|
|
95
|
+
runs only for a fully clean manifest: clear authored edges, then re-apply.
|
|
96
|
+
|
|
97
|
+
Drift (a resolved tool ref with no live EXPOSES edge) is a NOTICE, not a
|
|
98
|
+
warning: rejecting would make a fleet with one temporarily-down server
|
|
99
|
+
un-ingestable. ``strict_drift`` promotes drift to warnings for CI use.
|
|
100
|
+
"""
|
|
101
|
+
warnings: list[str] = []
|
|
102
|
+
notices: list[str] = []
|
|
103
|
+
# Detect duplicate policy ids BEFORE building deny_policies. Without this,
|
|
104
|
+
# a manifest with [Policy(id='x', effect='DENY'), Policy(id='x', effect='ALLOW')]
|
|
105
|
+
# passes deny-policy validation (set-based), then Phase 2 MERGE+SET on the
|
|
106
|
+
# same id silently overwrites effect to whichever was processed last —
|
|
107
|
+
# potentially leaving an ALLOW node that no longer matches the DENY-only
|
|
108
|
+
# classifier in _deny_evidence_for. Codex R3.5 review caught this.
|
|
109
|
+
seen_policy_ids: set[str] = set()
|
|
110
|
+
for p in gov.policies:
|
|
111
|
+
if p.id in seen_policy_ids:
|
|
112
|
+
warnings.append(
|
|
113
|
+
f"POLICY: duplicate id {p.id!r} in policies: list — "
|
|
114
|
+
f"each policy must be declared once"
|
|
115
|
+
)
|
|
116
|
+
seen_policy_ids.add(p.id)
|
|
117
|
+
known_policies = {p.id for p in gov.policies}
|
|
118
|
+
deny_policies = {p.id for p in gov.policies if p.effect == "DENY"}
|
|
119
|
+
known_agents = set(gov.agents) | {g.agent for g in gov.grants}
|
|
120
|
+
|
|
121
|
+
# --- Phase 1: validate + resolve (read-only) -------------------------
|
|
122
|
+
# Resolve once on this transaction; validation still walks the original
|
|
123
|
+
# manifest order so diagnostics and last-entry-wins semantics do not move.
|
|
124
|
+
refs = [g.tool for g in gov.grants] + [d.tool for d in gov.data_access]
|
|
125
|
+
refs.extend(ref for ref in gov.governed_by if not is_resource_ref(ref))
|
|
126
|
+
refs.extend(e.tool for e in gov.expected_exceptions)
|
|
127
|
+
resolved_refs = resolve_tool_refs(tx, refs)
|
|
128
|
+
resolved_grants: list[tuple] = []
|
|
129
|
+
for grant in gov.grants:
|
|
130
|
+
key, matches = _resolve(resolved_refs, grant.tool)
|
|
131
|
+
if key is None:
|
|
132
|
+
warnings.append(
|
|
133
|
+
f"CAN_CALL: tool {grant.tool!r} {_resolve_problem(grant.tool, matches)} "
|
|
134
|
+
f"(agent {grant.agent!r})"
|
|
135
|
+
)
|
|
136
|
+
else:
|
|
137
|
+
resolved_grants.append(
|
|
138
|
+
(grant.agent, key, grant.granted_by, edge_provenance_props(grant.provenance))
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
resolved_access: list[tuple] = []
|
|
142
|
+
authored_access: set[tuple[str, str]] = set() # (tool_key, resource_uri) for exception check
|
|
143
|
+
for da in gov.data_access:
|
|
144
|
+
key, matches = _resolve(resolved_refs, da.tool)
|
|
145
|
+
if key is None:
|
|
146
|
+
warnings.append(f"{da.mode}: tool {da.tool!r} {_resolve_problem(da.tool, matches)}")
|
|
147
|
+
else:
|
|
148
|
+
uri = normalize_resource_uri(da.resource)
|
|
149
|
+
resolved_access.append((key, da.mode, uri, edge_provenance_props(da.provenance)))
|
|
150
|
+
authored_access.add((key, uri))
|
|
151
|
+
|
|
152
|
+
# Resolve governed_by first so exception validation can see the full
|
|
153
|
+
# (resource→policy) and (tool→policy) reachability map.
|
|
154
|
+
resolved_governed: list[tuple] = [] # (kind, key_or_uri, list[(pid, prov_props)])
|
|
155
|
+
policy_resources: dict[str, set[str]] = {} # pid -> {uri, ...}
|
|
156
|
+
policy_tools: dict[str, set[str]] = {} # pid -> {tool_key, ...}
|
|
157
|
+
for node_ref, items in gov.governed_by.items():
|
|
158
|
+
bindings = _normalize_bindings(items)
|
|
159
|
+
for b in bindings:
|
|
160
|
+
if b.policy not in known_policies:
|
|
161
|
+
warnings.append(
|
|
162
|
+
f"GOVERNED_BY: policy {b.policy!r} not defined (node {node_ref!r})"
|
|
163
|
+
)
|
|
164
|
+
valid = [
|
|
165
|
+
(b.policy, edge_provenance_props(b.provenance))
|
|
166
|
+
for b in bindings
|
|
167
|
+
if b.policy in known_policies
|
|
168
|
+
]
|
|
169
|
+
if is_resource_ref(node_ref):
|
|
170
|
+
uri = normalize_resource_uri(node_ref)
|
|
171
|
+
resolved_governed.append(("resource", uri, valid))
|
|
172
|
+
for pid, _ in valid:
|
|
173
|
+
policy_resources.setdefault(pid, set()).add(uri)
|
|
174
|
+
else:
|
|
175
|
+
key, matches = _resolve(resolved_refs, node_ref)
|
|
176
|
+
if key is None:
|
|
177
|
+
warnings.append(
|
|
178
|
+
f"GOVERNED_BY: tool {node_ref!r} {_resolve_problem(node_ref, matches)}"
|
|
179
|
+
)
|
|
180
|
+
else:
|
|
181
|
+
resolved_governed.append(("tool", key, valid))
|
|
182
|
+
for pid, _ in valid:
|
|
183
|
+
policy_tools.setdefault(pid, set()).add(key)
|
|
184
|
+
|
|
185
|
+
# Reachability lookups for exception validation: which tools each agent
|
|
186
|
+
# holds a grant on, and which resources each tool actually touches.
|
|
187
|
+
agent_grant_tools: dict[str, set[str]] = {}
|
|
188
|
+
for a, k, _, _ in resolved_grants:
|
|
189
|
+
agent_grant_tools.setdefault(a, set()).add(k)
|
|
190
|
+
tool_resources: dict[str, set[str]] = {}
|
|
191
|
+
for k, _, uri, _ in resolved_access:
|
|
192
|
+
tool_resources.setdefault(k, set()).add(uri)
|
|
193
|
+
|
|
194
|
+
resolved_exceptions: list[tuple] = [] # (agent, tool_key, resource_or_None, policy, reason)
|
|
195
|
+
for exc in gov.expected_exceptions:
|
|
196
|
+
# Validate the agent against declared agents (or grant authors) so a
|
|
197
|
+
# typo like 'planer' doesn't silently create a dead-on-arrival
|
|
198
|
+
# exception node that fails to classify any row.
|
|
199
|
+
if exc.agent not in known_agents:
|
|
200
|
+
warnings.append(
|
|
201
|
+
f"EXPECTED_EXCEPTION: agent {exc.agent!r} not declared in agents: or grants: "
|
|
202
|
+
f"(tool {exc.tool!r})"
|
|
203
|
+
)
|
|
204
|
+
continue
|
|
205
|
+
if exc.policy not in known_policies:
|
|
206
|
+
warnings.append(
|
|
207
|
+
f"EXPECTED_EXCEPTION: policy {exc.policy!r} not defined "
|
|
208
|
+
f"(agent {exc.agent!r}, tool {exc.tool!r})"
|
|
209
|
+
)
|
|
210
|
+
continue
|
|
211
|
+
# Classification only joins to DENY policies (_deny_evidence_for matches
|
|
212
|
+
# `Policy {effect:'DENY'}`); an exception against an ALLOW policy can
|
|
213
|
+
# never fire. Codex R3 review caught this.
|
|
214
|
+
if exc.policy not in deny_policies:
|
|
215
|
+
warnings.append(
|
|
216
|
+
f"EXPECTED_EXCEPTION: policy {exc.policy!r} has effect=ALLOW; "
|
|
217
|
+
f"exceptions only apply to DENY policies "
|
|
218
|
+
f"(agent {exc.agent!r}, tool {exc.tool!r})"
|
|
219
|
+
)
|
|
220
|
+
continue
|
|
221
|
+
key, matches = _resolve(resolved_refs, exc.tool)
|
|
222
|
+
if key is None:
|
|
223
|
+
warnings.append(
|
|
224
|
+
f"EXPECTED_EXCEPTION: tool {exc.tool!r} {_resolve_problem(exc.tool, matches)} "
|
|
225
|
+
f"(agent {exc.agent!r})"
|
|
226
|
+
)
|
|
227
|
+
continue
|
|
228
|
+
# The exception only fires when the agent's deny_evidence row matches
|
|
229
|
+
# (agent, tool, resource?, policy). That requires:
|
|
230
|
+
# 1) agent holds a CAN_CALL grant on the tool — otherwise the agent
|
|
231
|
+
# will never appear in unsafe_callable_tools for that tool;
|
|
232
|
+
# 2) the (tool, policy) pair is actually reachable — either via a
|
|
233
|
+
# governed resource the tool touches, or by direct tool-level
|
|
234
|
+
# GOVERNED_BY. Without (2) no deny row will ever fire.
|
|
235
|
+
# Closing only the "wrong resource" slice (the previous round) wasn't
|
|
236
|
+
# enough — Codex flagged that wrong-policy and ungranted-agent
|
|
237
|
+
# exceptions still survived.
|
|
238
|
+
if key not in agent_grant_tools.get(exc.agent, set()):
|
|
239
|
+
warnings.append(
|
|
240
|
+
f"EXPECTED_EXCEPTION: agent {exc.agent!r} has no CAN_CALL grant on tool "
|
|
241
|
+
f"{exc.tool!r} — exception will never classify any row"
|
|
242
|
+
)
|
|
243
|
+
continue
|
|
244
|
+
resource_uri = normalize_resource_uri(exc.resource) if exc.resource else None
|
|
245
|
+
policy_resource_set = policy_resources.get(exc.policy, set())
|
|
246
|
+
policy_tool_set = policy_tools.get(exc.policy, set())
|
|
247
|
+
if resource_uri is not None:
|
|
248
|
+
# Specific resource: tool must READ/WRITE it AND that resource
|
|
249
|
+
# must be GOVERNED_BY this policy.
|
|
250
|
+
if (key, resource_uri) not in authored_access:
|
|
251
|
+
warnings.append(
|
|
252
|
+
f"EXPECTED_EXCEPTION: tool {exc.tool!r} has no READS/WRITES "
|
|
253
|
+
f"to resource {exc.resource!r} in this manifest "
|
|
254
|
+
f"(agent {exc.agent!r}, policy {exc.policy!r})"
|
|
255
|
+
)
|
|
256
|
+
continue
|
|
257
|
+
if resource_uri not in policy_resource_set:
|
|
258
|
+
warnings.append(
|
|
259
|
+
f"EXPECTED_EXCEPTION: resource {exc.resource!r} is not governed_by "
|
|
260
|
+
f"policy {exc.policy!r} in this manifest "
|
|
261
|
+
f"(agent {exc.agent!r}, tool {exc.tool!r})"
|
|
262
|
+
)
|
|
263
|
+
continue
|
|
264
|
+
else:
|
|
265
|
+
# Wildcard: tool must reach this policy via SOME path —
|
|
266
|
+
# a tool-touched resource governed by the policy, OR the tool
|
|
267
|
+
# itself directly governed by the policy.
|
|
268
|
+
reaches_via_resource = bool(
|
|
269
|
+
tool_resources.get(key, set()) & policy_resource_set
|
|
270
|
+
)
|
|
271
|
+
reaches_via_tool = key in policy_tool_set
|
|
272
|
+
if not (reaches_via_resource or reaches_via_tool):
|
|
273
|
+
warnings.append(
|
|
274
|
+
f"EXPECTED_EXCEPTION: tool {exc.tool!r} has no path to DENY policy "
|
|
275
|
+
f"{exc.policy!r} in this manifest "
|
|
276
|
+
f"(agent {exc.agent!r}) — wildcard exception is dead-on-arrival"
|
|
277
|
+
)
|
|
278
|
+
continue
|
|
279
|
+
resolved_exceptions.append(
|
|
280
|
+
(exc.agent, key, resource_uri, exc.policy, exc.reason)
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
# ADR-0002 drift contract: a ref can resolve to a Tool node that no
|
|
284
|
+
# server currently exposes (the loader keeps governed tools so a DENY
|
|
285
|
+
# never silently becomes ALLOW). Without this check, re-ingesting the
|
|
286
|
+
# same manifest silently re-grants against the stale node and the drift
|
|
287
|
+
# persists with zero signal in the authoring loop.
|
|
288
|
+
referenced_keys = {k for _, k, _, _ in resolved_grants}
|
|
289
|
+
referenced_keys |= {k for k, _, _, _ in resolved_access}
|
|
290
|
+
referenced_keys |= {ref for kind, ref, _ in resolved_governed if kind == "tool"}
|
|
291
|
+
referenced_keys |= {k for _, k, _, _, _ in resolved_exceptions}
|
|
292
|
+
if referenced_keys:
|
|
293
|
+
drifted = [
|
|
294
|
+
r["key"]
|
|
295
|
+
for r in tx.run(
|
|
296
|
+
"MATCH (t:Tool) WHERE t.key IN $keys "
|
|
297
|
+
"AND NOT EXISTS { MATCH (:MCPServer)-[:EXPOSES]->(t) } "
|
|
298
|
+
"RETURN t.key AS key ORDER BY key",
|
|
299
|
+
keys=sorted(referenced_keys),
|
|
300
|
+
)
|
|
301
|
+
]
|
|
302
|
+
for key in drifted:
|
|
303
|
+
(warnings if strict_drift else notices).append(
|
|
304
|
+
f"DRIFT: tool {key!r} resolves but no server currently exposes "
|
|
305
|
+
f"it — authored governance targets a stale tool. Remove it from "
|
|
306
|
+
f"the manifest, or re-crawl its server if it should still exist."
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
if warnings: # reject the whole manifest — do not mutate
|
|
310
|
+
return warnings, notices
|
|
311
|
+
|
|
312
|
+
# --- Phase 2: mutate (clean manifest only) ---------------------------
|
|
313
|
+
# Deliberate size tradeoff: this transaction scales with the total number
|
|
314
|
+
# of authored edges and cannot be batched — a partial apply could clear a
|
|
315
|
+
# DENY and leave a false ALLOW (see the module docstring / ADR-0002).
|
|
316
|
+
tx.run("MATCH ()-[r:CAN_CALL|READS|WRITES|GOVERNED_BY]->() DELETE r")
|
|
317
|
+
# ExpectedException nodes are pure authored bookkeeping — clear and re-apply
|
|
318
|
+
# alongside the edges so a removed exception promotes the row back to violation.
|
|
319
|
+
tx.run("MATCH (e:ExpectedException) DETACH DELETE e")
|
|
320
|
+
|
|
321
|
+
tx.run("UNWIND $ids AS id MERGE (:Agent {id:id})", ids=sorted(known_agents))
|
|
322
|
+
# Policy provenance lives on the GOVERNED_BY binding (per-edge), not the
|
|
323
|
+
# node — see GovernedByBinding. Policy nodes carry only declaration data.
|
|
324
|
+
# Null assignments drop the legacy ``prov_*`` properties left over from the
|
|
325
|
+
# pre-completion-round shape, so an upgrading graph cleans itself up.
|
|
326
|
+
if gov.policies:
|
|
327
|
+
tx.run(
|
|
328
|
+
"""
|
|
329
|
+
UNWIND $policies AS p
|
|
330
|
+
MERGE (pol:Policy {id:p.id})
|
|
331
|
+
SET pol.effect=p.effect, pol.scope=p.scope, pol.description=p.description
|
|
332
|
+
SET pol.prov_source=null, pol.prov_confidence=null, pol.prov_evidence=null
|
|
333
|
+
""",
|
|
334
|
+
policies=[p.model_dump() for p in gov.policies],
|
|
335
|
+
)
|
|
336
|
+
# Declarative manifest: prune Agent + Policy nodes the operator removed
|
|
337
|
+
# from this manifest. The new ``found`` / ``agent_found`` API treats
|
|
338
|
+
# node existence as a truth signal; leaking renamed-away nodes would
|
|
339
|
+
# silently make ``unsafe-tools old-agent`` look like a clean empty
|
|
340
|
+
# answer and ``blast-radius old-policy`` claim found:true. Edges were
|
|
341
|
+
# already cleared at the top of Phase 2, so DETACH DELETE is a no-op
|
|
342
|
+
# on the edge side; it just removes the now-orphan node.
|
|
343
|
+
policy_ids = [p.id for p in gov.policies]
|
|
344
|
+
tx.run(
|
|
345
|
+
"MATCH (a:Agent) WHERE NOT a.id IN $ids DETACH DELETE a",
|
|
346
|
+
ids=sorted(known_agents),
|
|
347
|
+
)
|
|
348
|
+
tx.run(
|
|
349
|
+
"MATCH (pol:Policy) WHERE NOT pol.id IN $ids DETACH DELETE pol",
|
|
350
|
+
ids=policy_ids,
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
# Deduplicate to last-entry-wins BEFORE batching: the old per-row loops
|
|
354
|
+
# deterministically left the later manifest entry's metadata on a
|
|
355
|
+
# duplicate logical edge, while UNWIND gives repeated SETs on one
|
|
356
|
+
# relationship an undefined winner. dict insertion order preserves the
|
|
357
|
+
# manifest order, and overwriting keeps the last occurrence.
|
|
358
|
+
resolved_grants = list({(row[0], row[1]): row for row in resolved_grants}.values())
|
|
359
|
+
resolved_access = list(
|
|
360
|
+
{(row[0], row[1], row[2]): row for row in resolved_access}.values()
|
|
361
|
+
)
|
|
362
|
+
if resolved_grants:
|
|
363
|
+
# One UNWIND batch instead of a round trip per grant (the same shape
|
|
364
|
+
# policies and exceptions already use).
|
|
365
|
+
tx.run(
|
|
366
|
+
"""
|
|
367
|
+
UNWIND $rows AS row
|
|
368
|
+
MERGE (a:Agent {id:row.agent})
|
|
369
|
+
WITH a, row MATCH (t:Tool {key:row.key})
|
|
370
|
+
MERGE (a)-[r:CAN_CALL]->(t)
|
|
371
|
+
SET r.granted_by=row.granted_by,
|
|
372
|
+
r.source=row.source, r.confidence=row.confidence, r.evidence=row.evidence
|
|
373
|
+
""",
|
|
374
|
+
rows=[
|
|
375
|
+
{"agent": agent, "key": key, "granted_by": granted_by, **prov}
|
|
376
|
+
for agent, key, granted_by, prov in resolved_grants
|
|
377
|
+
],
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
# mode is a validated Literal -> safe to inline as the relationship type;
|
|
381
|
+
# one UNWIND batch per relationship type instead of a round trip per edge.
|
|
382
|
+
for mode in ("READS", "WRITES"):
|
|
383
|
+
batch = [
|
|
384
|
+
{"key": key, "uri": uri, **prov}
|
|
385
|
+
for key, row_mode, uri, prov in resolved_access
|
|
386
|
+
if row_mode == mode
|
|
387
|
+
]
|
|
388
|
+
if not batch:
|
|
389
|
+
continue
|
|
390
|
+
tx.run(
|
|
391
|
+
f"""
|
|
392
|
+
UNWIND $rows AS row
|
|
393
|
+
MATCH (t:Tool {{key:row.key}})
|
|
394
|
+
MERGE (res:Resource {{uri:row.uri}})
|
|
395
|
+
MERGE (t)-[r:{mode}]->(res)
|
|
396
|
+
SET r.source=row.source, r.confidence=row.confidence, r.evidence=row.evidence
|
|
397
|
+
""",
|
|
398
|
+
rows=batch,
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
resolved_exceptions = list({row[:4]: row for row in resolved_exceptions}.values())
|
|
402
|
+
if resolved_exceptions:
|
|
403
|
+
tx.run(
|
|
404
|
+
"""
|
|
405
|
+
UNWIND $rows AS row
|
|
406
|
+
MERGE (e:ExpectedException {key:row.key})
|
|
407
|
+
SET e.agent=row.agent, e.tool_key=row.tool_key,
|
|
408
|
+
e.resource=row.resource, e.policy=row.policy, e.reason=row.reason
|
|
409
|
+
""",
|
|
410
|
+
rows=[
|
|
411
|
+
{
|
|
412
|
+
"key": exception_key(a, k, r, p),
|
|
413
|
+
"agent": a,
|
|
414
|
+
"tool_key": k,
|
|
415
|
+
"resource": r,
|
|
416
|
+
"policy": p,
|
|
417
|
+
"reason": reason,
|
|
418
|
+
}
|
|
419
|
+
for a, k, r, p, reason in resolved_exceptions
|
|
420
|
+
],
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
# GOVERNED_BY edges carry per-binding provenance: an author can cite WHY
|
|
424
|
+
# this resource (or tool) is bound to this policy. Bare-string entries in
|
|
425
|
+
# ``governed_by`` lift to bindings with no evidence (operator_asserted/high
|
|
426
|
+
# by default), so unbacked_edges still surfaces them as needing a citation.
|
|
427
|
+
resource_bindings = [
|
|
428
|
+
{"ref": ref, "pid": pid, **prov}
|
|
429
|
+
for kind, ref, bindings in resolved_governed
|
|
430
|
+
if kind == "resource"
|
|
431
|
+
for pid, prov in bindings
|
|
432
|
+
]
|
|
433
|
+
tool_bindings = [
|
|
434
|
+
{"ref": ref, "pid": pid, **prov}
|
|
435
|
+
for kind, ref, bindings in resolved_governed
|
|
436
|
+
if kind != "resource"
|
|
437
|
+
for pid, prov in bindings
|
|
438
|
+
]
|
|
439
|
+
# Same last-entry-wins rule for duplicate (node, policy) bindings.
|
|
440
|
+
resource_bindings = list(
|
|
441
|
+
{(row["ref"], row["pid"]): row for row in resource_bindings}.values()
|
|
442
|
+
)
|
|
443
|
+
tool_bindings = list(
|
|
444
|
+
{(row["ref"], row["pid"]): row for row in tool_bindings}.values()
|
|
445
|
+
)
|
|
446
|
+
if resource_bindings:
|
|
447
|
+
tx.run(
|
|
448
|
+
"""
|
|
449
|
+
UNWIND $rows AS row
|
|
450
|
+
MERGE (res:Resource {uri:row.ref})
|
|
451
|
+
WITH res, row MATCH (p:Policy {id:row.pid})
|
|
452
|
+
MERGE (res)-[r:GOVERNED_BY]->(p)
|
|
453
|
+
SET r.source=row.source, r.confidence=row.confidence, r.evidence=row.evidence
|
|
454
|
+
""",
|
|
455
|
+
rows=resource_bindings,
|
|
456
|
+
)
|
|
457
|
+
if tool_bindings:
|
|
458
|
+
tx.run(
|
|
459
|
+
"""
|
|
460
|
+
UNWIND $rows AS row
|
|
461
|
+
MATCH (t:Tool {key:row.ref})
|
|
462
|
+
MATCH (p:Policy {id:row.pid})
|
|
463
|
+
MERGE (t)-[r:GOVERNED_BY]->(p)
|
|
464
|
+
SET r.source=row.source, r.confidence=row.confidence, r.evidence=row.evidence
|
|
465
|
+
""",
|
|
466
|
+
rows=tool_bindings,
|
|
467
|
+
)
|
|
468
|
+
|
|
469
|
+
# ADR-0001: node existence is a truth signal, for Resources too. A
|
|
470
|
+
# resource the manifest stopped mentioning had its authored edges cleared
|
|
471
|
+
# above and nothing re-applied; leaving the node would make blast_radius
|
|
472
|
+
# answer found:true / impacted:[] for it — indistinguishable from a real
|
|
473
|
+
# but untouched resource. Crawled resources keep their PROVIDES edge and
|
|
474
|
+
# never match this degree-0 filter; the loader stays their owner.
|
|
475
|
+
tx.run("MATCH (res:Resource) WHERE NOT EXISTS { MATCH (res)--() } DELETE res")
|
|
476
|
+
|
|
477
|
+
# Same rule for Tools (ADR-0007). A tool kept only for its authored
|
|
478
|
+
# governance — typically after fleet reconciliation retired its server —
|
|
479
|
+
# loses its last edge when the manifest stops mentioning it. No crawl
|
|
480
|
+
# will ever reconcile a retired server's name again, so without this a
|
|
481
|
+
# degree-0 tool ghosts in bare-name resolution forever. Live crawled
|
|
482
|
+
# tools always carry EXPOSES and never match.
|
|
483
|
+
tx.run("MATCH (t:Tool) WHERE NOT EXISTS { MATCH (t)--() } DELETE t")
|
|
484
|
+
|
|
485
|
+
# The authored manifest is the source of truth. Persist its semantic
|
|
486
|
+
# digest in the same transaction as the derived graph and generation so a
|
|
487
|
+
# policy-bundle compiler can reject stale graph/file combinations.
|
|
488
|
+
tx.run(
|
|
489
|
+
"MERGE (m:GraphMeta {id:'singleton'}) "
|
|
490
|
+
"SET m.governance_digest=$digest",
|
|
491
|
+
digest=digest,
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
# ADR-0004: only a manifest that actually applied bumps the generation —
|
|
495
|
+
# the rejected path returned before Phase 2, leaving caches valid.
|
|
496
|
+
bump_generation(tx)
|
|
497
|
+
|
|
498
|
+
return [], notices
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def ingest_governance(gov: Governance, strict_drift: bool = False) -> IngestReport:
|
|
502
|
+
"""Apply the manifest atomically.
|
|
503
|
+
|
|
504
|
+
Non-empty ``report.warnings`` means rejected — graph untouched.
|
|
505
|
+
``report.notices`` are non-fatal drift advisories (see IngestReport).
|
|
506
|
+
"""
|
|
507
|
+
with session() as s:
|
|
508
|
+
warnings, notices = s.execute_write(
|
|
509
|
+
_ingest, gov, strict_drift, governance_digest(gov)
|
|
510
|
+
)
|
|
511
|
+
return IngestReport(warnings=warnings, notices=notices)
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def governance_counts() -> dict[str, int]:
|
|
515
|
+
"""Authored-governance tallies in one UNION ALL statement."""
|
|
516
|
+
query = """
|
|
517
|
+
MATCH (n:Agent) RETURN 'agent' AS name, count(n) AS count
|
|
518
|
+
UNION ALL
|
|
519
|
+
MATCH (n:Policy) RETURN 'policy' AS name, count(n) AS count
|
|
520
|
+
UNION ALL
|
|
521
|
+
MATCH ()-[r:CAN_CALL]->() RETURN 'can_call' AS name, count(r) AS count
|
|
522
|
+
UNION ALL
|
|
523
|
+
MATCH ()-[r:READS]->() RETURN 'reads' AS name, count(r) AS count
|
|
524
|
+
UNION ALL
|
|
525
|
+
MATCH ()-[r:WRITES]->() RETURN 'writes' AS name, count(r) AS count
|
|
526
|
+
UNION ALL
|
|
527
|
+
MATCH ()-[r:GOVERNED_BY]->() RETURN 'governed_by' AS name, count(r) AS count
|
|
528
|
+
UNION ALL
|
|
529
|
+
MATCH (n:ExpectedException) RETURN 'expected_exception' AS name, count(n) AS count
|
|
530
|
+
"""
|
|
531
|
+
names = (
|
|
532
|
+
"agent",
|
|
533
|
+
"policy",
|
|
534
|
+
"can_call",
|
|
535
|
+
"reads",
|
|
536
|
+
"writes",
|
|
537
|
+
"governed_by",
|
|
538
|
+
"expected_exception",
|
|
539
|
+
)
|
|
540
|
+
with session() as s:
|
|
541
|
+
counts = {r["name"]: r["count"] for r in s.run(query)}
|
|
542
|
+
return {name: counts.get(name, 0) for name in names}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Parse the authored governance manifest (governance.yaml)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from toolgraph.models import Governance
|
|
10
|
+
from toolgraph.yaml_loader import load_yaml_mapping
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def load_governance(path: Path) -> Governance:
|
|
14
|
+
data = load_yaml_mapping(path)
|
|
15
|
+
return Governance.model_validate(data)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def governance_digest(governance: Governance) -> str:
|
|
19
|
+
"""Stable semantic digest of the validated authored governance model."""
|
|
20
|
+
payload = json.dumps(
|
|
21
|
+
governance.model_dump(mode="json"),
|
|
22
|
+
ensure_ascii=False,
|
|
23
|
+
separators=(",", ":"),
|
|
24
|
+
sort_keys=True,
|
|
25
|
+
).encode("utf-8")
|
|
26
|
+
return hashlib.sha256(payload).hexdigest()
|