contextwall 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- context_firewall/__init__.py +3 -0
- context_firewall/analytics/__init__.py +5 -0
- context_firewall/analytics/engine.py +234 -0
- context_firewall/api/__init__.py +5 -0
- context_firewall/api/app.py +1020 -0
- context_firewall/api/models.py +54 -0
- context_firewall/classifier/__init__.py +5 -0
- context_firewall/classifier/classifier.py +165 -0
- context_firewall/cli/__init__.py +5 -0
- context_firewall/cli/main.py +513 -0
- context_firewall/compliance/__init__.py +1 -0
- context_firewall/compliance/baa.py +57 -0
- context_firewall/compliance/chain.py +84 -0
- context_firewall/compliance/control_mappings.py +92 -0
- context_firewall/compliance/export.py +280 -0
- context_firewall/compliance/keys.py +94 -0
- context_firewall/config.py +234 -0
- context_firewall/control_plane/__init__.py +0 -0
- context_firewall/control_plane/client.py +107 -0
- context_firewall/control_plane/models.py +72 -0
- context_firewall/control_plane/pusher.py +377 -0
- context_firewall/daemon/__init__.py +5 -0
- context_firewall/daemon/main.py +461 -0
- context_firewall/db/__init__.py +6 -0
- context_firewall/db/connection.py +30 -0
- context_firewall/db/migrations.py +408 -0
- context_firewall/entropy/__init__.py +5 -0
- context_firewall/entropy/engine.py +271 -0
- context_firewall/graph/__init__.py +5 -0
- context_firewall/graph/engine.py +162 -0
- context_firewall/lint/__init__.py +0 -0
- context_firewall/lint/engine.py +334 -0
- context_firewall/mcp/__init__.py +5 -0
- context_firewall/mcp/server.py +237 -0
- context_firewall/metrics.py +100 -0
- context_firewall/models.py +130 -0
- context_firewall/policy/__init__.py +10 -0
- context_firewall/policy/detectors/__init__.py +0 -0
- context_firewall/policy/detectors/injection.py +480 -0
- context_firewall/policy/dsl/__init__.py +0 -0
- context_firewall/policy/dsl/evaluator.py +208 -0
- context_firewall/policy/dsl/loader.py +150 -0
- context_firewall/policy/dsl/types.py +112 -0
- context_firewall/policy/engine.py +532 -0
- context_firewall/provenance/__init__.py +5 -0
- context_firewall/provenance/engine.py +658 -0
- context_firewall/provenance/models.py +91 -0
- context_firewall/proxy/__init__.py +1 -0
- context_firewall/proxy/router.py +519 -0
- context_firewall/proxy/scanner.py +214 -0
- context_firewall/proxy/tokens.py +156 -0
- context_firewall/runtime/__init__.py +5 -0
- context_firewall/runtime/engine.py +303 -0
- context_firewall/source/__init__.py +1 -0
- context_firewall/source/registry.py +299 -0
- context_firewall/source/types.py +12 -0
- context_firewall/synthesizer/__init__.py +5 -0
- context_firewall/synthesizer/synthesizer.py +255 -0
- context_firewall/trust/__init__.py +5 -0
- context_firewall/trust/engine.py +139 -0
- context_firewall/trust/signals.py +184 -0
- contextwall-0.1.0.dist-info/METADATA +431 -0
- contextwall-0.1.0.dist-info/RECORD +66 -0
- contextwall-0.1.0.dist-info/WHEEL +4 -0
- contextwall-0.1.0.dist-info/entry_points.txt +3 -0
- contextwall-0.1.0.dist-info/licenses/LICENSE +33 -0
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
"""Versioned SQLite migrations.
|
|
2
|
+
|
|
3
|
+
Each migration is a (version, description, sql_statements) tuple.
|
|
4
|
+
The runner applies only migrations whose version is higher than the
|
|
5
|
+
current schema_version stored in the DB - safe to run repeatedly.
|
|
6
|
+
|
|
7
|
+
Adding a migration:
|
|
8
|
+
1. Append a new entry to _MIGRATIONS with the next version number.
|
|
9
|
+
2. Never modify an existing migration entry (already applied on prod DBs).
|
|
10
|
+
3. Use ALTER TABLE ADD COLUMN (not DROP/RECREATE) for additive changes.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import logging
|
|
16
|
+
|
|
17
|
+
import aiosqlite
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
# ── Migration registry ─────────────────────────────────────────────────────────
|
|
22
|
+
# Each entry: (version: int, description: str, statements: list[str])
|
|
23
|
+
# Statements are executed inside a transaction; if any fails the migration
|
|
24
|
+
# is rolled back and startup aborts.
|
|
25
|
+
|
|
26
|
+
_MIGRATIONS: list[tuple[int, str, list[str]]] = [
|
|
27
|
+
(
|
|
28
|
+
1,
|
|
29
|
+
"initial schema",
|
|
30
|
+
[
|
|
31
|
+
"""
|
|
32
|
+
CREATE TABLE IF NOT EXISTS runtime_signals (
|
|
33
|
+
node_id TEXT PRIMARY KEY,
|
|
34
|
+
invocation_count INTEGER NOT NULL DEFAULT 0,
|
|
35
|
+
exception_count INTEGER NOT NULL DEFAULT 0,
|
|
36
|
+
exception_rate REAL NOT NULL DEFAULT 0.0,
|
|
37
|
+
p50_latency_ms REAL,
|
|
38
|
+
p95_latency_ms REAL,
|
|
39
|
+
p99_latency_ms REAL,
|
|
40
|
+
last_observed_at DATETIME,
|
|
41
|
+
latency_degraded BOOLEAN NOT NULL DEFAULT FALSE,
|
|
42
|
+
exception_rate_high BOOLEAN NOT NULL DEFAULT FALSE,
|
|
43
|
+
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
44
|
+
)
|
|
45
|
+
""",
|
|
46
|
+
"""
|
|
47
|
+
CREATE TABLE IF NOT EXISTS service_map (
|
|
48
|
+
caller_service TEXT NOT NULL,
|
|
49
|
+
callee_service TEXT NOT NULL,
|
|
50
|
+
observed_count INTEGER NOT NULL DEFAULT 1,
|
|
51
|
+
last_seen_at DATETIME NOT NULL,
|
|
52
|
+
PRIMARY KEY (caller_service, callee_service)
|
|
53
|
+
)
|
|
54
|
+
""",
|
|
55
|
+
"""
|
|
56
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
57
|
+
session_id TEXT PRIMARY KEY,
|
|
58
|
+
agent_id TEXT,
|
|
59
|
+
model_version TEXT,
|
|
60
|
+
repository_root TEXT,
|
|
61
|
+
client_type TEXT,
|
|
62
|
+
status TEXT NOT NULL DEFAULT 'open',
|
|
63
|
+
started_at DATETIME NOT NULL,
|
|
64
|
+
closed_at DATETIME,
|
|
65
|
+
request_count INTEGER NOT NULL DEFAULT 0,
|
|
66
|
+
total_tokens INTEGER NOT NULL DEFAULT 0,
|
|
67
|
+
avg_outcome_score REAL
|
|
68
|
+
)
|
|
69
|
+
""",
|
|
70
|
+
"""
|
|
71
|
+
CREATE TABLE IF NOT EXISTS provenance_events (
|
|
72
|
+
event_id TEXT PRIMARY KEY,
|
|
73
|
+
event_type TEXT NOT NULL,
|
|
74
|
+
session_id TEXT NOT NULL,
|
|
75
|
+
request_id TEXT NOT NULL,
|
|
76
|
+
occurred_at DATETIME NOT NULL,
|
|
77
|
+
payload TEXT NOT NULL,
|
|
78
|
+
FOREIGN KEY (session_id) REFERENCES sessions(session_id)
|
|
79
|
+
)
|
|
80
|
+
""",
|
|
81
|
+
"CREATE INDEX IF NOT EXISTS idx_provenance_session ON provenance_events(session_id, occurred_at)",
|
|
82
|
+
"CREATE INDEX IF NOT EXISTS idx_provenance_request ON provenance_events(request_id, occurred_at)",
|
|
83
|
+
"""
|
|
84
|
+
CREATE TABLE IF NOT EXISTS policy_enforcement_events (
|
|
85
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
86
|
+
session_id TEXT NOT NULL,
|
|
87
|
+
request_id TEXT NOT NULL,
|
|
88
|
+
rule_name TEXT NOT NULL,
|
|
89
|
+
action TEXT NOT NULL,
|
|
90
|
+
file_path TEXT NOT NULL,
|
|
91
|
+
node_id TEXT,
|
|
92
|
+
line_numbers TEXT,
|
|
93
|
+
reason TEXT NOT NULL,
|
|
94
|
+
pattern_name TEXT,
|
|
95
|
+
occurred_at DATETIME NOT NULL
|
|
96
|
+
)
|
|
97
|
+
""",
|
|
98
|
+
"""
|
|
99
|
+
CREATE TABLE IF NOT EXISTS analytics_snapshots (
|
|
100
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
101
|
+
metric_type TEXT NOT NULL,
|
|
102
|
+
granularity TEXT NOT NULL,
|
|
103
|
+
key TEXT NOT NULL,
|
|
104
|
+
payload TEXT NOT NULL,
|
|
105
|
+
computed_at DATETIME NOT NULL,
|
|
106
|
+
window_start DATETIME,
|
|
107
|
+
window_end DATETIME
|
|
108
|
+
)
|
|
109
|
+
""",
|
|
110
|
+
"CREATE INDEX IF NOT EXISTS idx_analytics_type_key ON analytics_snapshots(metric_type, key, computed_at DESC)",
|
|
111
|
+
"""
|
|
112
|
+
CREATE TABLE IF NOT EXISTS entropy_snapshots (
|
|
113
|
+
node_id TEXT NOT NULL,
|
|
114
|
+
file_path TEXT NOT NULL,
|
|
115
|
+
entropy_score REAL NOT NULL,
|
|
116
|
+
active_signals TEXT,
|
|
117
|
+
snapshot_at DATETIME NOT NULL,
|
|
118
|
+
PRIMARY KEY (node_id, snapshot_at)
|
|
119
|
+
)
|
|
120
|
+
""",
|
|
121
|
+
"""
|
|
122
|
+
CREATE TABLE IF NOT EXISTS trust_score_snapshots (
|
|
123
|
+
node_id TEXT NOT NULL,
|
|
124
|
+
file_path TEXT NOT NULL,
|
|
125
|
+
trust_score REAL NOT NULL,
|
|
126
|
+
signal_breakdown TEXT,
|
|
127
|
+
snapshot_at DATETIME NOT NULL,
|
|
128
|
+
trigger_reason TEXT,
|
|
129
|
+
PRIMARY KEY (node_id, snapshot_at)
|
|
130
|
+
)
|
|
131
|
+
""",
|
|
132
|
+
"""
|
|
133
|
+
CREATE TABLE IF NOT EXISTS trust_signal_cache (
|
|
134
|
+
node_id TEXT PRIMARY KEY,
|
|
135
|
+
structural_score REAL,
|
|
136
|
+
runtime_score REAL,
|
|
137
|
+
freshness_score REAL,
|
|
138
|
+
stability_score REAL,
|
|
139
|
+
verification_score REAL,
|
|
140
|
+
consistency_score REAL,
|
|
141
|
+
injection_risk REAL,
|
|
142
|
+
entropy_score REAL,
|
|
143
|
+
cached_at DATETIME NOT NULL
|
|
144
|
+
)
|
|
145
|
+
""",
|
|
146
|
+
"""
|
|
147
|
+
CREATE TABLE IF NOT EXISTS outcome_signals (
|
|
148
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
149
|
+
session_id TEXT NOT NULL,
|
|
150
|
+
request_id TEXT NOT NULL,
|
|
151
|
+
outcome_type TEXT NOT NULL,
|
|
152
|
+
success BOOLEAN NOT NULL,
|
|
153
|
+
score REAL NOT NULL,
|
|
154
|
+
node_ids TEXT NOT NULL,
|
|
155
|
+
recorded_at DATETIME NOT NULL
|
|
156
|
+
)
|
|
157
|
+
""",
|
|
158
|
+
"""
|
|
159
|
+
CREATE TABLE IF NOT EXISTS weight_profiles (
|
|
160
|
+
profile_name TEXT NOT NULL,
|
|
161
|
+
structural_relevance REAL NOT NULL,
|
|
162
|
+
runtime_evidence REAL NOT NULL,
|
|
163
|
+
freshness REAL NOT NULL,
|
|
164
|
+
stability REAL NOT NULL,
|
|
165
|
+
verification REAL NOT NULL,
|
|
166
|
+
consistency REAL NOT NULL,
|
|
167
|
+
injection_risk REAL NOT NULL,
|
|
168
|
+
entropy_contribution REAL NOT NULL,
|
|
169
|
+
updated_at DATETIME NOT NULL,
|
|
170
|
+
PRIMARY KEY (profile_name)
|
|
171
|
+
)
|
|
172
|
+
""",
|
|
173
|
+
"""
|
|
174
|
+
CREATE TABLE IF NOT EXISTS entropy_annotations (
|
|
175
|
+
node_id TEXT PRIMARY KEY,
|
|
176
|
+
file_path TEXT NOT NULL,
|
|
177
|
+
entropy_score REAL NOT NULL,
|
|
178
|
+
signals TEXT NOT NULL,
|
|
179
|
+
reasons TEXT NOT NULL,
|
|
180
|
+
computed_at DATETIME NOT NULL,
|
|
181
|
+
stale BOOLEAN NOT NULL DEFAULT FALSE
|
|
182
|
+
)
|
|
183
|
+
""",
|
|
184
|
+
"""
|
|
185
|
+
CREATE TABLE IF NOT EXISTS git_metadata (
|
|
186
|
+
file_path TEXT PRIMARY KEY,
|
|
187
|
+
last_commit_at DATETIME,
|
|
188
|
+
commit_count_30d INTEGER NOT NULL DEFAULT 0,
|
|
189
|
+
author_count INTEGER NOT NULL DEFAULT 0,
|
|
190
|
+
indexed_at DATETIME NOT NULL
|
|
191
|
+
)
|
|
192
|
+
""",
|
|
193
|
+
"""
|
|
194
|
+
CREATE TABLE IF NOT EXISTS provenance_chain (
|
|
195
|
+
event_id TEXT PRIMARY KEY,
|
|
196
|
+
sequence INTEGER NOT NULL,
|
|
197
|
+
prev_hash TEXT NOT NULL,
|
|
198
|
+
entry_hash TEXT NOT NULL,
|
|
199
|
+
chained_at DATETIME NOT NULL,
|
|
200
|
+
FOREIGN KEY (event_id) REFERENCES provenance_events(event_id)
|
|
201
|
+
)
|
|
202
|
+
""",
|
|
203
|
+
"CREATE UNIQUE INDEX IF NOT EXISTS idx_chain_sequence ON provenance_chain(sequence)",
|
|
204
|
+
"""
|
|
205
|
+
CREATE TABLE IF NOT EXISTS compliance_bundles (
|
|
206
|
+
bundle_id TEXT PRIMARY KEY,
|
|
207
|
+
scope_json TEXT NOT NULL,
|
|
208
|
+
signature TEXT NOT NULL DEFAULT '',
|
|
209
|
+
generated_at DATETIME NOT NULL
|
|
210
|
+
)
|
|
211
|
+
""",
|
|
212
|
+
"""
|
|
213
|
+
CREATE TABLE IF NOT EXISTS key_store (
|
|
214
|
+
key_id TEXT PRIMARY KEY,
|
|
215
|
+
public_key TEXT NOT NULL,
|
|
216
|
+
algorithm TEXT NOT NULL DEFAULT 'Ed25519',
|
|
217
|
+
created_at DATETIME NOT NULL
|
|
218
|
+
)
|
|
219
|
+
""",
|
|
220
|
+
],
|
|
221
|
+
),
|
|
222
|
+
(
|
|
223
|
+
2,
|
|
224
|
+
"source trust registry",
|
|
225
|
+
[
|
|
226
|
+
"""
|
|
227
|
+
CREATE TABLE IF NOT EXISTS sources (
|
|
228
|
+
id TEXT PRIMARY KEY,
|
|
229
|
+
source_type TEXT NOT NULL,
|
|
230
|
+
trust_tier TEXT NOT NULL DEFAULT 'untrusted',
|
|
231
|
+
owner TEXT NOT NULL DEFAULT '',
|
|
232
|
+
region TEXT NOT NULL DEFAULT '',
|
|
233
|
+
data_classification TEXT NOT NULL DEFAULT 'internal',
|
|
234
|
+
compliance_scope TEXT NOT NULL DEFAULT '[]',
|
|
235
|
+
config_json TEXT NOT NULL DEFAULT '{}',
|
|
236
|
+
created_at DATETIME NOT NULL,
|
|
237
|
+
deleted_at DATETIME
|
|
238
|
+
)
|
|
239
|
+
""",
|
|
240
|
+
],
|
|
241
|
+
),
|
|
242
|
+
(
|
|
243
|
+
3,
|
|
244
|
+
"transparent proxy tokens",
|
|
245
|
+
[
|
|
246
|
+
"""
|
|
247
|
+
CREATE TABLE IF NOT EXISTS proxy_tokens (
|
|
248
|
+
key_id TEXT PRIMARY KEY,
|
|
249
|
+
key_hash TEXT NOT NULL UNIQUE,
|
|
250
|
+
project_id TEXT NOT NULL,
|
|
251
|
+
project_name TEXT NOT NULL,
|
|
252
|
+
provider TEXT NOT NULL DEFAULT 'any',
|
|
253
|
+
upstream_key TEXT NOT NULL,
|
|
254
|
+
scopes TEXT NOT NULL DEFAULT '["proxy"]',
|
|
255
|
+
created_at TEXT NOT NULL,
|
|
256
|
+
revoked INTEGER NOT NULL DEFAULT 0
|
|
257
|
+
)
|
|
258
|
+
""",
|
|
259
|
+
],
|
|
260
|
+
),
|
|
261
|
+
(
|
|
262
|
+
4,
|
|
263
|
+
"policy enforcement index for analytics",
|
|
264
|
+
[
|
|
265
|
+
"CREATE INDEX IF NOT EXISTS idx_enforcement_rule ON policy_enforcement_events(rule_name, occurred_at DESC)",
|
|
266
|
+
"CREATE INDEX IF NOT EXISTS idx_enforcement_session ON policy_enforcement_events(session_id, occurred_at DESC)",
|
|
267
|
+
],
|
|
268
|
+
),
|
|
269
|
+
(
|
|
270
|
+
5,
|
|
271
|
+
"lint findings table",
|
|
272
|
+
[
|
|
273
|
+
"""
|
|
274
|
+
CREATE TABLE IF NOT EXISTS lint_findings (
|
|
275
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
276
|
+
ran_at TEXT NOT NULL,
|
|
277
|
+
window_days INTEGER NOT NULL DEFAULT 30,
|
|
278
|
+
category TEXT NOT NULL,
|
|
279
|
+
severity TEXT NOT NULL,
|
|
280
|
+
subject TEXT NOT NULL,
|
|
281
|
+
detail TEXT NOT NULL DEFAULT '',
|
|
282
|
+
suggestion TEXT NOT NULL DEFAULT '',
|
|
283
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
284
|
+
)
|
|
285
|
+
""",
|
|
286
|
+
"CREATE INDEX IF NOT EXISTS idx_lint_ran_at ON lint_findings(ran_at DESC)",
|
|
287
|
+
"CREATE INDEX IF NOT EXISTS idx_lint_severity ON lint_findings(severity, category)",
|
|
288
|
+
],
|
|
289
|
+
),
|
|
290
|
+
(
|
|
291
|
+
6,
|
|
292
|
+
"source enforcement penalties",
|
|
293
|
+
[
|
|
294
|
+
# Track cumulative violation history per source for trust score compounding.
|
|
295
|
+
# penalty_score is a [0,1] float: 0 = clean, 1 = maximum distrust.
|
|
296
|
+
# It decays toward 0 over time and compounds on each new deny event.
|
|
297
|
+
"""
|
|
298
|
+
CREATE TABLE IF NOT EXISTS source_enforcement_penalties (
|
|
299
|
+
source_id TEXT PRIMARY KEY,
|
|
300
|
+
violation_count INTEGER NOT NULL DEFAULT 0,
|
|
301
|
+
penalty_score REAL NOT NULL DEFAULT 0.0,
|
|
302
|
+
last_violation_at DATETIME NOT NULL,
|
|
303
|
+
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
304
|
+
)
|
|
305
|
+
""",
|
|
306
|
+
"CREATE INDEX IF NOT EXISTS idx_sep_updated ON source_enforcement_penalties(updated_at DESC)",
|
|
307
|
+
# Add source_id to enforcement events so blocks can be traced back to sources.
|
|
308
|
+
"ALTER TABLE policy_enforcement_events ADD COLUMN source_id TEXT NOT NULL DEFAULT ''",
|
|
309
|
+
],
|
|
310
|
+
),
|
|
311
|
+
(
|
|
312
|
+
7,
|
|
313
|
+
"sources: rename source_type column to type",
|
|
314
|
+
[
|
|
315
|
+
# Migration 2 created the sources table with column 'source_type', but all
|
|
316
|
+
# registry code uses 'type'. SQLite has no RENAME COLUMN before 3.25, so
|
|
317
|
+
# we recreate the table. COALESCE handles DBs where _migrate() already added
|
|
318
|
+
# a 'type' column alongside the original 'source_type'.
|
|
319
|
+
"""
|
|
320
|
+
CREATE TABLE IF NOT EXISTS sources_v7 (
|
|
321
|
+
id TEXT PRIMARY KEY,
|
|
322
|
+
type TEXT NOT NULL DEFAULT 'unknown',
|
|
323
|
+
trust_tier TEXT NOT NULL DEFAULT 'untrusted',
|
|
324
|
+
owner TEXT NOT NULL DEFAULT '',
|
|
325
|
+
region TEXT NOT NULL DEFAULT '',
|
|
326
|
+
data_classification TEXT NOT NULL DEFAULT '',
|
|
327
|
+
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
328
|
+
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
329
|
+
config TEXT NOT NULL DEFAULT '{}',
|
|
330
|
+
deleted_at DATETIME
|
|
331
|
+
)
|
|
332
|
+
""",
|
|
333
|
+
"""
|
|
334
|
+
INSERT OR IGNORE INTO sources_v7
|
|
335
|
+
(id, type, trust_tier, owner, region, data_classification,
|
|
336
|
+
created_at, updated_at, config, deleted_at)
|
|
337
|
+
SELECT
|
|
338
|
+
id,
|
|
339
|
+
COALESCE(source_type, 'unknown'),
|
|
340
|
+
COALESCE(trust_tier, 'untrusted'),
|
|
341
|
+
COALESCE(owner, ''),
|
|
342
|
+
COALESCE(region, ''),
|
|
343
|
+
COALESCE(data_classification, ''),
|
|
344
|
+
COALESCE(created_at, CURRENT_TIMESTAMP),
|
|
345
|
+
CURRENT_TIMESTAMP,
|
|
346
|
+
COALESCE(config_json, '{}'),
|
|
347
|
+
deleted_at
|
|
348
|
+
FROM sources
|
|
349
|
+
""",
|
|
350
|
+
"DROP TABLE sources",
|
|
351
|
+
"ALTER TABLE sources_v7 RENAME TO sources",
|
|
352
|
+
"CREATE INDEX IF NOT EXISTS idx_sources_tier ON sources(trust_tier) WHERE deleted_at IS NULL",
|
|
353
|
+
],
|
|
354
|
+
),
|
|
355
|
+
]
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
# ── Runner ─────────────────────────────────────────────────────────────────────
|
|
359
|
+
|
|
360
|
+
async def run_migrations(db_path: str) -> None:
|
|
361
|
+
async with aiosqlite.connect(db_path) as db:
|
|
362
|
+
db.row_factory = aiosqlite.Row
|
|
363
|
+
await db.execute("PRAGMA journal_mode=WAL")
|
|
364
|
+
await db.execute("PRAGMA foreign_keys=ON")
|
|
365
|
+
|
|
366
|
+
# Bootstrap the version tracking table
|
|
367
|
+
await db.execute("""
|
|
368
|
+
CREATE TABLE IF NOT EXISTS schema_version (
|
|
369
|
+
version INTEGER NOT NULL,
|
|
370
|
+
description TEXT NOT NULL,
|
|
371
|
+
applied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
372
|
+
)
|
|
373
|
+
""")
|
|
374
|
+
await db.commit()
|
|
375
|
+
|
|
376
|
+
async with db.execute("SELECT MAX(version) AS v FROM schema_version") as cur:
|
|
377
|
+
row = await cur.fetchone()
|
|
378
|
+
current = row["v"] if row["v"] is not None else 0
|
|
379
|
+
|
|
380
|
+
pending = [(v, desc, stmts) for v, desc, stmts in _MIGRATIONS if v > current]
|
|
381
|
+
if not pending:
|
|
382
|
+
logger.debug("schema up to date at version %d", current)
|
|
383
|
+
return
|
|
384
|
+
|
|
385
|
+
for version, description, statements in pending:
|
|
386
|
+
logger.info("applying migration %d: %s", version, description)
|
|
387
|
+
try:
|
|
388
|
+
for stmt in statements:
|
|
389
|
+
await db.execute(stmt)
|
|
390
|
+
await db.execute(
|
|
391
|
+
"INSERT INTO schema_version (version, description) VALUES (?, ?)",
|
|
392
|
+
(version, description),
|
|
393
|
+
)
|
|
394
|
+
await db.commit()
|
|
395
|
+
logger.info("migration %d applied", version)
|
|
396
|
+
except Exception as e:
|
|
397
|
+
await db.rollback()
|
|
398
|
+
logger.error(
|
|
399
|
+
"migration %d failed - rolling back: %s", version, e
|
|
400
|
+
)
|
|
401
|
+
raise RuntimeError(f"Migration {version} ({description}) failed: {e}") from e
|
|
402
|
+
|
|
403
|
+
logger.info(
|
|
404
|
+
"migrations complete: %d → %d (%d applied)",
|
|
405
|
+
current,
|
|
406
|
+
pending[-1][0],
|
|
407
|
+
len(pending),
|
|
408
|
+
)
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""Context Entropy Engine - computes entropy annotations offline."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import logging
|
|
8
|
+
import re
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import aiosqlite
|
|
13
|
+
|
|
14
|
+
from context_firewall.config import Config
|
|
15
|
+
from context_firewall.models import EntropyAnnotation, SubsystemHealth
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
# Heuristics: these patterns signal expected divergence, not entropy
|
|
20
|
+
_SKIP_PATTERNS = [
|
|
21
|
+
re.compile(r"//\s*cre:skip"),
|
|
22
|
+
re.compile(r"#\s*cre:skip"),
|
|
23
|
+
re.compile(r"(?i)legacy"),
|
|
24
|
+
re.compile(r"(?i)feature.?flag"),
|
|
25
|
+
re.compile(r"(?i)retry"),
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
_NAMING_INCONSISTENCY_PAIRS = [
|
|
29
|
+
(re.compile(r"\bget\w+\b"), re.compile(r"\bfetch\w+\b")),
|
|
30
|
+
(re.compile(r"\bcreate\w+\b"), re.compile(r"\bnew\w+\b")),
|
|
31
|
+
(re.compile(r"\bdelete\w+\b"), re.compile(r"\bremove\w+\b")),
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _has_skip_annotation(content: str) -> bool:
|
|
36
|
+
return any(p.search(content) for p in _SKIP_PATTERNS)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _compute_naming_inconsistency(content: str) -> float:
|
|
40
|
+
"""Detect mixed naming conventions (get/fetch, create/new, delete/remove)."""
|
|
41
|
+
score = 0.0
|
|
42
|
+
for pattern_a, pattern_b in _NAMING_INCONSISTENCY_PAIRS:
|
|
43
|
+
has_a = bool(pattern_a.search(content))
|
|
44
|
+
has_b = bool(pattern_b.search(content))
|
|
45
|
+
if has_a and has_b:
|
|
46
|
+
score += 0.33
|
|
47
|
+
return min(1.0, score)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _compute_dead_code_references(content: str) -> float:
|
|
51
|
+
"""Proxy: deprecated/todo/fixme markers as dead code signal."""
|
|
52
|
+
markers = re.findall(r"(?i)(deprecated|todo|fixme|hack|xxx|dead.?code)", content)
|
|
53
|
+
return min(1.0, len(markers) * 0.15)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _compute_contradictory_documentation(content: str) -> float:
|
|
57
|
+
"""Structural contradiction: docstring describes opposite of what code does."""
|
|
58
|
+
doc_lines = re.findall(r'""".*?"""', content, re.DOTALL)
|
|
59
|
+
negations = 0
|
|
60
|
+
for doc in doc_lines:
|
|
61
|
+
negations += len(re.findall(r"\b(not|never|no longer|deprecated|do not|don\'t)\b", doc, re.I))
|
|
62
|
+
return min(1.0, negations * 0.2)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _compute_semantic_duplication(content_a: str, content_b: str) -> float:
|
|
66
|
+
"""Jaccard similarity on significant token sets between two files."""
|
|
67
|
+
tokens_a = set(t for t in re.findall(r"[a-zA-Z_]\w{3,}", content_a.lower()) if len(t) > 4)
|
|
68
|
+
tokens_b = set(t for t in re.findall(r"[a-zA-Z_]\w{3,}", content_b.lower()) if len(t) > 4)
|
|
69
|
+
if not tokens_a or not tokens_b:
|
|
70
|
+
return 0.0
|
|
71
|
+
intersection = tokens_a & tokens_b
|
|
72
|
+
union = tokens_a | tokens_b
|
|
73
|
+
return len(intersection) / len(union)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _compute_divergent_execution_paths(content: str) -> float:
|
|
77
|
+
"""Estimate branching complexity from structural patterns."""
|
|
78
|
+
lines = content.split("\n")
|
|
79
|
+
total = max(1, len(lines))
|
|
80
|
+
# Count branching constructs normalized to file length
|
|
81
|
+
branches = 0
|
|
82
|
+
for line in lines:
|
|
83
|
+
stripped = line.lstrip()
|
|
84
|
+
if re.match(r"(if |elif |else:|except |except:|finally:|case )", stripped):
|
|
85
|
+
branches += 1
|
|
86
|
+
elif re.match(r"(for |while )", stripped):
|
|
87
|
+
branches += 0.5
|
|
88
|
+
# Also penalize high cyclomatic density: > 1 branch per 8 lines is high
|
|
89
|
+
density = branches / (total / 8)
|
|
90
|
+
return min(1.0, density * 0.4)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _compute_runtime_inconsistency(content: str) -> float:
|
|
94
|
+
"""Detect non-deterministic sources: random, time, env vars, network."""
|
|
95
|
+
patterns = [
|
|
96
|
+
r"\brandom\b", r"\brandint\b", r"\brandrange\b", r"\buuid\b",
|
|
97
|
+
r"\bdatetime\.now\b", r"\btime\.time\b", r"\btime\.sleep\b",
|
|
98
|
+
r"\bos\.environ\b", r"\bgetenv\b",
|
|
99
|
+
r"\bsocket\.", r"\brequests\.", r"\bhttpx\.", r"\baiohttp\.",
|
|
100
|
+
r"\bopen\(.*[rwa]", r"\bsubprocess\.",
|
|
101
|
+
]
|
|
102
|
+
hits = sum(1 for p in patterns if re.search(p, content, re.I))
|
|
103
|
+
return min(1.0, hits * 0.12)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _aggregate_entropy(signals: dict[str, float]) -> float:
|
|
107
|
+
weights = {
|
|
108
|
+
"semantic_duplication": 0.25,
|
|
109
|
+
"divergent_execution_paths": 0.20,
|
|
110
|
+
"contradictory_documentation": 0.20,
|
|
111
|
+
"dead_code_references": 0.15,
|
|
112
|
+
"naming_inconsistency": 0.10,
|
|
113
|
+
"runtime_inconsistency": 0.10,
|
|
114
|
+
}
|
|
115
|
+
total = sum(weights.get(k, 0.0) * v for k, v in signals.items())
|
|
116
|
+
return max(0.0, min(1.0, total))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class ContextEntropyEngine:
|
|
120
|
+
name = "context_entropy_engine"
|
|
121
|
+
critical = False
|
|
122
|
+
|
|
123
|
+
def __init__(self) -> None:
|
|
124
|
+
self._config: Config | None = None
|
|
125
|
+
self._db: aiosqlite.Connection | None = None
|
|
126
|
+
|
|
127
|
+
async def init(self, config: Config) -> None:
|
|
128
|
+
self._config = config
|
|
129
|
+
from context_firewall.db.connection import get_db
|
|
130
|
+
self._db = await get_db()
|
|
131
|
+
logger.info("ContextEntropyEngine initialized")
|
|
132
|
+
|
|
133
|
+
def health_check(self) -> SubsystemHealth:
|
|
134
|
+
return SubsystemHealth(name=self.name, healthy=True)
|
|
135
|
+
|
|
136
|
+
async def shutdown(self) -> None:
|
|
137
|
+
self._db = None
|
|
138
|
+
|
|
139
|
+
async def get_entropy(self, node_id: str) -> float:
|
|
140
|
+
"""Hot-path: O(1) lookup from pre-computed annotations."""
|
|
141
|
+
if self._db is None:
|
|
142
|
+
return self._config.entropy.default_score if self._config else 0.30
|
|
143
|
+
try:
|
|
144
|
+
async with self._db.execute(
|
|
145
|
+
"SELECT entropy_score, stale FROM entropy_annotations WHERE node_id = ?",
|
|
146
|
+
(node_id,),
|
|
147
|
+
) as cursor:
|
|
148
|
+
row = await cursor.fetchone()
|
|
149
|
+
if row is None:
|
|
150
|
+
return self._config.entropy.default_score if self._config else 0.30
|
|
151
|
+
score = float(row["entropy_score"])
|
|
152
|
+
if row["stale"]:
|
|
153
|
+
score = min(1.0, score + 0.15)
|
|
154
|
+
return score
|
|
155
|
+
except Exception:
|
|
156
|
+
logger.warning("entropy score lookup failed for node %s", node_id, exc_info=True)
|
|
157
|
+
return 0.30
|
|
158
|
+
|
|
159
|
+
async def run_analysis(self, repository_root: str) -> int:
|
|
160
|
+
"""Offline: compute entropy for all files in repository_root."""
|
|
161
|
+
root = Path(repository_root)
|
|
162
|
+
annotated = 0
|
|
163
|
+
exts = ("*.py", "*.ts", "*.go", "*.tsx", "*.js")
|
|
164
|
+
files = []
|
|
165
|
+
for ext in exts:
|
|
166
|
+
files.extend(root.rglob(ext))
|
|
167
|
+
|
|
168
|
+
# Load contents once so we can do cross-file duplication comparisons
|
|
169
|
+
contents: list[tuple[str, str]] = []
|
|
170
|
+
for file_path in files:
|
|
171
|
+
try:
|
|
172
|
+
content = file_path.read_text(errors="ignore")
|
|
173
|
+
rel = str(file_path.relative_to(root))
|
|
174
|
+
contents.append((rel, content))
|
|
175
|
+
except Exception as e:
|
|
176
|
+
logger.debug("could not read %s: %s", file_path, e)
|
|
177
|
+
|
|
178
|
+
for i, (rel_path, content) in enumerate(contents):
|
|
179
|
+
try:
|
|
180
|
+
# Use adjacent file for duplication comparison (avoids O(N^2) over all files)
|
|
181
|
+
sibling_content = contents[i - 1][1] if i > 0 else ""
|
|
182
|
+
annotation = await self._analyze_file(rel_path, content, sibling_content)
|
|
183
|
+
await self._persist(annotation)
|
|
184
|
+
annotated += 1
|
|
185
|
+
except Exception as e:
|
|
186
|
+
logger.debug("entropy analysis skipped", extra={"file": rel_path, "error": str(e)})
|
|
187
|
+
logger.info("entropy analysis complete", extra={"annotated": annotated})
|
|
188
|
+
return annotated
|
|
189
|
+
|
|
190
|
+
async def _analyze_file(
|
|
191
|
+
self, file_path: str, content: str, sibling_content: str = ""
|
|
192
|
+
) -> EntropyAnnotation:
|
|
193
|
+
if _has_skip_annotation(content):
|
|
194
|
+
signals = {k: 0.0 for k in [
|
|
195
|
+
"semantic_duplication", "divergent_execution_paths",
|
|
196
|
+
"contradictory_documentation", "dead_code_references",
|
|
197
|
+
"naming_inconsistency", "runtime_inconsistency",
|
|
198
|
+
]}
|
|
199
|
+
return EntropyAnnotation(
|
|
200
|
+
node_id=file_path,
|
|
201
|
+
file_path=file_path,
|
|
202
|
+
entropy_score=0.0,
|
|
203
|
+
signals=signals,
|
|
204
|
+
reasons=["cre:skip annotation present"],
|
|
205
|
+
computed_at=datetime.now(timezone.utc),
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
signals: dict[str, float] = {}
|
|
209
|
+
reasons: list[str] = []
|
|
210
|
+
|
|
211
|
+
signals["naming_inconsistency"] = _compute_naming_inconsistency(content)
|
|
212
|
+
if signals["naming_inconsistency"] > 0.3:
|
|
213
|
+
reasons.append("mixed naming conventions detected")
|
|
214
|
+
|
|
215
|
+
signals["dead_code_references"] = _compute_dead_code_references(content)
|
|
216
|
+
if signals["dead_code_references"] > 0.3:
|
|
217
|
+
reasons.append("deprecated/todo markers found")
|
|
218
|
+
|
|
219
|
+
signals["contradictory_documentation"] = _compute_contradictory_documentation(content)
|
|
220
|
+
if signals["contradictory_documentation"] > 0.3:
|
|
221
|
+
reasons.append("potential doc/code contradiction")
|
|
222
|
+
|
|
223
|
+
signals["semantic_duplication"] = (
|
|
224
|
+
_compute_semantic_duplication(content, sibling_content) if sibling_content else 0.0
|
|
225
|
+
)
|
|
226
|
+
if signals["semantic_duplication"] > 0.55:
|
|
227
|
+
reasons.append("high token overlap with adjacent file")
|
|
228
|
+
|
|
229
|
+
signals["divergent_execution_paths"] = _compute_divergent_execution_paths(content)
|
|
230
|
+
if signals["divergent_execution_paths"] > 0.5:
|
|
231
|
+
reasons.append("high branching complexity")
|
|
232
|
+
|
|
233
|
+
signals["runtime_inconsistency"] = _compute_runtime_inconsistency(content)
|
|
234
|
+
if signals["runtime_inconsistency"] > 0.4:
|
|
235
|
+
reasons.append("non-deterministic runtime sources detected")
|
|
236
|
+
|
|
237
|
+
entropy_score = _aggregate_entropy(signals)
|
|
238
|
+
return EntropyAnnotation(
|
|
239
|
+
node_id=file_path,
|
|
240
|
+
file_path=file_path,
|
|
241
|
+
entropy_score=entropy_score,
|
|
242
|
+
signals=signals,
|
|
243
|
+
reasons=reasons,
|
|
244
|
+
computed_at=datetime.now(timezone.utc),
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
async def _persist(self, annotation: EntropyAnnotation) -> None:
|
|
248
|
+
if self._db is None:
|
|
249
|
+
return
|
|
250
|
+
await self._db.execute(
|
|
251
|
+
"""
|
|
252
|
+
INSERT INTO entropy_annotations
|
|
253
|
+
(node_id, file_path, entropy_score, signals, reasons, computed_at, stale)
|
|
254
|
+
VALUES (?, ?, ?, ?, ?, ?, FALSE)
|
|
255
|
+
ON CONFLICT(node_id) DO UPDATE SET
|
|
256
|
+
entropy_score = excluded.entropy_score,
|
|
257
|
+
signals = excluded.signals,
|
|
258
|
+
reasons = excluded.reasons,
|
|
259
|
+
computed_at = excluded.computed_at,
|
|
260
|
+
stale = FALSE
|
|
261
|
+
""",
|
|
262
|
+
(
|
|
263
|
+
annotation.node_id,
|
|
264
|
+
annotation.file_path,
|
|
265
|
+
annotation.entropy_score,
|
|
266
|
+
json.dumps(annotation.signals),
|
|
267
|
+
json.dumps(annotation.reasons),
|
|
268
|
+
annotation.computed_at.isoformat(),
|
|
269
|
+
),
|
|
270
|
+
)
|
|
271
|
+
await self._db.commit()
|