codegraph-engine 2.2.1__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codegraph/__init__.py +1 -1
- codegraph/agent_brain.py +172 -2
- codegraph/agent_capabilities.py +564 -6
- codegraph/agent_rules.py +1 -1
- codegraph/api_drift.py +455 -0
- codegraph/async_queue/__init__.py +50 -0
- codegraph/async_queue/detectors/__init__.py +48 -0
- codegraph/async_queue/detectors/celery.py +160 -0
- codegraph/async_queue/detectors/kafka.py +168 -0
- codegraph/async_queue/detectors/rabbitmq.py +116 -0
- codegraph/async_queue/detectors/redis.py +130 -0
- codegraph/async_queue/detectors/sqs.py +135 -0
- codegraph/async_queue/models.py +161 -0
- codegraph/async_queue/otel_models.py +74 -0
- codegraph/async_queue/otel_parser.py +337 -0
- codegraph/async_queue/reconciler.py +427 -0
- codegraph/async_queue/scanner.py +291 -0
- codegraph/async_queue/traversal.py +447 -0
- codegraph/branch_comparison.py +197 -0
- codegraph/change_impact.py +547 -0
- codegraph/cli.py +992 -3
- codegraph/cli_output.py +17 -2
- codegraph/context.py +38 -11
- codegraph/context_freshness.py +351 -0
- codegraph/database/__init__.py +14 -0
- codegraph/database/extractor.py +712 -17
- codegraph/database/interrogation.py +388 -2
- codegraph/database/schema_drift.py +862 -0
- codegraph/epistemic.py +7 -0
- codegraph/evidence_contract.py +21 -0
- codegraph/frameworks.py +171 -6
- codegraph/frameworks_nestjs.py +502 -0
- codegraph/frameworks_nextjs.py +337 -0
- codegraph/frameworks_react.py +457 -0
- codegraph/freshness.py +2 -0
- codegraph/frontend_html.py +109 -0
- codegraph/git.py +2 -0
- codegraph/git_state.py +545 -0
- codegraph/indexing/__init__.py +7 -1
- codegraph/indexing/indexer.py +291 -38
- codegraph/indexing/models.py +6 -0
- codegraph/indexing/parser.py +239 -0
- codegraph/indexing/pool.py +136 -0
- codegraph/indexing/scanner.py +6 -0
- codegraph/indexing/templates.py +418 -0
- codegraph/interrogation.py +128 -20
- codegraph/mcp/server.py +258 -27
- codegraph/models.py +7 -0
- codegraph/monorepo.py +3 -0
- codegraph/optimizer.py +42 -5
- codegraph/process_lifecycle.py +31 -13
- codegraph/refactor/__init__.py +36 -0
- codegraph/refactor/models.py +114 -0
- codegraph/refactor/renamer.py +844 -0
- codegraph/refactor/transactions.py +403 -0
- codegraph/resolver.py +1202 -159
- codegraph/retrieval_policy.py +6 -0
- codegraph/route_topology.py +172 -0
- codegraph/runtime/__init__.py +19 -0
- codegraph/runtime/collector.py +155 -0
- codegraph/runtime/db_watcher.py +191 -0
- codegraph/runtime/ingestor.py +30 -10
- codegraph/runtime/port_inspector.py +304 -0
- codegraph/runtime/runner.py +155 -6
- codegraph/runtime/trace_linker.py +175 -0
- codegraph/search/hybrid.py +26 -0
- codegraph/semantic_conflicts.py +270 -0
- codegraph/semantic_decorators.py +34 -3
- codegraph/structural_diff.py +542 -0
- codegraph/styles.py +144 -0
- codegraph/symbol_history.py +277 -0
- codegraph/tsconfig.py +236 -0
- codegraph/ui/__init__.py +6 -0
- codegraph/ui/server.py +715 -0
- codegraph/watcher.py +323 -0
- {codegraph_engine-2.2.1.dist-info → codegraph_engine-3.0.0.dist-info}/METADATA +257 -39
- codegraph_engine-3.0.0.dist-info/RECORD +129 -0
- codegraph_engine-2.2.1.dist-info/RECORD +0 -87
- {codegraph_engine-2.2.1.dist-info → codegraph_engine-3.0.0.dist-info}/WHEEL +0 -0
- {codegraph_engine-2.2.1.dist-info → codegraph_engine-3.0.0.dist-info}/entry_points.txt +0 -0
- {codegraph_engine-2.2.1.dist-info → codegraph_engine-3.0.0.dist-info}/licenses/LICENSE +0 -0
- {codegraph_engine-2.2.1.dist-info → codegraph_engine-3.0.0.dist-info}/top_level.txt +0 -0
codegraph/retrieval_policy.py
CHANGED
|
@@ -84,6 +84,8 @@ _register(RetrievalPolicy(
|
|
|
84
84
|
"CALLS", "DISPATCHES_TO", "RESOLVES_DEPENDENCY", "INJECTS", "PROVIDES",
|
|
85
85
|
"DEPENDS_ON", "IMPORTS", "TESTS", "POSSIBLE_CALLS",
|
|
86
86
|
"DEFINES", "CONTAINS", "PARALLEL_IMPLEMENTATION",
|
|
87
|
+
"DISPATCHES_TASK", "TRIGGERS_SIGNAL", "HANDLES_SIGNAL", "RENDERS_TEMPLATE",
|
|
88
|
+
"DISPATCHES_FORWARD", "TOOL_HANDLER", "PIPELINE_STEP",
|
|
87
89
|
}),
|
|
88
90
|
preferred_flow=("ENTRYPOINT", "HANDLER", "SERVICE", "DATA", "TEST"),
|
|
89
91
|
include_callers=True,
|
|
@@ -104,6 +106,8 @@ _register(RetrievalPolicy(
|
|
|
104
106
|
"POSSIBLE_CALLS", "EXTENDS", "IMPLEMENTS", "PARALLEL_IMPLEMENTATION",
|
|
105
107
|
"MOUNTS", "HANDLED_BY", "ROUTE_HANDLER",
|
|
106
108
|
"REGISTERS", "DISPATCHES_TO", "EVENT_LISTENER", "TASK_HANDLER", "COMMAND_HANDLER",
|
|
109
|
+
"DISPATCHES_TASK", "TRIGGERS_SIGNAL", "HANDLES_SIGNAL", "RENDERS_TEMPLATE",
|
|
110
|
+
"DISPATCHES_FORWARD", "TOOL_HANDLER", "PIPELINE_STEP",
|
|
107
111
|
"INJECTS", "PROVIDES", "RESOLVES_DEPENDENCY", "CONFIGURES",
|
|
108
112
|
"DEPENDS_ON_PACKAGE", "TESTS",
|
|
109
113
|
}),
|
|
@@ -126,6 +130,8 @@ _register(RetrievalPolicy(
|
|
|
126
130
|
"POSSIBLE_CALLS", "DEFINES", "PARALLEL_IMPLEMENTATION",
|
|
127
131
|
"HANDLED_BY", "MOUNTS", "ROUTE_HANDLER",
|
|
128
132
|
"REGISTERS", "DISPATCHES_TO", "EVENT_LISTENER", "TASK_HANDLER", "COMMAND_HANDLER",
|
|
133
|
+
"DISPATCHES_TASK", "TRIGGERS_SIGNAL", "HANDLES_SIGNAL", "RENDERS_TEMPLATE",
|
|
134
|
+
"DISPATCHES_FORWARD", "TOOL_HANDLER", "PIPELINE_STEP",
|
|
129
135
|
"INJECTS", "PROVIDES", "RESOLVES_DEPENDENCY", "CONFIGURES",
|
|
130
136
|
"DEPENDS_ON_PACKAGE",
|
|
131
137
|
}),
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""Universal Parameterized Route Topology and Cross-Language Route Matching.
|
|
2
|
+
|
|
3
|
+
Bridges frontend fetch/axios/SWR/React-Query client calls with backend route declarations
|
|
4
|
+
across disparate frameworks (React, Next.js, Express, NestJS, Flask, FastAPI, Django).
|
|
5
|
+
|
|
6
|
+
Supported Syntax Conventions:
|
|
7
|
+
- JavaScript/TypeScript: `${param}`, `:param`, `[param]`, `[...slug]`
|
|
8
|
+
- Python Flask: `<param>`, `<int:param>`, `<path:param>`, `<uuid:param>`
|
|
9
|
+
- Python FastAPI: `{param}`
|
|
10
|
+
- Express/NestJS: `:param`
|
|
11
|
+
- Django: `<int:param>`, `<str:param>`, `<slug:param>`
|
|
12
|
+
- Literal client instances: `/users/42/orders` -> matches `/users/:id/orders`
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
_PARAM_SEGMENT_TOKEN = "{*}"
|
|
21
|
+
|
|
22
|
+
# Common API Gateway / Reverse Proxy prefixes used in frontend apps
|
|
23
|
+
_KNOWN_PROXY_PREFIXES = (
|
|
24
|
+
("api", "v1"),
|
|
25
|
+
("api", "v2"),
|
|
26
|
+
("api",),
|
|
27
|
+
("v1",),
|
|
28
|
+
("v2",),
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# Regex to detect parameter segments across all framework dialects
|
|
32
|
+
_IS_PARAM_SEGMENT = re.compile(
|
|
33
|
+
r"""^(?:
|
|
34
|
+
\$\{[^}]+\} | # JS/TS template literal: ${userId}
|
|
35
|
+
:[a-zA-Z0-9_$]+ | # Express / NestJS: :userId
|
|
36
|
+
\{[a-zA-Z0-9_$]+\} | # FastAPI / Next.js: {userId}
|
|
37
|
+
\[[a-zA-Z0-9_$]+\] | # Next.js: [userId]
|
|
38
|
+
\[\.\.\.[a-zA-Z0-9_$]+\] | # Next.js catch-all: [...slug]
|
|
39
|
+
<[a-zA-Z0-9_:]+> | # Flask / Django: <int:user_id>, <user_id>
|
|
40
|
+
\d+ | # Literal integer in client URL: 42, 101
|
|
41
|
+
[0-9a-fA-F-]{36} # Literal UUID in client URL
|
|
42
|
+
)$""",
|
|
43
|
+
re.VERBOSE,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class NormalizedRoute:
|
|
49
|
+
method: str
|
|
50
|
+
raw_path: str
|
|
51
|
+
segments: tuple[str, ...]
|
|
52
|
+
proxy_stripped_segments: tuple[str, ...]
|
|
53
|
+
is_parameterized: bool
|
|
54
|
+
|
|
55
|
+
def as_dict(self) -> dict[str, Any]:
|
|
56
|
+
return {
|
|
57
|
+
"method": self.method,
|
|
58
|
+
"raw_path": self.raw_path,
|
|
59
|
+
"segments": list(self.segments),
|
|
60
|
+
"proxy_stripped_segments": list(self.proxy_stripped_segments),
|
|
61
|
+
"is_parameterized": self.is_parameterized,
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def normalize_route(method: str, path: str) -> NormalizedRoute:
|
|
66
|
+
"""Normalize a raw route into a canonical topological representation.
|
|
67
|
+
|
|
68
|
+
Strips query strings, hashes, leading/trailing whitespace and slashes,
|
|
69
|
+
normalizes all parameter styles into '{*}', and computes proxy-stripped segments.
|
|
70
|
+
"""
|
|
71
|
+
clean_method = (method or "GET").strip().upper()
|
|
72
|
+
# Strip query parameters (?foo=bar) and URL fragments (#anchor)
|
|
73
|
+
clean_path = path.strip().split("?")[0].split("#")[0]
|
|
74
|
+
# Remove leading and trailing slashes
|
|
75
|
+
trimmed = clean_path.strip("/")
|
|
76
|
+
|
|
77
|
+
if not trimmed:
|
|
78
|
+
return NormalizedRoute(
|
|
79
|
+
method=clean_method,
|
|
80
|
+
raw_path=path,
|
|
81
|
+
segments=("/",),
|
|
82
|
+
proxy_stripped_segments=("/",),
|
|
83
|
+
is_parameterized=False,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
raw_parts = [p for p in trimmed.split("/") if p]
|
|
87
|
+
normalized_parts: list[str] = []
|
|
88
|
+
is_param = False
|
|
89
|
+
|
|
90
|
+
for part in raw_parts:
|
|
91
|
+
if _IS_PARAM_SEGMENT.match(part):
|
|
92
|
+
normalized_parts.append(_PARAM_SEGMENT_TOKEN)
|
|
93
|
+
is_param = True
|
|
94
|
+
elif "${" in part:
|
|
95
|
+
# Embedded template interpolation: e.g. "user-${id}"
|
|
96
|
+
normalized_parts.append(_PARAM_SEGMENT_TOKEN)
|
|
97
|
+
is_param = True
|
|
98
|
+
else:
|
|
99
|
+
normalized_parts.append(part.lower())
|
|
100
|
+
|
|
101
|
+
segments_tuple = tuple(normalized_parts)
|
|
102
|
+
|
|
103
|
+
# Compute proxy-stripped segments
|
|
104
|
+
stripped_parts = list(segments_tuple)
|
|
105
|
+
for prefix in _KNOWN_PROXY_PREFIXES:
|
|
106
|
+
if len(stripped_parts) > len(prefix) and tuple(stripped_parts[: len(prefix)]) == prefix:
|
|
107
|
+
stripped_parts = stripped_parts[len(prefix) :]
|
|
108
|
+
break
|
|
109
|
+
|
|
110
|
+
proxy_stripped_tuple = tuple(stripped_parts) if stripped_parts else ("/",)
|
|
111
|
+
|
|
112
|
+
return NormalizedRoute(
|
|
113
|
+
method=clean_method,
|
|
114
|
+
raw_path=path,
|
|
115
|
+
segments=segments_tuple,
|
|
116
|
+
proxy_stripped_segments=proxy_stripped_tuple,
|
|
117
|
+
is_parameterized=is_param,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def match_route_topology(
|
|
122
|
+
client_route: NormalizedRoute,
|
|
123
|
+
backend_route: NormalizedRoute,
|
|
124
|
+
allow_proxy_prefix: bool = True,
|
|
125
|
+
match_methods: bool = True,
|
|
126
|
+
) -> tuple[bool, str]:
|
|
127
|
+
"""Deterministically match a client-side route against a backend route definition.
|
|
128
|
+
|
|
129
|
+
Returns (matched, match_type):
|
|
130
|
+
- "EXACT": Segments match exactly with parameter wildcards.
|
|
131
|
+
- "PROXY_PREFIX": Client calls with /api or /v1 prefix matching backend root route.
|
|
132
|
+
- "BACKEND_MOUNTED": Backend route has prefix (e.g. mounted blueprint) matching client.
|
|
133
|
+
- "": No match.
|
|
134
|
+
"""
|
|
135
|
+
# 1. Check HTTP method compatibility
|
|
136
|
+
if match_methods and client_route.method and backend_route.method:
|
|
137
|
+
client_m = client_route.method
|
|
138
|
+
backend_m = backend_route.method
|
|
139
|
+
if client_m != "*" and backend_m != "*" and client_m != backend_m:
|
|
140
|
+
return False, ""
|
|
141
|
+
|
|
142
|
+
# 2. Exact topological match
|
|
143
|
+
if client_route.segments == backend_route.segments:
|
|
144
|
+
return True, "EXACT"
|
|
145
|
+
|
|
146
|
+
# 3. Proxy prefix match (e.g., Client: ['api', 'orders'] vs Backend: ['orders'])
|
|
147
|
+
if allow_proxy_prefix:
|
|
148
|
+
if client_route.proxy_stripped_segments == backend_route.segments:
|
|
149
|
+
return True, "PROXY_PREFIX"
|
|
150
|
+
|
|
151
|
+
# Reverse proxy prefix: Backend declared /api/orders, client called /orders (or vice-versa)
|
|
152
|
+
if client_route.segments == backend_route.proxy_stripped_segments:
|
|
153
|
+
return True, "PROXY_PREFIX"
|
|
154
|
+
|
|
155
|
+
if client_route.proxy_stripped_segments == backend_route.proxy_stripped_segments:
|
|
156
|
+
if client_route.proxy_stripped_segments != ("/",):
|
|
157
|
+
return True, "PROXY_PREFIX"
|
|
158
|
+
|
|
159
|
+
# 4. Suffix / sub-path mount match (e.g. Blueprint mounted at /api/shop and client calls /api/shop/checkout)
|
|
160
|
+
# Check if backend route is a suffix of client route or vice versa with identical parameter topology
|
|
161
|
+
c_segs = client_route.proxy_stripped_segments
|
|
162
|
+
b_segs = backend_route.proxy_stripped_segments
|
|
163
|
+
if len(c_segs) == len(b_segs):
|
|
164
|
+
matched = True
|
|
165
|
+
for cs, bs in zip(c_segs, b_segs, strict=True):
|
|
166
|
+
if cs != bs and cs != _PARAM_SEGMENT_TOKEN and bs != _PARAM_SEGMENT_TOKEN:
|
|
167
|
+
matched = False
|
|
168
|
+
break
|
|
169
|
+
if matched:
|
|
170
|
+
return True, "TOPOLOGICAL_PARAM_MATCH"
|
|
171
|
+
|
|
172
|
+
return False, ""
|
codegraph/runtime/__init__.py
CHANGED
|
@@ -23,6 +23,16 @@ __all__ = [
|
|
|
23
23
|
"ReconciliationStatus",
|
|
24
24
|
"RuntimeAggregatedEdge",
|
|
25
25
|
"RuntimeEvent",
|
|
26
|
+
"ServiceEndpoint",
|
|
27
|
+
"discover_live_listening_ports",
|
|
28
|
+
"discover_static_configured_ports",
|
|
29
|
+
"resolve_port_to_service",
|
|
30
|
+
"SchemaDriftItem",
|
|
31
|
+
"SchemaDriftReport",
|
|
32
|
+
"detect_schema_drift",
|
|
33
|
+
"RuntimeCollectorDaemon",
|
|
34
|
+
"DistributedTraceReport",
|
|
35
|
+
"link_distributed_trace",
|
|
26
36
|
"get_runtime_trace",
|
|
27
37
|
"ingest_runtime_traces",
|
|
28
38
|
"parse_json_runtime_events",
|
|
@@ -30,3 +40,12 @@ __all__ = [
|
|
|
30
40
|
"parse_sql_query_logs",
|
|
31
41
|
"reconcile_static_runtime",
|
|
32
42
|
]
|
|
43
|
+
from .collector import RuntimeCollectorDaemon
|
|
44
|
+
from .db_watcher import SchemaDriftItem, SchemaDriftReport, detect_schema_drift
|
|
45
|
+
from .port_inspector import (
|
|
46
|
+
ServiceEndpoint,
|
|
47
|
+
discover_live_listening_ports,
|
|
48
|
+
discover_static_configured_ports,
|
|
49
|
+
resolve_port_to_service,
|
|
50
|
+
)
|
|
51
|
+
from .trace_linker import DistributedTraceReport, link_distributed_trace
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Zero-Heat, Push-Based OpenTelemetry (OTLP) & Dev Telemetry Collector.
|
|
2
|
+
|
|
3
|
+
Listens for OTLP HTTP traces from localhost dev servers:
|
|
4
|
+
- POST /v1/traces (Standard OpenTelemetry protobuf-in-JSON or OTLP JSON)
|
|
5
|
+
- POST /api/telemetry (Lightweight JSON / JSONL dev events)
|
|
6
|
+
- GET /healthz (Health probe)
|
|
7
|
+
|
|
8
|
+
Efficiency & Anti-Heat Invariants:
|
|
9
|
+
1. Purely event-driven socket select; 0.0% CPU when no requests are being processed.
|
|
10
|
+
2. In-memory bounded queue with ring-buffer semantics (capped at max_events=1,000) to prevent memory ballooning.
|
|
11
|
+
3. Strict secrecy redaction: Authorization, Cookie, and passwords stripped before SQLite persistence.
|
|
12
|
+
4. Graceful shutdown: Closes sockets cleanly without leaking ports.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import sqlite3
|
|
18
|
+
import threading
|
|
19
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from codegraph.runtime.ingestor import ingest_runtime_traces
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class TelemetryRequestHandler(BaseHTTPRequestHandler):
|
|
27
|
+
"""Zero-overhead HTTP request handler for OTLP spans and dev telemetry."""
|
|
28
|
+
|
|
29
|
+
server: TelemetryServer # Type annotation for custom server
|
|
30
|
+
|
|
31
|
+
def log_message(self, format: str, *args: Any) -> None:
|
|
32
|
+
"""Suppress standard stderr access logging to prevent noisy terminal output."""
|
|
33
|
+
return
|
|
34
|
+
|
|
35
|
+
def do_GET(self) -> None:
|
|
36
|
+
if self.path in ("/healthz", "/health"):
|
|
37
|
+
self.send_response(200)
|
|
38
|
+
self.send_header("Content-Type", "application/json")
|
|
39
|
+
self.end_headers()
|
|
40
|
+
self.wfile.write(b'{"status":"ok","engine":"codegraph-runtime-collector"}')
|
|
41
|
+
return
|
|
42
|
+
self.send_response(404)
|
|
43
|
+
self.end_headers()
|
|
44
|
+
|
|
45
|
+
def do_POST(self) -> None:
|
|
46
|
+
if self.path not in ("/v1/traces", "/api/telemetry", "/telemetry"):
|
|
47
|
+
self.send_response(404)
|
|
48
|
+
self.end_headers()
|
|
49
|
+
return
|
|
50
|
+
|
|
51
|
+
content_length = int(self.headers.get("Content-Length", 0))
|
|
52
|
+
if content_length <= 0 or content_length > 10 * 1024 * 1024: # Max 10MB payload
|
|
53
|
+
self.send_response(400)
|
|
54
|
+
self.end_headers()
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
try:
|
|
58
|
+
body_bytes = self.rfile.read(content_length)
|
|
59
|
+
body_text = body_bytes.decode("utf-8", errors="replace")
|
|
60
|
+
parsed_json = json.loads(body_text)
|
|
61
|
+
|
|
62
|
+
# Ingest into CodeGraph SQLite database
|
|
63
|
+
with self.server.db_lock:
|
|
64
|
+
ingest_runtime_traces(
|
|
65
|
+
self.server.db_connection,
|
|
66
|
+
parsed_json,
|
|
67
|
+
repository=self.server.repo_root,
|
|
68
|
+
format="otel" if "/traces" in self.path else "json",
|
|
69
|
+
max_events=1000,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
self.send_response(200)
|
|
73
|
+
self.send_header("Content-Type", "application/json")
|
|
74
|
+
self.end_headers()
|
|
75
|
+
self.wfile.write(b'{"partialSuccess":{}}')
|
|
76
|
+
except Exception as exc:
|
|
77
|
+
self.send_response(500)
|
|
78
|
+
self.send_header("Content-Type", "application/json")
|
|
79
|
+
self.end_headers()
|
|
80
|
+
err_msg = json.dumps({"error": str(exc)}).encode("utf-8")
|
|
81
|
+
self.wfile.write(err_msg)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class TelemetryServer(HTTPServer):
|
|
85
|
+
"""Custom HTTPServer maintaining db connection and repo references."""
|
|
86
|
+
|
|
87
|
+
def __init__(
|
|
88
|
+
self,
|
|
89
|
+
server_address: tuple[str, int],
|
|
90
|
+
db_connection: sqlite3.Connection,
|
|
91
|
+
repo_root: Path,
|
|
92
|
+
) -> None:
|
|
93
|
+
super().__init__(server_address, TelemetryRequestHandler)
|
|
94
|
+
self.db_connection = db_connection
|
|
95
|
+
self.repo_root = repo_root
|
|
96
|
+
self.db_lock = threading.Lock()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class RuntimeCollectorDaemon:
|
|
100
|
+
"""Manages lifecycle of background OTLP collector."""
|
|
101
|
+
|
|
102
|
+
def __init__(
|
|
103
|
+
self,
|
|
104
|
+
db_connection: sqlite3.Connection,
|
|
105
|
+
repo_root: Path,
|
|
106
|
+
port: int = 4318,
|
|
107
|
+
host: str = "127.0.0.1",
|
|
108
|
+
) -> None:
|
|
109
|
+
self.db_connection = db_connection
|
|
110
|
+
self.repo_root = repo_root
|
|
111
|
+
self.port = port
|
|
112
|
+
self.host = host
|
|
113
|
+
self._server: TelemetryServer | None = None
|
|
114
|
+
self._thread: threading.Thread | None = None
|
|
115
|
+
self._is_running = False
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def is_running(self) -> bool:
|
|
119
|
+
return self._is_running
|
|
120
|
+
|
|
121
|
+
def start(self) -> bool:
|
|
122
|
+
"""Start the collector in a low-priority background daemon thread."""
|
|
123
|
+
if self._is_running:
|
|
124
|
+
return True
|
|
125
|
+
try:
|
|
126
|
+
self._server = TelemetryServer(
|
|
127
|
+
(self.host, self.port),
|
|
128
|
+
self.db_connection,
|
|
129
|
+
self.repo_root,
|
|
130
|
+
)
|
|
131
|
+
self._thread = threading.Thread(
|
|
132
|
+
target=self._server.serve_forever,
|
|
133
|
+
name="codegraph-otlp-collector",
|
|
134
|
+
daemon=True,
|
|
135
|
+
)
|
|
136
|
+
self._thread.start()
|
|
137
|
+
self._is_running = True
|
|
138
|
+
return True
|
|
139
|
+
except OSError:
|
|
140
|
+
# Port might be in use or unavailable; fail closed gracefully without crashing
|
|
141
|
+
self._is_running = False
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
def stop(self) -> None:
|
|
145
|
+
"""Shutdown the collector cleanly."""
|
|
146
|
+
if not self._is_running or not self._server:
|
|
147
|
+
return
|
|
148
|
+
try:
|
|
149
|
+
self._server.shutdown()
|
|
150
|
+
self._server.server_close()
|
|
151
|
+
except Exception:
|
|
152
|
+
pass
|
|
153
|
+
finally:
|
|
154
|
+
self._is_running = False
|
|
155
|
+
self._server = None
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Automatic Database Schema Sync & Migration Drift Watcher.
|
|
2
|
+
|
|
3
|
+
Continuously tracks:
|
|
4
|
+
1. Migration directory updates (Prisma, Drizzle, Alembic, raw SQL migrations).
|
|
5
|
+
2. Live database schema version cookies (PRAGMA schema_version, _prisma_migrations, alembic_version).
|
|
6
|
+
3. Schema drift detection: flags discrepancy between static ORM models and database state.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import sqlite3
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class SchemaDriftItem:
|
|
20
|
+
kind: str # MISSING_COLUMN | MISSING_TABLE | TYPE_MISMATCH | NULLABILITY_MISMATCH
|
|
21
|
+
table_name: str
|
|
22
|
+
column_name: str = ""
|
|
23
|
+
db_type: str = ""
|
|
24
|
+
model_type: str = ""
|
|
25
|
+
message: str = ""
|
|
26
|
+
|
|
27
|
+
def as_dict(self) -> dict[str, Any]:
|
|
28
|
+
return {
|
|
29
|
+
"kind": self.kind,
|
|
30
|
+
"table_name": self.table_name,
|
|
31
|
+
"column_name": self.column_name,
|
|
32
|
+
"db_type": self.db_type,
|
|
33
|
+
"model_type": self.model_type,
|
|
34
|
+
"message": self.message,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class SchemaDriftReport:
|
|
40
|
+
is_drifted: bool
|
|
41
|
+
schema_version: str
|
|
42
|
+
migration_hash: str
|
|
43
|
+
drifts: tuple[SchemaDriftItem, ...] = ()
|
|
44
|
+
migration_files_count: int = 0
|
|
45
|
+
latest_migration_name: str = ""
|
|
46
|
+
|
|
47
|
+
def as_dict(self) -> dict[str, Any]:
|
|
48
|
+
return {
|
|
49
|
+
"is_drifted": self.is_drifted,
|
|
50
|
+
"schema_version": self.schema_version,
|
|
51
|
+
"migration_hash": self.migration_hash,
|
|
52
|
+
"drifts": [d.as_dict() for d in self.drifts],
|
|
53
|
+
"migration_files_count": self.migration_files_count,
|
|
54
|
+
"latest_migration_name": self.latest_migration_name,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def compute_migration_directory_state(repo_root: Path) -> tuple[str, int, str]:
|
|
59
|
+
"""Scan migration directories and compute combined hash, count, and latest migration name."""
|
|
60
|
+
migration_patterns = (
|
|
61
|
+
"**/migrations/*.sql",
|
|
62
|
+
"**/prisma/migrations/**/*.sql",
|
|
63
|
+
"**/drizzle/*.sql",
|
|
64
|
+
"**/alembic/versions/*.py",
|
|
65
|
+
)
|
|
66
|
+
migration_files: list[Path] = []
|
|
67
|
+
for pattern in migration_patterns:
|
|
68
|
+
migration_files.extend(repo_root.glob(pattern))
|
|
69
|
+
|
|
70
|
+
migration_files = [f for f in migration_files if "node_modules" not in f.parts and ".git" not in f.parts]
|
|
71
|
+
if not migration_files:
|
|
72
|
+
return "", 0, ""
|
|
73
|
+
|
|
74
|
+
migration_files.sort(key=lambda f: f.name)
|
|
75
|
+
hasher = hashlib.sha256()
|
|
76
|
+
for f in migration_files:
|
|
77
|
+
try:
|
|
78
|
+
hasher.update(f.name.encode())
|
|
79
|
+
hasher.update(str(f.stat().st_mtime_ns).encode())
|
|
80
|
+
except OSError:
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
latest_name = migration_files[-1].name
|
|
84
|
+
return hasher.hexdigest()[:16], len(migration_files), latest_name
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def check_sqlite_schema_version(db_path: Path) -> int:
|
|
88
|
+
"""Read PRAGMA schema_version from SQLite database."""
|
|
89
|
+
if not db_path.exists():
|
|
90
|
+
return 0
|
|
91
|
+
try:
|
|
92
|
+
con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
|
93
|
+
cur = con.execute("PRAGMA schema_version")
|
|
94
|
+
val = cur.fetchone()
|
|
95
|
+
con.close()
|
|
96
|
+
return int(val[0]) if val else 0
|
|
97
|
+
except Exception:
|
|
98
|
+
return 0
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
_DRIFT_CACHE: dict[str, tuple[float, SchemaDriftReport]] = {}
|
|
102
|
+
_DRIFT_CACHE_TTL_SEC = 5.0
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def detect_schema_drift(
|
|
106
|
+
con: sqlite3.Connection,
|
|
107
|
+
repo_root: Path,
|
|
108
|
+
) -> SchemaDriftReport:
|
|
109
|
+
"""Compare indexed DB models against migration state with 5-second TTL cache (Zero idle CPU)."""
|
|
110
|
+
global _DRIFT_CACHE
|
|
111
|
+
cache_key = str(repo_root)
|
|
112
|
+
now = time.monotonic()
|
|
113
|
+
if cache_key in _DRIFT_CACHE:
|
|
114
|
+
last_t, cached_report = _DRIFT_CACHE[cache_key]
|
|
115
|
+
if now - last_t < _DRIFT_CACHE_TTL_SEC:
|
|
116
|
+
return cached_report
|
|
117
|
+
|
|
118
|
+
mig_hash, mig_count, latest_mig = compute_migration_directory_state(repo_root)
|
|
119
|
+
|
|
120
|
+
# 1. Fetch tables and columns registered in CodeGraph
|
|
121
|
+
db_tables_rows = con.execute("SELECT table_name, name, dialect FROM db_entities WHERE kind = 'Table'").fetchall()
|
|
122
|
+
indexed_tables = {str(r["table_name"] or r["name"]).lower(): str(r["dialect"]) for r in db_tables_rows}
|
|
123
|
+
|
|
124
|
+
indexed_cols: dict[str, set[str]] = {}
|
|
125
|
+
col_rows = con.execute("SELECT table_name, name, data_type FROM db_entities WHERE kind = 'Column'").fetchall()
|
|
126
|
+
for r in col_rows:
|
|
127
|
+
t_name = str(r["table_name"] or "").lower()
|
|
128
|
+
if t_name not in indexed_cols:
|
|
129
|
+
indexed_cols[t_name] = set()
|
|
130
|
+
c_name = str(r["name"] or "").lower()
|
|
131
|
+
if c_name:
|
|
132
|
+
indexed_cols[t_name].add(c_name)
|
|
133
|
+
|
|
134
|
+
drifts: list[SchemaDriftItem] = []
|
|
135
|
+
|
|
136
|
+
# 2. If a local SQLite database exists in repo (e.g. dev.db, local.db, test.db), check live tables
|
|
137
|
+
local_dbs = list(repo_root.glob("*.db")) + list(repo_root.glob("*.sqlite")) + list(repo_root.glob("*.sqlite3"))
|
|
138
|
+
local_dbs = [d for d in local_dbs if not d.name.startswith(".codegraph")]
|
|
139
|
+
|
|
140
|
+
schema_version_str = "0"
|
|
141
|
+
if local_dbs:
|
|
142
|
+
target_db = local_dbs[0]
|
|
143
|
+
schema_version_str = str(check_sqlite_schema_version(target_db))
|
|
144
|
+
try:
|
|
145
|
+
live_con = sqlite3.connect(f"file:{target_db}?mode=ro", uri=True)
|
|
146
|
+
live_tables = [
|
|
147
|
+
row[0].lower()
|
|
148
|
+
for row in live_con.execute(
|
|
149
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
|
|
150
|
+
).fetchall()
|
|
151
|
+
]
|
|
152
|
+
for lt in live_tables:
|
|
153
|
+
if lt not in indexed_tables and not lt.startswith(("_prisma", "alembic")):
|
|
154
|
+
drifts.append(
|
|
155
|
+
SchemaDriftItem(
|
|
156
|
+
kind="MISSING_TABLE",
|
|
157
|
+
table_name=lt,
|
|
158
|
+
message=f"Table '{lt}' exists in live database but has no corresponding code model.",
|
|
159
|
+
)
|
|
160
|
+
)
|
|
161
|
+
else:
|
|
162
|
+
# check columns
|
|
163
|
+
live_cols = [
|
|
164
|
+
crow[1].lower()
|
|
165
|
+
for crow in live_con.execute(f"PRAGMA table_info('{lt}')").fetchall()
|
|
166
|
+
]
|
|
167
|
+
known_c = indexed_cols.get(lt, set())
|
|
168
|
+
for lc in live_cols:
|
|
169
|
+
if lc not in known_c:
|
|
170
|
+
drifts.append(
|
|
171
|
+
SchemaDriftItem(
|
|
172
|
+
kind="MISSING_COLUMN",
|
|
173
|
+
table_name=lt,
|
|
174
|
+
column_name=lc,
|
|
175
|
+
message=f"Column '{lt}.{lc}' exists in live database but is missing in code model.",
|
|
176
|
+
)
|
|
177
|
+
)
|
|
178
|
+
live_con.close()
|
|
179
|
+
except Exception:
|
|
180
|
+
pass
|
|
181
|
+
|
|
182
|
+
report = SchemaDriftReport(
|
|
183
|
+
is_drifted=len(drifts) > 0,
|
|
184
|
+
schema_version=schema_version_str,
|
|
185
|
+
migration_hash=mig_hash,
|
|
186
|
+
drifts=tuple(drifts),
|
|
187
|
+
migration_files_count=mig_count,
|
|
188
|
+
latest_migration_name=latest_mig,
|
|
189
|
+
)
|
|
190
|
+
_DRIFT_CACHE[cache_key] = (now, report)
|
|
191
|
+
return report
|
codegraph/runtime/ingestor.py
CHANGED
|
@@ -460,8 +460,14 @@ def ingest_runtime_traces(
|
|
|
460
460
|
spans_by_id: dict[tuple[str, str], RuntimeEvent] = {(e.trace_id, e.span_id): e for e in events}
|
|
461
461
|
|
|
462
462
|
for ev in events:
|
|
463
|
-
|
|
464
|
-
|
|
463
|
+
parent_ev = spans_by_id.get((ev.trace_id, ev.parent_span_id)) if ev.parent_span_id else None
|
|
464
|
+
route_path_eff = ev.route_path or (parent_ev.route_path if parent_ev else "")
|
|
465
|
+
http_method_eff = ev.http_method or (parent_ev.http_method if parent_ev else "")
|
|
466
|
+
handler_sym = ev.handler_symbol or (parent_ev.handler_symbol if parent_ev else "")
|
|
467
|
+
caller_sym = ev.caller_symbol or handler_sym or ev.service_name or (parent_ev.caller_symbol if parent_ev else "")
|
|
468
|
+
|
|
469
|
+
caller_resolved = _resolve_symbol_in_db(con, caller_sym)
|
|
470
|
+
handler_resolved = _resolve_symbol_in_db(con, handler_sym) if handler_sym else caller_resolved
|
|
465
471
|
callee_resolved = _resolve_symbol_in_db(con, ev.callee_symbol) if ev.callee_symbol else ""
|
|
466
472
|
tbl_resolved = _resolve_table_in_db(con, ev.db_table) if ev.db_table else ""
|
|
467
473
|
|
|
@@ -479,8 +485,8 @@ def ingest_runtime_traces(
|
|
|
479
485
|
ev.parent_span_id,
|
|
480
486
|
ev.timestamp,
|
|
481
487
|
ev.source_format,
|
|
482
|
-
|
|
483
|
-
redact_secrets(
|
|
488
|
+
http_method_eff,
|
|
489
|
+
redact_secrets(route_path_eff),
|
|
484
490
|
handler_resolved,
|
|
485
491
|
caller_resolved,
|
|
486
492
|
callee_resolved,
|
|
@@ -498,22 +504,21 @@ def ingest_runtime_traces(
|
|
|
498
504
|
)
|
|
499
505
|
|
|
500
506
|
# 1. Route -> Handler runtime edge
|
|
501
|
-
if
|
|
502
|
-
ep_label = f"ENDPOINT:{
|
|
507
|
+
if route_path_eff and handler_resolved:
|
|
508
|
+
ep_label = f"ENDPOINT:{http_method_eff or 'ANY'}:{route_path_eff}"
|
|
503
509
|
_upsert_runtime_edge(
|
|
504
510
|
con,
|
|
505
511
|
source=ep_label,
|
|
506
512
|
target=handler_resolved,
|
|
507
513
|
relationship="HANDLED_BY",
|
|
508
|
-
operation=
|
|
514
|
+
operation=http_method_eff or "HTTP",
|
|
509
515
|
ev=ev,
|
|
510
516
|
next_gen=next_gen,
|
|
511
517
|
)
|
|
512
518
|
edges_upserted += 1
|
|
513
519
|
|
|
514
520
|
# 2. Parent span caller -> child span caller edge
|
|
515
|
-
if ev.parent_span_id and
|
|
516
|
-
parent_ev = spans_by_id[(ev.trace_id, ev.parent_span_id)]
|
|
521
|
+
if ev.parent_span_id and parent_ev:
|
|
517
522
|
parent_sym = _resolve_symbol_in_db(con, parent_ev.caller_symbol or parent_ev.handler_symbol)
|
|
518
523
|
if parent_sym and caller_resolved and parent_sym != caller_resolved:
|
|
519
524
|
_upsert_runtime_edge(
|
|
@@ -542,7 +547,7 @@ def ingest_runtime_traces(
|
|
|
542
547
|
|
|
543
548
|
# 4. Caller / Service -> Database Table & Column runtime edges
|
|
544
549
|
if ev.db_table and tbl_resolved:
|
|
545
|
-
src_actor = caller_resolved or ev.service_name or (f"ENDPOINT:{
|
|
550
|
+
src_actor = caller_resolved or handler_resolved or ev.service_name or (f"ENDPOINT:{http_method_eff or 'ANY'}:{route_path_eff}" if route_path_eff else "runtime.anonymous")
|
|
546
551
|
op_upper = ev.db_operation.upper()
|
|
547
552
|
rel = "READS_TABLE" if op_upper in ("SELECT", "READ", "FIND", "GET") else "WRITES_TABLE"
|
|
548
553
|
_upsert_runtime_edge(
|
|
@@ -556,6 +561,21 @@ def ingest_runtime_traces(
|
|
|
556
561
|
)
|
|
557
562
|
edges_upserted += 1
|
|
558
563
|
|
|
564
|
+
# Also create edge from route endpoint if available and distinct
|
|
565
|
+
if route_path_eff:
|
|
566
|
+
route_actor = f"ENDPOINT:{http_method_eff or 'ANY'}:{route_path_eff}"
|
|
567
|
+
if route_actor != src_actor:
|
|
568
|
+
_upsert_runtime_edge(
|
|
569
|
+
con,
|
|
570
|
+
source=route_actor,
|
|
571
|
+
target=tbl_resolved,
|
|
572
|
+
relationship=rel,
|
|
573
|
+
operation=op_upper or "QUERY",
|
|
574
|
+
ev=ev,
|
|
575
|
+
next_gen=next_gen,
|
|
576
|
+
)
|
|
577
|
+
edges_upserted += 1
|
|
578
|
+
|
|
559
579
|
col_rel = "READS_COLUMN" if rel == "READS_TABLE" else "WRITES_COLUMN"
|
|
560
580
|
for col in ev.db_columns:
|
|
561
581
|
col_cid = f"{tbl_resolved}.{col.lower()}"
|