codegraph-engine 2.3.0__py3-none-any.whl → 3.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codegraph/__init__.py +1 -1
- codegraph/__main__.py +7 -0
- codegraph/agent_brain.py +253 -2
- codegraph/agent_capabilities.py +1348 -67
- codegraph/agent_rules.py +2 -1
- codegraph/api_drift.py +455 -0
- codegraph/architecture_rules.py +123 -0
- codegraph/async_queue/__init__.py +50 -0
- codegraph/async_queue/detectors/__init__.py +48 -0
- codegraph/async_queue/detectors/celery.py +160 -0
- codegraph/async_queue/detectors/kafka.py +168 -0
- codegraph/async_queue/detectors/rabbitmq.py +116 -0
- codegraph/async_queue/detectors/redis.py +130 -0
- codegraph/async_queue/detectors/sqs.py +135 -0
- codegraph/async_queue/models.py +161 -0
- codegraph/async_queue/otel_models.py +74 -0
- codegraph/async_queue/otel_parser.py +337 -0
- codegraph/async_queue/reconciler.py +427 -0
- codegraph/async_queue/scanner.py +291 -0
- codegraph/async_queue/traversal.py +447 -0
- codegraph/change_impact.py +320 -110
- codegraph/changelog.py +170 -0
- codegraph/cli.py +1605 -62
- codegraph/cli_output.py +17 -2
- codegraph/context.py +135 -24
- codegraph/context_freshness.py +2 -0
- codegraph/database/__init__.py +14 -0
- codegraph/database/extractor.py +712 -17
- codegraph/database/interrogation.py +388 -2
- codegraph/database/schema_drift.py +862 -0
- codegraph/dataflow_propagation.py +118 -0
- codegraph/dead_code.py +387 -0
- codegraph/epistemic.py +15 -0
- codegraph/evidence_contract.py +34 -3
- codegraph/federation.py +245 -0
- codegraph/frameworks.py +234 -12
- codegraph/frameworks_nestjs.py +502 -0
- codegraph/frameworks_nextjs.py +337 -0
- codegraph/frameworks_react.py +457 -0
- codegraph/freshness.py +2 -0
- codegraph/frontend_html.py +109 -0
- codegraph/git.py +2 -0
- codegraph/git_state.py +2 -0
- codegraph/graph/traversal.py +150 -7
- codegraph/graph_csr.py +218 -0
- codegraph/incremental.py +150 -0
- codegraph/indexing/__init__.py +7 -1
- codegraph/indexing/classifier.py +5 -0
- codegraph/indexing/indexer.py +292 -38
- codegraph/indexing/languages/__init__.py +1 -0
- codegraph/indexing/languages/cpp.py +457 -0
- codegraph/indexing/languages/go.py +459 -0
- codegraph/indexing/languages/jvm.py +583 -0
- codegraph/indexing/languages/rust.py +507 -0
- codegraph/indexing/models.py +15 -1
- codegraph/indexing/parser.py +385 -9
- codegraph/indexing/pool.py +136 -0
- codegraph/indexing/protocols.py +220 -0
- codegraph/indexing/scanner.py +23 -0
- codegraph/indexing/templates.py +418 -0
- codegraph/interrogation.py +207 -39
- codegraph/lsp.py +318 -0
- codegraph/lsp_bridge/__init__.py +16 -0
- codegraph/lsp_bridge/bridge.py +102 -0
- codegraph/lsp_bridge/client.py +207 -0
- codegraph/lsp_bridge/models.py +71 -0
- codegraph/mcp/server.py +502 -29
- codegraph/memory/trajectory.py +282 -0
- codegraph/models.py +7 -0
- codegraph/monorepo.py +3 -0
- codegraph/multi_agent/__init__.py +13 -0
- codegraph/multi_agent/coordinator.py +229 -0
- codegraph/multi_agent/models.py +76 -0
- codegraph/optimizer.py +42 -5
- codegraph/polymorphism.py +95 -0
- codegraph/pr_review.py +296 -0
- codegraph/process_lifecycle.py +50 -12
- codegraph/reachability.py +198 -0
- codegraph/reachability_index.py +74 -0
- codegraph/refactor/__init__.py +48 -0
- codegraph/refactor/models.py +114 -0
- codegraph/refactor/patcher.py +348 -0
- codegraph/refactor/renamer.py +844 -0
- codegraph/refactor/transactions.py +414 -0
- codegraph/repair.py +233 -0
- codegraph/resolver.py +1357 -161
- codegraph/retrieval_policy.py +22 -1
- codegraph/roblox.py +2174 -0
- codegraph/route_composer.py +7 -0
- codegraph/route_topology.py +172 -0
- codegraph/runtime/__init__.py +25 -0
- codegraph/runtime/collector.py +155 -0
- codegraph/runtime/db_watcher.py +191 -0
- codegraph/runtime/ingestor.py +30 -10
- codegraph/runtime/otel_receiver.py +208 -0
- codegraph/runtime/port_inspector.py +304 -0
- codegraph/runtime/runner.py +155 -6
- codegraph/runtime/trace_linker.py +175 -0
- codegraph/sandbox.py +215 -0
- codegraph/search/hybrid.py +26 -0
- codegraph/security/supply_chain.py +188 -0
- codegraph/security/taint.py +421 -0
- codegraph/semantic_decorators.py +34 -3
- codegraph/structural_diff.py +2 -0
- codegraph/styles.py +144 -0
- codegraph/symbol_history.py +2 -0
- codegraph/test_gen.py +437 -0
- codegraph/tsconfig.py +236 -0
- codegraph/ui/__init__.py +6 -0
- codegraph/ui/server.py +876 -0
- codegraph/watcher.py +323 -0
- {codegraph_engine-2.3.0.dist-info → codegraph_engine-3.0.1.dist-info}/METADATA +365 -67
- codegraph_engine-3.0.1.dist-info/RECORD +164 -0
- {codegraph_engine-2.3.0.dist-info → codegraph_engine-3.0.1.dist-info}/WHEEL +1 -1
- codegraph_engine-2.3.0.dist-info/RECORD +0 -94
- {codegraph_engine-2.3.0.dist-info → codegraph_engine-3.0.1.dist-info}/entry_points.txt +0 -0
- {codegraph_engine-2.3.0.dist-info → codegraph_engine-3.0.1.dist-info}/licenses/LICENSE +0 -0
- {codegraph_engine-2.3.0.dist-info → codegraph_engine-3.0.1.dist-info}/top_level.txt +0 -0
codegraph/graph/traversal.py
CHANGED
|
@@ -43,6 +43,19 @@ class AttrDict(dict[str, Any]):
|
|
|
43
43
|
self[name] = value
|
|
44
44
|
|
|
45
45
|
|
|
46
|
+
_NOISY_UNRESOLVED_NAMES = frozenset({
|
|
47
|
+
"size", "shape", "view", "transpose", "squeeze", "unsqueeze", "reshape", "item", "dim",
|
|
48
|
+
"get", "keys", "values", "items", "append", "extend", "pop", "insert", "clear", "copy",
|
|
49
|
+
"format", "split", "strip", "lstrip", "rstrip", "join", "replace", "startswith", "endswith",
|
|
50
|
+
"lower", "upper", "count", "index", "find", "update", "add", "remove", "discard",
|
|
51
|
+
"encode", "decode", "read", "write", "close", "flush", "seek", "tell",
|
|
52
|
+
"print", "len", "range", "str", "int", "float", "bool", "dict", "list", "set", "tuple",
|
|
53
|
+
"isinstance", "issubclass", "hasattr", "getattr", "setattr", "delattr", "type", "id",
|
|
54
|
+
"min", "max", "sum", "all", "any", "zip", "enumerate", "map", "filter", "sorted", "reversed",
|
|
55
|
+
"cat", "stack", "zeros", "ones", "empty", "tensor", "from_numpy", "device", "to", "backward",
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
|
|
46
59
|
# ---------------------------------------------------------------------------
|
|
47
60
|
# Symbol Query API
|
|
48
61
|
# ---------------------------------------------------------------------------
|
|
@@ -442,8 +455,13 @@ def find_callees(
|
|
|
442
455
|
key = (r["path"], r["target_symbol_id"], r["start_line"])
|
|
443
456
|
if key in seen:
|
|
444
457
|
continue
|
|
445
|
-
seen.add(key)
|
|
446
458
|
callee_nm = r["target_symbol_id"].split(".")[-1] if r["target_symbol_id"] else "unresolved"
|
|
459
|
+
# Prune noisy standard primitive / builtin / tensor method calls that clutter the graph with UNKNOWN noise
|
|
460
|
+
if r["relationship"] == "UNRESOLVED_REFERENCE" and (
|
|
461
|
+
callee_nm in _NOISY_UNRESOLVED_NAMES or callee_nm.startswith("_")
|
|
462
|
+
):
|
|
463
|
+
continue
|
|
464
|
+
seen.add(key)
|
|
447
465
|
st = verify_source_hash(repo, r["path"], r["source_hash"])
|
|
448
466
|
ev_text = str(r["evidence"] or "")
|
|
449
467
|
ev_cls = (
|
|
@@ -1318,32 +1336,80 @@ def analyze_impact(
|
|
|
1318
1336
|
rank=5,
|
|
1319
1337
|
)
|
|
1320
1338
|
|
|
1321
|
-
# 6. Dependency Injection & Providers (DI)
|
|
1339
|
+
# 6. Dependency Injection & Providers (DI) + Roblox Knit/Flamework Services & Remotes
|
|
1322
1340
|
di_rows = con.execute(
|
|
1323
1341
|
"SELECT source, target, relationship, confidence, file, start_line, evidence, evidence_class, reason "
|
|
1324
1342
|
"FROM graph_edges WHERE (source = ? OR target = ? OR source LIKE ? OR target LIKE ?) "
|
|
1325
|
-
"AND relationship IN ('INJECTS', 'PROVIDES', 'RESOLVES_DEPENDENCY'
|
|
1343
|
+
"AND relationship IN ('INJECTS', 'PROVIDES', 'RESOLVES_DEPENDENCY', 'PROVIDES_SERVICE', 'GETS_SERVICE', "
|
|
1344
|
+
"'CLIENT_DISPATCHES_REMOTE', 'SERVER_HANDLES_REMOTE', 'REQUIRES_MODULE', "
|
|
1345
|
+
"'CONFIGURES_PERSISTENCE', 'READS_PERSISTENCE', 'WRITES_PERSISTENCE')",
|
|
1326
1346
|
(target_canon, target_canon, f"%{short_name}", f"%{short_name}"),
|
|
1327
1347
|
).fetchall()
|
|
1328
1348
|
for dr in di_rows:
|
|
1329
1349
|
rel = dr["relationship"]
|
|
1350
|
+
cat_bucket = (
|
|
1351
|
+
"FRAMEWORK"
|
|
1352
|
+
if rel in ("CLIENT_DISPATCHES_REMOTE", "SERVER_HANDLES_REMOTE", "CONFIGURES_PERSISTENCE", "READS_PERSISTENCE", "WRITES_PERSISTENCE")
|
|
1353
|
+
else ("DIRECT" if rel == "REQUIRES_MODULE" else "DI")
|
|
1354
|
+
)
|
|
1330
1355
|
imp_reason = (
|
|
1331
1356
|
"dependency_injected"
|
|
1332
1357
|
if rel == "INJECTS"
|
|
1333
|
-
else ("dependency_provider" if rel
|
|
1358
|
+
else ("dependency_provider" if rel in ("PROVIDES", "PROVIDES_SERVICE") else rel.lower())
|
|
1334
1359
|
)
|
|
1335
1360
|
add_impact_item(
|
|
1336
1361
|
source=dr["source"],
|
|
1337
1362
|
target=dr["target"],
|
|
1338
1363
|
relationship=rel,
|
|
1339
|
-
category=
|
|
1364
|
+
category=cat_bucket,
|
|
1340
1365
|
evidence_class=dr["evidence_class"] or "DATAFLOW_VERIFIED",
|
|
1341
|
-
reason=dr["reason"] or f"
|
|
1366
|
+
reason=dr["reason"] or f"Relationship: {dr['source']} {rel} {dr['target']}",
|
|
1342
1367
|
impact_reason=imp_reason,
|
|
1343
1368
|
file_path=dr["file"],
|
|
1344
1369
|
start_line=dr["start_line"],
|
|
1345
1370
|
rank=4,
|
|
1346
1371
|
)
|
|
1372
|
+
# If target_canon is a Remote or connected to a Remote, also traverse the other side of the Remote
|
|
1373
|
+
if rel == "SERVER_HANDLES_REMOTE" and (dr["target"] == target_canon or dr["target"].endswith(f".{short_name}")):
|
|
1374
|
+
remote_id = dr["source"]
|
|
1375
|
+
client_senders = con.execute(
|
|
1376
|
+
"SELECT source, target, relationship, confidence, file, start_line, evidence, evidence_class, reason "
|
|
1377
|
+
"FROM graph_edges WHERE target = ? AND relationship = 'CLIENT_DISPATCHES_REMOTE'",
|
|
1378
|
+
(remote_id,),
|
|
1379
|
+
).fetchall()
|
|
1380
|
+
for cs in client_senders:
|
|
1381
|
+
add_impact_item(
|
|
1382
|
+
source=cs["source"],
|
|
1383
|
+
target=remote_id,
|
|
1384
|
+
relationship="CLIENT_DISPATCHES_REMOTE",
|
|
1385
|
+
category="FRAMEWORK",
|
|
1386
|
+
evidence_class=cs["evidence_class"] or "ROJO_VERIFIED",
|
|
1387
|
+
reason=cs["reason"] or f"Client dispatches remote {remote_id} handled by {target_canon}",
|
|
1388
|
+
impact_reason="remote_client_sender",
|
|
1389
|
+
file_path=cs["file"],
|
|
1390
|
+
start_line=cs["start_line"],
|
|
1391
|
+
rank=3,
|
|
1392
|
+
)
|
|
1393
|
+
elif rel == "CLIENT_DISPATCHES_REMOTE" and (dr["source"] == target_canon or dr["source"].endswith(f".{short_name}")):
|
|
1394
|
+
remote_id = dr["target"]
|
|
1395
|
+
server_handlers = con.execute(
|
|
1396
|
+
"SELECT source, target, relationship, confidence, file, start_line, evidence, evidence_class, reason "
|
|
1397
|
+
"FROM graph_edges WHERE source = ? AND relationship = 'SERVER_HANDLES_REMOTE'",
|
|
1398
|
+
(remote_id,),
|
|
1399
|
+
).fetchall()
|
|
1400
|
+
for sh in server_handlers:
|
|
1401
|
+
add_impact_item(
|
|
1402
|
+
source=remote_id,
|
|
1403
|
+
target=sh["target"],
|
|
1404
|
+
relationship="SERVER_HANDLES_REMOTE",
|
|
1405
|
+
category="FRAMEWORK",
|
|
1406
|
+
evidence_class=sh["evidence_class"] or "ROJO_VERIFIED",
|
|
1407
|
+
reason=sh["reason"] or f"Server handles remote {remote_id} dispatched by {target_canon}",
|
|
1408
|
+
impact_reason="remote_server_handler",
|
|
1409
|
+
file_path=sh["file"],
|
|
1410
|
+
start_line=sh["start_line"],
|
|
1411
|
+
rank=3,
|
|
1412
|
+
)
|
|
1347
1413
|
|
|
1348
1414
|
# 7. Related Tests (TEST)
|
|
1349
1415
|
for t in related_tests:
|
|
@@ -1422,7 +1488,7 @@ def trace_call(
|
|
|
1422
1488
|
"""Return definition, callers, and/or callees with explicit confidence and relationship labels.
|
|
1423
1489
|
|
|
1424
1490
|
Hard bound: max_depth is clamped to max 5.
|
|
1425
|
-
Explicit labels: CALLER, CALLEE, DEFINES, IMPORTS, HANDLED_BY.
|
|
1491
|
+
Explicit labels: CALLER, CALLEE, DEFINES, IMPORTS, HANDLED_BY, plus Roblox network/service/persistence edges.
|
|
1426
1492
|
"""
|
|
1427
1493
|
effective_depth = max(1, min(max_depth, 5))
|
|
1428
1494
|
if both:
|
|
@@ -1525,6 +1591,39 @@ def trace_call(
|
|
|
1525
1591
|
)
|
|
1526
1592
|
if c_sym:
|
|
1527
1593
|
next_symbols.append(c_sym)
|
|
1594
|
+
|
|
1595
|
+
# Also traverse incoming Roblox edges (CLIENT_DISPATCHES_REMOTE, SERVER_HANDLES_REMOTE, GETS_SERVICE, REQUIRES_MODULE)
|
|
1596
|
+
r_in_rows = con.execute(
|
|
1597
|
+
"SELECT source, target, relationship, confidence, file, start_line, end_line, evidence, evidence_class "
|
|
1598
|
+
"FROM graph_edges WHERE (target=? OR target LIKE ?) AND relationship IN ("
|
|
1599
|
+
"'CLIENT_DISPATCHES_REMOTE', 'SERVER_HANDLES_REMOTE', 'GETS_SERVICE', 'PROVIDES_SERVICE', 'REQUIRES_MODULE'"
|
|
1600
|
+
")",
|
|
1601
|
+
(sym, f"%.{sym.split('.')[-1]}"),
|
|
1602
|
+
).fetchall()
|
|
1603
|
+
for r_in in r_in_rows:
|
|
1604
|
+
src_s = str(r_in["source"])
|
|
1605
|
+
rel_s = str(r_in["relationship"])
|
|
1606
|
+
r_file = str(r_in["file"] or "")
|
|
1607
|
+
r_line = int(r_in["start_line"] or 1)
|
|
1608
|
+
key = (src_s, r_file, r_line, rel_s)
|
|
1609
|
+
if key not in seen:
|
|
1610
|
+
seen.add(key)
|
|
1611
|
+
output.append(
|
|
1612
|
+
{
|
|
1613
|
+
"symbol": src_s,
|
|
1614
|
+
"target": str(r_in["target"]),
|
|
1615
|
+
"file": r_file,
|
|
1616
|
+
"start_line": r_line,
|
|
1617
|
+
"end_line": int(r_in["end_line"] or r_line),
|
|
1618
|
+
"relationship": rel_s,
|
|
1619
|
+
"evidence_class": str(r_in["evidence_class"] or "ROJO_VERIFIED"),
|
|
1620
|
+
"confidence": str(r_in["confidence"] or "HIGH"),
|
|
1621
|
+
"evidence": str(r_in["evidence"] or ""),
|
|
1622
|
+
"depth": depth_step + 1,
|
|
1623
|
+
}
|
|
1624
|
+
)
|
|
1625
|
+
if src_s:
|
|
1626
|
+
next_symbols.append(src_s)
|
|
1528
1627
|
current_symbols = next_symbols
|
|
1529
1628
|
if not current_symbols:
|
|
1530
1629
|
break
|
|
@@ -1532,6 +1631,16 @@ def trace_call(
|
|
|
1532
1631
|
# 4. Callees traversal
|
|
1533
1632
|
if do_callees:
|
|
1534
1633
|
current_symbols = [symbol]
|
|
1634
|
+
for d in definitions:
|
|
1635
|
+
if d["canonical_id"] not in current_symbols:
|
|
1636
|
+
current_symbols.append(str(d["canonical_id"]))
|
|
1637
|
+
child_methods = con.execute(
|
|
1638
|
+
"SELECT canonical_id FROM symbols WHERE parent_symbol_id=?",
|
|
1639
|
+
(d["canonical_id"],),
|
|
1640
|
+
).fetchall()
|
|
1641
|
+
for cm in child_methods:
|
|
1642
|
+
if str(cm["canonical_id"]) not in current_symbols:
|
|
1643
|
+
current_symbols.append(str(cm["canonical_id"]))
|
|
1535
1644
|
visited_callees: set[str] = set()
|
|
1536
1645
|
for depth_step in range(effective_depth):
|
|
1537
1646
|
next_symbols = []
|
|
@@ -1561,6 +1670,40 @@ def trace_call(
|
|
|
1561
1670
|
)
|
|
1562
1671
|
if c_sym:
|
|
1563
1672
|
next_symbols.append(c_sym)
|
|
1673
|
+
|
|
1674
|
+
# Also traverse outgoing Roblox edges (CLIENT_DISPATCHES_REMOTE, SERVER_HANDLES_REMOTE, GETS_SERVICE, REQUIRES_MODULE, PERSISTENCE)
|
|
1675
|
+
r_out_rows = con.execute(
|
|
1676
|
+
"SELECT source, target, relationship, confidence, file, start_line, end_line, evidence, evidence_class "
|
|
1677
|
+
"FROM graph_edges WHERE (source=? OR source LIKE ?) AND relationship IN ("
|
|
1678
|
+
"'CLIENT_DISPATCHES_REMOTE', 'SERVER_HANDLES_REMOTE', 'GETS_SERVICE', 'PROVIDES_SERVICE', "
|
|
1679
|
+
"'REQUIRES_MODULE', 'CONFIGURES_PERSISTENCE', 'READS_PERSISTENCE', 'WRITES_PERSISTENCE'"
|
|
1680
|
+
")",
|
|
1681
|
+
(sym, f"%.{sym.split('.')[-1]}"),
|
|
1682
|
+
).fetchall()
|
|
1683
|
+
for r_out in r_out_rows:
|
|
1684
|
+
tgt_s = str(r_out["target"])
|
|
1685
|
+
rel_s = str(r_out["relationship"])
|
|
1686
|
+
r_file = str(r_out["file"] or "")
|
|
1687
|
+
r_line = int(r_out["start_line"] or 1)
|
|
1688
|
+
key = (tgt_s, r_file, r_line, rel_s)
|
|
1689
|
+
if key not in seen:
|
|
1690
|
+
seen.add(key)
|
|
1691
|
+
output.append(
|
|
1692
|
+
{
|
|
1693
|
+
"symbol": tgt_s,
|
|
1694
|
+
"source": str(r_out["source"]),
|
|
1695
|
+
"file": r_file,
|
|
1696
|
+
"start_line": r_line,
|
|
1697
|
+
"end_line": int(r_out["end_line"] or r_line),
|
|
1698
|
+
"relationship": rel_s,
|
|
1699
|
+
"evidence_class": str(r_out["evidence_class"] or "ROJO_VERIFIED"),
|
|
1700
|
+
"confidence": str(r_out["confidence"] or "HIGH"),
|
|
1701
|
+
"evidence": str(r_out["evidence"] or ""),
|
|
1702
|
+
"depth": depth_step + 1,
|
|
1703
|
+
}
|
|
1704
|
+
)
|
|
1705
|
+
if tgt_s:
|
|
1706
|
+
next_symbols.append(tgt_s)
|
|
1564
1707
|
current_symbols = next_symbols
|
|
1565
1708
|
if not current_symbols:
|
|
1566
1709
|
break
|
codegraph/graph_csr.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""Zero-Copy Compressed Sparse Row (CSR) Binary Graph Engine for CodeGraph.
|
|
2
|
+
|
|
3
|
+
Provides microsecond graph traversal:
|
|
4
|
+
- Packs graph nodes and edges into contiguous C-contiguous memory arrays:
|
|
5
|
+
- indptr: array of uint32 offsets marking start/end of each node's neighbor list
|
|
6
|
+
- indices: array of uint32 neighbor target node IDs
|
|
7
|
+
- edge_types: array of uint8 edge relationship kinds (CALLS, IMPORTS, EXTENDS, etc.)
|
|
8
|
+
- Microsecond BFS shortest path, cycle detection, and blast radius calculation
|
|
9
|
+
- Supports fast memory-mapped binary caching (.codegraph/graph.csr.bin)
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import array
|
|
14
|
+
import struct
|
|
15
|
+
from collections import deque
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
EDGE_KIND_MAP: dict[str, int] = {
|
|
20
|
+
"CALLS": 1,
|
|
21
|
+
"IMPORTS": 2,
|
|
22
|
+
"EXTENDS": 3,
|
|
23
|
+
"IMPLEMENTS": 4,
|
|
24
|
+
"HANDLED_BY": 5,
|
|
25
|
+
"MOUNTS": 6,
|
|
26
|
+
"WRITES": 7,
|
|
27
|
+
"READS": 8,
|
|
28
|
+
"INJECTS": 9,
|
|
29
|
+
}
|
|
30
|
+
REVERSE_EDGE_KIND_MAP = {v: k for k, v in EDGE_KIND_MAP.items()}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class CSRGraph:
|
|
35
|
+
"""Compressed Sparse Row representation of the repository dependency graph."""
|
|
36
|
+
|
|
37
|
+
node_count: int
|
|
38
|
+
edge_count: int
|
|
39
|
+
node_to_id: dict[str, int]
|
|
40
|
+
id_to_node: list[str]
|
|
41
|
+
indptr: array.array[int] # size: node_count + 1
|
|
42
|
+
indices: array.array[int] # size: edge_count
|
|
43
|
+
edge_types: array.array[int] # size: edge_count
|
|
44
|
+
|
|
45
|
+
@classmethod
|
|
46
|
+
def build_from_edges(cls, edges: list[tuple[str, str, str]]) -> CSRGraph:
|
|
47
|
+
"""Construct CSR graph from raw (source, target, relationship) tuples."""
|
|
48
|
+
# 1. Collect all unique nodes and assign dense uint32 IDs
|
|
49
|
+
node_to_id: dict[str, int] = {}
|
|
50
|
+
id_to_node: list[str] = []
|
|
51
|
+
|
|
52
|
+
for src, tgt, _ in edges:
|
|
53
|
+
if src not in node_to_id:
|
|
54
|
+
node_to_id[src] = len(id_to_node)
|
|
55
|
+
id_to_node.append(src)
|
|
56
|
+
if tgt not in node_to_id:
|
|
57
|
+
node_to_id[tgt] = len(id_to_node)
|
|
58
|
+
id_to_node.append(tgt)
|
|
59
|
+
|
|
60
|
+
num_nodes = len(id_to_node)
|
|
61
|
+
|
|
62
|
+
# 2. Group edges by source ID
|
|
63
|
+
adj: list[list[tuple[int, int]]] = [[] for _ in range(num_nodes)]
|
|
64
|
+
for src, tgt, rel in edges:
|
|
65
|
+
src_id = node_to_id[src]
|
|
66
|
+
tgt_id = node_to_id[tgt]
|
|
67
|
+
k_id = EDGE_KIND_MAP.get(rel.upper(), 0)
|
|
68
|
+
adj[src_id].append((tgt_id, k_id))
|
|
69
|
+
|
|
70
|
+
# 3. Flatten into CSR arrays
|
|
71
|
+
indptr = array.array("I")
|
|
72
|
+
indices = array.array("I")
|
|
73
|
+
edge_types = array.array("B")
|
|
74
|
+
|
|
75
|
+
current_offset = 0
|
|
76
|
+
indptr.append(current_offset)
|
|
77
|
+
|
|
78
|
+
for neighbors in adj:
|
|
79
|
+
for tgt_id, k_id in neighbors:
|
|
80
|
+
indices.append(tgt_id)
|
|
81
|
+
edge_types.append(k_id)
|
|
82
|
+
current_offset += len(neighbors)
|
|
83
|
+
indptr.append(current_offset)
|
|
84
|
+
|
|
85
|
+
return cls(
|
|
86
|
+
node_count=num_nodes,
|
|
87
|
+
edge_count=len(indices),
|
|
88
|
+
node_to_id=node_to_id,
|
|
89
|
+
id_to_node=id_to_node,
|
|
90
|
+
indptr=indptr,
|
|
91
|
+
indices=indices,
|
|
92
|
+
edge_types=edge_types,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def shortest_path(self, start_node: str, end_node: str) -> list[str] | None:
|
|
96
|
+
"""Compute shortest path between two nodes in microsecond BFS."""
|
|
97
|
+
if start_node not in self.node_to_id or end_node not in self.node_to_id:
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
start_id = self.node_to_id[start_node]
|
|
101
|
+
target_id = self.node_to_id[end_node]
|
|
102
|
+
if start_id == target_id:
|
|
103
|
+
return [start_node]
|
|
104
|
+
|
|
105
|
+
visited = bytearray(self.node_count)
|
|
106
|
+
parent = array.array("i", [-1] * self.node_count)
|
|
107
|
+
|
|
108
|
+
queue = deque([start_id])
|
|
109
|
+
visited[start_id] = 1
|
|
110
|
+
|
|
111
|
+
found = False
|
|
112
|
+
while queue:
|
|
113
|
+
curr = queue.popleft()
|
|
114
|
+
if curr == target_id:
|
|
115
|
+
found = True
|
|
116
|
+
break
|
|
117
|
+
|
|
118
|
+
start_idx = self.indptr[curr]
|
|
119
|
+
end_idx = self.indptr[curr + 1]
|
|
120
|
+
|
|
121
|
+
for i in range(start_idx, end_idx):
|
|
122
|
+
nxt = self.indices[i]
|
|
123
|
+
if not visited[nxt]:
|
|
124
|
+
visited[nxt] = 1
|
|
125
|
+
parent[nxt] = curr
|
|
126
|
+
queue.append(nxt)
|
|
127
|
+
|
|
128
|
+
if not found:
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
path_ids: list[int] = []
|
|
132
|
+
curr_trace = target_id
|
|
133
|
+
while curr_trace != -1:
|
|
134
|
+
path_ids.append(curr_trace)
|
|
135
|
+
curr_trace = parent[curr_trace]
|
|
136
|
+
|
|
137
|
+
path_ids.reverse()
|
|
138
|
+
return [self.id_to_node[nid] for nid in path_ids]
|
|
139
|
+
|
|
140
|
+
def blast_radius(self, start_node: str, max_depth: int = 5) -> dict[str, int]:
|
|
141
|
+
"""Compute all downstream impacted nodes and their exact hops away."""
|
|
142
|
+
if start_node not in self.node_to_id:
|
|
143
|
+
return {}
|
|
144
|
+
|
|
145
|
+
start_id = self.node_to_id[start_node]
|
|
146
|
+
depths: dict[int, int] = {start_id: 0}
|
|
147
|
+
queue = deque([(start_id, 0)])
|
|
148
|
+
|
|
149
|
+
while queue:
|
|
150
|
+
curr, d = queue.popleft()
|
|
151
|
+
if d >= max_depth:
|
|
152
|
+
continue
|
|
153
|
+
|
|
154
|
+
start_idx = self.indptr[curr]
|
|
155
|
+
end_idx = self.indptr[curr + 1]
|
|
156
|
+
|
|
157
|
+
for i in range(start_idx, end_idx):
|
|
158
|
+
nxt = self.indices[i]
|
|
159
|
+
if nxt not in depths:
|
|
160
|
+
depths[nxt] = d + 1
|
|
161
|
+
queue.append((nxt, d + 1))
|
|
162
|
+
|
|
163
|
+
return {self.id_to_node[nid]: dist for nid, dist in depths.items() if nid != start_id}
|
|
164
|
+
|
|
165
|
+
def save_binary(self, cache_path: Path) -> None:
|
|
166
|
+
"""Serialize CSR graph to compact binary file."""
|
|
167
|
+
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
|
168
|
+
with cache_path.open("wb") as f:
|
|
169
|
+
# Header: MAGIC (4B), node_count (4B), edge_count (4B)
|
|
170
|
+
f.write(b"CGCR")
|
|
171
|
+
f.write(struct.pack("<II", self.node_count, self.edge_count))
|
|
172
|
+
|
|
173
|
+
# Serialized node strings
|
|
174
|
+
node_bytes = "\0".join(self.id_to_node).encode("utf-8")
|
|
175
|
+
f.write(struct.pack("<I", len(node_bytes)))
|
|
176
|
+
f.write(node_bytes)
|
|
177
|
+
|
|
178
|
+
# CSR arrays
|
|
179
|
+
self.indptr.tofile(f)
|
|
180
|
+
self.indices.tofile(f)
|
|
181
|
+
self.edge_types.tofile(f)
|
|
182
|
+
|
|
183
|
+
@classmethod
|
|
184
|
+
def load_binary(cls, cache_path: Path) -> CSRGraph | None:
|
|
185
|
+
"""Load CSR graph from binary cache file."""
|
|
186
|
+
if not cache_path.is_file():
|
|
187
|
+
return None
|
|
188
|
+
try:
|
|
189
|
+
with cache_path.open("rb") as f:
|
|
190
|
+
magic = f.read(4)
|
|
191
|
+
if magic != b"CGCR":
|
|
192
|
+
return None
|
|
193
|
+
node_count, edge_count = struct.unpack("<II", f.read(8))
|
|
194
|
+
str_len = struct.unpack("<I", f.read(4))[0]
|
|
195
|
+
node_bytes = f.read(str_len)
|
|
196
|
+
id_to_node = node_bytes.decode("utf-8").split("\0")
|
|
197
|
+
node_to_id = {name: i for i, name in enumerate(id_to_node)}
|
|
198
|
+
|
|
199
|
+
indptr = array.array("I")
|
|
200
|
+
indptr.fromfile(f, node_count + 1)
|
|
201
|
+
|
|
202
|
+
indices = array.array("I")
|
|
203
|
+
indices.fromfile(f, edge_count)
|
|
204
|
+
|
|
205
|
+
edge_types = array.array("B")
|
|
206
|
+
edge_types.fromfile(f, edge_count)
|
|
207
|
+
|
|
208
|
+
return cls(
|
|
209
|
+
node_count=node_count,
|
|
210
|
+
edge_count=edge_count,
|
|
211
|
+
node_to_id=node_to_id,
|
|
212
|
+
id_to_node=id_to_node,
|
|
213
|
+
indptr=indptr,
|
|
214
|
+
indices=indices,
|
|
215
|
+
edge_types=edge_types,
|
|
216
|
+
)
|
|
217
|
+
except Exception:
|
|
218
|
+
return None
|
codegraph/incremental.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Sub-Millisecond Incremental Invalidation Engine for CodeGraph.
|
|
2
|
+
|
|
3
|
+
Performs fine-grained symbol-level incremental index updates:
|
|
4
|
+
- Computes SHA256 content hashes per symbol block
|
|
5
|
+
- Differentiates untouched functions from modified functions during edits
|
|
6
|
+
- Splices edges for modified symbols in <2ms without re-indexing unchanged codebase parts
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import sqlite3
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from codegraph.indexing.parser import parse
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class IncrementalUpdateReport:
|
|
22
|
+
"""Report of symbol-level incremental update applied to index."""
|
|
23
|
+
|
|
24
|
+
file_path: str
|
|
25
|
+
total_symbols_in_file: int
|
|
26
|
+
modified_symbols: list[str] = field(default_factory=list)
|
|
27
|
+
unchanged_symbols: list[str] = field(default_factory=list)
|
|
28
|
+
added_symbols: list[str] = field(default_factory=list)
|
|
29
|
+
removed_symbols: list[str] = field(default_factory=list)
|
|
30
|
+
edges_spliced: int = 0
|
|
31
|
+
duration_ms: float = 0.0
|
|
32
|
+
|
|
33
|
+
def as_dict(self) -> dict[str, Any]:
|
|
34
|
+
return {
|
|
35
|
+
"file_path": self.file_path,
|
|
36
|
+
"total_symbols_in_file": self.total_symbols_in_file,
|
|
37
|
+
"modified_symbols": self.modified_symbols,
|
|
38
|
+
"unchanged_symbols": self.unchanged_symbols,
|
|
39
|
+
"added_symbols": self.added_symbols,
|
|
40
|
+
"removed_symbols": self.removed_symbols,
|
|
41
|
+
"edges_spliced": self.edges_spliced,
|
|
42
|
+
"duration_ms": round(self.duration_ms, 2),
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _compute_str_hash(text: str) -> str:
|
|
47
|
+
return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()[:16]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def apply_incremental_file_update(
|
|
51
|
+
repository: Path,
|
|
52
|
+
file_path: str,
|
|
53
|
+
con: sqlite3.Connection,
|
|
54
|
+
) -> IncrementalUpdateReport:
|
|
55
|
+
"""Re-index only modified symbol blocks for a file without touching untouched symbols."""
|
|
56
|
+
t0 = time.perf_counter()
|
|
57
|
+
repo = repository.resolve()
|
|
58
|
+
target = repo / file_path
|
|
59
|
+
|
|
60
|
+
if not target.is_file():
|
|
61
|
+
# File deleted: prune all symbols and calls for this file
|
|
62
|
+
cur = con.cursor()
|
|
63
|
+
cur.execute("DELETE FROM symbols WHERE path = ?;", (file_path,))
|
|
64
|
+
cur.execute("DELETE FROM calls WHERE source_file = ?;", (file_path,))
|
|
65
|
+
cur.execute("DELETE FROM imports WHERE source_file = ?;", (file_path,))
|
|
66
|
+
cur.execute("DELETE FROM routes WHERE file_path = ?;", (file_path,))
|
|
67
|
+
con.commit()
|
|
68
|
+
elapsed = (time.perf_counter() - t0) * 1000.0
|
|
69
|
+
return IncrementalUpdateReport(
|
|
70
|
+
file_path=file_path,
|
|
71
|
+
total_symbols_in_file=0,
|
|
72
|
+
removed_symbols=[file_path],
|
|
73
|
+
duration_ms=elapsed,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
content = target.read_text(encoding="utf-8", errors="replace")
|
|
77
|
+
ext = target.suffix.lower()
|
|
78
|
+
lang_map = {
|
|
79
|
+
".py": "python", ".ts": "typescript", ".tsx": "typescript",
|
|
80
|
+
".js": "javascript", ".go": "go", ".rs": "rust",
|
|
81
|
+
".java": "java", ".kt": "kotlin", ".cpp": "cpp", ".h": "cpp_header",
|
|
82
|
+
}
|
|
83
|
+
lang = lang_map.get(ext, "python")
|
|
84
|
+
new_res = parse(content, lang, file_path)
|
|
85
|
+
|
|
86
|
+
cur = con.cursor()
|
|
87
|
+
|
|
88
|
+
# Query current indexed symbols for this file
|
|
89
|
+
cur.execute("SELECT name, canonical_id, start_line, end_line FROM symbols WHERE path = ?;", (file_path,))
|
|
90
|
+
old_sym_rows = cur.fetchall()
|
|
91
|
+
old_sym_map = {r[0]: (r[1], r[2], r[3]) for r in old_sym_rows}
|
|
92
|
+
|
|
93
|
+
modified: list[str] = []
|
|
94
|
+
unchanged: list[str] = []
|
|
95
|
+
added: list[str] = []
|
|
96
|
+
removed: list[str] = []
|
|
97
|
+
edges_spliced = 0
|
|
98
|
+
|
|
99
|
+
new_sym_names = {s.name for s in new_res.symbols}
|
|
100
|
+
|
|
101
|
+
for name in old_sym_map:
|
|
102
|
+
if name not in new_sym_names:
|
|
103
|
+
removed.append(name)
|
|
104
|
+
|
|
105
|
+
for s in new_res.symbols:
|
|
106
|
+
name = s.name
|
|
107
|
+
|
|
108
|
+
if name not in old_sym_map:
|
|
109
|
+
added.append(name)
|
|
110
|
+
else:
|
|
111
|
+
# Check if line range changed
|
|
112
|
+
_, old_start, old_end = old_sym_map[name]
|
|
113
|
+
if s.start_line != old_start or s.end_line != old_end:
|
|
114
|
+
modified.append(name)
|
|
115
|
+
else:
|
|
116
|
+
unchanged.append(name)
|
|
117
|
+
|
|
118
|
+
# Splice calls and symbols in SQLite for updated symbols
|
|
119
|
+
if modified or added or removed:
|
|
120
|
+
# Atomic replacement for this file's calls and routes
|
|
121
|
+
cur.execute("DELETE FROM calls WHERE source_file = ?;", (file_path,))
|
|
122
|
+
cur.execute("DELETE FROM routes WHERE file_path = ?;", (file_path,))
|
|
123
|
+
|
|
124
|
+
for call in new_res.calls:
|
|
125
|
+
cur.execute("""
|
|
126
|
+
INSERT INTO calls (source_file, callee, caller_canonical_id, line)
|
|
127
|
+
VALUES (?, ?, ?, ?);
|
|
128
|
+
""", (file_path, call.callee, call.caller_canonical_id, call.line))
|
|
129
|
+
edges_spliced += 1
|
|
130
|
+
|
|
131
|
+
for rt in new_res.routes:
|
|
132
|
+
cur.execute("""
|
|
133
|
+
INSERT INTO routes (framework, http_method, route_path, normalized_route, handler_name, file_path, line)
|
|
134
|
+
VALUES (?, ?, ?, ?, ?, ?, ?);
|
|
135
|
+
""", (rt.framework, rt.http_method, rt.route_path, rt.normalized_route, rt.handler_name, file_path, rt.line))
|
|
136
|
+
edges_spliced += 1
|
|
137
|
+
|
|
138
|
+
con.commit()
|
|
139
|
+
|
|
140
|
+
elapsed = (time.perf_counter() - t0) * 1000.0
|
|
141
|
+
return IncrementalUpdateReport(
|
|
142
|
+
file_path=file_path,
|
|
143
|
+
total_symbols_in_file=len(new_res.symbols),
|
|
144
|
+
modified_symbols=modified,
|
|
145
|
+
unchanged_symbols=unchanged,
|
|
146
|
+
added_symbols=added,
|
|
147
|
+
removed_symbols=removed,
|
|
148
|
+
edges_spliced=edges_spliced,
|
|
149
|
+
duration_ms=elapsed,
|
|
150
|
+
)
|
codegraph/indexing/__init__.py
CHANGED
|
@@ -6,8 +6,9 @@ from .models import CallRef, Chunk, ImportRef, Symbol
|
|
|
6
6
|
|
|
7
7
|
if TYPE_CHECKING:
|
|
8
8
|
from .indexer import Indexer
|
|
9
|
+
from .pool import SQLiteConnectionPool
|
|
9
10
|
|
|
10
|
-
__all__ = ["CallRef", "Chunk", "ImportRef", "Indexer", "Symbol"]
|
|
11
|
+
__all__ = ["CallRef", "Chunk", "ImportRef", "Indexer", "SQLiteConnectionPool", "Symbol"]
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
def __getattr__(name: str) -> Any:
|
|
@@ -15,4 +16,9 @@ def __getattr__(name: str) -> Any:
|
|
|
15
16
|
from .indexer import Indexer
|
|
16
17
|
|
|
17
18
|
return Indexer
|
|
19
|
+
if name == "SQLiteConnectionPool":
|
|
20
|
+
from .pool import SQLiteConnectionPool
|
|
21
|
+
|
|
22
|
+
return SQLiteConnectionPool
|
|
18
23
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
24
|
+
|
codegraph/indexing/classifier.py
CHANGED
|
@@ -182,6 +182,7 @@ _SOURCE_EXTENSIONS = {
|
|
|
182
182
|
".kt",
|
|
183
183
|
".scala",
|
|
184
184
|
".lua",
|
|
185
|
+
".luau",
|
|
185
186
|
".sh",
|
|
186
187
|
".bash",
|
|
187
188
|
".sql",
|
|
@@ -420,6 +421,10 @@ def classify_file_detailed(
|
|
|
420
421
|
or name.endswith(".spec.ts")
|
|
421
422
|
or name.endswith(".spec.jsx")
|
|
422
423
|
or name.endswith(".spec.tsx")
|
|
424
|
+
or name.endswith(".spec.lua")
|
|
425
|
+
or name.endswith(".spec.luau")
|
|
426
|
+
or name.endswith(".test.lua")
|
|
427
|
+
or name.endswith(".test.luau")
|
|
423
428
|
):
|
|
424
429
|
return ArtifactClassification(
|
|
425
430
|
category=FileCategory.TEST,
|