entrygraph 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. entrygraph/__init__.py +87 -0
  2. entrygraph/__main__.py +8 -0
  3. entrygraph/_version.py +24 -0
  4. entrygraph/api.py +549 -0
  5. entrygraph/cli/__init__.py +0 -0
  6. entrygraph/cli/main.py +387 -0
  7. entrygraph/cli/render.py +136 -0
  8. entrygraph/data/sinks/csharp.toml +106 -0
  9. entrygraph/data/sinks/go.toml +87 -0
  10. entrygraph/data/sinks/java.toml +92 -0
  11. entrygraph/data/sinks/javascript.toml +112 -0
  12. entrygraph/data/sinks/lib_javascript.toml +34 -0
  13. entrygraph/data/sinks/lib_python.toml +39 -0
  14. entrygraph/data/sinks/php.toml +125 -0
  15. entrygraph/data/sinks/python.toml +160 -0
  16. entrygraph/data/sinks/ruby.toml +102 -0
  17. entrygraph/data/sinks/rust.toml +68 -0
  18. entrygraph/db/__init__.py +0 -0
  19. entrygraph/db/engine.py +34 -0
  20. entrygraph/db/meta.py +70 -0
  21. entrygraph/db/models.py +176 -0
  22. entrygraph/db/queries.py +155 -0
  23. entrygraph/detect/__init__.py +0 -0
  24. entrygraph/detect/entrypoints/__init__.py +22 -0
  25. entrygraph/detect/entrypoints/base.py +124 -0
  26. entrygraph/detect/entrypoints/configs.py +139 -0
  27. entrygraph/detect/entrypoints/csharp.py +156 -0
  28. entrygraph/detect/entrypoints/golang.py +158 -0
  29. entrygraph/detect/entrypoints/java.py +187 -0
  30. entrygraph/detect/entrypoints/javascript.py +211 -0
  31. entrygraph/detect/entrypoints/php.py +133 -0
  32. entrygraph/detect/entrypoints/python.py +335 -0
  33. entrygraph/detect/entrypoints/ruby.py +147 -0
  34. entrygraph/detect/entrypoints/rust.py +153 -0
  35. entrygraph/detect/frameworks.py +369 -0
  36. entrygraph/detect/manifests.py +234 -0
  37. entrygraph/detect/taint.py +224 -0
  38. entrygraph/errors.py +27 -0
  39. entrygraph/extract/__init__.py +0 -0
  40. entrygraph/extract/base.py +51 -0
  41. entrygraph/extract/csharp.py +502 -0
  42. entrygraph/extract/golang.py +342 -0
  43. entrygraph/extract/ir.py +105 -0
  44. entrygraph/extract/java.py +329 -0
  45. entrygraph/extract/javascript.py +400 -0
  46. entrygraph/extract/php.py +426 -0
  47. entrygraph/extract/python.py +390 -0
  48. entrygraph/extract/registry.py +43 -0
  49. entrygraph/extract/ruby.py +321 -0
  50. entrygraph/extract/rust.py +482 -0
  51. entrygraph/fs/__init__.py +0 -0
  52. entrygraph/fs/hashing.py +78 -0
  53. entrygraph/fs/lang.py +134 -0
  54. entrygraph/fs/walker.py +167 -0
  55. entrygraph/graph/__init__.py +0 -0
  56. entrygraph/graph/adjacency.py +146 -0
  57. entrygraph/graph/cte.py +123 -0
  58. entrygraph/graph/scoring.py +101 -0
  59. entrygraph/kinds.py +51 -0
  60. entrygraph/parsing/__init__.py +0 -0
  61. entrygraph/parsing/parsers.py +49 -0
  62. entrygraph/parsing/queries.py +39 -0
  63. entrygraph/pipeline/__init__.py +0 -0
  64. entrygraph/pipeline/scanner.py +506 -0
  65. entrygraph/pipeline/worker.py +49 -0
  66. entrygraph/pipeline/writer.py +41 -0
  67. entrygraph/py.typed +0 -0
  68. entrygraph/queries/csharp/calls.scm +4 -0
  69. entrygraph/queries/csharp/definitions.scm +29 -0
  70. entrygraph/queries/csharp/imports.scm +4 -0
  71. entrygraph/queries/go/calls.scm +2 -0
  72. entrygraph/queries/go/definitions.scm +24 -0
  73. entrygraph/queries/go/imports.scm +1 -0
  74. entrygraph/queries/java/calls.scm +2 -0
  75. entrygraph/queries/java/definitions.scm +14 -0
  76. entrygraph/queries/java/imports.scm +2 -0
  77. entrygraph/queries/javascript/calls.scm +4 -0
  78. entrygraph/queries/javascript/definitions.scm +4 -0
  79. entrygraph/queries/javascript/imports.scm +6 -0
  80. entrygraph/queries/php/calls.scm +8 -0
  81. entrygraph/queries/php/definitions.scm +24 -0
  82. entrygraph/queries/php/imports.scm +1 -0
  83. entrygraph/queries/python/calls.scm +2 -0
  84. entrygraph/queries/python/definitions.scm +11 -0
  85. entrygraph/queries/python/imports.scm +2 -0
  86. entrygraph/queries/ruby/calls.scm +4 -0
  87. entrygraph/queries/ruby/definitions.scm +20 -0
  88. entrygraph/queries/ruby/imports.scm +7 -0
  89. entrygraph/queries/rust/calls.scm +5 -0
  90. entrygraph/queries/rust/definitions.scm +26 -0
  91. entrygraph/queries/rust/imports.scm +4 -0
  92. entrygraph/resolve/__init__.py +0 -0
  93. entrygraph/resolve/externals.py +61 -0
  94. entrygraph/resolve/hierarchy.py +152 -0
  95. entrygraph/resolve/resolver.py +275 -0
  96. entrygraph/resolve/symbol_table.py +48 -0
  97. entrygraph/results.py +138 -0
  98. entrygraph-0.1.0.dist-info/METADATA +204 -0
  99. entrygraph-0.1.0.dist-info/RECORD +102 -0
  100. entrygraph-0.1.0.dist-info/WHEEL +4 -0
  101. entrygraph-0.1.0.dist-info/entry_points.txt +2 -0
  102. entrygraph-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,506 @@
1
+ """Index orchestrator: walk -> diff -> parse+extract -> resolve -> detect -> write.
2
+
3
+ One code path serves both full and incremental indexing. Incremental wipes only
4
+ changed/deleted files, re-extracts them, resolves their references against a
5
+ symbol table seeded from the surviving DB symbols, and heals edges (in either
6
+ direction) that now bind to newly-created symbols. The result is identical to a
7
+ full re-index.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import os
14
+ import time
15
+ from concurrent.futures import ProcessPoolExecutor
16
+ from datetime import datetime, timezone
17
+ from pathlib import Path
18
+
19
+ from sqlalchemy import Engine, delete, select, update
20
+ from sqlalchemy.orm import Session
21
+
22
+ from entrygraph.db.meta import ensure_schema
23
+ from entrygraph.db.models import Detection, Edge, Entrypoint, File, Repository, Symbol
24
+ from entrygraph.detect import entrypoints as entrypoint_rules
25
+ from entrygraph.detect.frameworks import detect_frameworks
26
+ from entrygraph.detect.manifests import parse_manifests
27
+ from entrygraph.detect.taint import SinkRegistry, registry_for_repo
28
+ from entrygraph.extract.ir import FileExtraction
29
+ from entrygraph.fs.hashing import FileState, diff_files
30
+ from entrygraph.fs.walker import RepoLanguageProfile, WalkedFile, walk_repo
31
+ from entrygraph.kinds import Confidence, EdgeKind, SymbolKind
32
+ from entrygraph.pipeline.worker import extract_batch, extract_one
33
+ from entrygraph.pipeline.writer import IdAllocator, bulk_insert
34
+ from entrygraph.resolve.externals import ExternalRegistry
35
+ from entrygraph.resolve.hierarchy import resolve_hierarchy
36
+ from entrygraph.resolve.resolver import FileResolver
37
+ from entrygraph.resolve.symbol_table import SymbolTable
38
+ from entrygraph.results import IndexStats
39
+
40
+ _PARALLEL_THRESHOLD = 200 # files; below this the pool overhead isn't worth it
41
+ _BATCH = 24
42
+
43
+
44
+ def index_repository(
45
+ root: str | Path,
46
+ engine: Engine,
47
+ *,
48
+ incremental: bool = False,
49
+ paranoid: bool = False,
50
+ max_workers: int | None = None,
51
+ ) -> IndexStats:
52
+ started = time.monotonic()
53
+ root = Path(root).resolve()
54
+ fresh = ensure_schema(engine)
55
+ if fresh:
56
+ incremental = False
57
+
58
+ walked, profile = walk_repo(root)
59
+ manifests = parse_manifests(root)
60
+ sink_registry = registry_for_repo(root)
61
+
62
+ with Session(engine) as session:
63
+ repo = _load_or_create_repo(session, root, incremental)
64
+ known = _known_file_states(session, repo.id) if incremental else {}
65
+ diff = diff_files(walked, known, paranoid=paranoid)
66
+
67
+ if incremental:
68
+ deleted = _wipe_files(session, repo.id, [*[w.path for w in diff.changed],
69
+ *diff.deleted_paths])
70
+ else:
71
+ _wipe_repo_graph(session, repo.id)
72
+ deleted = 0
73
+ session.flush()
74
+
75
+ alloc = IdAllocator(session)
76
+ table = SymbolTable()
77
+ if incremental:
78
+ _load_existing_symbols(session, repo.id, table)
79
+
80
+ to_index = diff.to_index
81
+ extractions = _parse_phase(to_index, max_workers)
82
+
83
+ # ---- persist file rows (skipped + indexed + unchanged metadata) ----
84
+ file_id_by_path = _write_files(session, repo, walked, diff, alloc, incremental)
85
+
86
+ # ---- framework detection + entrypoint rules ----
87
+ frameworks, detected_names = _detect_frameworks(
88
+ manifests, extractions, walked, profile
89
+ )
90
+ for _path, x, _pkg in extractions:
91
+ for rule in entrypoint_rules.rules_for(x.language, detected_names):
92
+ x.entrypoint_hints.extend(rule.match(x))
93
+
94
+ # ---- symbols ----
95
+ symbol_id_by_qname, module_ids = _write_symbols(
96
+ session, extractions, file_id_by_path, alloc, table
97
+ )
98
+
99
+ # ---- class hierarchy (parents + re-exports) before edge resolution ----
100
+ resolve_hierarchy(extractions, table)
101
+
102
+ # ---- resolve references -> edges + entrypoints ----
103
+ externals = ExternalRegistry(lambda: alloc.take(Symbol))
104
+ if incremental:
105
+ externals.preload(_existing_externals(session))
106
+ new_qnames = set(symbol_id_by_qname) | {x.module_path for _p, x, _pkg in extractions}
107
+ edge_count, entrypoint_count = _write_edges_and_entrypoints(
108
+ session, extractions, file_id_by_path, module_ids, symbol_id_by_qname,
109
+ table, externals, alloc, sink_registry,
110
+ )
111
+ entrypoint_count += _write_config_entrypoints(
112
+ session, root, symbol_id_by_qname, table, alloc, incremental
113
+ )
114
+
115
+ if incremental:
116
+ _heal_dangling_edges(session, table, new_qnames)
117
+ _gc_orphan_externals(session)
118
+
119
+ # ---- detections ----
120
+ _write_detections(session, repo, profile, frameworks)
121
+
122
+ repo.file_count = len(walked)
123
+ repo.symbol_count = _count_symbols(session)
124
+ session.commit()
125
+
126
+ return IndexStats(
127
+ files_scanned=len(walked),
128
+ files_indexed=len(diff.to_index),
129
+ files_skipped=sum(1 for w in walked if w.skip_reason),
130
+ files_deleted=deleted,
131
+ symbols=repo.symbol_count,
132
+ edges=edge_count,
133
+ entrypoints=entrypoint_count,
134
+ duration_seconds=round(time.monotonic() - started, 3),
135
+ )
136
+
137
+
138
+ # ---------------- phase helpers ----------------
139
+
140
+ def _pool_context():
141
+ """multiprocessing context for the parse pool.
142
+
143
+ Prefer ``fork`` where available (POSIX): unlike the ``spawn`` default on
144
+ macOS/Windows, fork does not re-import the caller's ``__main__`` module, so
145
+ ``CodeGraph.index()`` works from any calling context — scripts without an
146
+ ``if __name__ == "__main__"`` guard, notebooks, web request handlers — rather
147
+ than raising a bootstrapping RuntimeError and re-running top-level code in
148
+ every worker. tree-sitter parsers are created lazily inside each worker, so
149
+ there is no pre-fork C state to corrupt.
150
+ """
151
+ import multiprocessing as mp
152
+
153
+ if "fork" in mp.get_all_start_methods():
154
+ return mp.get_context("fork")
155
+ return mp.get_context() # spawn (Windows); callers must guard __main__
156
+
157
+
158
+ def _parse_phase(
159
+ to_index: list[WalkedFile], max_workers: int | None
160
+ ) -> list[tuple[str, FileExtraction, bool]]:
161
+ if not to_index:
162
+ return []
163
+ workers = max_workers if max_workers is not None else (os.cpu_count() or 2)
164
+ if len(to_index) < _PARALLEL_THRESHOLD or workers <= 1:
165
+ results = []
166
+ for wf in to_index:
167
+ result = extract_one(wf)
168
+ if result is not None:
169
+ results.append(result)
170
+ return results
171
+
172
+ batches = [to_index[i : i + _BATCH] for i in range(0, len(to_index), _BATCH)]
173
+ results = []
174
+ with ProcessPoolExecutor(max_workers=workers, mp_context=_pool_context()) as pool:
175
+ for batch_result in pool.map(extract_batch, batches):
176
+ results.extend(batch_result)
177
+ return results
178
+
179
+
180
+ def _load_or_create_repo(session: Session, root: Path, incremental: bool) -> Repository:
181
+ repo = session.execute(
182
+ select(Repository).where(Repository.root_path == str(root))
183
+ ).scalars().first()
184
+ if repo is None:
185
+ repo = Repository(id=1, root_path=str(root), index_generation=0)
186
+ session.add(repo)
187
+ session.flush()
188
+ repo.indexed_at = datetime.now(timezone.utc)
189
+ repo.index_generation += 1
190
+ return repo
191
+
192
+
193
+ def _known_file_states(session: Session, repo_id: int) -> dict[str, FileState]:
194
+ rows = session.execute(
195
+ select(File.path, File.content_hash, File.size_bytes, File.mtime_ns).where(
196
+ File.repo_id == repo_id
197
+ )
198
+ )
199
+ return {p: FileState(h, s, m) for p, h, s, m in rows}
200
+
201
+
202
+ def _wipe_repo_graph(session: Session, repo_id: int) -> None:
203
+ """Full-reindex: clear the graph for this repo, keep the repo row + meta."""
204
+ file_ids = select(File.id).where(File.repo_id == repo_id)
205
+ symbol_ids = select(Symbol.id).where(Symbol.file_id.in_(file_ids))
206
+ session.execute(delete(Edge).where(Edge.src_file_id.in_(file_ids)))
207
+ session.execute(delete(Entrypoint).where(Entrypoint.symbol_id.in_(symbol_ids)))
208
+ session.execute(delete(Symbol)) # includes external placeholders (file_id NULL)
209
+ session.execute(delete(File).where(File.repo_id == repo_id))
210
+ session.execute(delete(Detection).where(Detection.repo_id == repo_id))
211
+
212
+
213
+ def _wipe_files(session: Session, repo_id: int, paths: list[str]) -> int:
214
+ if not paths:
215
+ return 0
216
+ file_ids = list(
217
+ session.execute(
218
+ select(File.id).where(File.repo_id == repo_id, File.path.in_(paths))
219
+ ).scalars()
220
+ )
221
+ if not file_ids:
222
+ return 0
223
+ # edges owned by these files; symbols cascade to entrypoints and SET NULL on
224
+ # inbound edges (degrading them to unresolved but keeping dst_qname).
225
+ session.execute(delete(Edge).where(Edge.src_file_id.in_(file_ids)))
226
+ session.execute(delete(Symbol).where(Symbol.file_id.in_(file_ids)))
227
+ session.execute(delete(File).where(File.id.in_(file_ids)))
228
+ return len(file_ids)
229
+
230
+
231
+ def _load_existing_symbols(session: Session, repo_id: int, table: SymbolTable) -> None:
232
+ rows = session.execute(
233
+ select(Symbol.id, Symbol.qname, Symbol.name, Symbol.kind)
234
+ )
235
+ for sid, qname, name, kind in rows:
236
+ if kind is SymbolKind.MODULE:
237
+ table.add_module(qname, sid)
238
+ elif kind is not SymbolKind.EXTERNAL:
239
+ table.add_symbol(sid, qname, name, kind)
240
+ # class bases/parents of surviving classes, from inherit + implement edges.
241
+ # dst_qname is the already-resolved parent FQN, so it feeds both class_bases
242
+ # (raw text, legacy) and class_parents (the transitive ancestor walk).
243
+ base_rows = session.execute(
244
+ select(Symbol.qname, Edge.dst_qname).join(Edge, Edge.src_symbol_id == Symbol.id).where(
245
+ Edge.kind.in_((EdgeKind.INHERITS, EdgeKind.IMPLEMENTS))
246
+ )
247
+ )
248
+ for class_qname, base_qname in base_rows:
249
+ table.class_bases.setdefault(class_qname, []).append(base_qname)
250
+ if base_qname in table.by_fqn: # project parent -> walkable ancestor
251
+ table.class_parents.setdefault(class_qname, []).append(base_qname)
252
+
253
+
254
+ def _existing_externals(session: Session) -> dict[str, int]:
255
+ rows = session.execute(
256
+ select(Symbol.qname, Symbol.id).where(Symbol.kind == SymbolKind.EXTERNAL)
257
+ )
258
+ return {qname: sid for qname, sid in rows}
259
+
260
+
261
+ def _write_files(
262
+ session: Session,
263
+ repo: Repository,
264
+ walked: list[WalkedFile],
265
+ diff,
266
+ alloc: IdAllocator,
267
+ incremental: bool,
268
+ ) -> dict[str, int]:
269
+ """Insert File rows for added/changed (+ skipped) files; return path->id.
270
+
271
+ On incremental, unchanged files keep their existing rows; we look up their
272
+ ids so symbols/edges can still reference them if needed.
273
+ """
274
+ file_id_by_path: dict[str, int] = {}
275
+ if incremental:
276
+ for path, fid in session.execute(
277
+ select(File.path, File.id).where(File.repo_id == repo.id)
278
+ ):
279
+ file_id_by_path[path] = fid
280
+
281
+ reindexed = {w.path for w in diff.to_index} | {
282
+ w.path for w in walked if w.skip_reason and w.path not in file_id_by_path
283
+ }
284
+ new_rows = []
285
+ for wf in walked:
286
+ if wf.path in file_id_by_path and wf.path not in {w.path for w in diff.to_index}:
287
+ continue # unchanged; row already present
288
+ if wf.path not in reindexed and not wf.skip_reason:
289
+ continue
290
+ file_id = alloc.take(File)
291
+ file_id_by_path[wf.path] = file_id
292
+ new_rows.append(
293
+ {
294
+ "id": file_id,
295
+ "repo_id": repo.id,
296
+ "path": wf.path,
297
+ "language": wf.language,
298
+ "content_hash": diff.hashes.get(wf.path, ""),
299
+ "size_bytes": wf.size_bytes,
300
+ "mtime_ns": wf.mtime_ns,
301
+ "generation": repo.index_generation,
302
+ "skip_reason": wf.skip_reason,
303
+ }
304
+ )
305
+ bulk_insert(session, File, new_rows)
306
+ return file_id_by_path
307
+
308
+
309
+ def _detect_frameworks(manifests, extractions, walked, profile: RepoLanguageProfile):
310
+ import_signals = {
311
+ (x.language, value)
312
+ for _p, x, _pkg in extractions
313
+ for kind, value in x.framework_signals
314
+ if kind == "import"
315
+ }
316
+ symbol_names = {raw.name for _p, x, _pkg in extractions for raw in x.symbols}
317
+ languages_present = profile.extractable_languages()
318
+ if languages_present & {"typescript", "tsx"}:
319
+ languages_present.add("javascript")
320
+ frameworks = detect_frameworks(
321
+ manifests, import_signals, [w.path for w in walked], symbol_names,
322
+ languages_present=languages_present,
323
+ )
324
+ return frameworks, {fw.name for fw in frameworks}
325
+
326
+
327
+ def _write_symbols(session, extractions, file_id_by_path, alloc, table):
328
+ symbol_rows: list[dict] = []
329
+ module_ids: dict[str, int] = {}
330
+ for path, x, _pkg in extractions:
331
+ module_id = alloc.take(Symbol)
332
+ module_ids[path] = module_id
333
+ table.add_module(x.module_path, module_id)
334
+ symbol_rows.append(
335
+ {
336
+ "id": module_id, "file_id": file_id_by_path[path], "kind": SymbolKind.MODULE,
337
+ "name": x.module_path.rsplit(".", 1)[-1], "qname": x.module_path,
338
+ "parent_id": None, "start_line": 1, "end_line": 0, "start_col": 0,
339
+ "signature": None, "docstring": None, "is_exported": True,
340
+ }
341
+ )
342
+
343
+ symbol_id_by_qname: dict[str, int] = {}
344
+ for path, x, _pkg in extractions:
345
+ for raw in x.symbols:
346
+ symbol_id = alloc.take(Symbol)
347
+ symbol_id_by_qname[raw.qualified_name] = symbol_id
348
+ table.add_symbol(symbol_id, raw.qualified_name, raw.name, raw.kind)
349
+ if raw.kind is SymbolKind.CLASS and raw.bases:
350
+ table.class_bases[raw.qualified_name] = raw.bases
351
+ symbol_rows.append(
352
+ {
353
+ "id": symbol_id, "file_id": file_id_by_path[path], "kind": raw.kind,
354
+ "name": raw.name, "qname": raw.qualified_name, "parent_id": None,
355
+ "start_line": raw.span.start_line, "end_line": raw.span.end_line,
356
+ "start_col": raw.span.start_col, "signature": raw.signature,
357
+ "docstring": raw.docstring, "is_exported": raw.is_exported,
358
+ }
359
+ )
360
+ for row in symbol_rows:
361
+ qname = row["qname"]
362
+ if row["kind"] is not SymbolKind.MODULE and "." in qname:
363
+ parent_q = qname.rsplit(".", 1)[0]
364
+ row["parent_id"] = symbol_id_by_qname.get(parent_q) or table.module_symbol_ids.get(parent_q)
365
+ # A row's parent has one fewer qname segment, so inserting shallowest-first
366
+ # guarantees the self-referential parent_id FK is satisfied within the batch.
367
+ symbol_rows.sort(key=lambda r: r["qname"].count("."))
368
+ bulk_insert(session, Symbol, symbol_rows)
369
+ return symbol_id_by_qname, module_ids
370
+
371
+
372
+ def _write_edges_and_entrypoints(
373
+ session, extractions, file_id_by_path, module_ids, symbol_id_by_qname,
374
+ table, externals, alloc, sink_registry: SinkRegistry,
375
+ ):
376
+ edge_rows: list[dict] = []
377
+ entrypoint_rows: list[dict] = []
378
+
379
+ for path, x, is_package in extractions:
380
+ resolver = FileResolver(x, module_ids[path], table, externals, is_package)
381
+ file_id = file_id_by_path[path]
382
+ for edge in resolver.resolve():
383
+ sink_id = (
384
+ sink_registry.match(edge.dst_qname, edge.arg_preview)
385
+ if edge.kind is EdgeKind.CALLS else None
386
+ )
387
+ edge_rows.append(
388
+ {
389
+ "id": alloc.take(Edge), "kind": edge.kind,
390
+ "src_symbol_id": edge.src_symbol_id, "dst_symbol_id": edge.dst_symbol_id,
391
+ "dst_qname": edge.dst_qname, "src_file_id": file_id, "line": edge.line,
392
+ "confidence": int(edge.confidence), "arg_preview": edge.arg_preview,
393
+ "sink_id": sink_id, "via": edge.via,
394
+ }
395
+ )
396
+ for hint in x.entrypoint_hints:
397
+ symbol_id = symbol_id_by_qname.get(hint.handler_qualified_name or "") or module_ids[path]
398
+ entrypoint_rows.append(
399
+ {
400
+ "id": alloc.take(Entrypoint), "kind": hint.kind, "framework": hint.framework,
401
+ "symbol_id": symbol_id, "route": hint.route,
402
+ "http_method": ",".join(hint.http_methods) or None,
403
+ "extra": json.dumps(hint.metadata) if hint.metadata else None,
404
+ }
405
+ )
406
+ bulk_insert(session, Symbol, externals.new_rows)
407
+ bulk_insert(session, Edge, edge_rows)
408
+ bulk_insert(session, Entrypoint, entrypoint_rows)
409
+ return len(edge_rows), len(entrypoint_rows)
410
+
411
+
412
+ def _write_config_entrypoints(
413
+ session, root, symbol_id_by_qname: dict[str, int], table: SymbolTable,
414
+ alloc: IdAllocator, incremental: bool,
415
+ ) -> int:
416
+ """Scan serverless/SAM/Procfile/Dockerfile and bind their handlers to symbols.
417
+
418
+ Config files aren't tracked in the files table, so on incremental runs their
419
+ entrypoints are fully deleted and re-derived each time.
420
+ """
421
+ from entrygraph.detect.entrypoints.configs import (
422
+ CONFIG_FRAMEWORKS,
423
+ bind_handler,
424
+ scan_config_entrypoints,
425
+ )
426
+
427
+ if incremental:
428
+ session.execute(
429
+ delete(Entrypoint).where(Entrypoint.framework.in_(CONFIG_FRAMEWORKS))
430
+ )
431
+
432
+ rows = []
433
+ for hint in scan_config_entrypoints(root):
434
+ symbol_id = bind_handler(hint.handler_ref, symbol_id_by_qname, table.module_symbol_ids)
435
+ if symbol_id is None: # non-nullable FK: skip unbindable handlers
436
+ continue
437
+ rows.append(
438
+ {
439
+ "id": alloc.take(Entrypoint), "kind": hint.kind, "framework": hint.framework,
440
+ "symbol_id": symbol_id, "route": hint.route, "http_method": None,
441
+ "extra": json.dumps(hint.metadata) if hint.metadata else None,
442
+ }
443
+ )
444
+ bulk_insert(session, Entrypoint, rows)
445
+ return len(rows)
446
+
447
+
448
+ def _heal_dangling_edges(session, table: SymbolTable, new_qnames: set[str]) -> None:
449
+ """Re-bind edges left NULL (degraded on wipe, or targeting a not-yet-existing
450
+ symbol) whose dst_qname now names a freshly-created symbol."""
451
+ dangling = session.execute(
452
+ select(Edge.id, Edge.dst_qname).where(
453
+ Edge.dst_symbol_id.is_(None), Edge.dst_qname.in_(new_qnames)
454
+ )
455
+ ).all()
456
+ updates = []
457
+ for edge_id, dst_qname in dangling:
458
+ target = table.by_fqn.get(dst_qname)
459
+ if target is not None:
460
+ updates.append({"id": edge_id, "dst_symbol_id": target,
461
+ "confidence": int(Confidence.IMPORT)})
462
+ if updates:
463
+ session.execute(update(Edge), updates)
464
+
465
+
466
+ def _gc_orphan_externals(session) -> None:
467
+ """Drop external placeholder symbols no edge points at anymore.
468
+
469
+ Full re-index never creates them; deleting them keeps an incremental graph
470
+ byte-identical to a full one after a file's last reference disappears.
471
+ """
472
+ referenced = select(Edge.dst_symbol_id).where(Edge.dst_symbol_id.is_not(None))
473
+ session.execute(
474
+ delete(Symbol).where(
475
+ Symbol.kind == SymbolKind.EXTERNAL, Symbol.id.not_in(referenced)
476
+ )
477
+ )
478
+
479
+
480
+ def _write_detections(session, repo, profile: RepoLanguageProfile, frameworks) -> None:
481
+ session.execute(delete(Detection).where(Detection.repo_id == repo.id))
482
+ rows = [
483
+ {
484
+ "repo_id": repo.id, "category": "language", "name": stat.name, "version": None,
485
+ "confidence": 1.0,
486
+ "evidence": json.dumps({"files": stat.file_count, "bytes": stat.byte_count,
487
+ "percent": round(stat.percent, 2)}),
488
+ }
489
+ for stat in profile.stats()
490
+ ]
491
+ rows.extend(
492
+ {
493
+ "repo_id": repo.id, "category": "framework", "name": fw.name, "version": None,
494
+ "confidence": fw.confidence,
495
+ "evidence": json.dumps({"language": fw.language, "signals": list(fw.evidence)}),
496
+ }
497
+ for fw in frameworks
498
+ )
499
+ if rows:
500
+ bulk_insert(session, Detection, rows)
501
+
502
+
503
+ def _count_symbols(session) -> int:
504
+ from sqlalchemy import func
505
+
506
+ return session.execute(select(func.count(Symbol.id))).scalar() or 0
@@ -0,0 +1,49 @@
1
+ """Process-pool extraction worker.
2
+
3
+ Workers parse and extract files into plain-data IR. tree-sitter parsers and
4
+ compiled queries are lazily cached per process (see parsing.parsers /
5
+ parsing.queries), so nothing unpicklable crosses the pool boundary — inputs are
6
+ WalkedFile records, outputs are FileExtraction dataclasses.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from pathlib import Path
12
+
13
+ from entrygraph.extract.base import FileContext
14
+ from entrygraph.extract.ir import FileExtraction
15
+ from entrygraph.extract.registry import extractor_for
16
+ from entrygraph.fs.walker import WalkedFile
17
+ from entrygraph.parsing.parsers import parse, supported
18
+
19
+
20
+ def extract_one(walked: WalkedFile) -> tuple[str, FileExtraction, bool] | None:
21
+ """Parse+extract one file -> (path, extraction, is_package), or None to skip."""
22
+ if walked.skip_reason or not supported(walked.language or ""):
23
+ return None
24
+ extractor = extractor_for(walked.language)
25
+ if extractor is None:
26
+ return None
27
+ try:
28
+ source = Path(walked.abs_path).read_bytes()
29
+ except OSError:
30
+ return None
31
+ module_path, is_package = extractor.module_path_for(walked.path)
32
+ tree = parse(walked.language, source)
33
+ ctx = FileContext(
34
+ path=walked.path,
35
+ language=walked.language,
36
+ module_path=module_path,
37
+ source=source,
38
+ is_package=is_package,
39
+ )
40
+ return walked.path, extractor.extract(tree, ctx), is_package
41
+
42
+
43
+ def extract_batch(batch: list[WalkedFile]) -> list[tuple[str, FileExtraction, bool]]:
44
+ results = []
45
+ for walked in batch:
46
+ result = extract_one(walked)
47
+ if result is not None:
48
+ results.append(result)
49
+ return results
@@ -0,0 +1,41 @@
1
+ """The only bridge between extraction IR and the ORM.
2
+
3
+ All inserts are SQLAlchemy 2.0 ORM-enabled bulk inserts —
4
+ ``session.execute(insert(Model), list_of_dicts)`` — with application-assigned
5
+ integer primary keys so edge rows can reference symbol ids without RETURNING
6
+ round-trips. This is the fastest write path that still goes through the ORM.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import Iterable, Type
12
+
13
+ from sqlalchemy import func, insert, select
14
+ from sqlalchemy.orm import Session
15
+
16
+ from entrygraph.db.models import Base, Edge, Entrypoint, File, Symbol
17
+
18
+ _BATCH = 5000
19
+
20
+
21
+ class IdAllocator:
22
+ """Per-table monotonically increasing ids, seeded from MAX(id)."""
23
+
24
+ def __init__(self, session: Session) -> None:
25
+ self._next: dict[str, int] = {}
26
+ for model in (File, Symbol, Edge, Entrypoint):
27
+ current = session.execute(select(func.max(model.id))).scalar() or 0
28
+ self._next[model.__tablename__] = current + 1
29
+
30
+ def take(self, model: Type[Base]) -> int:
31
+ table = model.__tablename__
32
+ value = self._next[table]
33
+ self._next[table] = value + 1
34
+ return value
35
+
36
+
37
+ def bulk_insert(session: Session, model: Type[Base], rows: Iterable[dict]) -> int:
38
+ rows = list(rows)
39
+ for start in range(0, len(rows), _BATCH):
40
+ session.execute(insert(model), rows[start : start + _BATCH])
41
+ return len(rows)
entrygraph/py.typed ADDED
File without changes
@@ -0,0 +1,4 @@
1
+ ; Method calls and object creations. The shaper reads the `function` /
2
+ ; `type` fields to derive callee text, receiver, and constructed-type name.
3
+ (invocation_expression) @call
4
+ (object_creation_expression) @new
@@ -0,0 +1,29 @@
1
+ ; Harvest-level queries: find definition nodes; the C# shaper computes
2
+ ; qualified names (from the enclosing namespace), kinds, attributes (as
3
+ ; decorators), modifiers, and supertypes by walking from these.
4
+
5
+ (class_declaration
6
+ name: (identifier) @name) @def.class
7
+
8
+ (interface_declaration
9
+ name: (identifier) @name) @def.interface
10
+
11
+ (struct_declaration
12
+ name: (identifier) @name) @def.struct
13
+
14
+ (record_declaration
15
+ name: (identifier) @name) @def.record
16
+
17
+ (method_declaration
18
+ name: (identifier) @name) @def.method
19
+
20
+ (constructor_declaration
21
+ name: (identifier) @name) @def.constructor
22
+
23
+ (local_function_statement
24
+ name: (identifier) @name) @def.local_function
25
+
26
+ (property_declaration
27
+ name: (identifier) @name) @def.property
28
+
29
+ (field_declaration) @def.field
@@ -0,0 +1,4 @@
1
+ ; C# `using` directives: plain (`using System.Diagnostics;`), aliased
2
+ ; (`using Alias = System.Collections.Generic;`), and static
3
+ ; (`using static System.Math;`). The shaper distinguishes them.
4
+ (using_directive) @import
@@ -0,0 +1,2 @@
1
+ (call_expression) @call
2
+ (composite_literal type: (qualified_type) @composite.type) @composite
@@ -0,0 +1,24 @@
1
+ ; Harvest-level queries: find definition nodes; the Go shaper computes
2
+ ; qualified names, kinds, and receivers by walking from these.
3
+
4
+ (function_declaration
5
+ name: (identifier) @name) @def.function
6
+
7
+ (method_declaration
8
+ name: (field_identifier) @name) @def.method
9
+
10
+ (type_declaration
11
+ (type_spec
12
+ name: (type_identifier) @name
13
+ type: (struct_type)) ) @def.struct
14
+
15
+ (type_declaration
16
+ (type_spec
17
+ name: (type_identifier) @name
18
+ type: (interface_type)) ) @def.interface
19
+
20
+ (const_declaration (const_spec name: (identifier) @name)) @def.const
21
+
22
+ (var_declaration (var_spec name: (identifier) @name)) @def.var
23
+
24
+ (field_declaration name: (field_identifier) @name) @def.field
@@ -0,0 +1 @@
1
+ (import_declaration) @import
@@ -0,0 +1,2 @@
1
+ (method_invocation) @call
2
+ (object_creation_expression) @new