entrygraph 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- entrygraph/__init__.py +87 -0
- entrygraph/__main__.py +8 -0
- entrygraph/_version.py +24 -0
- entrygraph/api.py +549 -0
- entrygraph/cli/__init__.py +0 -0
- entrygraph/cli/main.py +387 -0
- entrygraph/cli/render.py +136 -0
- entrygraph/data/sinks/csharp.toml +106 -0
- entrygraph/data/sinks/go.toml +87 -0
- entrygraph/data/sinks/java.toml +92 -0
- entrygraph/data/sinks/javascript.toml +112 -0
- entrygraph/data/sinks/lib_javascript.toml +34 -0
- entrygraph/data/sinks/lib_python.toml +39 -0
- entrygraph/data/sinks/php.toml +125 -0
- entrygraph/data/sinks/python.toml +160 -0
- entrygraph/data/sinks/ruby.toml +102 -0
- entrygraph/data/sinks/rust.toml +68 -0
- entrygraph/db/__init__.py +0 -0
- entrygraph/db/engine.py +34 -0
- entrygraph/db/meta.py +70 -0
- entrygraph/db/models.py +176 -0
- entrygraph/db/queries.py +155 -0
- entrygraph/detect/__init__.py +0 -0
- entrygraph/detect/entrypoints/__init__.py +22 -0
- entrygraph/detect/entrypoints/base.py +124 -0
- entrygraph/detect/entrypoints/configs.py +139 -0
- entrygraph/detect/entrypoints/csharp.py +156 -0
- entrygraph/detect/entrypoints/golang.py +158 -0
- entrygraph/detect/entrypoints/java.py +187 -0
- entrygraph/detect/entrypoints/javascript.py +211 -0
- entrygraph/detect/entrypoints/php.py +133 -0
- entrygraph/detect/entrypoints/python.py +335 -0
- entrygraph/detect/entrypoints/ruby.py +147 -0
- entrygraph/detect/entrypoints/rust.py +153 -0
- entrygraph/detect/frameworks.py +369 -0
- entrygraph/detect/manifests.py +234 -0
- entrygraph/detect/taint.py +224 -0
- entrygraph/errors.py +27 -0
- entrygraph/extract/__init__.py +0 -0
- entrygraph/extract/base.py +51 -0
- entrygraph/extract/csharp.py +502 -0
- entrygraph/extract/golang.py +342 -0
- entrygraph/extract/ir.py +105 -0
- entrygraph/extract/java.py +329 -0
- entrygraph/extract/javascript.py +400 -0
- entrygraph/extract/php.py +426 -0
- entrygraph/extract/python.py +390 -0
- entrygraph/extract/registry.py +43 -0
- entrygraph/extract/ruby.py +321 -0
- entrygraph/extract/rust.py +482 -0
- entrygraph/fs/__init__.py +0 -0
- entrygraph/fs/hashing.py +78 -0
- entrygraph/fs/lang.py +134 -0
- entrygraph/fs/walker.py +167 -0
- entrygraph/graph/__init__.py +0 -0
- entrygraph/graph/adjacency.py +146 -0
- entrygraph/graph/cte.py +123 -0
- entrygraph/graph/scoring.py +101 -0
- entrygraph/kinds.py +51 -0
- entrygraph/parsing/__init__.py +0 -0
- entrygraph/parsing/parsers.py +49 -0
- entrygraph/parsing/queries.py +39 -0
- entrygraph/pipeline/__init__.py +0 -0
- entrygraph/pipeline/scanner.py +506 -0
- entrygraph/pipeline/worker.py +49 -0
- entrygraph/pipeline/writer.py +41 -0
- entrygraph/py.typed +0 -0
- entrygraph/queries/csharp/calls.scm +4 -0
- entrygraph/queries/csharp/definitions.scm +29 -0
- entrygraph/queries/csharp/imports.scm +4 -0
- entrygraph/queries/go/calls.scm +2 -0
- entrygraph/queries/go/definitions.scm +24 -0
- entrygraph/queries/go/imports.scm +1 -0
- entrygraph/queries/java/calls.scm +2 -0
- entrygraph/queries/java/definitions.scm +14 -0
- entrygraph/queries/java/imports.scm +2 -0
- entrygraph/queries/javascript/calls.scm +4 -0
- entrygraph/queries/javascript/definitions.scm +4 -0
- entrygraph/queries/javascript/imports.scm +6 -0
- entrygraph/queries/php/calls.scm +8 -0
- entrygraph/queries/php/definitions.scm +24 -0
- entrygraph/queries/php/imports.scm +1 -0
- entrygraph/queries/python/calls.scm +2 -0
- entrygraph/queries/python/definitions.scm +11 -0
- entrygraph/queries/python/imports.scm +2 -0
- entrygraph/queries/ruby/calls.scm +4 -0
- entrygraph/queries/ruby/definitions.scm +20 -0
- entrygraph/queries/ruby/imports.scm +7 -0
- entrygraph/queries/rust/calls.scm +5 -0
- entrygraph/queries/rust/definitions.scm +26 -0
- entrygraph/queries/rust/imports.scm +4 -0
- entrygraph/resolve/__init__.py +0 -0
- entrygraph/resolve/externals.py +61 -0
- entrygraph/resolve/hierarchy.py +152 -0
- entrygraph/resolve/resolver.py +275 -0
- entrygraph/resolve/symbol_table.py +48 -0
- entrygraph/results.py +138 -0
- entrygraph-0.1.0.dist-info/METADATA +204 -0
- entrygraph-0.1.0.dist-info/RECORD +102 -0
- entrygraph-0.1.0.dist-info/WHEEL +4 -0
- entrygraph-0.1.0.dist-info/entry_points.txt +2 -0
- entrygraph-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,506 @@
|
|
|
1
|
+
"""Index orchestrator: walk -> diff -> parse+extract -> resolve -> detect -> write.
|
|
2
|
+
|
|
3
|
+
One code path serves both full and incremental indexing. Incremental wipes only
|
|
4
|
+
changed/deleted files, re-extracts them, resolves their references against a
|
|
5
|
+
symbol table seeded from the surviving DB symbols, and heals edges (in either
|
|
6
|
+
direction) that now bind to newly-created symbols. The result is identical to a
|
|
7
|
+
full re-index.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import time
|
|
15
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
16
|
+
from datetime import datetime, timezone
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from sqlalchemy import Engine, delete, select, update
|
|
20
|
+
from sqlalchemy.orm import Session
|
|
21
|
+
|
|
22
|
+
from entrygraph.db.meta import ensure_schema
|
|
23
|
+
from entrygraph.db.models import Detection, Edge, Entrypoint, File, Repository, Symbol
|
|
24
|
+
from entrygraph.detect import entrypoints as entrypoint_rules
|
|
25
|
+
from entrygraph.detect.frameworks import detect_frameworks
|
|
26
|
+
from entrygraph.detect.manifests import parse_manifests
|
|
27
|
+
from entrygraph.detect.taint import SinkRegistry, registry_for_repo
|
|
28
|
+
from entrygraph.extract.ir import FileExtraction
|
|
29
|
+
from entrygraph.fs.hashing import FileState, diff_files
|
|
30
|
+
from entrygraph.fs.walker import RepoLanguageProfile, WalkedFile, walk_repo
|
|
31
|
+
from entrygraph.kinds import Confidence, EdgeKind, SymbolKind
|
|
32
|
+
from entrygraph.pipeline.worker import extract_batch, extract_one
|
|
33
|
+
from entrygraph.pipeline.writer import IdAllocator, bulk_insert
|
|
34
|
+
from entrygraph.resolve.externals import ExternalRegistry
|
|
35
|
+
from entrygraph.resolve.hierarchy import resolve_hierarchy
|
|
36
|
+
from entrygraph.resolve.resolver import FileResolver
|
|
37
|
+
from entrygraph.resolve.symbol_table import SymbolTable
|
|
38
|
+
from entrygraph.results import IndexStats
|
|
39
|
+
|
|
40
|
+
_PARALLEL_THRESHOLD = 200 # files; below this the pool overhead isn't worth it
|
|
41
|
+
_BATCH = 24
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def index_repository(
|
|
45
|
+
root: str | Path,
|
|
46
|
+
engine: Engine,
|
|
47
|
+
*,
|
|
48
|
+
incremental: bool = False,
|
|
49
|
+
paranoid: bool = False,
|
|
50
|
+
max_workers: int | None = None,
|
|
51
|
+
) -> IndexStats:
|
|
52
|
+
started = time.monotonic()
|
|
53
|
+
root = Path(root).resolve()
|
|
54
|
+
fresh = ensure_schema(engine)
|
|
55
|
+
if fresh:
|
|
56
|
+
incremental = False
|
|
57
|
+
|
|
58
|
+
walked, profile = walk_repo(root)
|
|
59
|
+
manifests = parse_manifests(root)
|
|
60
|
+
sink_registry = registry_for_repo(root)
|
|
61
|
+
|
|
62
|
+
with Session(engine) as session:
|
|
63
|
+
repo = _load_or_create_repo(session, root, incremental)
|
|
64
|
+
known = _known_file_states(session, repo.id) if incremental else {}
|
|
65
|
+
diff = diff_files(walked, known, paranoid=paranoid)
|
|
66
|
+
|
|
67
|
+
if incremental:
|
|
68
|
+
deleted = _wipe_files(session, repo.id, [*[w.path for w in diff.changed],
|
|
69
|
+
*diff.deleted_paths])
|
|
70
|
+
else:
|
|
71
|
+
_wipe_repo_graph(session, repo.id)
|
|
72
|
+
deleted = 0
|
|
73
|
+
session.flush()
|
|
74
|
+
|
|
75
|
+
alloc = IdAllocator(session)
|
|
76
|
+
table = SymbolTable()
|
|
77
|
+
if incremental:
|
|
78
|
+
_load_existing_symbols(session, repo.id, table)
|
|
79
|
+
|
|
80
|
+
to_index = diff.to_index
|
|
81
|
+
extractions = _parse_phase(to_index, max_workers)
|
|
82
|
+
|
|
83
|
+
# ---- persist file rows (skipped + indexed + unchanged metadata) ----
|
|
84
|
+
file_id_by_path = _write_files(session, repo, walked, diff, alloc, incremental)
|
|
85
|
+
|
|
86
|
+
# ---- framework detection + entrypoint rules ----
|
|
87
|
+
frameworks, detected_names = _detect_frameworks(
|
|
88
|
+
manifests, extractions, walked, profile
|
|
89
|
+
)
|
|
90
|
+
for _path, x, _pkg in extractions:
|
|
91
|
+
for rule in entrypoint_rules.rules_for(x.language, detected_names):
|
|
92
|
+
x.entrypoint_hints.extend(rule.match(x))
|
|
93
|
+
|
|
94
|
+
# ---- symbols ----
|
|
95
|
+
symbol_id_by_qname, module_ids = _write_symbols(
|
|
96
|
+
session, extractions, file_id_by_path, alloc, table
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# ---- class hierarchy (parents + re-exports) before edge resolution ----
|
|
100
|
+
resolve_hierarchy(extractions, table)
|
|
101
|
+
|
|
102
|
+
# ---- resolve references -> edges + entrypoints ----
|
|
103
|
+
externals = ExternalRegistry(lambda: alloc.take(Symbol))
|
|
104
|
+
if incremental:
|
|
105
|
+
externals.preload(_existing_externals(session))
|
|
106
|
+
new_qnames = set(symbol_id_by_qname) | {x.module_path for _p, x, _pkg in extractions}
|
|
107
|
+
edge_count, entrypoint_count = _write_edges_and_entrypoints(
|
|
108
|
+
session, extractions, file_id_by_path, module_ids, symbol_id_by_qname,
|
|
109
|
+
table, externals, alloc, sink_registry,
|
|
110
|
+
)
|
|
111
|
+
entrypoint_count += _write_config_entrypoints(
|
|
112
|
+
session, root, symbol_id_by_qname, table, alloc, incremental
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
if incremental:
|
|
116
|
+
_heal_dangling_edges(session, table, new_qnames)
|
|
117
|
+
_gc_orphan_externals(session)
|
|
118
|
+
|
|
119
|
+
# ---- detections ----
|
|
120
|
+
_write_detections(session, repo, profile, frameworks)
|
|
121
|
+
|
|
122
|
+
repo.file_count = len(walked)
|
|
123
|
+
repo.symbol_count = _count_symbols(session)
|
|
124
|
+
session.commit()
|
|
125
|
+
|
|
126
|
+
return IndexStats(
|
|
127
|
+
files_scanned=len(walked),
|
|
128
|
+
files_indexed=len(diff.to_index),
|
|
129
|
+
files_skipped=sum(1 for w in walked if w.skip_reason),
|
|
130
|
+
files_deleted=deleted,
|
|
131
|
+
symbols=repo.symbol_count,
|
|
132
|
+
edges=edge_count,
|
|
133
|
+
entrypoints=entrypoint_count,
|
|
134
|
+
duration_seconds=round(time.monotonic() - started, 3),
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# ---------------- phase helpers ----------------
|
|
139
|
+
|
|
140
|
+
def _pool_context():
|
|
141
|
+
"""multiprocessing context for the parse pool.
|
|
142
|
+
|
|
143
|
+
Prefer ``fork`` where available (POSIX): unlike the ``spawn`` default on
|
|
144
|
+
macOS/Windows, fork does not re-import the caller's ``__main__`` module, so
|
|
145
|
+
``CodeGraph.index()`` works from any calling context — scripts without an
|
|
146
|
+
``if __name__ == "__main__"`` guard, notebooks, web request handlers — rather
|
|
147
|
+
than raising a bootstrapping RuntimeError and re-running top-level code in
|
|
148
|
+
every worker. tree-sitter parsers are created lazily inside each worker, so
|
|
149
|
+
there is no pre-fork C state to corrupt.
|
|
150
|
+
"""
|
|
151
|
+
import multiprocessing as mp
|
|
152
|
+
|
|
153
|
+
if "fork" in mp.get_all_start_methods():
|
|
154
|
+
return mp.get_context("fork")
|
|
155
|
+
return mp.get_context() # spawn (Windows); callers must guard __main__
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _parse_phase(
|
|
159
|
+
to_index: list[WalkedFile], max_workers: int | None
|
|
160
|
+
) -> list[tuple[str, FileExtraction, bool]]:
|
|
161
|
+
if not to_index:
|
|
162
|
+
return []
|
|
163
|
+
workers = max_workers if max_workers is not None else (os.cpu_count() or 2)
|
|
164
|
+
if len(to_index) < _PARALLEL_THRESHOLD or workers <= 1:
|
|
165
|
+
results = []
|
|
166
|
+
for wf in to_index:
|
|
167
|
+
result = extract_one(wf)
|
|
168
|
+
if result is not None:
|
|
169
|
+
results.append(result)
|
|
170
|
+
return results
|
|
171
|
+
|
|
172
|
+
batches = [to_index[i : i + _BATCH] for i in range(0, len(to_index), _BATCH)]
|
|
173
|
+
results = []
|
|
174
|
+
with ProcessPoolExecutor(max_workers=workers, mp_context=_pool_context()) as pool:
|
|
175
|
+
for batch_result in pool.map(extract_batch, batches):
|
|
176
|
+
results.extend(batch_result)
|
|
177
|
+
return results
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _load_or_create_repo(session: Session, root: Path, incremental: bool) -> Repository:
|
|
181
|
+
repo = session.execute(
|
|
182
|
+
select(Repository).where(Repository.root_path == str(root))
|
|
183
|
+
).scalars().first()
|
|
184
|
+
if repo is None:
|
|
185
|
+
repo = Repository(id=1, root_path=str(root), index_generation=0)
|
|
186
|
+
session.add(repo)
|
|
187
|
+
session.flush()
|
|
188
|
+
repo.indexed_at = datetime.now(timezone.utc)
|
|
189
|
+
repo.index_generation += 1
|
|
190
|
+
return repo
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _known_file_states(session: Session, repo_id: int) -> dict[str, FileState]:
|
|
194
|
+
rows = session.execute(
|
|
195
|
+
select(File.path, File.content_hash, File.size_bytes, File.mtime_ns).where(
|
|
196
|
+
File.repo_id == repo_id
|
|
197
|
+
)
|
|
198
|
+
)
|
|
199
|
+
return {p: FileState(h, s, m) for p, h, s, m in rows}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _wipe_repo_graph(session: Session, repo_id: int) -> None:
|
|
203
|
+
"""Full-reindex: clear the graph for this repo, keep the repo row + meta."""
|
|
204
|
+
file_ids = select(File.id).where(File.repo_id == repo_id)
|
|
205
|
+
symbol_ids = select(Symbol.id).where(Symbol.file_id.in_(file_ids))
|
|
206
|
+
session.execute(delete(Edge).where(Edge.src_file_id.in_(file_ids)))
|
|
207
|
+
session.execute(delete(Entrypoint).where(Entrypoint.symbol_id.in_(symbol_ids)))
|
|
208
|
+
session.execute(delete(Symbol)) # includes external placeholders (file_id NULL)
|
|
209
|
+
session.execute(delete(File).where(File.repo_id == repo_id))
|
|
210
|
+
session.execute(delete(Detection).where(Detection.repo_id == repo_id))
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _wipe_files(session: Session, repo_id: int, paths: list[str]) -> int:
|
|
214
|
+
if not paths:
|
|
215
|
+
return 0
|
|
216
|
+
file_ids = list(
|
|
217
|
+
session.execute(
|
|
218
|
+
select(File.id).where(File.repo_id == repo_id, File.path.in_(paths))
|
|
219
|
+
).scalars()
|
|
220
|
+
)
|
|
221
|
+
if not file_ids:
|
|
222
|
+
return 0
|
|
223
|
+
# edges owned by these files; symbols cascade to entrypoints and SET NULL on
|
|
224
|
+
# inbound edges (degrading them to unresolved but keeping dst_qname).
|
|
225
|
+
session.execute(delete(Edge).where(Edge.src_file_id.in_(file_ids)))
|
|
226
|
+
session.execute(delete(Symbol).where(Symbol.file_id.in_(file_ids)))
|
|
227
|
+
session.execute(delete(File).where(File.id.in_(file_ids)))
|
|
228
|
+
return len(file_ids)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _load_existing_symbols(session: Session, repo_id: int, table: SymbolTable) -> None:
|
|
232
|
+
rows = session.execute(
|
|
233
|
+
select(Symbol.id, Symbol.qname, Symbol.name, Symbol.kind)
|
|
234
|
+
)
|
|
235
|
+
for sid, qname, name, kind in rows:
|
|
236
|
+
if kind is SymbolKind.MODULE:
|
|
237
|
+
table.add_module(qname, sid)
|
|
238
|
+
elif kind is not SymbolKind.EXTERNAL:
|
|
239
|
+
table.add_symbol(sid, qname, name, kind)
|
|
240
|
+
# class bases/parents of surviving classes, from inherit + implement edges.
|
|
241
|
+
# dst_qname is the already-resolved parent FQN, so it feeds both class_bases
|
|
242
|
+
# (raw text, legacy) and class_parents (the transitive ancestor walk).
|
|
243
|
+
base_rows = session.execute(
|
|
244
|
+
select(Symbol.qname, Edge.dst_qname).join(Edge, Edge.src_symbol_id == Symbol.id).where(
|
|
245
|
+
Edge.kind.in_((EdgeKind.INHERITS, EdgeKind.IMPLEMENTS))
|
|
246
|
+
)
|
|
247
|
+
)
|
|
248
|
+
for class_qname, base_qname in base_rows:
|
|
249
|
+
table.class_bases.setdefault(class_qname, []).append(base_qname)
|
|
250
|
+
if base_qname in table.by_fqn: # project parent -> walkable ancestor
|
|
251
|
+
table.class_parents.setdefault(class_qname, []).append(base_qname)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _existing_externals(session: Session) -> dict[str, int]:
|
|
255
|
+
rows = session.execute(
|
|
256
|
+
select(Symbol.qname, Symbol.id).where(Symbol.kind == SymbolKind.EXTERNAL)
|
|
257
|
+
)
|
|
258
|
+
return {qname: sid for qname, sid in rows}
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _write_files(
|
|
262
|
+
session: Session,
|
|
263
|
+
repo: Repository,
|
|
264
|
+
walked: list[WalkedFile],
|
|
265
|
+
diff,
|
|
266
|
+
alloc: IdAllocator,
|
|
267
|
+
incremental: bool,
|
|
268
|
+
) -> dict[str, int]:
|
|
269
|
+
"""Insert File rows for added/changed (+ skipped) files; return path->id.
|
|
270
|
+
|
|
271
|
+
On incremental, unchanged files keep their existing rows; we look up their
|
|
272
|
+
ids so symbols/edges can still reference them if needed.
|
|
273
|
+
"""
|
|
274
|
+
file_id_by_path: dict[str, int] = {}
|
|
275
|
+
if incremental:
|
|
276
|
+
for path, fid in session.execute(
|
|
277
|
+
select(File.path, File.id).where(File.repo_id == repo.id)
|
|
278
|
+
):
|
|
279
|
+
file_id_by_path[path] = fid
|
|
280
|
+
|
|
281
|
+
reindexed = {w.path for w in diff.to_index} | {
|
|
282
|
+
w.path for w in walked if w.skip_reason and w.path not in file_id_by_path
|
|
283
|
+
}
|
|
284
|
+
new_rows = []
|
|
285
|
+
for wf in walked:
|
|
286
|
+
if wf.path in file_id_by_path and wf.path not in {w.path for w in diff.to_index}:
|
|
287
|
+
continue # unchanged; row already present
|
|
288
|
+
if wf.path not in reindexed and not wf.skip_reason:
|
|
289
|
+
continue
|
|
290
|
+
file_id = alloc.take(File)
|
|
291
|
+
file_id_by_path[wf.path] = file_id
|
|
292
|
+
new_rows.append(
|
|
293
|
+
{
|
|
294
|
+
"id": file_id,
|
|
295
|
+
"repo_id": repo.id,
|
|
296
|
+
"path": wf.path,
|
|
297
|
+
"language": wf.language,
|
|
298
|
+
"content_hash": diff.hashes.get(wf.path, ""),
|
|
299
|
+
"size_bytes": wf.size_bytes,
|
|
300
|
+
"mtime_ns": wf.mtime_ns,
|
|
301
|
+
"generation": repo.index_generation,
|
|
302
|
+
"skip_reason": wf.skip_reason,
|
|
303
|
+
}
|
|
304
|
+
)
|
|
305
|
+
bulk_insert(session, File, new_rows)
|
|
306
|
+
return file_id_by_path
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _detect_frameworks(manifests, extractions, walked, profile: RepoLanguageProfile):
|
|
310
|
+
import_signals = {
|
|
311
|
+
(x.language, value)
|
|
312
|
+
for _p, x, _pkg in extractions
|
|
313
|
+
for kind, value in x.framework_signals
|
|
314
|
+
if kind == "import"
|
|
315
|
+
}
|
|
316
|
+
symbol_names = {raw.name for _p, x, _pkg in extractions for raw in x.symbols}
|
|
317
|
+
languages_present = profile.extractable_languages()
|
|
318
|
+
if languages_present & {"typescript", "tsx"}:
|
|
319
|
+
languages_present.add("javascript")
|
|
320
|
+
frameworks = detect_frameworks(
|
|
321
|
+
manifests, import_signals, [w.path for w in walked], symbol_names,
|
|
322
|
+
languages_present=languages_present,
|
|
323
|
+
)
|
|
324
|
+
return frameworks, {fw.name for fw in frameworks}
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _write_symbols(session, extractions, file_id_by_path, alloc, table):
|
|
328
|
+
symbol_rows: list[dict] = []
|
|
329
|
+
module_ids: dict[str, int] = {}
|
|
330
|
+
for path, x, _pkg in extractions:
|
|
331
|
+
module_id = alloc.take(Symbol)
|
|
332
|
+
module_ids[path] = module_id
|
|
333
|
+
table.add_module(x.module_path, module_id)
|
|
334
|
+
symbol_rows.append(
|
|
335
|
+
{
|
|
336
|
+
"id": module_id, "file_id": file_id_by_path[path], "kind": SymbolKind.MODULE,
|
|
337
|
+
"name": x.module_path.rsplit(".", 1)[-1], "qname": x.module_path,
|
|
338
|
+
"parent_id": None, "start_line": 1, "end_line": 0, "start_col": 0,
|
|
339
|
+
"signature": None, "docstring": None, "is_exported": True,
|
|
340
|
+
}
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
symbol_id_by_qname: dict[str, int] = {}
|
|
344
|
+
for path, x, _pkg in extractions:
|
|
345
|
+
for raw in x.symbols:
|
|
346
|
+
symbol_id = alloc.take(Symbol)
|
|
347
|
+
symbol_id_by_qname[raw.qualified_name] = symbol_id
|
|
348
|
+
table.add_symbol(symbol_id, raw.qualified_name, raw.name, raw.kind)
|
|
349
|
+
if raw.kind is SymbolKind.CLASS and raw.bases:
|
|
350
|
+
table.class_bases[raw.qualified_name] = raw.bases
|
|
351
|
+
symbol_rows.append(
|
|
352
|
+
{
|
|
353
|
+
"id": symbol_id, "file_id": file_id_by_path[path], "kind": raw.kind,
|
|
354
|
+
"name": raw.name, "qname": raw.qualified_name, "parent_id": None,
|
|
355
|
+
"start_line": raw.span.start_line, "end_line": raw.span.end_line,
|
|
356
|
+
"start_col": raw.span.start_col, "signature": raw.signature,
|
|
357
|
+
"docstring": raw.docstring, "is_exported": raw.is_exported,
|
|
358
|
+
}
|
|
359
|
+
)
|
|
360
|
+
for row in symbol_rows:
|
|
361
|
+
qname = row["qname"]
|
|
362
|
+
if row["kind"] is not SymbolKind.MODULE and "." in qname:
|
|
363
|
+
parent_q = qname.rsplit(".", 1)[0]
|
|
364
|
+
row["parent_id"] = symbol_id_by_qname.get(parent_q) or table.module_symbol_ids.get(parent_q)
|
|
365
|
+
# A row's parent has one fewer qname segment, so inserting shallowest-first
|
|
366
|
+
# guarantees the self-referential parent_id FK is satisfied within the batch.
|
|
367
|
+
symbol_rows.sort(key=lambda r: r["qname"].count("."))
|
|
368
|
+
bulk_insert(session, Symbol, symbol_rows)
|
|
369
|
+
return symbol_id_by_qname, module_ids
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _write_edges_and_entrypoints(
|
|
373
|
+
session, extractions, file_id_by_path, module_ids, symbol_id_by_qname,
|
|
374
|
+
table, externals, alloc, sink_registry: SinkRegistry,
|
|
375
|
+
):
|
|
376
|
+
edge_rows: list[dict] = []
|
|
377
|
+
entrypoint_rows: list[dict] = []
|
|
378
|
+
|
|
379
|
+
for path, x, is_package in extractions:
|
|
380
|
+
resolver = FileResolver(x, module_ids[path], table, externals, is_package)
|
|
381
|
+
file_id = file_id_by_path[path]
|
|
382
|
+
for edge in resolver.resolve():
|
|
383
|
+
sink_id = (
|
|
384
|
+
sink_registry.match(edge.dst_qname, edge.arg_preview)
|
|
385
|
+
if edge.kind is EdgeKind.CALLS else None
|
|
386
|
+
)
|
|
387
|
+
edge_rows.append(
|
|
388
|
+
{
|
|
389
|
+
"id": alloc.take(Edge), "kind": edge.kind,
|
|
390
|
+
"src_symbol_id": edge.src_symbol_id, "dst_symbol_id": edge.dst_symbol_id,
|
|
391
|
+
"dst_qname": edge.dst_qname, "src_file_id": file_id, "line": edge.line,
|
|
392
|
+
"confidence": int(edge.confidence), "arg_preview": edge.arg_preview,
|
|
393
|
+
"sink_id": sink_id, "via": edge.via,
|
|
394
|
+
}
|
|
395
|
+
)
|
|
396
|
+
for hint in x.entrypoint_hints:
|
|
397
|
+
symbol_id = symbol_id_by_qname.get(hint.handler_qualified_name or "") or module_ids[path]
|
|
398
|
+
entrypoint_rows.append(
|
|
399
|
+
{
|
|
400
|
+
"id": alloc.take(Entrypoint), "kind": hint.kind, "framework": hint.framework,
|
|
401
|
+
"symbol_id": symbol_id, "route": hint.route,
|
|
402
|
+
"http_method": ",".join(hint.http_methods) or None,
|
|
403
|
+
"extra": json.dumps(hint.metadata) if hint.metadata else None,
|
|
404
|
+
}
|
|
405
|
+
)
|
|
406
|
+
bulk_insert(session, Symbol, externals.new_rows)
|
|
407
|
+
bulk_insert(session, Edge, edge_rows)
|
|
408
|
+
bulk_insert(session, Entrypoint, entrypoint_rows)
|
|
409
|
+
return len(edge_rows), len(entrypoint_rows)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _write_config_entrypoints(
|
|
413
|
+
session, root, symbol_id_by_qname: dict[str, int], table: SymbolTable,
|
|
414
|
+
alloc: IdAllocator, incremental: bool,
|
|
415
|
+
) -> int:
|
|
416
|
+
"""Scan serverless/SAM/Procfile/Dockerfile and bind their handlers to symbols.
|
|
417
|
+
|
|
418
|
+
Config files aren't tracked in the files table, so on incremental runs their
|
|
419
|
+
entrypoints are fully deleted and re-derived each time.
|
|
420
|
+
"""
|
|
421
|
+
from entrygraph.detect.entrypoints.configs import (
|
|
422
|
+
CONFIG_FRAMEWORKS,
|
|
423
|
+
bind_handler,
|
|
424
|
+
scan_config_entrypoints,
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
if incremental:
|
|
428
|
+
session.execute(
|
|
429
|
+
delete(Entrypoint).where(Entrypoint.framework.in_(CONFIG_FRAMEWORKS))
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
rows = []
|
|
433
|
+
for hint in scan_config_entrypoints(root):
|
|
434
|
+
symbol_id = bind_handler(hint.handler_ref, symbol_id_by_qname, table.module_symbol_ids)
|
|
435
|
+
if symbol_id is None: # non-nullable FK: skip unbindable handlers
|
|
436
|
+
continue
|
|
437
|
+
rows.append(
|
|
438
|
+
{
|
|
439
|
+
"id": alloc.take(Entrypoint), "kind": hint.kind, "framework": hint.framework,
|
|
440
|
+
"symbol_id": symbol_id, "route": hint.route, "http_method": None,
|
|
441
|
+
"extra": json.dumps(hint.metadata) if hint.metadata else None,
|
|
442
|
+
}
|
|
443
|
+
)
|
|
444
|
+
bulk_insert(session, Entrypoint, rows)
|
|
445
|
+
return len(rows)
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _heal_dangling_edges(session, table: SymbolTable, new_qnames: set[str]) -> None:
|
|
449
|
+
"""Re-bind edges left NULL (degraded on wipe, or targeting a not-yet-existing
|
|
450
|
+
symbol) whose dst_qname now names a freshly-created symbol."""
|
|
451
|
+
dangling = session.execute(
|
|
452
|
+
select(Edge.id, Edge.dst_qname).where(
|
|
453
|
+
Edge.dst_symbol_id.is_(None), Edge.dst_qname.in_(new_qnames)
|
|
454
|
+
)
|
|
455
|
+
).all()
|
|
456
|
+
updates = []
|
|
457
|
+
for edge_id, dst_qname in dangling:
|
|
458
|
+
target = table.by_fqn.get(dst_qname)
|
|
459
|
+
if target is not None:
|
|
460
|
+
updates.append({"id": edge_id, "dst_symbol_id": target,
|
|
461
|
+
"confidence": int(Confidence.IMPORT)})
|
|
462
|
+
if updates:
|
|
463
|
+
session.execute(update(Edge), updates)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _gc_orphan_externals(session) -> None:
|
|
467
|
+
"""Drop external placeholder symbols no edge points at anymore.
|
|
468
|
+
|
|
469
|
+
Full re-index never creates them; deleting them keeps an incremental graph
|
|
470
|
+
byte-identical to a full one after a file's last reference disappears.
|
|
471
|
+
"""
|
|
472
|
+
referenced = select(Edge.dst_symbol_id).where(Edge.dst_symbol_id.is_not(None))
|
|
473
|
+
session.execute(
|
|
474
|
+
delete(Symbol).where(
|
|
475
|
+
Symbol.kind == SymbolKind.EXTERNAL, Symbol.id.not_in(referenced)
|
|
476
|
+
)
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _write_detections(session, repo, profile: RepoLanguageProfile, frameworks) -> None:
|
|
481
|
+
session.execute(delete(Detection).where(Detection.repo_id == repo.id))
|
|
482
|
+
rows = [
|
|
483
|
+
{
|
|
484
|
+
"repo_id": repo.id, "category": "language", "name": stat.name, "version": None,
|
|
485
|
+
"confidence": 1.0,
|
|
486
|
+
"evidence": json.dumps({"files": stat.file_count, "bytes": stat.byte_count,
|
|
487
|
+
"percent": round(stat.percent, 2)}),
|
|
488
|
+
}
|
|
489
|
+
for stat in profile.stats()
|
|
490
|
+
]
|
|
491
|
+
rows.extend(
|
|
492
|
+
{
|
|
493
|
+
"repo_id": repo.id, "category": "framework", "name": fw.name, "version": None,
|
|
494
|
+
"confidence": fw.confidence,
|
|
495
|
+
"evidence": json.dumps({"language": fw.language, "signals": list(fw.evidence)}),
|
|
496
|
+
}
|
|
497
|
+
for fw in frameworks
|
|
498
|
+
)
|
|
499
|
+
if rows:
|
|
500
|
+
bulk_insert(session, Detection, rows)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _count_symbols(session) -> int:
|
|
504
|
+
from sqlalchemy import func
|
|
505
|
+
|
|
506
|
+
return session.execute(select(func.count(Symbol.id))).scalar() or 0
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Process-pool extraction worker.
|
|
2
|
+
|
|
3
|
+
Workers parse and extract files into plain-data IR. tree-sitter parsers and
|
|
4
|
+
compiled queries are lazily cached per process (see parsing.parsers /
|
|
5
|
+
parsing.queries), so nothing unpicklable crosses the pool boundary — inputs are
|
|
6
|
+
WalkedFile records, outputs are FileExtraction dataclasses.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from entrygraph.extract.base import FileContext
|
|
14
|
+
from entrygraph.extract.ir import FileExtraction
|
|
15
|
+
from entrygraph.extract.registry import extractor_for
|
|
16
|
+
from entrygraph.fs.walker import WalkedFile
|
|
17
|
+
from entrygraph.parsing.parsers import parse, supported
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def extract_one(walked: WalkedFile) -> tuple[str, FileExtraction, bool] | None:
|
|
21
|
+
"""Parse+extract one file -> (path, extraction, is_package), or None to skip."""
|
|
22
|
+
if walked.skip_reason or not supported(walked.language or ""):
|
|
23
|
+
return None
|
|
24
|
+
extractor = extractor_for(walked.language)
|
|
25
|
+
if extractor is None:
|
|
26
|
+
return None
|
|
27
|
+
try:
|
|
28
|
+
source = Path(walked.abs_path).read_bytes()
|
|
29
|
+
except OSError:
|
|
30
|
+
return None
|
|
31
|
+
module_path, is_package = extractor.module_path_for(walked.path)
|
|
32
|
+
tree = parse(walked.language, source)
|
|
33
|
+
ctx = FileContext(
|
|
34
|
+
path=walked.path,
|
|
35
|
+
language=walked.language,
|
|
36
|
+
module_path=module_path,
|
|
37
|
+
source=source,
|
|
38
|
+
is_package=is_package,
|
|
39
|
+
)
|
|
40
|
+
return walked.path, extractor.extract(tree, ctx), is_package
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def extract_batch(batch: list[WalkedFile]) -> list[tuple[str, FileExtraction, bool]]:
|
|
44
|
+
results = []
|
|
45
|
+
for walked in batch:
|
|
46
|
+
result = extract_one(walked)
|
|
47
|
+
if result is not None:
|
|
48
|
+
results.append(result)
|
|
49
|
+
return results
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""The only bridge between extraction IR and the ORM.
|
|
2
|
+
|
|
3
|
+
All inserts are SQLAlchemy 2.0 ORM-enabled bulk inserts —
|
|
4
|
+
``session.execute(insert(Model), list_of_dicts)`` — with application-assigned
|
|
5
|
+
integer primary keys so edge rows can reference symbol ids without RETURNING
|
|
6
|
+
round-trips. This is the fastest write path that still goes through the ORM.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Iterable, Type
|
|
12
|
+
|
|
13
|
+
from sqlalchemy import func, insert, select
|
|
14
|
+
from sqlalchemy.orm import Session
|
|
15
|
+
|
|
16
|
+
from entrygraph.db.models import Base, Edge, Entrypoint, File, Symbol
|
|
17
|
+
|
|
18
|
+
_BATCH = 5000
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class IdAllocator:
|
|
22
|
+
"""Per-table monotonically increasing ids, seeded from MAX(id)."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, session: Session) -> None:
|
|
25
|
+
self._next: dict[str, int] = {}
|
|
26
|
+
for model in (File, Symbol, Edge, Entrypoint):
|
|
27
|
+
current = session.execute(select(func.max(model.id))).scalar() or 0
|
|
28
|
+
self._next[model.__tablename__] = current + 1
|
|
29
|
+
|
|
30
|
+
def take(self, model: Type[Base]) -> int:
|
|
31
|
+
table = model.__tablename__
|
|
32
|
+
value = self._next[table]
|
|
33
|
+
self._next[table] = value + 1
|
|
34
|
+
return value
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def bulk_insert(session: Session, model: Type[Base], rows: Iterable[dict]) -> int:
|
|
38
|
+
rows = list(rows)
|
|
39
|
+
for start in range(0, len(rows), _BATCH):
|
|
40
|
+
session.execute(insert(model), rows[start : start + _BATCH])
|
|
41
|
+
return len(rows)
|
entrygraph/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
; Harvest-level queries: find definition nodes; the C# shaper computes
|
|
2
|
+
; qualified names (from the enclosing namespace), kinds, attributes (as
|
|
3
|
+
; decorators), modifiers, and supertypes by walking from these.
|
|
4
|
+
|
|
5
|
+
(class_declaration
|
|
6
|
+
name: (identifier) @name) @def.class
|
|
7
|
+
|
|
8
|
+
(interface_declaration
|
|
9
|
+
name: (identifier) @name) @def.interface
|
|
10
|
+
|
|
11
|
+
(struct_declaration
|
|
12
|
+
name: (identifier) @name) @def.struct
|
|
13
|
+
|
|
14
|
+
(record_declaration
|
|
15
|
+
name: (identifier) @name) @def.record
|
|
16
|
+
|
|
17
|
+
(method_declaration
|
|
18
|
+
name: (identifier) @name) @def.method
|
|
19
|
+
|
|
20
|
+
(constructor_declaration
|
|
21
|
+
name: (identifier) @name) @def.constructor
|
|
22
|
+
|
|
23
|
+
(local_function_statement
|
|
24
|
+
name: (identifier) @name) @def.local_function
|
|
25
|
+
|
|
26
|
+
(property_declaration
|
|
27
|
+
name: (identifier) @name) @def.property
|
|
28
|
+
|
|
29
|
+
(field_declaration) @def.field
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
; Harvest-level queries: find definition nodes; the Go shaper computes
|
|
2
|
+
; qualified names, kinds, and receivers by walking from these.
|
|
3
|
+
|
|
4
|
+
(function_declaration
|
|
5
|
+
name: (identifier) @name) @def.function
|
|
6
|
+
|
|
7
|
+
(method_declaration
|
|
8
|
+
name: (field_identifier) @name) @def.method
|
|
9
|
+
|
|
10
|
+
(type_declaration
|
|
11
|
+
(type_spec
|
|
12
|
+
name: (type_identifier) @name
|
|
13
|
+
type: (struct_type)) ) @def.struct
|
|
14
|
+
|
|
15
|
+
(type_declaration
|
|
16
|
+
(type_spec
|
|
17
|
+
name: (type_identifier) @name
|
|
18
|
+
type: (interface_type)) ) @def.interface
|
|
19
|
+
|
|
20
|
+
(const_declaration (const_spec name: (identifier) @name)) @def.const
|
|
21
|
+
|
|
22
|
+
(var_declaration (var_spec name: (identifier) @name)) @def.var
|
|
23
|
+
|
|
24
|
+
(field_declaration name: (field_identifier) @name) @def.field
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
(import_declaration) @import
|