auryn-csharp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- auryn/ingestors/csharp/__init__.py +5 -0
- auryn/ingestors/csharp/config.py +38 -0
- auryn/ingestors/csharp/file_parser.py +614 -0
- auryn/ingestors/csharp/git_sync.py +448 -0
- auryn/ingestors/csharp/grammar.py +17 -0
- auryn/ingestors/csharp/ids.py +20 -0
- auryn/ingestors/csharp/ingestor.py +142 -0
- auryn/ingestors/csharp/kendo.py +287 -0
- auryn/ingestors/csharp/plugin.py +49 -0
- auryn/ingestors/csharp/raw_model.py +137 -0
- auryn/ingestors/csharp/resolution.py +793 -0
- auryn/ingestors/csharp/roslyn.py +197 -0
- auryn/ingestors/csharp/roslyn_extractor/Extractor.cs +944 -0
- auryn/ingestors/csharp/roslyn_extractor/Ids.cs +86 -0
- auryn/ingestors/csharp/roslyn_extractor/Model.cs +40 -0
- auryn/ingestors/csharp/roslyn_extractor/Program.cs +126 -0
- auryn/ingestors/csharp/roslyn_extractor/RoslynExtractor.csproj +18 -0
- auryn/ingestors/csharp/schema.py +81 -0
- auryn_csharp-0.1.0.dist-info/METADATA +12 -0
- auryn_csharp-0.1.0.dist-info/RECORD +23 -0
- auryn_csharp-0.1.0.dist-info/WHEEL +4 -0
- auryn_csharp-0.1.0.dist-info/entry_points.txt +5 -0
- auryn_csharp-0.1.0.dist-info/licenses/LICENSE +557 -0
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
"""Git-tracked C# ingestion: moves a project's graph to whatever commit/branch/tag
|
|
2
|
+
is requested -- forward, backward, or across branches -- by fully recomputing the
|
|
3
|
+
target state and diffing it against whatever the graph currently holds (see
|
|
4
|
+
graph_sync.compute_delta). See the design discussion recorded in this feature's
|
|
5
|
+
plan for why a source-level (git diff) approach was rejected: this ingestor's
|
|
6
|
+
resolution is global, so a file-level diff can't tell which *other*, untouched
|
|
7
|
+
files need re-examining.
|
|
8
|
+
|
|
9
|
+
Not a BaseIngestor subclass -- its lifecycle (a persistent repo cache, a
|
|
10
|
+
`commit_sha` marker on the Project node, real deletions) doesn't fit
|
|
11
|
+
BaseIngestor.run()'s stateless "extract a snapshot, merge it in, never delete"
|
|
12
|
+
contract.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
import pickle
|
|
20
|
+
import time
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
from auryn.core import progress
|
|
25
|
+
from auryn.core.git_source import DEFAULT_CACHE_ROOT, GitRepoCache, cache_home, repos_cache_dir
|
|
26
|
+
from auryn.core.parallel import make_process_pool
|
|
27
|
+
from auryn.core.graph_client import GraphWriter
|
|
28
|
+
from auryn.core.graph_model import GraphEdge, GraphNode
|
|
29
|
+
from auryn.core.graph_sync import stream_apply
|
|
30
|
+
from auryn.core.registry import LanguageIngestResult
|
|
31
|
+
from auryn.ingestors.csharp import kendo, resolution, roslyn
|
|
32
|
+
from auryn.ingestors.csharp.config import project_id
|
|
33
|
+
from auryn.ingestors.csharp.file_parser import parse_source
|
|
34
|
+
from auryn.ingestors.csharp.plugin import EXCLUDED_DIR_NAMES as _EXCLUDED_DIR_NAMES
|
|
35
|
+
from auryn.ingestors.csharp.raw_model import RawFile
|
|
36
|
+
from auryn.ingestors.csharp.schema import NODE_SCHEMAS
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger(__name__)
|
|
39
|
+
|
|
40
|
+
DEFAULT_ENGINE = "roslyn"
|
|
41
|
+
"""The C# extraction engine used by git-sync and the ingestion endpoint by default.
|
|
42
|
+
`roslyn` gives semantic analysis + DI resolution (see roslyn.py); it falls back to
|
|
43
|
+
`treesitter` automatically when the .NET SDK / extractor tool isn't available."""
|
|
44
|
+
|
|
45
|
+
# Two-phase depth marker, uniform with the Python engine (ast=fast < jedi=deep): the fast
|
|
46
|
+
# tree-sitter pass and the deep Roslyn pass target the SAME commit, so a bare commit_sha check
|
|
47
|
+
# would let the fast pass' sha short-circuit the deep upgrade. We record how deep the graph
|
|
48
|
+
# already is ("fast" < "deep") and skip only when it's already at least as deep as this engine
|
|
49
|
+
# would make it -- so `--deep off` runs tree-sitter only, `--deep inline` runs Roslyn right after,
|
|
50
|
+
# yet a genuine re-ingest of an unchanged, already-deep project stays a no-op.
|
|
51
|
+
_DEPTH_RANK = {None: 0, "": 0, "fast": 1, "deep": 2}
|
|
52
|
+
|
|
53
|
+
_PARSE_CACHE_VERSION = 2
|
|
54
|
+
"""Bump when what `parse_source` extracts into a `RawFile` changes shape, so a stale
|
|
55
|
+
on-disk pickle cache (which bypasses dataclass defaults on unpickle) can't feed back
|
|
56
|
+
`RawFile`s missing a newly added field. v2 = the `doc` property was added. The version is
|
|
57
|
+
part of the cache dir name, so bumping it simply starts a fresh cache."""
|
|
58
|
+
|
|
59
|
+
# DEFAULT_CACHE_ROOT and repos_cache_dir now live in core/git_source.py (shared infra,
|
|
60
|
+
# so nothing in core has to import an ingestor to find them); re-exported here so the
|
|
61
|
+
# viewer and CLI that already import them from this module keep working unchanged.
|
|
62
|
+
__all__ = ["DEFAULT_CACHE_ROOT", "repos_cache_dir", "GitCSharpProjectConfig", "SyncStats", "sync", "sync_at"]
|
|
63
|
+
|
|
64
|
+
# _EXCLUDED_DIR_NAMES / the C# suffixes ((.cs, .cshtml)) are defined once in plugin.py (the
|
|
65
|
+
# import-light registration module) and imported above -- a single source of truth shared by
|
|
66
|
+
# the lazy GitLanguage registration and the .cs walk below.
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True)
|
|
70
|
+
class GitCSharpProjectConfig:
|
|
71
|
+
repo_uri: str
|
|
72
|
+
project_name: str
|
|
73
|
+
solution_name: str | None = None
|
|
74
|
+
target_framework: str | None = None
|
|
75
|
+
dto_name_suffixes: tuple[str, ...] = ("Dto", "Request", "Response", "ViewModel")
|
|
76
|
+
workers: int | None = None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class SyncStats:
|
|
81
|
+
nodes_added: int = 0
|
|
82
|
+
nodes_updated: int = 0
|
|
83
|
+
nodes_removed: int = 0
|
|
84
|
+
edges_added: int = 0
|
|
85
|
+
edges_updated: int = 0
|
|
86
|
+
edges_removed: int = 0
|
|
87
|
+
commit_sha: str = ""
|
|
88
|
+
elapsed_seconds: float = 0.0
|
|
89
|
+
files_parsed: int = 0
|
|
90
|
+
""".cs files actually parsed this sync -- i.e. blob shas the parse cache had
|
|
91
|
+
never seen before, regardless of path/commit/branch."""
|
|
92
|
+
files_cached: int = 0
|
|
93
|
+
""".cs files reused verbatim from the parse cache (identical content already
|
|
94
|
+
parsed under some other path/commit/branch)."""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def sync(
|
|
98
|
+
writer: GraphWriter,
|
|
99
|
+
config: GitCSharpProjectConfig,
|
|
100
|
+
ref: str | None,
|
|
101
|
+
cache_root: Path | None = None,
|
|
102
|
+
engine: str = DEFAULT_ENGINE,
|
|
103
|
+
) -> SyncStats:
|
|
104
|
+
"""Ingest (first run) or move (every run after) a git-tracked C# project's graph
|
|
105
|
+
to `ref`. Fetches the repo, resolves `ref` to a commit sha, then delegates the
|
|
106
|
+
per-commit work to sync_at()."""
|
|
107
|
+
repo = GitRepoCache(config.repo_uri, repos_cache_dir(cache_root))
|
|
108
|
+
repo.fetch()
|
|
109
|
+
target_sha = repo.resolve_ref(ref)
|
|
110
|
+
return sync_at(writer, config, repo, target_sha, ref, cache_root, engine)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def sync_at(
|
|
114
|
+
writer: GraphWriter,
|
|
115
|
+
config: GitCSharpProjectConfig,
|
|
116
|
+
repo: GitRepoCache,
|
|
117
|
+
target_sha: str,
|
|
118
|
+
ref: str | None,
|
|
119
|
+
cache_root: Path | None = None,
|
|
120
|
+
engine: str = DEFAULT_ENGINE,
|
|
121
|
+
) -> SyncStats:
|
|
122
|
+
"""The per-commit half of sync(), given an already-fetched `repo` and an
|
|
123
|
+
already-resolved `target_sha`. Split out so the multi-language ingestion
|
|
124
|
+
endpoint (core/ingest.py) can fetch and resolve a repo once, then drive
|
|
125
|
+
several languages against that same commit without each one re-fetching.
|
|
126
|
+
|
|
127
|
+
`engine` selects the C# extractor: "roslyn" (semantic + DI resolution, the
|
|
128
|
+
default) or "treesitter" (syntax-only fallback). "roslyn" falls back to
|
|
129
|
+
tree-sitter automatically when the .NET SDK / extractor tool is unavailable."""
|
|
130
|
+
started = time.monotonic()
|
|
131
|
+
proj_id = project_id(config.project_name, config.solution_name)
|
|
132
|
+
|
|
133
|
+
# setup_schema() must run before any read below -- FalkorDB's RO_QUERY raises
|
|
134
|
+
# ("Invalid graph operation on empty key") against a graph nothing has ever
|
|
135
|
+
# been written to; creating an index is what brings the graph key into
|
|
136
|
+
# existence, same ordering BaseIngestor.run() already relies on.
|
|
137
|
+
writer.setup_schema(NODE_SCHEMAS)
|
|
138
|
+
|
|
139
|
+
current_sha = writer.get_property("Project", {"id": proj_id}, "commit_sha")
|
|
140
|
+
current_depth = writer.get_property("Project", {"id": proj_id}, "depth")
|
|
141
|
+
requested_depth = "deep" if engine == "roslyn" else "fast"
|
|
142
|
+
if current_sha == target_sha and _DEPTH_RANK.get(current_depth, 0) >= _DEPTH_RANK[requested_depth]:
|
|
143
|
+
return SyncStats(commit_sha=target_sha, elapsed_seconds=time.monotonic() - started)
|
|
144
|
+
|
|
145
|
+
if engine == "roslyn":
|
|
146
|
+
try:
|
|
147
|
+
# A genuine same-commit fast->deep UPGRADE (tree-sitter just ran at this commit) must PRESERVE
|
|
148
|
+
# the fast pass's extern:// call placeholders: Roslyn can occasionally fail to bind a call the
|
|
149
|
+
# syntactic fast pass represented as a placeholder, and deleting it would stop a later producer
|
|
150
|
+
# ingest from healing the cross-repo reference (PR #17 review). A standalone Roslyn resync is
|
|
151
|
+
# authoritative instead, so it still diffs stale extern edges away.
|
|
152
|
+
is_upgrade = current_sha == target_sha and current_depth == "fast"
|
|
153
|
+
return _roslyn_sync_at(writer, config, repo, target_sha, ref, proj_id, started,
|
|
154
|
+
diff_extern_edges=not is_upgrade)
|
|
155
|
+
except roslyn.RoslynUnavailable as e:
|
|
156
|
+
logger.warning("Roslyn engine unavailable (%s); falling back to tree-sitter", e)
|
|
157
|
+
return _treesitter_sync_at(writer, config, repo, target_sha, ref, cache_root, proj_id, started)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _project_items(
|
|
161
|
+
config: GitCSharpProjectConfig, proj_id: str, target_sha: str, ref: str | None, depth: str
|
|
162
|
+
) -> list[GraphNode | GraphEdge]:
|
|
163
|
+
"""The Project (and optional Solution) root nodes -- identical across engines, so
|
|
164
|
+
the viewer's git-projects panel, the depth-aware short-circuit, and re-sync all
|
|
165
|
+
behave the same however the project was extracted. `depth` is "fast" (tree-sitter) or
|
|
166
|
+
"deep" (Roslyn); it records how deep this graph is for the two-phase short-circuit."""
|
|
167
|
+
items: list[GraphNode | GraphEdge] = [
|
|
168
|
+
GraphNode(
|
|
169
|
+
"Project",
|
|
170
|
+
{"id": proj_id},
|
|
171
|
+
{
|
|
172
|
+
"name": config.project_name,
|
|
173
|
+
"solution_name": config.solution_name,
|
|
174
|
+
"target_framework": config.target_framework,
|
|
175
|
+
"repo_uri": config.repo_uri,
|
|
176
|
+
"commit_sha": target_sha,
|
|
177
|
+
"depth": depth, # "fast" (tree-sitter) or "deep" (Roslyn) -- see sync_at short-circuit
|
|
178
|
+
"ref": ref,
|
|
179
|
+
"dto_name_suffixes": list(config.dto_name_suffixes),
|
|
180
|
+
},
|
|
181
|
+
)
|
|
182
|
+
]
|
|
183
|
+
if config.solution_name:
|
|
184
|
+
solution_id = f"csharp://{config.solution_name}"
|
|
185
|
+
items.append(GraphNode("Solution", {"id": solution_id}, {"name": config.solution_name}))
|
|
186
|
+
items.append(GraphEdge("HAS_PROJECT", "Solution", {"id": solution_id}, "Project", {"id": proj_id}))
|
|
187
|
+
return items
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _prune_stale_endpoint_edges(writer: GraphWriter, proj_id: str, target_items: list[GraphNode | GraphEdge]) -> None:
|
|
191
|
+
"""Delete THIS project's declared seam edges to a route-keyed (`extern://http/`) endpoint that the
|
|
192
|
+
freshly-computed target no longer emits.
|
|
193
|
+
|
|
194
|
+
`stream_apply`'s delta only tracks edges whose BOTH endpoints share the project id-prefix (see
|
|
195
|
+
read_fingerprints), but a declared endpoint is deliberately route-keyed OUTSIDE any project's
|
|
196
|
+
prefix so callers in every repo converge on one node (ADR 0006 dedup). That puts its `EXPOSES`
|
|
197
|
+
(Method -> endpoint) and `ACCEPTS`/`RETURNS` (endpoint -> DTO) edges out of the delta's sight: an
|
|
198
|
+
endpoint REMOVED with its method is cleaned by the method's DETACH, but an in-place route/verb
|
|
199
|
+
CHANGE (method persists, its route attribute changes -> new endpoint id) would leave the old edge
|
|
200
|
+
behind, the old endpoint would never orphan, and a duplicate would accumulate. This prunes those
|
|
201
|
+
stale edges; the now-orphaned endpoint node is reclaimed by the http-endpoints stitch's GC.
|
|
202
|
+
|
|
203
|
+
Idempotent: on an unchanged resync the keep-set equals what the graph holds, so nothing is deleted.
|
|
204
|
+
Only this project's edges are touched (Method / DTO scoped to `proj_id`), and never the stitch's own
|
|
205
|
+
`http-stitch` EXPOSES (which it rebuilds every run)."""
|
|
206
|
+
keep_exposes = [[e.from_key["id"], e.to_key["id"]]
|
|
207
|
+
for e in target_items if isinstance(e, GraphEdge)
|
|
208
|
+
and e.rel_type == "EXPOSES_ENDPOINT" and str(e.to_key["id"]).startswith("extern://http/")]
|
|
209
|
+
keep_dto = [[e.rel_type, e.from_key["id"], e.to_key["id"]]
|
|
210
|
+
for e in target_items if isinstance(e, GraphEdge)
|
|
211
|
+
and e.rel_type in ("ACCEPTS", "RETURNS") and str(e.from_key["id"]).startswith("extern://http/")]
|
|
212
|
+
writer.query(
|
|
213
|
+
"MATCH (m)-[r:EXPOSES_ENDPOINT]->(e:APIEndpoint) "
|
|
214
|
+
"WHERE m.id STARTS WITH $p AND e.id STARTS WITH 'extern://http/' "
|
|
215
|
+
"AND coalesce(r.provenance, '') <> 'http-stitch' "
|
|
216
|
+
"AND NOT [m.id, e.id] IN $keep DELETE r",
|
|
217
|
+
{"p": proj_id, "keep": keep_exposes},
|
|
218
|
+
)
|
|
219
|
+
writer.query(
|
|
220
|
+
"MATCH (e:APIEndpoint)-[r:ACCEPTS|RETURNS]->(d) "
|
|
221
|
+
"WHERE e.id STARTS WITH 'extern://http/' AND d.id STARTS WITH $p "
|
|
222
|
+
"AND NOT [type(r), e.id, d.id] IN $keep DELETE r",
|
|
223
|
+
{"p": proj_id, "keep": keep_dto},
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _apply(
|
|
228
|
+
writer: GraphWriter, proj_id: str, target_items: list[GraphNode | GraphEdge], target_sha: str, started: float,
|
|
229
|
+
files_parsed: int, files_cached: int, *, diff_extern_edges: bool = True
|
|
230
|
+
) -> SyncStats:
|
|
231
|
+
# Cross-repo reconciliation (ADR 0004): fold placeholder<->definer pairs among the linkages
|
|
232
|
+
# this ingest touched -- targeted index seeks, no scan. No-op unless the graph holds >1 repo.
|
|
233
|
+
from auryn.core.linkage import reconcile_linkages
|
|
234
|
+
|
|
235
|
+
# kendo.extract_ui inspects the target to derive UI edges, so csharp (unlike the streaming
|
|
236
|
+
# resolvers) builds target_items up front. stream_apply still avoids the second full copy
|
|
237
|
+
# compute_delta would make; it iterates the list twice (nodes pass, then edges pass), so a
|
|
238
|
+
# fresh iterator each call is required -- hence `lambda: iter(target_items)`. `diff_extern_edges`
|
|
239
|
+
# is False only for a same-commit fast->deep Roslyn upgrade (preserve the fast pass's extern://
|
|
240
|
+
# placeholders Roslyn may not re-author); the fast/standalone pass is authoritative (True).
|
|
241
|
+
labels = [schema.label for schema in NODE_SCHEMAS]
|
|
242
|
+
stats, reals, placeholders = stream_apply(writer, proj_id, labels, lambda: iter(target_items),
|
|
243
|
+
diff_extern_edges=diff_extern_edges)
|
|
244
|
+
reconcile_linkages(writer, reals, placeholders)
|
|
245
|
+
_prune_stale_endpoint_edges(writer, proj_id, target_items)
|
|
246
|
+
|
|
247
|
+
return SyncStats(
|
|
248
|
+
nodes_added=stats.nodes_added,
|
|
249
|
+
nodes_updated=stats.nodes_updated,
|
|
250
|
+
nodes_removed=stats.nodes_removed,
|
|
251
|
+
edges_added=stats.edges_added,
|
|
252
|
+
edges_updated=stats.edges_updated,
|
|
253
|
+
edges_removed=stats.edges_removed,
|
|
254
|
+
commit_sha=target_sha,
|
|
255
|
+
elapsed_seconds=time.monotonic() - started,
|
|
256
|
+
files_parsed=files_parsed,
|
|
257
|
+
files_cached=files_cached,
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _treesitter_sync_at(
|
|
262
|
+
writer: GraphWriter,
|
|
263
|
+
config: GitCSharpProjectConfig,
|
|
264
|
+
repo: GitRepoCache,
|
|
265
|
+
target_sha: str,
|
|
266
|
+
ref: str | None,
|
|
267
|
+
cache_root: Path | None,
|
|
268
|
+
proj_id: str,
|
|
269
|
+
started: float,
|
|
270
|
+
) -> SyncStats:
|
|
271
|
+
workers = config.workers or os.cpu_count() or 1
|
|
272
|
+
parse_cache_dir = (cache_root or cache_home()) / f"parse_cache_csharp.v{_PARSE_CACHE_VERSION}"
|
|
273
|
+
raw_files, files_parsed, files_cached = _build_raw_files(repo, target_sha, parse_cache_dir, workers)
|
|
274
|
+
|
|
275
|
+
target_items = _project_items(config, proj_id, target_sha, ref, "fast")
|
|
276
|
+
target_items.extend(resolution.resolve(raw_files, proj_id, config.dto_name_suffixes))
|
|
277
|
+
target_items.extend(kendo.extract_ui(repo, target_sha, proj_id, target_items))
|
|
278
|
+
return _apply(writer, proj_id, target_items, target_sha, started, files_parsed, files_cached)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _roslyn_sync_at(
|
|
282
|
+
writer: GraphWriter,
|
|
283
|
+
config: GitCSharpProjectConfig,
|
|
284
|
+
repo: GitRepoCache,
|
|
285
|
+
target_sha: str,
|
|
286
|
+
ref: str | None,
|
|
287
|
+
proj_id: str,
|
|
288
|
+
started: float,
|
|
289
|
+
*,
|
|
290
|
+
diff_extern_edges: bool = True,
|
|
291
|
+
) -> SyncStats:
|
|
292
|
+
"""Get a real directory for the tree at `target_sha` and run the Roslyn extractor
|
|
293
|
+
over it. The tool needs real files on disk (it builds a Compilation), unlike
|
|
294
|
+
tree-sitter's blob streaming, so there's no per-file blob cache here -- the whole
|
|
295
|
+
solution is re-analyzed each sync (see the plan's scale note). `working_tree`
|
|
296
|
+
materializes a temp checkout for a git repo and yields the directory itself for a
|
|
297
|
+
local source, so this path is identical either way."""
|
|
298
|
+
with repo.working_tree(target_sha) as workdir:
|
|
299
|
+
items, metrics = roslyn.extract(
|
|
300
|
+
str(workdir),
|
|
301
|
+
config.project_name,
|
|
302
|
+
solution_name=config.solution_name,
|
|
303
|
+
commit_sha=target_sha,
|
|
304
|
+
target_framework=config.target_framework,
|
|
305
|
+
dto_suffixes=config.dto_name_suffixes,
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
# Swap the tool's root nodes for our canonical ones (full props: repo_uri, ref,
|
|
309
|
+
# dto_name_suffixes) so the Project node matches the tree-sitter engine's exactly.
|
|
310
|
+
body = [item for item in items if not _is_root_item(item)]
|
|
311
|
+
target_items = _project_items(config, proj_id, target_sha, ref, "deep") + body
|
|
312
|
+
target_items.extend(kendo.extract_ui(repo, target_sha, proj_id, target_items))
|
|
313
|
+
return _apply(writer, proj_id, target_items, target_sha, started, metrics.get("filesParsed", 0), 0,
|
|
314
|
+
diff_extern_edges=diff_extern_edges)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _is_root_item(item: GraphNode | GraphEdge) -> bool:
|
|
318
|
+
if isinstance(item, GraphNode):
|
|
319
|
+
return item.label in ("Project", "Solution")
|
|
320
|
+
return item.rel_type == "HAS_PROJECT"
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _parse_blob(entry: tuple[str, bytes]) -> RawFile:
|
|
324
|
+
path, source = entry
|
|
325
|
+
return parse_source(source, path)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _cache_path(parse_cache_dir: Path, blob_sha: str) -> Path:
|
|
329
|
+
return parse_cache_dir / f"{blob_sha}.pickle"
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _load_cached(parse_cache_dir: Path, blob_sha: str) -> RawFile | None:
|
|
333
|
+
path = _cache_path(parse_cache_dir, blob_sha)
|
|
334
|
+
if not path.exists():
|
|
335
|
+
return None
|
|
336
|
+
with path.open("rb") as f:
|
|
337
|
+
return pickle.load(f)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _store_cached(parse_cache_dir: Path, blob_sha: str, raw_file: RawFile) -> None:
|
|
341
|
+
parse_cache_dir.mkdir(parents=True, exist_ok=True)
|
|
342
|
+
with _cache_path(parse_cache_dir, blob_sha).open("wb") as f:
|
|
343
|
+
pickle.dump(raw_file, f)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _build_raw_files(
|
|
347
|
+
repo: GitRepoCache, sha: str, parse_cache_dir: Path, workers: int
|
|
348
|
+
) -> tuple[list[RawFile], int, int]:
|
|
349
|
+
"""Every .cs file at `sha`, parsed. Content identical to something already
|
|
350
|
+
parsed under any other path/commit/branch is never re-parsed -- the cache is
|
|
351
|
+
keyed by blob sha, not path, so a pure rename is a cache hit too (just
|
|
352
|
+
re-stamped with the current tree's path for that blob). Returns
|
|
353
|
+
(raw_files, files_parsed, files_cached) -- the last two are pure reporting,
|
|
354
|
+
surfaced on SyncStats so a sync's actual cache effectiveness is visible rather
|
|
355
|
+
than just inferred from wall time."""
|
|
356
|
+
entries = repo.list_files(sha, ".cs", _EXCLUDED_DIR_NAMES)
|
|
357
|
+
|
|
358
|
+
raw_files: list[RawFile] = []
|
|
359
|
+
to_parse: list[tuple[str, str]] = []
|
|
360
|
+
for path, blob_sha in entries:
|
|
361
|
+
cached = _load_cached(parse_cache_dir, blob_sha)
|
|
362
|
+
if cached is not None:
|
|
363
|
+
cached.path = path
|
|
364
|
+
raw_files.append(cached)
|
|
365
|
+
else:
|
|
366
|
+
to_parse.append((path, blob_sha))
|
|
367
|
+
|
|
368
|
+
files_cached = len(raw_files)
|
|
369
|
+
if not to_parse:
|
|
370
|
+
return raw_files, 0, files_cached
|
|
371
|
+
|
|
372
|
+
blobs = [(path, repo.read_blob(blob_sha)) for path, blob_sha in to_parse]
|
|
373
|
+
if progress.verbose_enabled():
|
|
374
|
+
# Verbose: parse sequentially, logging each file *before* parsing it, so a native crash
|
|
375
|
+
# (BrokenProcessPool) leaves the offending file as the last logged line -- a worker pool
|
|
376
|
+
# can't pinpoint it (several files in flight at once). See the PL/SQL equivalent.
|
|
377
|
+
logger.info("Verbose: parsing %d file(s) sequentially so a crash pinpoints the file", len(blobs))
|
|
378
|
+
parsed = []
|
|
379
|
+
for i, b in enumerate(blobs, 1):
|
|
380
|
+
progress.log_file(logger, b[0], i, len(blobs))
|
|
381
|
+
parsed.append(_parse_blob(b))
|
|
382
|
+
elif workers == 1 or len(blobs) <= 1:
|
|
383
|
+
parsed = [_parse_blob(b) for b in blobs]
|
|
384
|
+
else:
|
|
385
|
+
# Preload the parse module (parse_source's home) into the forkserver control process so
|
|
386
|
+
# workers inherit its tree-sitter import via COW rather than re-importing -- see core.parallel.
|
|
387
|
+
with make_process_pool(workers, preload=["auryn.ingestors.csharp.file_parser"]) as pool:
|
|
388
|
+
parsed = list(pool.map(_parse_blob, blobs))
|
|
389
|
+
|
|
390
|
+
for (_, blob_sha), raw_file in zip(to_parse, parsed):
|
|
391
|
+
_store_cached(parse_cache_dir, blob_sha, raw_file)
|
|
392
|
+
raw_files.append(raw_file)
|
|
393
|
+
|
|
394
|
+
return raw_files, len(to_parse), files_cached
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _git_language_sync(
|
|
398
|
+
writer: GraphWriter,
|
|
399
|
+
repo: GitRepoCache,
|
|
400
|
+
target_sha: str,
|
|
401
|
+
*,
|
|
402
|
+
engine: str = DEFAULT_ENGINE,
|
|
403
|
+
repo_uri: str,
|
|
404
|
+
project_name: str,
|
|
405
|
+
solution_name: str | None,
|
|
406
|
+
ref: str | None,
|
|
407
|
+
cache_root: Path | None = None,
|
|
408
|
+
) -> LanguageIngestResult:
|
|
409
|
+
"""GitLanguage adapter for the multi-language ingestion endpoint: builds a
|
|
410
|
+
GitCSharpProjectConfig from the endpoint's language-agnostic inputs (the
|
|
411
|
+
C#-specific options -- dto suffixes, target framework, workers -- take their
|
|
412
|
+
defaults), runs sync_at() against the already-fetched repo/commit with `engine`, and
|
|
413
|
+
maps SyncStats onto the uniform LanguageIngestResult. `engine` picks the phase:
|
|
414
|
+
"treesitter" (fast) or "roslyn" (deep) -- both target the same commit and Project id, so
|
|
415
|
+
running roslyn after treesitter is an in-place upgrade via recompute-and-diff."""
|
|
416
|
+
config = GitCSharpProjectConfig(
|
|
417
|
+
repo_uri=repo_uri, project_name=project_name, solution_name=solution_name
|
|
418
|
+
)
|
|
419
|
+
s = sync_at(writer, config, repo, target_sha, ref, cache_root, engine=engine)
|
|
420
|
+
return LanguageIngestResult(
|
|
421
|
+
nodes_added=s.nodes_added,
|
|
422
|
+
nodes_updated=s.nodes_updated,
|
|
423
|
+
nodes_removed=s.nodes_removed,
|
|
424
|
+
edges_added=s.edges_added,
|
|
425
|
+
edges_updated=s.edges_updated,
|
|
426
|
+
edges_removed=s.edges_removed,
|
|
427
|
+
files_parsed=s.files_parsed,
|
|
428
|
+
files_cached=s.files_cached,
|
|
429
|
+
commit_sha=s.commit_sha,
|
|
430
|
+
elapsed_seconds=s.elapsed_seconds,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def treesitter_language_sync(writer, repo, target_sha, **kwargs) -> LanguageIngestResult:
|
|
435
|
+
"""Fast phase (tree-sitter, no .NET SDK) -- the GitLanguage's `sync`. See plugin.py."""
|
|
436
|
+
return _git_language_sync(writer, repo, target_sha, engine="treesitter", **kwargs)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def roslyn_language_sync(writer, repo, target_sha, **kwargs) -> LanguageIngestResult:
|
|
440
|
+
"""Deep phase (Roslyn: semantic analysis + DI resolution) -- the GitLanguage's `deep_sync`.
|
|
441
|
+
Layers over the fast pass in place (same Project id / recompute-and-diff); falls back to
|
|
442
|
+
tree-sitter automatically when the .NET SDK / extractor tool is unavailable."""
|
|
443
|
+
return _git_language_sync(writer, repo, target_sha, engine="roslyn", **kwargs)
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
# The GitLanguage is registered by the import-light plugin.py (the entry-point target),
|
|
447
|
+
# whose lazy `sync`/`deep_sync` thunks call these -- so importing this heavy engine module
|
|
448
|
+
# (tree-sitter, the Roslyn extractor) is deferred until a source has C# files.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""tree-sitter-c-sharp Language/Parser setup, in one place so file_parser.py and
|
|
2
|
+
tests always get the exact same configured parser.
|
|
3
|
+
|
|
4
|
+
Unlike PL/SQL's ANTLR grammar, tree-sitter-c-sharp ships a pre-built PyPI package --
|
|
5
|
+
no codegen/vendoring needed (see the C# ingestor plan for why this was chosen over
|
|
6
|
+
Roslyn)."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import tree_sitter_c_sharp as tscsharp
|
|
11
|
+
from tree_sitter import Language, Parser
|
|
12
|
+
|
|
13
|
+
LANGUAGE = Language(tscsharp.language())
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def new_parser() -> Parser:
|
|
17
|
+
return Parser(LANGUAGE)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Cross-repo linkage keys for the C# graph (ADR 0004).
|
|
2
|
+
|
|
3
|
+
C#'s node ids are project-scoped (see config.py / resolution.py) -- rooted under
|
|
4
|
+
`csharp://<solution>/<project>` -- so two repos that both define, say, `Acme.Billing.Charge`
|
|
5
|
+
keep distinct real nodes and never accidentally merge on id. The *linkage key* is the
|
|
6
|
+
complementary, project-INDEPENDENT handle: the fully-qualified name a consumer writes at a
|
|
7
|
+
call site (`Namespace.Type.Method`, e.g. `System.Console.WriteLine` or `Acme.Billing.Charge`).
|
|
8
|
+
It is stamped as the `linkage_key` property on every externally-visible real Method, and used
|
|
9
|
+
as the id of the placeholder Method that stands in for an unresolved call. When the defining
|
|
10
|
+
repo is later ingested into the same graph, the placeholder and the real method agree on this
|
|
11
|
+
one string and reconciliation folds them (core/linkage.py).
|
|
12
|
+
|
|
13
|
+
Name-based only: a consumer can't know a callee's parameter types, so the key never embeds
|
|
14
|
+
them (unlike the project-scoped Method id, whose overload disambiguation does)."""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def extern_method_key(qualified_name: str) -> str:
|
|
20
|
+
return f"extern://csharp/{qualified_name}"
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Extracts Namespace/File/Class/Interface/Struct/Enum/Record/Delegate/DTO/Method/
|
|
2
|
+
Property/Field/Event/Parameter/EnumMember/APIEndpoint structure from a directory of
|
|
3
|
+
C# source files.
|
|
4
|
+
|
|
5
|
+
Phase 1 (parsing every file) runs in parallel across a ProcessPoolExecutor -- see the
|
|
6
|
+
C# ingestor plan for why a process pool rather than threads (the GIL means the
|
|
7
|
+
AST-walking cost in file_parser.py wouldn't parallelize under threads anyway). Phase
|
|
8
|
+
2 (everything needing cross-file information: EXTENDS/IMPLEMENTS, DTO/API endpoint
|
|
9
|
+
classification, call/type-reference resolution) then runs sequentially, in
|
|
10
|
+
resolution.py.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import logging
|
|
16
|
+
import os
|
|
17
|
+
from collections.abc import Iterator
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
import click
|
|
21
|
+
|
|
22
|
+
from auryn.core.base_ingestor import BaseIngestor
|
|
23
|
+
from auryn.core.graph_client import GraphWriter, connect
|
|
24
|
+
from auryn.core.graph_model import GraphEdge, GraphNode
|
|
25
|
+
from auryn.core.parallel import make_process_pool
|
|
26
|
+
from auryn.core.registry import register_ingestor
|
|
27
|
+
from auryn.ingestors.csharp import resolution
|
|
28
|
+
from auryn.ingestors.csharp.config import CSharpSourceConfig, project_id
|
|
29
|
+
from auryn.ingestors.csharp.file_parser import parse_file
|
|
30
|
+
from auryn.ingestors.csharp.raw_model import RawFile
|
|
31
|
+
from auryn.ingestors.csharp.schema import EDGE_SCHEMAS, NODE_SCHEMAS
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
_EXCLUDED_DIR_NAMES = {"bin", "obj"}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@register_ingestor("csharp")
|
|
39
|
+
class CSharpIngestor(BaseIngestor):
|
|
40
|
+
node_schemas = NODE_SCHEMAS
|
|
41
|
+
edge_schemas = EDGE_SCHEMAS
|
|
42
|
+
|
|
43
|
+
def __init__(self, config: CSharpSourceConfig) -> None:
|
|
44
|
+
self._config = config
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
def build_cli_command(cls) -> click.Command:
|
|
48
|
+
@click.command(
|
|
49
|
+
"csharp",
|
|
50
|
+
help="Ingest a C# codebase (classes, interfaces, methods, API endpoints, DTOs, ...).",
|
|
51
|
+
)
|
|
52
|
+
@click.option(
|
|
53
|
+
"--path",
|
|
54
|
+
required=True,
|
|
55
|
+
type=click.Path(exists=True, file_okay=False),
|
|
56
|
+
help="Root directory to scan for .cs files (recursively).",
|
|
57
|
+
)
|
|
58
|
+
@click.option(
|
|
59
|
+
"--project-name", required=True, help="Logical name for this project, used to build the Project node's id."
|
|
60
|
+
)
|
|
61
|
+
@click.option("--solution-name", default=None, help="Optional: wraps the Project in a Solution node.")
|
|
62
|
+
@click.option("--target-framework", default=None, help="Free-text, e.g. net8.0 -- informational only.")
|
|
63
|
+
@click.option(
|
|
64
|
+
"--dto-suffix",
|
|
65
|
+
"dto_suffixes",
|
|
66
|
+
multiple=True,
|
|
67
|
+
help="DTO name-suffix heuristic (repeatable). Default: Dto, Request, Response, ViewModel.",
|
|
68
|
+
)
|
|
69
|
+
@click.option(
|
|
70
|
+
"--workers", default=None, type=int, help="Process pool size for parallel per-file parsing. Default: os.cpu_count()."
|
|
71
|
+
)
|
|
72
|
+
@click.pass_context
|
|
73
|
+
def command(
|
|
74
|
+
ctx: click.Context,
|
|
75
|
+
path: str,
|
|
76
|
+
project_name: str,
|
|
77
|
+
solution_name: str | None,
|
|
78
|
+
target_framework: str | None,
|
|
79
|
+
dto_suffixes: tuple[str, ...],
|
|
80
|
+
workers: int | None,
|
|
81
|
+
) -> None:
|
|
82
|
+
writer = GraphWriter(connect(ctx.obj["falkordb_config"]))
|
|
83
|
+
config = CSharpSourceConfig(
|
|
84
|
+
path=path,
|
|
85
|
+
project_name=project_name,
|
|
86
|
+
solution_name=solution_name,
|
|
87
|
+
target_framework=target_framework,
|
|
88
|
+
dto_name_suffixes=tuple(dto_suffixes) or CSharpSourceConfig.dto_name_suffixes,
|
|
89
|
+
workers=workers,
|
|
90
|
+
)
|
|
91
|
+
stats = cls(config).run(writer)
|
|
92
|
+
click.echo(
|
|
93
|
+
f"Merged {stats.nodes_merged} nodes, {stats.edges_merged} edges "
|
|
94
|
+
f"in {stats.elapsed_seconds:.2f}s"
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
return command
|
|
98
|
+
|
|
99
|
+
def extract(self) -> Iterator[GraphNode | GraphEdge]:
|
|
100
|
+
root = Path(self._config.path).resolve()
|
|
101
|
+
files = self._discover_files(root)
|
|
102
|
+
logger.info("Found %d .cs files under %s", len(files), root)
|
|
103
|
+
|
|
104
|
+
raw_files = self._parse_files(files, root)
|
|
105
|
+
|
|
106
|
+
proj_id = project_id(self._config.project_name, self._config.solution_name)
|
|
107
|
+
yield GraphNode(
|
|
108
|
+
"Project",
|
|
109
|
+
{"id": proj_id},
|
|
110
|
+
{"name": self._config.project_name, "target_framework": self._config.target_framework},
|
|
111
|
+
)
|
|
112
|
+
if self._config.solution_name:
|
|
113
|
+
# The Project node must already be merged before this edge -- MERGE's
|
|
114
|
+
# MATCH silently no-ops (with a logged warning) if either endpoint is
|
|
115
|
+
# missing, same ordering lesson as the SQL ingestor's FK pass.
|
|
116
|
+
solution_id = f"csharp://{self._config.solution_name}"
|
|
117
|
+
yield GraphNode("Solution", {"id": solution_id}, {"name": self._config.solution_name})
|
|
118
|
+
yield GraphEdge("HAS_PROJECT", "Solution", {"id": solution_id}, "Project", {"id": proj_id})
|
|
119
|
+
|
|
120
|
+
yield from resolution.resolve(raw_files, proj_id, self._config.dto_name_suffixes)
|
|
121
|
+
|
|
122
|
+
def _discover_files(self, root: Path) -> list[Path]:
|
|
123
|
+
return sorted(
|
|
124
|
+
p
|
|
125
|
+
for p in root.rglob("*.cs")
|
|
126
|
+
if not _EXCLUDED_DIR_NAMES & set(p.relative_to(root).parts[:-1])
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
def _parse_files(self, files: list[Path], root: Path) -> list[RawFile]:
|
|
130
|
+
workers = self._config.workers or os.cpu_count() or 1
|
|
131
|
+
paths = [str(f) for f in files]
|
|
132
|
+
if workers == 1 or len(paths) <= 1:
|
|
133
|
+
raw_files = [parse_file(p) for p in paths]
|
|
134
|
+
else:
|
|
135
|
+
# Preload the parse module (parse_file's home) into the forkserver control process so
|
|
136
|
+
# workers inherit its tree-sitter import via COW rather than re-importing -- see
|
|
137
|
+
# core.parallel.
|
|
138
|
+
with make_process_pool(workers, preload=["auryn.ingestors.csharp.file_parser"]) as pool:
|
|
139
|
+
raw_files = list(pool.map(parse_file, paths))
|
|
140
|
+
for raw_file in raw_files:
|
|
141
|
+
raw_file.path = Path(raw_file.path).relative_to(root).as_posix()
|
|
142
|
+
return raw_files
|