auryn-ruby 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- auryn/ingestors/ruby/__init__.py +0 -0
- auryn/ingestors/ruby/file_parser.py +375 -0
- auryn/ingestors/ruby/git_sync.py +134 -0
- auryn/ingestors/ruby/grammar.py +17 -0
- auryn/ingestors/ruby/ids.py +68 -0
- auryn/ingestors/ruby/plugin.py +38 -0
- auryn/ingestors/ruby/raw_model.py +126 -0
- auryn/ingestors/ruby/resolution.py +485 -0
- auryn/ingestors/ruby/schema.py +61 -0
- auryn_ruby-0.1.0.dist-info/METADATA +11 -0
- auryn_ruby-0.1.0.dist-info/RECORD +14 -0
- auryn_ruby-0.1.0.dist-info/WHEEL +4 -0
- auryn_ruby-0.1.0.dist-info/entry_points.txt +2 -0
- auryn_ruby-0.1.0.dist-info/licenses/LICENSE +557 -0
|
File without changes
|
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
"""parse_file(path) -> RawFile: the parallelizable phase-1 unit (pure function, no shared state
|
|
2
|
+
-- safe under a ProcessPoolExecutor; only picklable raw_model dataclasses are returned, no
|
|
3
|
+
tree-sitter node crosses the boundary). parse_source(source, path_label) holds the logic and
|
|
4
|
+
takes raw bytes, so a git blob can be parsed without touching disk.
|
|
5
|
+
|
|
6
|
+
Every node type / field name below was confirmed empirically against tree-sitter-ruby 0.23.1
|
|
7
|
+
(see this package's grammar probe), not assumed:
|
|
8
|
+
|
|
9
|
+
- namespaces: `module` (`name` = constant|scope_resolution, `body` = body_statement) and `class`
|
|
10
|
+
(same, plus optional `superclass` field whose named child is the base constant). `singleton_class`
|
|
11
|
+
(`class << self`, `value`+`body`) contributes class-level methods to the enclosing type.
|
|
12
|
+
- callables: `method` (`name` = identifier|operator|setter, optional `parameters`
|
|
13
|
+
= method_parameters, optional `body`) and `singleton_method` (`def self.x`/`def Obj.x`:
|
|
14
|
+
`object`+`name`+`parameters`+`body`) -- the latter is a class/singleton method.
|
|
15
|
+
- `method_parameters` children: `identifier` (required), `optional_parameter` (`name`+`value`),
|
|
16
|
+
`splat_parameter`/`hash_splat_parameter`/`block_parameter` (`name`), `keyword_parameter`
|
|
17
|
+
(`name`, optional `value`).
|
|
18
|
+
- calls: `call` (`method` = identifier|super|operator|setter, optional `receiver`, optional
|
|
19
|
+
`arguments` = argument_list, optional `block`). A receiverless no-arg no-paren call is a bare
|
|
20
|
+
`identifier` in statement position (Ruby dynamism) -- captured too, guarded by the set of
|
|
21
|
+
locals assigned in the same body. `super` is its own node when bare.
|
|
22
|
+
- class-body DSL calls: `include`/`prepend`/`extend Mod` -> mixins; `attr_accessor`/`attr_reader`/
|
|
23
|
+
`attr_writer :x` -> fields; `private`/`protected`/`public` toggle the current visibility.
|
|
24
|
+
- `require`/`require_relative "path"` -> the require graph. Constants: an `assignment` whose
|
|
25
|
+
`left` is a `constant` (uppercase-initial) -> RawConst.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
from auryn.core.docstrings import leading_comment
|
|
33
|
+
from auryn.ingestors.ruby.grammar import new_parser
|
|
34
|
+
from auryn.ingestors.ruby.raw_model import (
|
|
35
|
+
RawCall,
|
|
36
|
+
RawConst,
|
|
37
|
+
RawField,
|
|
38
|
+
RawFile,
|
|
39
|
+
RawInclude,
|
|
40
|
+
RawMethod,
|
|
41
|
+
RawModule,
|
|
42
|
+
RawParameter,
|
|
43
|
+
RawRequire,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# Node types that introduce a fresh statement context: a bare identifier directly under one of
|
|
47
|
+
# these is a receiverless "command" call, not an argument/condition/operand (Ruby dynamism).
|
|
48
|
+
_STATEMENT_CONTAINERS = frozenset({"body_statement", "then", "else", "begin", "ensure"})
|
|
49
|
+
# Pseudo-keyword bare identifiers that are never method calls we want to record.
|
|
50
|
+
_BARE_SKIP = frozenset({
|
|
51
|
+
"self", "nil", "true", "false", "__method__", "__FILE__", "__dir__",
|
|
52
|
+
"private", "protected", "public", "super", "yield", "return", "break", "next", "redo", "retry",
|
|
53
|
+
})
|
|
54
|
+
_MIXIN_METHODS = {"include": "include", "prepend": "prepend", "extend": "extend"}
|
|
55
|
+
_ATTR_METHODS = {"attr_accessor": "accessor", "attr_reader": "reader", "attr_writer": "writer"}
|
|
56
|
+
_VISIBILITY_MARKERS = frozenset({"private", "protected", "public"})
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def parse_file(path: str) -> RawFile:
|
|
60
|
+
return parse_source(Path(path).read_bytes(), path)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def parse_source(source: bytes, path_label: str) -> RawFile:
|
|
64
|
+
tree = new_parser().parse(source)
|
|
65
|
+
|
|
66
|
+
def txt(node) -> str:
|
|
67
|
+
return source[node.start_byte : node.end_byte].decode("utf-8", "replace") if node else ""
|
|
68
|
+
|
|
69
|
+
def lines(node) -> tuple[int, int]:
|
|
70
|
+
return node.start_point[0] + 1, node.end_point[0] + 1
|
|
71
|
+
|
|
72
|
+
def fieldn(node, name):
|
|
73
|
+
return node.child_by_field_name(name) if node is not None else None
|
|
74
|
+
|
|
75
|
+
def doc_of(node) -> str:
|
|
76
|
+
"""The shared `doc` (leading `#`/`=begin` comment), cleaned via core.docstrings.
|
|
77
|
+
|
|
78
|
+
tree-sitter-ruby hoists the leading comment of a block's *first* statement out of the
|
|
79
|
+
`body_statement` and up onto the enclosing `class`/`module` node (so the first method's
|
|
80
|
+
`prev_named_sibling` is None). When that happens, escape up to the body_statement so its
|
|
81
|
+
preceding comment sibling is visible; otherwise the walk is the normal one."""
|
|
82
|
+
anchor = node
|
|
83
|
+
if anchor.prev_named_sibling is None and anchor.parent is not None and anchor.parent.type == "body_statement":
|
|
84
|
+
anchor = anchor.parent
|
|
85
|
+
return leading_comment(anchor, source)
|
|
86
|
+
|
|
87
|
+
def const_name(node) -> str:
|
|
88
|
+
"""The written name of a `constant` or `scope_resolution` (`A::B`) node, collapsed."""
|
|
89
|
+
return " ".join(txt(node).split()) if node is not None else ""
|
|
90
|
+
|
|
91
|
+
def string_literal(node):
|
|
92
|
+
"""The text inside a `string` argument node, or None if it isn't a plain string."""
|
|
93
|
+
if node is None or node.type != "string":
|
|
94
|
+
return None
|
|
95
|
+
parts = [txt(c) for c in node.named_children if c.type == "string_content"]
|
|
96
|
+
return "".join(parts) if parts else txt(node).strip("\"'")
|
|
97
|
+
|
|
98
|
+
# -- parameters -------------------------------------------------------------------------
|
|
99
|
+
_PARAM_KINDS = {
|
|
100
|
+
"splat_parameter": "splat",
|
|
101
|
+
"hash_splat_parameter": "hash_splat",
|
|
102
|
+
"block_parameter": "block",
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
def parameters(params_node) -> list[RawParameter]:
|
|
106
|
+
if params_node is None:
|
|
107
|
+
return []
|
|
108
|
+
out: list[RawParameter] = []
|
|
109
|
+
pos = 0
|
|
110
|
+
for c in params_node.named_children:
|
|
111
|
+
t = c.type
|
|
112
|
+
if t == "identifier":
|
|
113
|
+
out.append(RawParameter(name=txt(c), kind="required", position=pos))
|
|
114
|
+
elif t == "optional_parameter":
|
|
115
|
+
out.append(RawParameter(
|
|
116
|
+
name=txt(fieldn(c, "name")), kind="optional", position=pos,
|
|
117
|
+
default=txt(fieldn(c, "value")) or None,
|
|
118
|
+
))
|
|
119
|
+
elif t == "keyword_parameter":
|
|
120
|
+
val = fieldn(c, "value")
|
|
121
|
+
out.append(RawParameter(
|
|
122
|
+
name=txt(fieldn(c, "name")),
|
|
123
|
+
kind="keyword_optional" if val is not None else "keyword",
|
|
124
|
+
position=pos, default=txt(val) if val is not None else None,
|
|
125
|
+
))
|
|
126
|
+
elif t in _PARAM_KINDS:
|
|
127
|
+
nm = fieldn(c, "name")
|
|
128
|
+
out.append(RawParameter(name=txt(nm) if nm is not None else "", kind=_PARAM_KINDS[t], position=pos))
|
|
129
|
+
else:
|
|
130
|
+
continue
|
|
131
|
+
pos += 1
|
|
132
|
+
return out
|
|
133
|
+
|
|
134
|
+
def signature(name: str, params: list[RawParameter]) -> str:
|
|
135
|
+
pieces = []
|
|
136
|
+
for p in params:
|
|
137
|
+
if p.kind == "splat":
|
|
138
|
+
pieces.append(f"*{p.name}")
|
|
139
|
+
elif p.kind == "hash_splat":
|
|
140
|
+
pieces.append(f"**{p.name}")
|
|
141
|
+
elif p.kind == "block":
|
|
142
|
+
pieces.append(f"&{p.name}")
|
|
143
|
+
elif p.kind == "keyword":
|
|
144
|
+
pieces.append(f"{p.name}:")
|
|
145
|
+
elif p.kind == "keyword_optional":
|
|
146
|
+
pieces.append(f"{p.name}: {p.default}")
|
|
147
|
+
elif p.kind == "optional":
|
|
148
|
+
pieces.append(f"{p.name} = {p.default}")
|
|
149
|
+
else:
|
|
150
|
+
pieces.append(p.name)
|
|
151
|
+
return f"{name}({', '.join(pieces)})"
|
|
152
|
+
|
|
153
|
+
# -- calls ------------------------------------------------------------------------------
|
|
154
|
+
def method_name_text(node) -> str:
|
|
155
|
+
"""The method name of a `method`/`singleton_method`/`call` -- an identifier, an operator
|
|
156
|
+
(`==`, `[]`), or a setter (`x=`, whose child holds the base name)."""
|
|
157
|
+
if node is None:
|
|
158
|
+
return ""
|
|
159
|
+
if node.type == "setter":
|
|
160
|
+
base = fieldn(node, "name")
|
|
161
|
+
return f"{txt(base)}=" if base is not None else txt(node)
|
|
162
|
+
return txt(node)
|
|
163
|
+
|
|
164
|
+
def _local_names(body) -> set[str]:
|
|
165
|
+
"""Names bound as locals in `body`: assignment/operator_assignment targets, rescue
|
|
166
|
+
variables, and block/for parameters -- so a bare-identifier command-call scan can skip a
|
|
167
|
+
local read (which is syntactically identical to a paren-less call)."""
|
|
168
|
+
locals_: set[str] = set()
|
|
169
|
+
stack = [body]
|
|
170
|
+
while stack:
|
|
171
|
+
n = stack.pop()
|
|
172
|
+
t = n.type
|
|
173
|
+
if t in ("assignment", "operator_assignment"):
|
|
174
|
+
left = fieldn(n, "left")
|
|
175
|
+
if left is not None and left.type == "identifier":
|
|
176
|
+
locals_.add(txt(left))
|
|
177
|
+
elif t == "exception_variable":
|
|
178
|
+
for g in n.named_children:
|
|
179
|
+
if g.type == "identifier":
|
|
180
|
+
locals_.add(txt(g))
|
|
181
|
+
elif t in ("block_parameters", "method_parameters"):
|
|
182
|
+
for g in n.named_children:
|
|
183
|
+
nm = fieldn(g, "name") if g.type != "identifier" else g
|
|
184
|
+
if nm is not None and nm.type == "identifier":
|
|
185
|
+
locals_.add(txt(nm))
|
|
186
|
+
stack.extend(n.children)
|
|
187
|
+
return locals_
|
|
188
|
+
|
|
189
|
+
def _receiver_call(node) -> RawCall | None:
|
|
190
|
+
"""A `call` node -> RawCall (or None to drop an unnameable/complex receiver)."""
|
|
191
|
+
method = fieldn(node, "method")
|
|
192
|
+
if method is not None and method.type == "super":
|
|
193
|
+
return RawCall(name="super", kind="super")
|
|
194
|
+
name = method_name_text(method)
|
|
195
|
+
if not name:
|
|
196
|
+
return None
|
|
197
|
+
recv = fieldn(node, "receiver")
|
|
198
|
+
if recv is None:
|
|
199
|
+
return RawCall(name=name, kind="command")
|
|
200
|
+
rt = recv.type
|
|
201
|
+
if rt == "self":
|
|
202
|
+
return RawCall(name=name, kind="self")
|
|
203
|
+
if rt in ("constant", "scope_resolution"):
|
|
204
|
+
return RawCall(name=name, kind="constant", qualifier=const_name(recv))
|
|
205
|
+
if rt in ("identifier", "instance_variable", "class_variable", "global_variable"):
|
|
206
|
+
return RawCall(name=name, kind="instance", receiver=txt(recv))
|
|
207
|
+
return RawCall(name=name, kind="chain") # a.b.c(), literal receivers, ... -> resolution drops
|
|
208
|
+
|
|
209
|
+
def collect_calls(body) -> list[RawCall]:
|
|
210
|
+
if body is None:
|
|
211
|
+
return []
|
|
212
|
+
locals_ = _local_names(body)
|
|
213
|
+
out: list[RawCall] = []
|
|
214
|
+
seen: set = set()
|
|
215
|
+
|
|
216
|
+
def add(call: RawCall | None) -> None:
|
|
217
|
+
if call is None or not call.name:
|
|
218
|
+
return
|
|
219
|
+
key = (call.name, call.kind, call.receiver, call.qualifier)
|
|
220
|
+
if key not in seen:
|
|
221
|
+
seen.add(key)
|
|
222
|
+
out.append(call)
|
|
223
|
+
|
|
224
|
+
stack = [(body, None)]
|
|
225
|
+
while stack:
|
|
226
|
+
n, parent_type = stack.pop()
|
|
227
|
+
t = n.type
|
|
228
|
+
if t == "call":
|
|
229
|
+
add(_receiver_call(n))
|
|
230
|
+
elif t == "super": # a bare `super` (with args it parses as a `call`)
|
|
231
|
+
add(RawCall(name="super", kind="super"))
|
|
232
|
+
elif t == "identifier" and parent_type in _STATEMENT_CONTAINERS:
|
|
233
|
+
name = txt(n)
|
|
234
|
+
if name not in locals_ and name not in _BARE_SKIP:
|
|
235
|
+
add(RawCall(name=name, kind="command"))
|
|
236
|
+
for c in n.children:
|
|
237
|
+
stack.append((c, t))
|
|
238
|
+
return out
|
|
239
|
+
|
|
240
|
+
# -- methods ----------------------------------------------------------------------------
|
|
241
|
+
def build_method(node, visibility: str, force_singleton: bool = False) -> RawMethod:
|
|
242
|
+
is_singleton = force_singleton or node.type == "singleton_method"
|
|
243
|
+
name = method_name_text(fieldn(node, "name"))
|
|
244
|
+
params = parameters(fieldn(node, "parameters"))
|
|
245
|
+
ls, le = lines(node)
|
|
246
|
+
return RawMethod(
|
|
247
|
+
name=name,
|
|
248
|
+
kind="METHOD", # rewritten to FUNCTION for a top-level def in collect_scope
|
|
249
|
+
is_singleton=is_singleton,
|
|
250
|
+
visibility=visibility,
|
|
251
|
+
parameters=params,
|
|
252
|
+
calls=collect_calls(fieldn(node, "body")),
|
|
253
|
+
signature=signature(name, params),
|
|
254
|
+
source=txt(node),
|
|
255
|
+
doc=doc_of(node),
|
|
256
|
+
line_start=ls,
|
|
257
|
+
line_end=le,
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
# -- namespaces -------------------------------------------------------------------------
|
|
261
|
+
def build_namespace(node, kind: str) -> RawModule:
|
|
262
|
+
ls, le = lines(node)
|
|
263
|
+
rm = RawModule(
|
|
264
|
+
name=const_name(fieldn(node, "name")),
|
|
265
|
+
kind=kind,
|
|
266
|
+
source=txt(node),
|
|
267
|
+
doc=doc_of(node),
|
|
268
|
+
line_start=ls,
|
|
269
|
+
line_end=le,
|
|
270
|
+
)
|
|
271
|
+
if kind == "Class":
|
|
272
|
+
sc = fieldn(node, "superclass")
|
|
273
|
+
if sc is not None:
|
|
274
|
+
base = next((c for c in sc.named_children if c.type in ("constant", "scope_resolution")), None)
|
|
275
|
+
rm.superclass = const_name(base) if base is not None else None
|
|
276
|
+
collect_scope(fieldn(node, "body"), rm)
|
|
277
|
+
return rm
|
|
278
|
+
|
|
279
|
+
def _symbol_names(args_node) -> list[str]:
|
|
280
|
+
"""Symbol / string argument names from an `attr_*` argument list (`:name`, `"name"`)."""
|
|
281
|
+
out: list[str] = []
|
|
282
|
+
if args_node is None:
|
|
283
|
+
return out
|
|
284
|
+
for c in args_node.named_children:
|
|
285
|
+
if c.type in ("simple_symbol", "symbol"):
|
|
286
|
+
out.append(txt(c).lstrip(":").strip("\"'"))
|
|
287
|
+
elif c.type == "string":
|
|
288
|
+
s = string_literal(c)
|
|
289
|
+
if s:
|
|
290
|
+
out.append(s)
|
|
291
|
+
return out
|
|
292
|
+
|
|
293
|
+
def handle_dsl_call(node, target) -> bool:
|
|
294
|
+
"""Interpret a class/module-body `call` that is a mixin/attr/require DSL. Returns True if
|
|
295
|
+
it was consumed (so it isn't also treated as a plain statement call)."""
|
|
296
|
+
method = fieldn(node, "method")
|
|
297
|
+
if method is None or fieldn(node, "receiver") is not None:
|
|
298
|
+
return False
|
|
299
|
+
name = txt(method)
|
|
300
|
+
args = fieldn(node, "arguments")
|
|
301
|
+
if name in _MIXIN_METHODS and args is not None:
|
|
302
|
+
# A file-scope `include`/`extend`/`prepend` (target is a RawFile) has no class/module to
|
|
303
|
+
# attach to, so skip the append but still consume the call -- it's a mixin DSL, not a
|
|
304
|
+
# regular method call to track (mirrors the `requires` guard below).
|
|
305
|
+
if hasattr(target, "includes"):
|
|
306
|
+
for a in args.named_children:
|
|
307
|
+
if a.type in ("constant", "scope_resolution"):
|
|
308
|
+
target.includes.append(RawInclude(name=const_name(a), kind=_MIXIN_METHODS[name]))
|
|
309
|
+
return True
|
|
310
|
+
if name in _ATTR_METHODS and args is not None:
|
|
311
|
+
if hasattr(target, "fields"):
|
|
312
|
+
ls, le = lines(node)
|
|
313
|
+
for sym in _symbol_names(args):
|
|
314
|
+
target.fields.append(RawField(name=sym, access=_ATTR_METHODS[name], line_start=ls, line_end=le))
|
|
315
|
+
return True
|
|
316
|
+
if name in ("require", "require_relative") and args is not None and hasattr(target, "requires"):
|
|
317
|
+
s = string_literal(args.named_children[0]) if args.named_children else None
|
|
318
|
+
if s is not None:
|
|
319
|
+
target.requires.append(RawRequire(target=s, kind=name))
|
|
320
|
+
return True
|
|
321
|
+
return False
|
|
322
|
+
|
|
323
|
+
def collect_scope(container, target) -> None:
|
|
324
|
+
"""Fill `target` (a RawFile or RawModule) from the statements of `container` (a
|
|
325
|
+
body_statement / the program root). Tracks the running method visibility inside a class."""
|
|
326
|
+
if container is None:
|
|
327
|
+
return
|
|
328
|
+
visibility = "public"
|
|
329
|
+
is_namespace = isinstance(target, RawModule)
|
|
330
|
+
for child in container.named_children:
|
|
331
|
+
t = child.type
|
|
332
|
+
if t == "class":
|
|
333
|
+
target.classes.append(build_namespace(child, "Class"))
|
|
334
|
+
elif t == "module":
|
|
335
|
+
target.modules.append(build_namespace(child, "Module"))
|
|
336
|
+
elif t == "method":
|
|
337
|
+
m = build_method(child, visibility)
|
|
338
|
+
if not is_namespace:
|
|
339
|
+
m.kind = "FUNCTION" # a top-level def is a free function
|
|
340
|
+
target.methods.append(m)
|
|
341
|
+
elif t == "singleton_method":
|
|
342
|
+
target.methods.append(build_method(child, visibility))
|
|
343
|
+
elif t == "singleton_class": # `class << self` -> its defs are class methods
|
|
344
|
+
body = fieldn(child, "body")
|
|
345
|
+
if body is not None:
|
|
346
|
+
for m in body.named_children:
|
|
347
|
+
if m.type in ("method", "singleton_method"):
|
|
348
|
+
target.methods.append(build_method(m, "public", force_singleton=True))
|
|
349
|
+
elif t == "assignment":
|
|
350
|
+
left = fieldn(child, "left")
|
|
351
|
+
if left is not None and left.type == "constant":
|
|
352
|
+
ls, le = lines(child)
|
|
353
|
+
target.consts.append(RawConst(
|
|
354
|
+
name=txt(left), value=" ".join(txt(fieldn(child, "right")).split()),
|
|
355
|
+
line_start=ls, line_end=le,
|
|
356
|
+
))
|
|
357
|
+
elif t == "identifier" and txt(child) in _VISIBILITY_MARKERS:
|
|
358
|
+
visibility = txt(child) # `private` / `protected` / `public` toggle
|
|
359
|
+
elif t == "call":
|
|
360
|
+
if handle_dsl_call(child, target):
|
|
361
|
+
continue
|
|
362
|
+
# `private def foo; end` (a call wrapping a def) -> mark that method's visibility.
|
|
363
|
+
m = fieldn(child, "method")
|
|
364
|
+
args = fieldn(child, "arguments")
|
|
365
|
+
if m is not None and txt(m) in _VISIBILITY_MARKERS and args is not None:
|
|
366
|
+
inner = next((a for a in args.named_children if a.type in ("method", "singleton_method")), None)
|
|
367
|
+
if inner is not None:
|
|
368
|
+
rm = build_method(inner, txt(m))
|
|
369
|
+
if not is_namespace:
|
|
370
|
+
rm.kind = "FUNCTION"
|
|
371
|
+
target.methods.append(rm)
|
|
372
|
+
|
|
373
|
+
rf = RawFile(path=path_label)
|
|
374
|
+
collect_scope(tree.root_node, rf)
|
|
375
|
+
return rf
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Git-tracked (and directory) Ruby ingestion: moves a project's Ruby subgraph to a target
|
|
2
|
+
commit/tree by fully recomputing the target state and diffing it against the graph
|
|
3
|
+
(graph_sync.compute_delta) -- the same recompute-and-delta model as the Rust/Go/Python ingestors
|
|
4
|
+
(resolution is global -- the project-wide namespace/method index and merged reopened classes --
|
|
5
|
+
so a file-level diff can't tell which untouched files a change affects).
|
|
6
|
+
|
|
7
|
+
Fast engine only for now: tree-sitter-ruby streams each `.rb` blob (no disk materialization) ->
|
|
8
|
+
RawFile -> resolution.resolve(). A deep Ruby pass (a real loader / Sorbet / RBS-backed resolver for
|
|
9
|
+
method dispatch, receiver types, and cross-file constant resolution) would be added as the
|
|
10
|
+
GitLanguage's `deep_sync` (see plugin.py). Registered as the `ruby` GitLanguage for the
|
|
11
|
+
multi-language endpoint (core/ingest.py) in both ingest_repository (git) and ingest_directory
|
|
12
|
+
(local)."""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import itertools
|
|
17
|
+
import logging
|
|
18
|
+
import time
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
|
|
21
|
+
from auryn.core.parallel import parse_blobs_ordered
|
|
22
|
+
from auryn.core.graph_model import GraphNode
|
|
23
|
+
from auryn.core.graph_sync import stream_apply
|
|
24
|
+
from auryn.core.registry import LanguageIngestResult
|
|
25
|
+
from auryn.ingestors.ruby import resolution
|
|
26
|
+
from auryn.ingestors.ruby.file_parser import parse_source
|
|
27
|
+
from auryn.ingestors.ruby.ids import project_id
|
|
28
|
+
from auryn.ingestors.ruby.plugin import EXCLUDED_DIR_NAMES as _EXCLUDED_DIR_NAMES
|
|
29
|
+
from auryn.ingestors.ruby.plugin import FILE_SUFFIXES as _FILE_SUFFIXES
|
|
30
|
+
from auryn.ingestors.ruby.schema import NODE_SCHEMAS
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class SyncStats:
|
|
37
|
+
nodes_added: int = 0
|
|
38
|
+
nodes_updated: int = 0
|
|
39
|
+
nodes_removed: int = 0
|
|
40
|
+
edges_added: int = 0
|
|
41
|
+
edges_updated: int = 0
|
|
42
|
+
edges_removed: int = 0
|
|
43
|
+
commit_sha: str = ""
|
|
44
|
+
elapsed_seconds: float = 0.0
|
|
45
|
+
files_parsed: int = 0
|
|
46
|
+
files_cached: int = 0
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def sync_at(writer, source, target_sha: str, project_name: str, repo_uri: str, ref: str | None) -> SyncStats:
|
|
50
|
+
"""Recompute the whole Ruby subgraph at `target_sha` and delta it onto the graph. `source`
|
|
51
|
+
is duck-typed (GitRepoCache or LocalDirectorySource): only list_files/read_blob used."""
|
|
52
|
+
started = time.monotonic()
|
|
53
|
+
proj_id = project_id(project_name)
|
|
54
|
+
|
|
55
|
+
writer.setup_schema(NODE_SCHEMAS) # brings the graph key into existence before the first read
|
|
56
|
+
if writer.get_property("Project", {"id": proj_id}, "commit_sha") == target_sha:
|
|
57
|
+
return SyncStats(commit_sha=target_sha, elapsed_seconds=time.monotonic() - started)
|
|
58
|
+
|
|
59
|
+
entries = [
|
|
60
|
+
(path, blob_sha)
|
|
61
|
+
for suffix in _FILE_SUFFIXES
|
|
62
|
+
for path, blob_sha in source.list_files(target_sha, suffix, set(_EXCLUDED_DIR_NAMES))
|
|
63
|
+
]
|
|
64
|
+
files = parse_blobs_ordered(
|
|
65
|
+
entries, source.read_blob, parse_source,
|
|
66
|
+
preload=["auryn.ingestors.ruby.file_parser"], logger=logger,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
project_node = GraphNode(
|
|
70
|
+
"Project", {"id": proj_id}, {"name": project_name, "repo_uri": repo_uri, "commit_sha": target_sha, "ref": ref}
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
def make_target(): # re-iterable: stream_apply runs it once for nodes, once for edges
|
|
74
|
+
return itertools.chain([project_node], resolution.resolve(files, proj_id))
|
|
75
|
+
|
|
76
|
+
return _apply(writer, proj_id, make_target, target_sha, started, files_parsed=len(files))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _apply(writer, proj_id: str, make_target, target_sha: str, started: float, files_parsed: int) -> SyncStats:
|
|
80
|
+
# Cross-repo reconciliation (ADR 0004): fold placeholder<->definer pairs among the linkages
|
|
81
|
+
# this ingest touched -- targeted index seeks, no scan. No-op unless the graph holds >1 repo.
|
|
82
|
+
from auryn.core.linkage import reconcile_linkages
|
|
83
|
+
|
|
84
|
+
# stream_apply writes the delta without materialising the target graph, and returns the
|
|
85
|
+
# linkage strings collected inline (see graph_sync.stream_apply / core.linkage).
|
|
86
|
+
labels = [s.label for s in NODE_SCHEMAS]
|
|
87
|
+
stats, reals, placeholders = stream_apply(writer, proj_id, labels, make_target)
|
|
88
|
+
reconcile_linkages(writer, reals, placeholders)
|
|
89
|
+
|
|
90
|
+
return SyncStats(
|
|
91
|
+
nodes_added=stats.nodes_added,
|
|
92
|
+
nodes_updated=stats.nodes_updated,
|
|
93
|
+
nodes_removed=stats.nodes_removed,
|
|
94
|
+
edges_added=stats.edges_added,
|
|
95
|
+
edges_updated=stats.edges_updated,
|
|
96
|
+
edges_removed=stats.edges_removed,
|
|
97
|
+
commit_sha=target_sha,
|
|
98
|
+
elapsed_seconds=time.monotonic() - started,
|
|
99
|
+
files_parsed=files_parsed,
|
|
100
|
+
files_cached=0,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _git_language_sync(
|
|
105
|
+
writer,
|
|
106
|
+
source,
|
|
107
|
+
target_sha: str,
|
|
108
|
+
*,
|
|
109
|
+
repo_uri: str,
|
|
110
|
+
project_name: str,
|
|
111
|
+
solution_name: str | None,
|
|
112
|
+
ref: str | None,
|
|
113
|
+
cache_root=None,
|
|
114
|
+
) -> LanguageIngestResult:
|
|
115
|
+
"""GitLanguage adapter for the multi-language endpoint (Ruby has no solution/cache knobs --
|
|
116
|
+
accepted for the shared interface and ignored)."""
|
|
117
|
+
s = sync_at(writer, source, target_sha, project_name=project_name, repo_uri=repo_uri, ref=ref)
|
|
118
|
+
return LanguageIngestResult(
|
|
119
|
+
nodes_added=s.nodes_added,
|
|
120
|
+
nodes_updated=s.nodes_updated,
|
|
121
|
+
nodes_removed=s.nodes_removed,
|
|
122
|
+
edges_added=s.edges_added,
|
|
123
|
+
edges_updated=s.edges_updated,
|
|
124
|
+
edges_removed=s.edges_removed,
|
|
125
|
+
files_parsed=s.files_parsed,
|
|
126
|
+
files_cached=s.files_cached,
|
|
127
|
+
commit_sha=s.commit_sha,
|
|
128
|
+
elapsed_seconds=s.elapsed_seconds,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# The GitLanguage is registered by the import-light plugin.py (the entry-point target), whose
|
|
133
|
+
# lazy `sync` thunk calls `_git_language_sync` above -- so importing this tree-sitter engine
|
|
134
|
+
# module is deferred until a source actually contains Ruby files.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""tree-sitter-ruby Language/Parser setup, in one place so file_parser.py and tests always
|
|
2
|
+
get the exact same configured parser. Like tree-sitter-go/-rust/-java, it ships a pre-built
|
|
3
|
+
PyPI package -- no codegen/vendoring (contrast PL/SQL's vendored ANTLR grammar).
|
|
4
|
+
|
|
5
|
+
NOTE: tree-sitter-ruby exposes a plain `language()` (unlike tree-sitter-php, which bundles two
|
|
6
|
+
grammars behind `language_php()`/`language_php_only()`). Verified via `dir(tree_sitter_ruby)`."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import tree_sitter_ruby as tsrb
|
|
11
|
+
from tree_sitter import Language, Parser
|
|
12
|
+
|
|
13
|
+
LANGUAGE = Language(tsrb.language())
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def new_parser() -> Parser:
|
|
17
|
+
return Parser(LANGUAGE)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Canonical id scheme for the Ruby graph, mirroring the Rust/Python ingestors' shape (a
|
|
2
|
+
namespace tier over classes/modules with members) so a single graph can hold every language with
|
|
3
|
+
interchangeable node ids. A future deep (Ruby loader / Sorbet / RBS) engine would key nodes by
|
|
4
|
+
these same functions, so its output stays a pure delta over the fast pass.
|
|
5
|
+
|
|
6
|
+
Ids embed names and parameter *names* (never inferred types -- Ruby has none statically, and the
|
|
7
|
+
id-stability rule forbids volatile types in ids). Ruby has no method overloading, but a class
|
|
8
|
+
method and an instance method may share a name; the `#singleton` suffix keeps their Method ids
|
|
9
|
+
from colliding. Namespaces are keyed by their fully-qualified `::`-path (lowercased, `::`->`.`),
|
|
10
|
+
so a class reopened in two files lands on one node."""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def project_id(project_name: str) -> str:
|
|
16
|
+
return f"ruby://{project_name}"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _qual_slug(qualified: str) -> str:
|
|
20
|
+
# "A::B::Dog" -> "a.b.dog"; a plain "Dog" -> "dog".
|
|
21
|
+
return ".".join(p for p in qualified.replace("::", ".").split(".") if p).lower()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def class_id(proj_id: str, qualified: str) -> str:
|
|
25
|
+
return f"{proj_id}/class/{_qual_slug(qualified)}"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def module_id(proj_id: str, qualified: str) -> str:
|
|
29
|
+
return f"{proj_id}/module/{_qual_slug(qualified)}"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def file_id(proj_id: str, rel_path: str) -> str:
|
|
33
|
+
return f"{proj_id}/file/{rel_path.replace(chr(92), '/').lstrip('./')}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _sig(param_names: list[str]) -> str:
|
|
37
|
+
return ",".join((n or "").lower() for n in param_names)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def function_id(scope_id: str, name: str, param_names: list[str]) -> str:
|
|
41
|
+
return f"{scope_id}.func.{name.lower()}({_sig(param_names)})"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def method_id(owner_id: str, name: str, param_names: list[str], is_singleton: bool = False) -> str:
|
|
45
|
+
base = f"{owner_id}.method.{name.lower()}({_sig(param_names)})"
|
|
46
|
+
return f"{base}#singleton" if is_singleton else base
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def parameter_id(owner_id: str, position: int, name: str) -> str:
|
|
50
|
+
return f"{owner_id}.param.{position}.{name.lower()}"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def field_id(owner_id: str, name: str) -> str:
|
|
54
|
+
return f"{owner_id}.field.{name.lower()}"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def const_id(scope_id: str, name: str) -> str:
|
|
58
|
+
return f"{scope_id}.const.{name.lower()}"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# -- cross-repo linkage (ADR 0004) -------------------------------------------------------
|
|
62
|
+
# Project-INDEPENDENT key for an externally-visible symbol, in the form a *consumer* writes to
|
|
63
|
+
# reach it: `Class#method` (instance), `Class.method` (class/singleton), `Module.method`, or a
|
|
64
|
+
# bare `name` for a top-level function. It is BOTH the id of the placeholder standing in for an
|
|
65
|
+
# unresolved constant-receiver call AND the `linkage_key` stamped on a real definition, so
|
|
66
|
+
# ingesting the defining repo into the same graph heals a placeholder onto its definition.
|
|
67
|
+
def extern_symbol_key(qualified_name: str) -> str:
|
|
68
|
+
return f"extern://ruby/{qualified_name}"
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Import-light Ruby language registration -- the `auryn.languages` entry-point target.
|
|
2
|
+
|
|
3
|
+
Importing this module registers the Ruby GitLanguage without importing tree-sitter: the
|
|
4
|
+
GitLanguage's `sync` is a lazy thunk that imports auryn.ingestors.ruby.git_sync only when a
|
|
5
|
+
source actually contains `.rb` files. Single-phase today (fast tree-sitter-ruby); a deep pass --
|
|
6
|
+
a real Ruby loader / Sorbet / RBS-backed resolver for method dispatch, receiver types, mixin
|
|
7
|
+
ancestry, and cross-file constant resolution -- would slot in here as `deep_sync` (making Ruby
|
|
8
|
+
two-phase like Python). Ruby is a dynamic language (open classes, method_missing, metaprogramming)
|
|
9
|
+
so the fast pass is name-based and deliberately conservative: it emits an edge only when it can
|
|
10
|
+
name a target, never a guess (see resolution.py). Suffix/exclude constants live here (single
|
|
11
|
+
source of truth); git_sync imports them back."""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from auryn.core.registry import GitLanguage, register_git_language
|
|
16
|
+
|
|
17
|
+
LANGUAGE = "ruby"
|
|
18
|
+
FILE_SUFFIXES = (".rb",)
|
|
19
|
+
# VCS/editor plus the conventional Ruby non-source trees: bundled gems (`vendor`), transient
|
|
20
|
+
# `tmp`/`log`, and the JS toolchain's `node_modules` (common in Rails apps).
|
|
21
|
+
EXCLUDED_DIR_NAMES = frozenset({".git", "vendor", "node_modules", "tmp", "log"})
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _sync(*args, **kwargs):
|
|
25
|
+
"""Deferred until a source has Ruby files -- imports the tree-sitter-ruby engine."""
|
|
26
|
+
from auryn.ingestors.ruby.git_sync import _git_language_sync
|
|
27
|
+
|
|
28
|
+
return _git_language_sync(*args, **kwargs)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
register_git_language(
|
|
32
|
+
GitLanguage(
|
|
33
|
+
language=LANGUAGE,
|
|
34
|
+
file_suffixes=FILE_SUFFIXES,
|
|
35
|
+
excluded_dir_names=EXCLUDED_DIR_NAMES,
|
|
36
|
+
sync=_sync,
|
|
37
|
+
)
|
|
38
|
+
)
|