auryn-python 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,385 @@
1
+ """Phase 1 of the `ast` fallback engine: parse one .py file into the raw model.
2
+
3
+ Pure syntax, stdlib `ast` only -- no type inference, no cross-file work (that's
4
+ resolution.py). A file that doesn't parse (syntax error) yields an empty module rather
5
+ than aborting the ingest, mirroring the endpoint's "one bad subgraph never aborts the
6
+ source" contract.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import ast
12
+ import contextlib
13
+ import logging
14
+ import os
15
+ import signal
16
+ import threading
17
+
18
+ from auryn.core.docstrings import leading_comment_from_source
19
+ from auryn.ingestors.python import ids
20
+ from auryn.ingestors.python.raw_model import (
21
+ RawAttribute,
22
+ RawCall,
23
+ RawClass,
24
+ RawFunction,
25
+ RawImport,
26
+ RawModule,
27
+ RawParameter,
28
+ )
29
+
30
+ logger = logging.getLogger(__name__)
31
+
32
+
33
+ def _max_parse_seconds() -> int:
34
+ """Per-file parse budget (SIGALRM, Unix only). A pathological file -- typically a huge
35
+ generated module -- is skipped instead of stalling the whole ingest. Read from the env each
36
+ call so it's overridable per run; 0/negative disables the cap. Mirrors the PL/SQL ingestor's
37
+ --max-parse-seconds guard (see plsql/file_ingestor.parse_plsql_file)."""
38
+ try:
39
+ return int(os.environ.get("AURYN_PY_MAX_PARSE_SECONDS", "30"))
40
+ except ValueError:
41
+ return 30
42
+
43
+
44
+ class _ParseTimeout(BaseException):
45
+ """Raised from the per-file SIGALRM handler. A BaseException (not Exception) so the
46
+ per-node parse work's own error handling can't accidentally swallow it."""
47
+
48
+
49
+ @contextlib.contextmanager
50
+ def _parse_deadline(max_seconds: int):
51
+ """Bound the wrapped parse to `max_seconds` via SIGALRM, restoring any prior handler after.
52
+ A no-op where the cap is disabled, SIGALRM is unavailable (Windows), or we're off the main
53
+ thread -- signal.signal() only works on the main thread, and the deep=background path parses
54
+ in a worker thread. So behaviour is unchanged except a would-be-hanging file on the main
55
+ thread (the fast serial path, and future pool workers' own main thread) is interrupted."""
56
+ if (
57
+ max_seconds <= 0
58
+ or not hasattr(signal, "SIGALRM")
59
+ or threading.current_thread() is not threading.main_thread()
60
+ ):
61
+ yield
62
+ return
63
+
64
+ def _on_timeout(signum, frame):
65
+ raise _ParseTimeout
66
+
67
+ previous = signal.signal(signal.SIGALRM, _on_timeout)
68
+ signal.alarm(int(max_seconds))
69
+ try:
70
+ yield
71
+ finally:
72
+ signal.alarm(0)
73
+ signal.signal(signal.SIGALRM, previous)
74
+
75
+
76
+ def _line_starts(data: bytes) -> list[int]:
77
+ """Byte offset of the start of each line in `data` (0-based line index -> byte position).
78
+ Computed once per file so source extraction is O(1) per declaration instead of re-scanning."""
79
+ starts = [0]
80
+ i = data.find(b"\n")
81
+ while i != -1:
82
+ starts.append(i + 1)
83
+ i = data.find(b"\n", i + 1)
84
+ return starts
85
+
86
+
87
+ class _Source:
88
+ """A file's text prepared once for cheap, repeated source extraction.
89
+
90
+ `ast.get_source_segment` and `leading_comment_from_source` each re-split the whole file on
91
+ every call -- O(file) per declaration, i.e. O(n^2) over a module (373s on a 162k-line generated
92
+ file; see the parse-hang fix). This precomputes line byte-offsets and the split line list once,
93
+ turning both into O(1)-setup lookups. `segment` mirrors ast.get_source_segment's byte-offset
94
+ semantics (CPython col_offset is a UTF-8 byte offset)."""
95
+
96
+ __slots__ = ("text", "_data", "_starts", "_lines")
97
+
98
+ def __init__(self, text: str) -> None:
99
+ self.text = text
100
+ self._data = text.encode("utf-8")
101
+ self._starts = _line_starts(self._data)
102
+ self._lines = text.splitlines() # reused by leading_comment (no keepends, matches docstrings)
103
+
104
+ def segment(self, node: ast.AST) -> str:
105
+ end_lineno = getattr(node, "end_lineno", None)
106
+ end_col = getattr(node, "end_col_offset", None)
107
+ if end_lineno is None or end_col is None:
108
+ return ""
109
+ start = self._starts[node.lineno - 1] + node.col_offset
110
+ end = self._starts[end_lineno - 1] + end_col
111
+ return self._data[start:end].decode("utf-8", "replace")
112
+
113
+ def leading_comment(self, top_line: int) -> str:
114
+ return leading_comment_from_source(self.text, top_line, lines=self._lines)
115
+
116
+
117
+ def parse_module(source: bytes | str, path: str, package_dirs: frozenset[str] | None = None) -> RawModule:
118
+ dotted = ids.dotted_module_path(path, package_dirs)
119
+ module = RawModule(path=path, dotted=dotted, is_package=ids.is_package_init(path))
120
+ text = source.decode("utf-8", "replace") if isinstance(source, bytes) else source
121
+ try:
122
+ with _parse_deadline(_max_parse_seconds()):
123
+ tree = ast.parse(text, filename=path)
124
+ src = _Source(text)
125
+ for node in tree.body:
126
+ if isinstance(node, (ast.Import, ast.ImportFrom)):
127
+ module.imports.extend(_imports(node, dotted, module.is_package))
128
+ elif isinstance(node, ast.ClassDef):
129
+ module.classes.append(_class(node, src))
130
+ elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
131
+ module.functions.append(_function(node, src))
132
+ module.calls = _module_calls(tree)
133
+ return module
134
+ except SyntaxError as e:
135
+ # Broken (mid-edit) code: flag it so the resync can preserve the file's last-good
136
+ # subgraph instead of wiping it to this empty module. See core/incremental_scoped.
137
+ logger.warning("Skipping %s: could not parse (%s)", path, e)
138
+ module.parse_ok = False
139
+ return module
140
+ except _ParseTimeout:
141
+ # Pathological (usually generated) file: skip rather than stall the whole ingest.
142
+ # Return a fresh empty module marked broken -- same contract as a syntax error, so the
143
+ # resync preserves the file's last-good subgraph instead of half-parsed noise.
144
+ lines = text.count("\n") + 1
145
+ logger.warning(
146
+ "Skipping %s: parse exceeded %ss (%d lines) -- likely generated; marked broken",
147
+ path, _max_parse_seconds(), lines,
148
+ )
149
+ broken = RawModule(path=path, dotted=dotted, is_package=module.is_package)
150
+ broken.parse_ok = False
151
+ return broken
152
+
153
+
154
+ def parse_blob(args: tuple[str, bytes, frozenset[str] | None]) -> RawModule:
155
+ """ProcessPoolExecutor worker: parse one ``(path, source_bytes, package_dirs)`` tuple. A
156
+ module-level function (so it's picklable for the pool) living in this light module (so a
157
+ spawned worker imports cheaply -- not the heavy git_sync). It runs in the worker's own main
158
+ thread, so ``parse_module``'s per-file SIGALRM timeout still applies (the guard only no-ops
159
+ off the main thread). Mirrors the C# ingestor's ``_parse_blob`` worker."""
160
+ path, data, package_dirs = args
161
+ return parse_module(data, path, package_dirs)
162
+
163
+
164
+ # -- imports -----------------------------------------------------------------------
165
+
166
+
167
+ def _imports(node: ast.Import | ast.ImportFrom, current_dotted: str, is_package: bool) -> list[RawImport]:
168
+ out: list[RawImport] = []
169
+ if isinstance(node, ast.Import):
170
+ for alias in node.names:
171
+ local = alias.asname or alias.name.split(".")[0]
172
+ out.append(RawImport(local=local, target=alias.name, is_module=True))
173
+ return out
174
+
175
+ base = _resolve_from_base(node, current_dotted, is_package)
176
+ for alias in node.names:
177
+ if alias.name == "*":
178
+ continue
179
+ target = f"{base}.{alias.name}" if base else alias.name
180
+ out.append(RawImport(local=alias.asname or alias.name, target=target, is_module=False))
181
+ return out
182
+
183
+
184
+ def _resolve_from_base(node: ast.ImportFrom, current_dotted: str, is_package: bool) -> str:
185
+ """Absolute dotted prefix a `from ... import` names hang off, resolving relative
186
+ (`level > 0`) imports against the current module's package."""
187
+ if not node.level:
188
+ return node.module or ""
189
+ # The current module's package: for a regular module a.b.c it's a.b; for a package's
190
+ # own __init__ (dotted already == the package) it's the package itself.
191
+ package = current_dotted if is_package else current_dotted.rsplit(".", 1)[0] if "." in current_dotted else ""
192
+ # level 1 = current package, each extra level strips one more trailing segment.
193
+ for _ in range(node.level - 1):
194
+ package = package.rsplit(".", 1)[0] if "." in package else ""
195
+ if node.module:
196
+ return f"{package}.{node.module}" if package else node.module
197
+ return package
198
+
199
+
200
+ # -- classes & functions -----------------------------------------------------------
201
+
202
+
203
+ def _decl_top_line(node: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef) -> int:
204
+ """The declaration's first source line -- above any decorators, since a leading comment
205
+ sits above the whole `@decorator`/`def` block, not between the decorators and the def."""
206
+ return min([node.lineno, *(d.lineno for d in node.decorator_list)])
207
+
208
+
209
+ def _class(node: ast.ClassDef, src: _Source) -> RawClass:
210
+ cls = RawClass(
211
+ name=node.name,
212
+ bases=[ast.unparse(b) for b in node.bases],
213
+ decorators=[ast.unparse(d) for d in node.decorator_list],
214
+ line_start=node.lineno,
215
+ line_end=node.end_lineno or node.lineno,
216
+ source=src.segment(node),
217
+ doc=src.leading_comment(_decl_top_line(node)),
218
+ )
219
+ for stmt in node.body:
220
+ if isinstance(stmt, ast.ClassDef):
221
+ cls.nested.append(_class(stmt, src))
222
+ elif isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
223
+ cls.methods.append(_function(stmt, src, in_class=True))
224
+ elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name):
225
+ cls.class_attrs.append(
226
+ RawAttribute(stmt.target.id, ast.unparse(stmt.annotation), "class_body", True,
227
+ name_line=stmt.target.lineno, name_col=stmt.target.col_offset)
228
+ )
229
+ elif isinstance(stmt, ast.Assign):
230
+ for tgt in stmt.targets:
231
+ if isinstance(tgt, ast.Name):
232
+ cls.class_attrs.append(
233
+ RawAttribute(tgt.id, None, "class_body", True, name_line=tgt.lineno, name_col=tgt.col_offset)
234
+ )
235
+ return cls
236
+
237
+
238
+ def _function(node: ast.FunctionDef | ast.AsyncFunctionDef, src: _Source, *, in_class: bool = False) -> RawFunction:
239
+ fn = RawFunction(
240
+ name=node.name,
241
+ params=_params(node.args),
242
+ returns=ast.unparse(node.returns) if node.returns else None,
243
+ decorators=[ast.unparse(d) for d in node.decorator_list],
244
+ is_async=isinstance(node, ast.AsyncFunctionDef),
245
+ is_generator=_is_generator(node),
246
+ line_start=node.lineno,
247
+ line_end=node.end_lineno or node.lineno,
248
+ calls=_calls(node),
249
+ source=src.segment(node),
250
+ doc=src.leading_comment(_decl_top_line(node)),
251
+ )
252
+ if in_class and fn.params:
253
+ fn.self_attrs = _self_attrs(node, self_name=fn.params[0].name)
254
+ return fn
255
+
256
+
257
+ def _params(args: ast.arguments) -> list[RawParameter]:
258
+ params: list[RawParameter] = []
259
+ pos = 0
260
+
261
+ def add(arg: ast.arg, kind: str, has_default: bool):
262
+ nonlocal pos
263
+ params.append(
264
+ RawParameter(
265
+ name=arg.arg,
266
+ annotation=ast.unparse(arg.annotation) if arg.annotation else None,
267
+ kind=kind,
268
+ position=pos,
269
+ has_default=has_default,
270
+ name_line=arg.lineno,
271
+ name_col=arg.col_offset,
272
+ )
273
+ )
274
+ pos += 1
275
+
276
+ positional = list(args.posonlyargs) + list(args.args)
277
+ n_defaults = len(args.defaults)
278
+ first_default = len(positional) - n_defaults
279
+ for i, arg in enumerate(args.posonlyargs):
280
+ add(arg, "positional_only", i >= first_default)
281
+ for j, arg in enumerate(args.args):
282
+ add(arg, "positional_or_keyword", (len(args.posonlyargs) + j) >= first_default)
283
+ if args.vararg:
284
+ add(args.vararg, "var_positional", False)
285
+ for arg, default in zip(args.kwonlyargs, args.kw_defaults):
286
+ add(arg, "keyword_only", default is not None)
287
+ if args.kwarg:
288
+ add(args.kwarg, "var_keyword", False)
289
+ return params
290
+
291
+
292
+ def _is_generator(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:
293
+ for child in ast.walk(node):
294
+ if isinstance(child, (ast.Yield, ast.YieldFrom)):
295
+ return True
296
+ # Don't descend into nested functions' yields.
297
+ if child is not node and isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
298
+ return False
299
+ return False
300
+
301
+
302
+ def _raw_call(func: ast.expr) -> RawCall | None:
303
+ """Build a RawCall from a call's `func` expression -- a bare `name()` or a `receiver.name()`
304
+ attribute access; anything else (a call on a subscript/call result) yields None. Shared by
305
+ the function-body and module-scope collectors so both name a call site identically."""
306
+ if isinstance(func, ast.Name):
307
+ return RawCall(name=func.id, receiver=None, is_simple_name=True, line=func.lineno, col=func.col_offset)
308
+ if isinstance(func, ast.Attribute):
309
+ receiver = func.value.id if isinstance(func.value, ast.Name) else None
310
+ end_col = func.end_col_offset or 0
311
+ return RawCall(
312
+ name=func.attr, receiver=receiver, is_simple_name=False,
313
+ line=func.end_lineno or func.lineno, col=max(end_col - len(func.attr), 0),
314
+ )
315
+ return None
316
+
317
+
318
+ def _calls_in_body(body: list[ast.stmt]) -> list[RawCall]:
319
+ """Calls executed when a scope's OWN code runs: every call in these statements and the
320
+ control-flow nested in them (if/for/while/with/try), but NOT inside a nested def/class/lambda
321
+ body -- each of those is its own scope and owns its own calls.
322
+
323
+ Shared by a module body and a function/method body. Because it walks the *body* only, a
324
+ function's decorators and default-argument expressions -- which hang off the `def` node, not
325
+ its body, and execute in the ENCLOSING scope at definition time (`@deco`, `def f(x=compute())`)
326
+ -- are correctly excluded from the function's own calls. Document order; a call's arguments are
327
+ descended into so `print(load())` captures both."""
328
+ out: list[RawCall] = []
329
+
330
+ def walk(node: ast.AST) -> None:
331
+ for child in ast.iter_child_nodes(node):
332
+ if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)):
333
+ continue # a nested scope -- owns its own calls
334
+ if isinstance(child, ast.Call):
335
+ rc = _raw_call(child.func)
336
+ if rc is not None:
337
+ out.append(rc)
338
+ walk(child)
339
+
340
+ for stmt in body:
341
+ if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
342
+ continue
343
+ walk(stmt)
344
+ return out
345
+
346
+
347
+ def _calls(node: ast.FunctionDef | ast.AsyncFunctionDef) -> list[RawCall]:
348
+ """The calls a function/method body makes -- scope-respecting (see _calls_in_body): excludes
349
+ the decorators/defaults on the `def` (they run in the enclosing scope) and any nested def's
350
+ body (its own scope)."""
351
+ return _calls_in_body(node.body)
352
+
353
+
354
+ def _module_calls(tree: ast.Module) -> list[RawCall]:
355
+ """Call sites in module-scope (top-level) code -- what makes the Module node a CALLS source for
356
+ a script's driver code (e.g. `main()` under `if __name__ == '__main__':`). Same scope rules as
357
+ a function body."""
358
+ return _calls_in_body(tree.body)
359
+
360
+
361
+ def _self_attrs(node: ast.FunctionDef | ast.AsyncFunctionDef, self_name: str) -> list[RawAttribute]:
362
+ out: list[RawAttribute] = []
363
+
364
+ def record(target: ast.expr, annotation: str | None):
365
+ if (
366
+ isinstance(target, ast.Attribute)
367
+ and isinstance(target.value, ast.Name)
368
+ and target.value.id == self_name
369
+ ):
370
+ end_col = target.end_col_offset or 0
371
+ out.append(
372
+ RawAttribute(
373
+ target.attr, annotation, node.name, False,
374
+ name_line=target.end_lineno or target.lineno,
375
+ name_col=max(end_col - len(target.attr), 0),
376
+ )
377
+ )
378
+
379
+ for stmt in ast.walk(node):
380
+ if isinstance(stmt, ast.AnnAssign):
381
+ record(stmt.target, ast.unparse(stmt.annotation))
382
+ elif isinstance(stmt, ast.Assign):
383
+ for tgt in stmt.targets:
384
+ record(tgt, None)
385
+ return out