auryn-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- auryn/ingestors/pyframework/__init__.py +11 -0
- auryn/ingestors/pyframework/schema.py +20 -0
- auryn/ingestors/pyframework/synthesize.py +130 -0
- auryn/ingestors/python/GRAPH_MODEL.md +175 -0
- auryn/ingestors/python/LIMITATIONS.md +95 -0
- auryn/ingestors/python/__init__.py +5 -0
- auryn/ingestors/python/file_parser.py +385 -0
- auryn/ingestors/python/git_sync.py +419 -0
- auryn/ingestors/python/ids.py +160 -0
- auryn/ingestors/python/incremental.py +206 -0
- auryn/ingestors/python/jedi_engine.py +203 -0
- auryn/ingestors/python/plugin.py +49 -0
- auryn/ingestors/python/raw_model.py +110 -0
- auryn/ingestors/python/resolution.py +760 -0
- auryn/ingestors/python/schema.py +78 -0
- auryn_python-0.1.0.dist-info/METADATA +10 -0
- auryn_python-0.1.0.dist-info/RECORD +20 -0
- auryn_python-0.1.0.dist-info/WHEEL +4 -0
- auryn_python-0.1.0.dist-info/entry_points.txt +5 -0
- auryn_python-0.1.0.dist-info/licenses/LICENSE +557 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Python web-framework enrichment -- a registered graph enrichment (post-pass), not a
|
|
2
|
+
BaseIngestor or a CLI command. It runs after the language engines (via
|
|
3
|
+
core.ingest.ingest_repository/ingest_directory) and synthesizes Python route endpoints
|
|
4
|
+
(APIEndpoint + EXPOSES_ENDPOINT) from route decorators. See synthesize.py."""
|
|
5
|
+
|
|
6
|
+
from auryn.core.registry import GraphEnrichment, register_graph_enrichment
|
|
7
|
+
from auryn.ingestors.pyframework.synthesize import run as _run
|
|
8
|
+
|
|
9
|
+
register_graph_enrichment(GraphEnrichment(name="python-routes", run=_run))
|
|
10
|
+
|
|
11
|
+
__all__ = ["_run"]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Graph schema for the Python web-framework enrichment.
|
|
2
|
+
|
|
3
|
+
Reuses the C# tier's `APIEndpoint` label + `EXPOSES_ENDPOINT` edge on purpose, so a Python
|
|
4
|
+
route is a first-class entry point to the same agent-access tools (trace_entrypoint,
|
|
5
|
+
search by APIEndpoint) that already understand endpoints -- see synthesize.py. The Python
|
|
6
|
+
endpoints carry `provenance:"pyframework"` so this pass can recompute only its own layer
|
|
7
|
+
without touching any C# APIEndpoints in the same graph.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from auryn.core.graph_model import EdgeSchema, NodeSchema
|
|
13
|
+
|
|
14
|
+
NODE_SCHEMAS = [
|
|
15
|
+
NodeSchema(label="APIEndpoint", indexed_properties=("route",)),
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
EDGE_SCHEMAS = [
|
|
19
|
+
EdgeSchema(rel_type="EXPOSES_ENDPOINT"), # handler Function|Method -> APIEndpoint
|
|
20
|
+
]
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Python web-framework enrichment: a post-pass that turns route *decorators* already
|
|
2
|
+
captured on Function/Method nodes into `APIEndpoint` entry points + `EXPOSES_ENDPOINT`
|
|
3
|
+
edges -- the Python analogue of the Java Spring route pass (javaframework), and the thing that makes a
|
|
4
|
+
request->handler->call-closure golden path traceable (and gives the agent-access layer
|
|
5
|
+
real Python entry points to inventory).
|
|
6
|
+
|
|
7
|
+
It reads what the language engines already produced (the `decorators` string list on each
|
|
8
|
+
Function/Method), so no source re-parsing: `@app.post("/widgets")` and
|
|
9
|
+
`@app.route("/x", methods=["POST"])` are recognised for FastAPI/Starlette/Flask. The
|
|
10
|
+
decorated function becomes the endpoint's handler, so trace_entrypoint(endpoint) reaches
|
|
11
|
+
its whole CALLS closure.
|
|
12
|
+
|
|
13
|
+
Idempotent: it recomputes only its own layer -- APIEndpoint nodes tagged
|
|
14
|
+
`provenance:"pyframework"` (so C#'s own APIEndpoints in the same graph are untouched) and
|
|
15
|
+
their EXPOSES_ENDPOINT edges.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import ast
|
|
21
|
+
import logging
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
|
|
24
|
+
from auryn.core.git_source import GitRepoCache
|
|
25
|
+
from auryn.core.graph_client import GraphWriter
|
|
26
|
+
from auryn.core.graph_model import GraphEdge, GraphNode
|
|
27
|
+
from auryn.ingestors.pyframework.schema import NODE_SCHEMAS
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger("auryn.pyframework")
|
|
30
|
+
|
|
31
|
+
_PROVENANCE = "pyframework"
|
|
32
|
+
_METHOD_VERBS = {"get", "post", "put", "delete", "patch", "head", "options"}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class RouteInfo:
|
|
37
|
+
http_method: str
|
|
38
|
+
path: str
|
|
39
|
+
framework: str
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def parse_routes(decorator: str) -> list[RouteInfo]:
|
|
43
|
+
"""Extract route(s) from one decorator's source text. Handles verb decorators
|
|
44
|
+
(`app.get("/x")`, `router.post("/x")` -- FastAPI/Starlette/Flask 2) and Flask's
|
|
45
|
+
`app.route("/x", methods=["GET","POST"])` (one endpoint per method; default GET).
|
|
46
|
+
Returns [] for any non-route decorator."""
|
|
47
|
+
try:
|
|
48
|
+
expr = ast.parse(decorator, mode="eval").body
|
|
49
|
+
except SyntaxError:
|
|
50
|
+
return []
|
|
51
|
+
if not isinstance(expr, ast.Call) or not isinstance(expr.func, ast.Attribute):
|
|
52
|
+
return []
|
|
53
|
+
attr = expr.func.attr.lower()
|
|
54
|
+
path = _first_str_arg(expr)
|
|
55
|
+
if path is None:
|
|
56
|
+
return []
|
|
57
|
+
if attr in _METHOD_VERBS:
|
|
58
|
+
return [RouteInfo(attr.upper(), path, "fastapi")]
|
|
59
|
+
if attr == "route":
|
|
60
|
+
methods = _methods_kwarg(expr) or ["GET"]
|
|
61
|
+
return [RouteInfo(m, path, "flask") for m in methods]
|
|
62
|
+
return []
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _first_str_arg(call: ast.Call) -> str | None:
|
|
66
|
+
for arg in call.args:
|
|
67
|
+
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
|
68
|
+
return arg.value
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _methods_kwarg(call: ast.Call) -> list[str]:
|
|
73
|
+
for kw in call.keywords:
|
|
74
|
+
if kw.arg == "methods" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
75
|
+
return [e.value.upper() for e in kw.value.elts if isinstance(e, ast.Constant) and isinstance(e.value, str)]
|
|
76
|
+
return []
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def run(writer: GraphWriter, repo: GitRepoCache, target_sha: str) -> dict | None:
|
|
80
|
+
"""GraphEnrichment entrypoint. Reads Function/Method decorators from the graph and
|
|
81
|
+
synthesizes the endpoint layer. Returns None when there are no Python routes."""
|
|
82
|
+
writer.setup_schema(NODE_SCHEMAS)
|
|
83
|
+
_clear_prior_layer(writer)
|
|
84
|
+
|
|
85
|
+
from auryn.core.graph_client import semantic_label
|
|
86
|
+
|
|
87
|
+
handlers = writer.ro_query(
|
|
88
|
+
"MATCH (h) WHERE (h:Function OR h:Method) AND h.decorators IS NOT NULL AND size(h.decorators) > 0 "
|
|
89
|
+
f"RETURN {semantic_label('h')} AS label, h.id AS id, h.name AS name, h.decorators AS decorators"
|
|
90
|
+
).result_set
|
|
91
|
+
|
|
92
|
+
nodes: list[GraphNode] = []
|
|
93
|
+
edges: list[GraphEdge] = []
|
|
94
|
+
handler_ids: set[str] = set()
|
|
95
|
+
for label, handler_id, handler_name, decorators in handlers:
|
|
96
|
+
for decorator in decorators or []:
|
|
97
|
+
for route in parse_routes(decorator):
|
|
98
|
+
endpoint_id = f"pyapi://{handler_id}#{route.http_method}:{route.path}"
|
|
99
|
+
nodes.append(
|
|
100
|
+
GraphNode(
|
|
101
|
+
"APIEndpoint",
|
|
102
|
+
{"id": endpoint_id},
|
|
103
|
+
{
|
|
104
|
+
"route": route.path,
|
|
105
|
+
"http_method": route.http_method,
|
|
106
|
+
"framework": route.framework,
|
|
107
|
+
"name": f"{route.http_method} {route.path}",
|
|
108
|
+
"handler": handler_name,
|
|
109
|
+
"provenance": _PROVENANCE,
|
|
110
|
+
},
|
|
111
|
+
)
|
|
112
|
+
)
|
|
113
|
+
edges.append(GraphEdge("EXPOSES_ENDPOINT", label, {"id": handler_id}, "APIEndpoint", {"id": endpoint_id}))
|
|
114
|
+
handler_ids.add(handler_id)
|
|
115
|
+
|
|
116
|
+
if not nodes:
|
|
117
|
+
return None
|
|
118
|
+
writer.merge_nodes(nodes)
|
|
119
|
+
writer.merge_edges(edges)
|
|
120
|
+
|
|
121
|
+
report = {"endpoints": len(nodes), "handlers": len(handler_ids)}
|
|
122
|
+
logger.info("python routes: %d endpoints across %d handlers", report["endpoints"], report["handlers"])
|
|
123
|
+
return report
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _clear_prior_layer(writer: GraphWriter) -> None:
|
|
127
|
+
"""Recompute-from-scratch: DETACH DELETE removes this pass's own APIEndpoints (tagged
|
|
128
|
+
with our provenance) *and* their EXPOSES_ENDPOINT edges in one step -- scoped by
|
|
129
|
+
provenance so any C# APIEndpoints in the same graph are left alone."""
|
|
130
|
+
writer.query(f"MATCH (e:APIEndpoint {{provenance: '{_PROVENANCE}'}}) DETACH DELETE e")
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# Python graph model
|
|
2
|
+
|
|
3
|
+
The node/edge model both Python engines emit into the shared FalkorDB graph:
|
|
4
|
+
|
|
5
|
+
- **`ast`** — stdlib-only, zero extra deps. Structure + statically-resolvable
|
|
6
|
+
inheritance/calls; types are `annotated` or `unresolved`.
|
|
7
|
+
- **`jedi`** (default) — layers [Jedi](https://jedi.readthedocs.io/)'s in-process static
|
|
8
|
+
inference on top: inferred types and resolved calls, including receiver-typed ones.
|
|
9
|
+
|
|
10
|
+
Both use the same id scheme (`ids.py`) and the same labels/properties below, so their
|
|
11
|
+
output is interchangeable for the backbone; they differ only in how much they resolve,
|
|
12
|
+
which the provenance fields record honestly. Jedi is a pure-Python pip dependency — the
|
|
13
|
+
whole install is `uv sync`, no second runtime.
|
|
14
|
+
|
|
15
|
+
## Id scheme (`ids.py`)
|
|
16
|
+
|
|
17
|
+
Extends the C# `.kind.name` convention. Root is `python://{project}`.
|
|
18
|
+
|
|
19
|
+
| Kind | Id |
|
|
20
|
+
|---|---|
|
|
21
|
+
| Project | `python://{project}` |
|
|
22
|
+
| Package | `python://{project}/package/{dotted.pkg}` |
|
|
23
|
+
| Module | `python://{project}/module/{dotted.module}` |
|
|
24
|
+
| Class | `{module_id}.class.{name_lower}` |
|
|
25
|
+
| Function (module-level) | `{module_id}.func.{name_lower}({param_names})` |
|
|
26
|
+
| Method (class member) | `{class_id}.method.{name_lower}({param_names})` |
|
|
27
|
+
| Attribute | `{class_id}.attr.{name_lower}` |
|
|
28
|
+
| Parameter | `{owner_id}.param.{position}.{name_lower}` |
|
|
29
|
+
|
|
30
|
+
### Id-stability rule
|
|
31
|
+
|
|
32
|
+
Method/function ids embed parameter **names**, never inferred types. Types are volatile
|
|
33
|
+
across re-syncs (inference shifts as code around them changes); a type in an id would churn
|
|
34
|
+
the id and storm the delta. Param names are stable source facts, and they also disambiguate
|
|
35
|
+
`@property`/`@x.setter` pairs and `@overload` stubs.
|
|
36
|
+
|
|
37
|
+
## Provenance vocabulary
|
|
38
|
+
|
|
39
|
+
Because the ingestor is generic (0%→100% annotated in the same repo), every type/edge says
|
|
40
|
+
*how we know it*:
|
|
41
|
+
|
|
42
|
+
- **`type_source`** on any type-bearing node property — `"annotated"` (a written hint),
|
|
43
|
+
`"inferred"` (an engine deduced it — Jedi only), `"unresolved"` (unknown).
|
|
44
|
+
- **`resolution`** on semantic edges — `"resolved"` (statically bound), `"inferred"` (bound
|
|
45
|
+
via receiver-type inference — Jedi only), `"constructor"` (a CALLS edge standing for `Cls()`
|
|
46
|
+
instantiation, redirected to the class's `__init__`), `"external"` (a nameable stdlib/
|
|
47
|
+
third-party/sibling-repo symbol → a placeholder node, ADR 0004), `"unresolved"`.
|
|
48
|
+
|
|
49
|
+
The `ast` engine never emits `inferred`; that value is the Jedi engine's signal. (`constructor`
|
|
50
|
+
and `external` are emitted by both engines.)
|
|
51
|
+
|
|
52
|
+
## Nodes
|
|
53
|
+
|
|
54
|
+
Common: `line_start`, `line_end`. `Package`/`Module` carry `name` + `dotted_path` (+
|
|
55
|
+
`file_path` on Module). `Class`/`Function`/`Method` also carry `source` — the entity's
|
|
56
|
+
verbatim source text (the shared `source` property that the C / C# / PL-SQL ingestors also
|
|
57
|
+
write, so `get_context` returns an entity's code uniformly across languages; it is stripped
|
|
58
|
+
from lean list results by `serialize_node_lean`).
|
|
59
|
+
|
|
60
|
+
- **Class** — `name`, `qualified_name`, `bases` (declared), `mro` (resolved, ordered),
|
|
61
|
+
`unresolved_bases`, `is_abstract`, `is_protocol`, `is_dataclass`, `decorators`, `source`.
|
|
62
|
+
- **Function / Method** — `name`, `qualified_name`, `signature`, `return_type`,
|
|
63
|
+
`return_type_source`, `is_async`, `is_generator`, `decorators`, `unresolved_calls`,
|
|
64
|
+
`source`. Method also: `kind` (`instance|static|class|property`), `is_abstract`,
|
|
65
|
+
`overrides`.
|
|
66
|
+
- **Attribute** — `name`, `type`, `type_source`, `defined_in` (`class_body` | method name),
|
|
67
|
+
`is_classvar`.
|
|
68
|
+
- **Parameter** — `name`, `type`, `type_source`, `kind`, `position`, `has_default`.
|
|
69
|
+
|
|
70
|
+
*(Deferred: a `DTO` classification for `@dataclass`/pydantic/`TypedDict`; the `is_dataclass`
|
|
71
|
+
flag carries the signal until then.)*
|
|
72
|
+
|
|
73
|
+
## Edges
|
|
74
|
+
|
|
75
|
+
Containment (`HAS_PACKAGE`/`HAS_SUBPACKAGE`/`HAS_MODULE`/`HAS_CLASS`/`HAS_FUNCTION`/
|
|
76
|
+
`HAS_METHOD`/`HAS_ATTRIBUTE`/`HAS_PARAMETER`/`HAS_NESTED_CLASS`) carry no properties.
|
|
77
|
+
|
|
78
|
+
| Edge | properties |
|
|
79
|
+
|---|---|
|
|
80
|
+
| `EXTENDS` | `resolution`, `mro_index` |
|
|
81
|
+
| `IMPLEMENTS` | `resolution` — when the local base is an ABC/Protocol |
|
|
82
|
+
| `CALLS` | `resolution`, `receiver_type`, `argument_types`, `type_arguments` |
|
|
83
|
+
| `RETURNS` | `resolution` — only when the return type resolves to a local class |
|
|
84
|
+
|
|
85
|
+
A `CALLS` edge is normally `Function`/`Method` → `Function`/`Method`. The one exception: a
|
|
86
|
+
module's **top-level (module-scope) call statements** — a script's driver code, e.g. `main()`
|
|
87
|
+
under `if __name__ == "__main__":` — originate a `CALLS` edge from the `Module` (or `Package`,
|
|
88
|
+
for an `__init__.py`) node itself. So the `Module`/`Package` node is the only non-callable that
|
|
89
|
+
can be a `CALLS` source; calls inside a `def`/`class`/`lambda` belong to their own node, never
|
|
90
|
+
the module. (`argument_types`/`type_arguments` stay `[]` as for any call.)
|
|
91
|
+
|
|
92
|
+
Endpoints of semantic edges are **always project-local nodes**; external/stdlib types live
|
|
93
|
+
only in string properties (`bases`, `mro`, `return_type`).
|
|
94
|
+
|
|
95
|
+
### The three-way split for calls
|
|
96
|
+
|
|
97
|
+
A call site becomes exactly one of:
|
|
98
|
+
|
|
99
|
+
1. **In-project target** → a `CALLS` edge (`resolved` for a bare name, `inferred` when the
|
|
100
|
+
target came from receiver-type inference — the Jedi-only case, e.g. `conn.execute()`).
|
|
101
|
+
2. **External** (`json.dumps`, stdlib, third-party) → **dropped** (out of graph scope).
|
|
102
|
+
3. **Dynamic / unresolvable** (`getattr`, computed dispatch) → recorded on the caller's
|
|
103
|
+
**`unresolved_calls`** property, not an edge. The `ast` engine leaves this empty (it
|
|
104
|
+
can't tell dynamic from external without types); the Jedi engine populates it.
|
|
105
|
+
|
|
106
|
+
## Worked example
|
|
107
|
+
|
|
108
|
+
Source (`demo/python-auryn/sample/app/`, project `demo`):
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
# app/models.py # app/db.py # app/widget.py
|
|
112
|
+
class Entity: class Connection: from app.models import Entity
|
|
113
|
+
def __init__(self, def execute(self, sql: str): from app.db import Connection, get_connection
|
|
114
|
+
id: int): ...
|
|
115
|
+
self.id = id class Widget(Entity):
|
|
116
|
+
def get_connection() -> Connection: def __init__(self, id, name: str):
|
|
117
|
+
... self.name = name # inferred str
|
|
118
|
+
self.tags = [] # inferred list
|
|
119
|
+
def render(self):
|
|
120
|
+
conn = get_connection() # resolved call
|
|
121
|
+
conn.execute(self.name) # inferred call
|
|
122
|
+
json.dumps(self.tags) # external -> dropped
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
What the **Jedi engine** emits for the interesting slice (backbone elided):
|
|
126
|
+
|
|
127
|
+
```
|
|
128
|
+
Class Widget mro=[app.widget.Widget, app.models.Entity, object]
|
|
129
|
+
Attribute Widget.name type=str type_source=inferred
|
|
130
|
+
Attribute Widget.tags type=list type_source=inferred
|
|
131
|
+
Parameter Widget.__init__.id type=Unknown type_source=unresolved # unannotated + no call site in these 3 files
|
|
132
|
+
Parameter render.self type=Widget type_source=inferred
|
|
133
|
+
# (in the full sample, app/api.py's `Widget(widget_id, ...)` with `widget_id: int`
|
|
134
|
+
# lets Jedi infer that `id` param as `int`/inferred -- cross-file call-site inference)
|
|
135
|
+
|
|
136
|
+
Widget -EXTENDS {resolution:resolved, mro_index:1}-> Entity
|
|
137
|
+
render -CALLS {resolution:resolved}-> get_connection
|
|
138
|
+
render -CALLS {resolution:inferred, receiver_type:Connection}-> Connection.execute
|
|
139
|
+
get_connection -RETURNS {resolution:resolved}-> Connection
|
|
140
|
+
# json.dumps: no node, no edge (external)
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The **ast engine** emits the same backbone + `EXTENDS`/`get_connection`/`RETURNS`, but the
|
|
144
|
+
attribute/param types are `unresolved` (no inference) and there is **no** `conn.execute`
|
|
145
|
+
edge (receiver-typed — needs inference). That gap is exactly what Jedi fills.
|
|
146
|
+
|
|
147
|
+
## Web-framework enrichment (endpoints & the golden path)
|
|
148
|
+
|
|
149
|
+
A registered graph-enrichment pass (`ingestors/pyframework/`, name `python-routes`) runs
|
|
150
|
+
after the language engines and turns route **decorators** already captured on
|
|
151
|
+
Function/Method nodes into entry points — no source re-parsing:
|
|
152
|
+
|
|
153
|
+
- `@app.post("/widgets")`, `@router.get("/x/{id}")` (FastAPI/Starlette/Flask 2 verb style)
|
|
154
|
+
and `@app.route("/x", methods=["GET","POST"])` (Flask; one endpoint per method) →
|
|
155
|
+
- an **`APIEndpoint`** node (`route`, `http_method`, `framework`, `provenance:"pyframework"`)
|
|
156
|
+
reusing the C# tier's label, plus a **`(handler Function|Method)-[:EXPOSES_ENDPOINT]->(APIEndpoint)`** edge (same direction as C#).
|
|
157
|
+
|
|
158
|
+
The pass is idempotent and provenance-scoped, so re-runs rebuild only its own layer and any
|
|
159
|
+
C# `APIEndpoint`s in the same graph are untouched.
|
|
160
|
+
|
|
161
|
+
**Why it matters — the golden path.** Because the endpoint reuses the shared label/edge, the
|
|
162
|
+
agent-access layer treats a Python route as a first-class entry point:
|
|
163
|
+
`trace_entrypoint` now resolves an `APIEndpoint` entry to its handler(s) via
|
|
164
|
+
`EXPOSES_ENDPOINT`, then returns the handler's full `CALLS` closure. So for the demo's
|
|
165
|
+
`app/api.py`:
|
|
166
|
+
|
|
167
|
+
```
|
|
168
|
+
POST /widgets -EXPOSES_ENDPOINT<- create_widget() -CALLS-> get_connection(), Connection.execute()
|
|
169
|
+
trace_entrypoint(POST /widgets) -> { create_widget, get_connection, execute }
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
i.e. request → handler → call closure is one traceable path. (`impact_of` stays generic:
|
|
173
|
+
it already traverses `CALLS` from any handler/function.) Framework *dynamic dispatch*
|
|
174
|
+
beyond decorator routes — Django `urls.py`, DI/`Depends`, Celery tasks — is not yet
|
|
175
|
+
modelled; those are future enrichment passes.
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# Python ingestor — known limitations
|
|
2
|
+
|
|
3
|
+
Behaviours of the **Python** ingestor's resolution that are currently incomplete or intentionally
|
|
4
|
+
deferred. Each has a **tracking test** (most are `xfail`, so they run on every CI pass and will
|
|
5
|
+
*fail loudly the day the behaviour is fixed* — a prompt to promote the test and delete the entry
|
|
6
|
+
here). Add new limitations as they're discovered; remove them as they're closed.
|
|
7
|
+
|
|
8
|
+
Companion to [`GRAPH_MODEL.md`](GRAPH_MODEL.md) (the graph this ingestor produces) and
|
|
9
|
+
[ADR 0004](../../../../../../docs/adr/0004-cross-repo-symbol-resolution.md) (the cross-repo design).
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Cross-repo symbol resolution
|
|
14
|
+
|
|
15
|
+
When several repos share one graph, an outbound reference to a symbol defined in another repo is
|
|
16
|
+
emitted as a `resolved:false` placeholder keyed by a **linkage key**
|
|
17
|
+
(`extern://python/<module>::<name>`). Ingesting the defining repo *heals* the reference:
|
|
18
|
+
reconciliation matches the placeholder's key to the real definition's `linkage_key`, repoints the
|
|
19
|
+
edge, and deletes the placeholder.
|
|
20
|
+
|
|
21
|
+
Because the key is derived from **how the consumer names the symbol** and **which real node carries a
|
|
22
|
+
matching `linkage_key`**, healing fails wherever those two strings can't be made to agree. The cases
|
|
23
|
+
below don't heal. In every one the reference is *safe but incomplete*: the graph under-reports a
|
|
24
|
+
cross-repo edge rather than inventing a wrong one — so for a reachability/impact audit, treat "no
|
|
25
|
+
path found" **across a repo boundary** with these caveats in mind.
|
|
26
|
+
|
|
27
|
+
| # | Case | Effect | Tracking test |
|
|
28
|
+
|---|------|--------|---------------|
|
|
29
|
+
| 1 | **Re-export through `__init__.py`** | not detected as a match | `tests/e2e/test_cross_repo_entity_kinds.py::test_reexport_via_init_is_a_known_gap` |
|
|
30
|
+
| 2 | **Cross-repo class inheritance** | no node *or* edge emitted | `tests/e2e/test_cross_repo_entity_kinds.py::test_cross_repo_inheritance_is_a_known_gap` |
|
|
31
|
+
| 3 | **Fully-qualified call** `pkg.sub.fn()` | reference undetected | `tests/e2e/test_cross_repo_import_forms.py::test_fully_qualified_call_is_a_known_gap` |
|
|
32
|
+
| 4 | **Star import** `from x import *` | reference undetected | `tests/e2e/test_cross_repo_entity_kinds.py::test_star_import_is_a_known_gap` |
|
|
33
|
+
| 5 | **Receiver-typed method call** whose type is in another repo | reference undetected | — (inherent; see below) |
|
|
34
|
+
|
|
35
|
+
### 1. Re-export through `__init__.py` (highest impact)
|
|
36
|
+
A package exposes a symbol on its public surface — `billing/__init__.py` does
|
|
37
|
+
`from billing.core import charge` — so consumers write `from billing import charge`. The consumer
|
|
38
|
+
mints linkage `billing::charge`, but the real definition is keyed `billing.core::charge`. The strings
|
|
39
|
+
differ, so nothing heals. This is the standard public-API pattern, so it's the most likely to bite in
|
|
40
|
+
practice.
|
|
41
|
+
*Fix (deferred, non-trivial):* let a producer's re-export contribute an **alias linkage key**
|
|
42
|
+
(`billing::charge`) onto the real node — i.e. support multiple linkage keys per node, plus a reconcile
|
|
43
|
+
change to match any of them.
|
|
44
|
+
|
|
45
|
+
### 2. Cross-repo class inheritance
|
|
46
|
+
`class Wallet(Account)` where `Account` is defined in another repo. Unlike a cross-repo *call*, the
|
|
47
|
+
ingestor records an unresolved base only as a **string** in the class's `unresolved_bases` property —
|
|
48
|
+
it emits **no placeholder `Class` and no `EXTENDS`/`IMPLEMENTS` edge** — so there is nothing for
|
|
49
|
+
reconciliation to upgrade, even once both repos are present.
|
|
50
|
+
*Fix (deferred, wide blast radius):* emit a placeholder `Class` + `EXTENDS` edge for an unresolved
|
|
51
|
+
base (mirroring the CALLS-placeholder path; reconcile already heals `EXTENDS`). Not done because it
|
|
52
|
+
would also fire on every stdlib/third-party base (`Exception`, `ABC`, `Protocol`, …), adding a
|
|
53
|
+
placeholder node + edge for each across the whole graph and shifting many golden-test baselines.
|
|
54
|
+
|
|
55
|
+
### 3. Fully-qualified call `pkg.sub.fn()`
|
|
56
|
+
`import billing.core; billing.core.charge()`. The `ast` call extractor records only a single-`Name`
|
|
57
|
+
receiver, so the `billing.core` receiver chain is dropped; and `import billing.core` binds the local
|
|
58
|
+
name `billing`, not `billing.core`. *Fix:* multi-level receiver capture in the parser + matching in
|
|
59
|
+
call resolution. (Note the sibling forms `import billing.core as bc; bc.charge()` and
|
|
60
|
+
`from billing import core; core.charge()` **do** heal — see the passing matrix in
|
|
61
|
+
`tests/e2e/test_cross_repo_import_forms.py`.)
|
|
62
|
+
|
|
63
|
+
### 4. Star import `from x import *`
|
|
64
|
+
The parser skips `*` imports, so imported names are unbound and no reference is recorded. Resolving
|
|
65
|
+
`*` statically needs the producer's exported names, which aren't present when the consumer is
|
|
66
|
+
ingested. Low priority (discouraged pattern).
|
|
67
|
+
|
|
68
|
+
### 5. Receiver-typed method call, type defined in another repo
|
|
69
|
+
`def run(c: Connection): c.execute(...)` where `Connection` lives in another repo. Neither engine can
|
|
70
|
+
bind `c.execute` without the type's definition, which is absent at consumer-ingest time. Inherent to
|
|
71
|
+
the "producer absent" condition; no cheap fix.
|
|
72
|
+
|
|
73
|
+
### Reconciliation caveat (applies to all languages)
|
|
74
|
+
The reconcile near-miss **diagnostic** — the warning that flags an unresolved reference matching a
|
|
75
|
+
definition that differs only by a module prefix — only fires when the placeholder and the real
|
|
76
|
+
definition are touched in a **single** ingest (e.g. a whole tree indexed as one project). A producer
|
|
77
|
+
ingested in a separate pass from its consumer isn't compared. Lives in `core/linkage.py`; unit test
|
|
78
|
+
`tests/core/test_linkage_diagnostics.py`.
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## Recently addressed (for context; no longer limitations)
|
|
83
|
+
|
|
84
|
+
- **Package-root pollution of the linkage key** — a `src/` layout, a hyphenated distribution dir, or
|
|
85
|
+
any wrapper directory used to leak into the dotted module path and break every cross-repo match.
|
|
86
|
+
Fixed via import-root detection (`ids.dotted_module_path`). Tests: `tests/ingestors/python/test_ids.py`,
|
|
87
|
+
`tests/e2e/test_cross_repo_resolution.py`.
|
|
88
|
+
- **Import-call forms** — `from m import fn`, `from m import fn as p`, `import m as x; x.fn()`, and
|
|
89
|
+
`from pkg import sub; sub.fn()` all heal. Tests: `tests/e2e/test_cross_repo_import_forms.py`.
|
|
90
|
+
|
|
91
|
+
## See also (not Python-specific)
|
|
92
|
+
- **Cross-language** resolution (service → DB, Python → C, …) — not implemented; a later phase in
|
|
93
|
+
[ADR 0004](../../../../../../docs/adr/0004-cross-repo-symbol-resolution.md) (§Interactions / phasing item 5).
|
|
94
|
+
- **TypeScript** cross-repo *symbol* healing — limited by module identity (npm package name vs
|
|
95
|
+
repo-relative path); belongs to the TypeScript ingestor (ADR 0004 item 4).
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"""The Python language family. Registration is import-light: the `auryn.languages`
|
|
2
|
+
entry point targets the tiny `plugin` module (suffixes + a lazy GitLanguage), so discovery
|
|
3
|
+
never imports the `ast`/Jedi extraction engine in git_sync -- that happens only when a
|
|
4
|
+
source actually contains `.py` files. See plugin.py.
|
|
5
|
+
"""
|