auryn-rust 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- auryn/ingestors/rust/__init__.py +0 -0
- auryn/ingestors/rust/cargo.py +284 -0
- auryn/ingestors/rust/file_parser.py +449 -0
- auryn/ingestors/rust/git_sync.py +231 -0
- auryn/ingestors/rust/grammar.py +14 -0
- auryn/ingestors/rust/ids.py +99 -0
- auryn/ingestors/rust/incremental.py +260 -0
- auryn/ingestors/rust/plugin.py +34 -0
- auryn/ingestors/rust/raw_model.py +149 -0
- auryn/ingestors/rust/resolution.py +660 -0
- auryn/ingestors/rust/schema.py +77 -0
- auryn_rust-0.1.0.dist-info/METADATA +11 -0
- auryn_rust-0.1.0.dist-info/RECORD +16 -0
- auryn_rust-0.1.0.dist-info/WHEEL +4 -0
- auryn_rust-0.1.0.dist-info/entry_points.txt +2 -0
- auryn_rust-0.1.0.dist-info/licenses/LICENSE +557 -0
|
File without changes
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
"""Cargo awareness for the Rust ingestor (issue #5): a repo can hold many crates (a Cargo
|
|
2
|
+
workspace with several `src/` folders), and two crates may define the same module path + function
|
|
3
|
+
name. Without crate awareness those collapse onto one node. This module supplies the two pieces
|
|
4
|
+
that fix that and model the "runtime building blocks" a query can ask about:
|
|
5
|
+
|
|
6
|
+
1. **Path-derived crate/target identity** -- `crate_target_for_path`. A pure function of the file
|
|
7
|
+
path alone (the standard Cargo `src/` layout): no manifest read, no source access. That matters
|
|
8
|
+
because the incremental delta-sync engine's per-file callbacks get only a path (no source), so a
|
|
9
|
+
path-only rule is computed identically in a full ingest and an incremental resync -- the
|
|
10
|
+
precondition for `incremental == full` (the differential oracle). It is what scopes a file's
|
|
11
|
+
items under its crate, so `api::util::parse` and `core::util::parse` stop colliding.
|
|
12
|
+
|
|
13
|
+
2. **Manifest parsing** -- `parse_manifest`. Reads a `Cargo.toml` (edition, version, `[dependencies]`,
|
|
14
|
+
`[workspace]` members, target overrides). The manifest is the ONLY source of the rich tier
|
|
15
|
+
(editions, versions, crate-to-crate deps); because those facts aren't in any `.rs` file, they are
|
|
16
|
+
applied by a deterministic reconcile in git_sync (which has source), NOT by the delta callbacks.
|
|
17
|
+
|
|
18
|
+
The crate/target the id layer uses is deliberately layout-derived (not manifest-derived): a crate
|
|
19
|
+
is keyed by its package directory + target kind/name, all read off the path. Manifest data only
|
|
20
|
+
*enriches* the already-identified crate (canonical name, edition) and adds dependency edges.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import tomllib
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
|
|
28
|
+
# Target kinds -- the "runtime building blocks". A package (one Cargo.toml) produces one library
|
|
29
|
+
# crate and/or one or more binary crates, plus integration tests / examples / benches.
|
|
30
|
+
KIND_LIB = "lib"
|
|
31
|
+
KIND_BIN = "bin"
|
|
32
|
+
KIND_TEST = "test"
|
|
33
|
+
KIND_EXAMPLE = "example"
|
|
34
|
+
KIND_BENCH = "bench"
|
|
35
|
+
KIND_BUILD = "build-script"
|
|
36
|
+
|
|
37
|
+
# Directory (relative to a package root) that holds each non-library target kind.
|
|
38
|
+
_TARGET_DIRS = {"bin": KIND_BIN, "tests": KIND_TEST, "examples": KIND_EXAMPLE, "benches": KIND_BENCH}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _norm(path: str) -> str:
|
|
42
|
+
return path.replace("\\", "/").lstrip("./")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _stem(filename: str) -> str:
|
|
46
|
+
return filename[:-3] if filename.endswith(".rs") else filename
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class CrateTarget:
|
|
51
|
+
"""The crate a `.rs` file compiles into, identified purely from its path. `package_dir` is the
|
|
52
|
+
directory holding the owning `Cargo.toml` (""/repo-root for a single-crate repo); `kind` is one
|
|
53
|
+
of the KIND_* constants; `name` is the target's crate name (the default library/binary takes the
|
|
54
|
+
package directory's basename, which equals the Cargo package name in the overwhelmingly common
|
|
55
|
+
case -- the manifest reconcile overrides it with the declared name when they differ)."""
|
|
56
|
+
|
|
57
|
+
package_dir: str
|
|
58
|
+
kind: str
|
|
59
|
+
name: str
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def crate_target_for_path(path: str) -> CrateTarget:
|
|
63
|
+
"""Map a `.rs` file path to its crate/target, from the standard Cargo layout alone:
|
|
64
|
+
|
|
65
|
+
<pkg>/src/lib.rs -> (pkg, lib, <pkg-basename>)
|
|
66
|
+
<pkg>/src/main.rs -> (pkg, bin, <pkg-basename>) # the default binary
|
|
67
|
+
<pkg>/src/**/*.rs -> (pkg, lib, <pkg-basename>) # library modules
|
|
68
|
+
<pkg>/src/bin/tool.rs -> (pkg, bin, tool)
|
|
69
|
+
<pkg>/src/bin/tool/*.rs -> (pkg, bin, tool)
|
|
70
|
+
<pkg>/tests|examples|benches/x.rs -> (pkg, test|example|bench, x)
|
|
71
|
+
<pkg>/build.rs -> (pkg, build-script, build)
|
|
72
|
+
|
|
73
|
+
`pkg` is everything before the first `src/` (or `tests|examples|benches/`, or a top-level
|
|
74
|
+
`build.rs`). A repo without a `Cargo.toml`/`src` layout still resolves (the file's directory
|
|
75
|
+
becomes a lib crate) so ingestion never fails on an unconventional tree -- it just doesn't split.
|
|
76
|
+
"""
|
|
77
|
+
p = _norm(path)
|
|
78
|
+
parts = p.split("/")
|
|
79
|
+
|
|
80
|
+
# Non-src target dirs (tests/examples/benches) and the default library/binary under src/.
|
|
81
|
+
for i, seg in enumerate(parts[:-1]):
|
|
82
|
+
if seg == "src":
|
|
83
|
+
pkg = "/".join(parts[:i])
|
|
84
|
+
rest = parts[i + 1:]
|
|
85
|
+
return _target_under_src(pkg, rest)
|
|
86
|
+
if seg in _TARGET_DIRS:
|
|
87
|
+
pkg = "/".join(parts[:i])
|
|
88
|
+
return CrateTarget(pkg, _TARGET_DIRS[seg], _stem(parts[i + 1]))
|
|
89
|
+
|
|
90
|
+
# A top-level build script.
|
|
91
|
+
if parts[-1] == "build.rs":
|
|
92
|
+
return CrateTarget("/".join(parts[:-1]), KIND_BUILD, "build")
|
|
93
|
+
|
|
94
|
+
# No recognizable layout: the file's own directory is a (degenerate) lib crate -- keeps distinct
|
|
95
|
+
# directories distinct without inventing structure.
|
|
96
|
+
pkg = "/".join(parts[:-1])
|
|
97
|
+
return CrateTarget(pkg, KIND_LIB, _basename(pkg))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _target_under_src(pkg: str, rest: list[str]) -> CrateTarget:
|
|
101
|
+
default_name = _basename(pkg)
|
|
102
|
+
if rest[:1] == ["bin"] and len(rest) >= 2:
|
|
103
|
+
return CrateTarget(pkg, KIND_BIN, _stem(rest[1])) # src/bin/tool.rs or src/bin/tool/*.rs
|
|
104
|
+
if rest == ["main.rs"]:
|
|
105
|
+
return CrateTarget(pkg, KIND_BIN, default_name) # the package's default binary
|
|
106
|
+
return CrateTarget(pkg, KIND_LIB, default_name) # lib.rs and every other library module
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _basename(dir_path: str) -> str:
|
|
110
|
+
"""The crate name a default lib/bin takes: the package directory's basename, normalized the way
|
|
111
|
+
Cargo normalizes a crate name (hyphens become underscores). Empty (repo root) -> "crate"."""
|
|
112
|
+
base = dir_path.rsplit("/", 1)[-1] if dir_path else ""
|
|
113
|
+
return base.replace("-", "_") if base else "crate"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass
|
|
117
|
+
class CargoDependency:
|
|
118
|
+
name: str # the dependency crate name as written in [dependencies]
|
|
119
|
+
req: str # version requirement or "" for a path/workspace dep
|
|
120
|
+
path: str | None # relative path for a `{ path = ... }` dep, else None
|
|
121
|
+
kind: str # "normal" | "dev" | "build"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass
|
|
125
|
+
class PackageManifest:
|
|
126
|
+
"""The parsed facts a `Cargo.toml` adds on top of the path-derived crate identity. `dir` is the
|
|
127
|
+
manifest's directory (""/repo-root for a top-level manifest)."""
|
|
128
|
+
|
|
129
|
+
dir: str
|
|
130
|
+
name: str | None = None # [package].name (None for a virtual workspace-root manifest)
|
|
131
|
+
version: str | None = None
|
|
132
|
+
edition: str | None = None
|
|
133
|
+
is_workspace_root: bool = False
|
|
134
|
+
workspace_members: list[str] = field(default_factory=list) # globs, as written
|
|
135
|
+
# [workspace.package] inheritable defaults (a member with `edition.workspace = true` inherits them).
|
|
136
|
+
ws_package_edition: str | None = None
|
|
137
|
+
ws_package_version: str | None = None
|
|
138
|
+
dependencies: list[CargoDependency] = field(default_factory=list)
|
|
139
|
+
# Explicit [lib]/[[bin]] name overrides, if any (path overrides are not honored -- the layout
|
|
140
|
+
# heuristic covers the standard tree; a custom `path=` target degrades to the default mapping).
|
|
141
|
+
lib_name: str | None = None
|
|
142
|
+
bin_names: list[str] = field(default_factory=list)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def parse_manifest(dir_path: str, data: bytes) -> PackageManifest:
|
|
146
|
+
"""Parse one `Cargo.toml`. Never raises on a malformed manifest -- returns a best-effort
|
|
147
|
+
PackageManifest (a parse error yields an empty one) so a single bad file can't fail ingestion."""
|
|
148
|
+
m = PackageManifest(dir=_norm(dir_path))
|
|
149
|
+
try:
|
|
150
|
+
doc = tomllib.loads(data.decode("utf-8", "replace"))
|
|
151
|
+
except (tomllib.TOMLDecodeError, ValueError):
|
|
152
|
+
return m
|
|
153
|
+
|
|
154
|
+
pkg = doc.get("package")
|
|
155
|
+
if isinstance(pkg, dict):
|
|
156
|
+
m.name = _as_str(pkg.get("name"))
|
|
157
|
+
m.version = _as_str(pkg.get("version"))
|
|
158
|
+
# edition can be `edition = "2021"` or `edition.workspace = true` (inherited) -- capture the
|
|
159
|
+
# literal form; inheritance is resolved by the reconcile against the workspace root.
|
|
160
|
+
m.edition = _as_str(pkg.get("edition"))
|
|
161
|
+
|
|
162
|
+
ws = doc.get("workspace")
|
|
163
|
+
if isinstance(ws, dict):
|
|
164
|
+
m.is_workspace_root = True
|
|
165
|
+
members = ws.get("members")
|
|
166
|
+
if isinstance(members, list):
|
|
167
|
+
m.workspace_members = [s for s in members if isinstance(s, str)]
|
|
168
|
+
wspkg = ws.get("package")
|
|
169
|
+
if isinstance(wspkg, dict):
|
|
170
|
+
m.ws_package_edition = _as_str(wspkg.get("edition"))
|
|
171
|
+
m.ws_package_version = _as_str(wspkg.get("version"))
|
|
172
|
+
|
|
173
|
+
for table, kind in (("dependencies", "normal"), ("dev-dependencies", "dev"),
|
|
174
|
+
("build-dependencies", "build")):
|
|
175
|
+
deps = doc.get(table)
|
|
176
|
+
if isinstance(deps, dict):
|
|
177
|
+
for name, spec in deps.items():
|
|
178
|
+
m.dependencies.append(_dependency(name, spec, kind))
|
|
179
|
+
|
|
180
|
+
lib = doc.get("lib")
|
|
181
|
+
if isinstance(lib, dict):
|
|
182
|
+
m.lib_name = _as_str(lib.get("name"))
|
|
183
|
+
bins = doc.get("bin")
|
|
184
|
+
if isinstance(bins, list):
|
|
185
|
+
m.bin_names = [n for b in bins if isinstance(b, dict) for n in (_as_str(b.get("name")),) if n]
|
|
186
|
+
return m
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _dependency(name: str, spec: object, kind: str) -> CargoDependency:
|
|
190
|
+
if isinstance(spec, str):
|
|
191
|
+
return CargoDependency(name=name, req=spec, path=None, kind=kind)
|
|
192
|
+
if isinstance(spec, dict):
|
|
193
|
+
return CargoDependency(
|
|
194
|
+
name=_as_str(spec.get("package")) or name, # `foo = { package = "real" }` renames
|
|
195
|
+
req=_as_str(spec.get("version")) or "",
|
|
196
|
+
path=_as_str(spec.get("path")),
|
|
197
|
+
kind=kind,
|
|
198
|
+
)
|
|
199
|
+
return CargoDependency(name=name, req="", path=None, kind=kind)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _as_str(v: object) -> str | None:
|
|
203
|
+
return v if isinstance(v, str) else None
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# -- manifest model (the rich tier git_sync's reconcile applies) ---------------------------
|
|
207
|
+
|
|
208
|
+
@dataclass
|
|
209
|
+
class CargoPackage:
|
|
210
|
+
dir: str
|
|
211
|
+
name: str
|
|
212
|
+
version: str | None
|
|
213
|
+
edition: str | None
|
|
214
|
+
is_workspace_root: bool
|
|
215
|
+
external_dependencies: list[str] # dep names that do NOT resolve to an in-repo package (sorted)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@dataclass
|
|
219
|
+
class CargoModel:
|
|
220
|
+
"""The manifest-derived facts the reconcile layers onto the path-derived graph: per-package
|
|
221
|
+
metadata, and the intra-repo crate dependency graph (as package-directory pairs -- git_sync maps
|
|
222
|
+
them to Package node ids). External (crates.io) deps are recorded per-package as names, not edges
|
|
223
|
+
(the depended-on crate isn't in this graph)."""
|
|
224
|
+
|
|
225
|
+
packages: list[CargoPackage]
|
|
226
|
+
depends_on: list[tuple[str, str]] # (dependent_dir, dependency_dir), intra-repo, deduped+sorted
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def build_cargo_model(manifests: list[PackageManifest]) -> CargoModel:
|
|
230
|
+
"""Turn parsed manifests into the reconcile model. Resolves `[workspace.package]` edition/version
|
|
231
|
+
inheritance, and classifies each runtime dependency as intra-repo (a path dep, or a name that
|
|
232
|
+
matches a workspace member's package name -> a DEPENDS_ON edge) or external (-> a name on the
|
|
233
|
+
package). Dev/build deps are excluded -- DEPENDS_ON models the runtime building blocks."""
|
|
234
|
+
ws_edition = next((m.ws_package_edition for m in manifests if m.ws_package_edition), None)
|
|
235
|
+
ws_version = next((m.ws_package_version for m in manifests if m.ws_package_version), None)
|
|
236
|
+
dir_by_name = {m.name: m.dir for m in manifests if m.name}
|
|
237
|
+
|
|
238
|
+
packages: list[CargoPackage] = []
|
|
239
|
+
depends_on: set[tuple[str, str]] = set()
|
|
240
|
+
for m in manifests:
|
|
241
|
+
if m.name is None:
|
|
242
|
+
continue # a virtual workspace-root manifest (no [package]) contributes no Package node
|
|
243
|
+
external: list[str] = []
|
|
244
|
+
for d in m.dependencies:
|
|
245
|
+
if d.kind != "normal":
|
|
246
|
+
continue
|
|
247
|
+
target = _dependency_dir(m.dir, d, dir_by_name)
|
|
248
|
+
if target is None:
|
|
249
|
+
external.append(d.name)
|
|
250
|
+
elif target != m.dir: # ignore a pathological self-dep
|
|
251
|
+
depends_on.add((m.dir, target))
|
|
252
|
+
packages.append(CargoPackage(
|
|
253
|
+
dir=m.dir, name=m.name, version=m.version or ws_version,
|
|
254
|
+
edition=m.edition or ws_edition, is_workspace_root=m.is_workspace_root,
|
|
255
|
+
external_dependencies=sorted(set(external)),
|
|
256
|
+
))
|
|
257
|
+
return CargoModel(packages=packages, depends_on=sorted(depends_on))
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _dependency_dir(from_dir: str, dep: CargoDependency, dir_by_name: dict[str, str]) -> str | None:
|
|
261
|
+
"""The in-repo package directory a dependency points at, or None if it's external. A `path` dep
|
|
262
|
+
is resolved relative to the depending manifest's directory; otherwise a name match against a
|
|
263
|
+
workspace member's package name wins (the common `foo = { workspace = true }` / member case)."""
|
|
264
|
+
if dep.path is not None:
|
|
265
|
+
return _join_rel(from_dir, dep.path)
|
|
266
|
+
return dir_by_name.get(dep.name)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _join_rel(base_dir: str, rel: str) -> str:
|
|
270
|
+
"""Normalize `base_dir/rel` with `.`/`..` resolved, without touching the filesystem -- so a
|
|
271
|
+
`{ path = "../core" }` dep resolves to the same repo-relative directory a full and an incremental
|
|
272
|
+
sync both see."""
|
|
273
|
+
parts = [p for p in base_dir.split("/") if p] if base_dir else []
|
|
274
|
+
# NB: split on the raw path -- do NOT run it through _norm(), whose lstrip("./") would eat a
|
|
275
|
+
# leading ".." (the very segment a `path = "../core"` dep needs to climb out on).
|
|
276
|
+
for seg in rel.replace("\\", "/").split("/"):
|
|
277
|
+
if seg in ("", "."):
|
|
278
|
+
continue
|
|
279
|
+
if seg == "..":
|
|
280
|
+
if parts:
|
|
281
|
+
parts.pop()
|
|
282
|
+
else:
|
|
283
|
+
parts.append(seg)
|
|
284
|
+
return "/".join(parts)
|