typedframes 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {typedframes-0.3.0 → typedframes-0.3.1}/PKG-INFO +27 -12
- {typedframes-0.3.0 → typedframes-0.3.1}/README.md +26 -11
- {typedframes-0.3.0 → typedframes-0.3.1}/pyproject.toml +1 -1
- {typedframes-0.3.0 → typedframes-0.3.1}/rust/Cargo.lock +1 -1
- {typedframes-0.3.0 → typedframes-0.3.1}/rust/Cargo.toml +1 -1
- {typedframes-0.3.0 → typedframes-0.3.1}/rust/src/lib.rs +460 -76
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/__init__.py +1 -1
- {typedframes-0.3.0 → typedframes-0.3.1}/rust/README.md +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/rust/benches/parser_bench.rs +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/rust/src/main.rs +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/rust/tests/integration_test.rs +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/base_schema.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/cli.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/column.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/column_group.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/column_group_error.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/column_set.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/missing_dependency_error.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/mypy.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/pandas.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/pandera.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/polars.py +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/py.typed +0 -0
- {typedframes-0.3.0 → typedframes-0.3.1}/src/typedframes/schema_algebra.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: typedframes
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -42,7 +42,7 @@ Project-URL: Repository, https://github.com/w-martin/typedframes
|
|
|
42
42
|
|
|
43
43
|
> ⚠️ **Project Status: Proof of Concept**
|
|
44
44
|
>
|
|
45
|
-
> `typedframes` (v0.3.
|
|
45
|
+
> `typedframes` (v0.3.1) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
|
|
46
46
|
> integrations work, but expect rough edges. The codebase prioritizes demonstrating the viability of static DataFrame
|
|
47
47
|
> schema validation over production-grade stability.
|
|
48
48
|
>
|
|
@@ -323,6 +323,20 @@ The contract is resolved *transitively*: if a function only forwards its paramet
|
|
|
323
323
|
requirements, catching a missing column even when no single function in the chain touches it directly.
|
|
324
324
|
Column-list slices (`df[["a", "b"]]`) contribute to the contract too.
|
|
325
325
|
|
|
326
|
+
**Known limitations:**
|
|
327
|
+
- Cross-file delegate/schema resolution follows `from module import name`, plain `import module` +
|
|
328
|
+
`module.helper(df)` attribute access, and `from module import *` wildcard imports. A dotted import with no
|
|
329
|
+
alias (`import a.b.c`) only binds the first segment (`a`), matching Python's own binding rules, so a
|
|
330
|
+
deeply nested submodule accessed without an alias is not tracked.
|
|
331
|
+
- Contract inference is a single top-to-bottom pass over a function body, not full control-flow analysis.
|
|
332
|
+
Deeply nested control flow (nested `try`/`except`, `match`, comprehensions) may under-report a function's
|
|
333
|
+
true requirements.
|
|
334
|
+
- A cycle in the delegate graph (mutually- or self-delegating helpers) contributes only each function's own
|
|
335
|
+
direct requirements to the cycle, not the full transitive union — conservative rather than exhaustive.
|
|
336
|
+
- If two plainly-imported modules both define a same-named function, an attribute-style delegate call
|
|
337
|
+
(`module.helper(df)`) resolves to whichever one is found first — the checker doesn't disambiguate by which
|
|
338
|
+
module the call site actually used.
|
|
339
|
+
|
|
326
340
|
### See Also
|
|
327
341
|
|
|
328
342
|
- [`examples/inference_example.py`](examples/inference_example.py) — single-file walkthrough of all four inference
|
|
@@ -436,17 +450,18 @@ code reference.
|
|
|
436
450
|
|
|
437
451
|
Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
|
|
438
452
|
|
|
439
|
-
**Benchmark results** (
|
|
453
|
+
**Benchmark results** (20 runs, 3 warmup, caches cleared between runs):
|
|
454
|
+
*2026-07-13 · Darwin 25.5.0 · arm · CPython 3.14.4 · 64GiB RAM · Great Expectations pinned @ 1.9.3*
|
|
440
455
|
|
|
441
|
-
| Tool | Version | What it does | typedframes (13 files) | great_expectations (
|
|
456
|
+
| Tool | Version | What it does | typedframes (13 files) | great_expectations (482 files) |
|
|
442
457
|
|------|---------|--------------|------------------------|--------------------------------|
|
|
443
|
-
| typedframes | 0.3.
|
|
444
|
-
| ruff | 0.15.21 | Linter (no type checking) |
|
|
445
|
-
| ty | 0.0.58 | Type checker |
|
|
446
|
-
| pyrefly | 1.1.1 | Type checker |
|
|
447
|
-
| mypy | 2.2.0 | Type checker (no plugin) |
|
|
448
|
-
| mypy + typedframes | 2.2.0 | Type checker + column checker |
|
|
449
|
-
| pyright | 1.1.411 | Type checker |
|
|
458
|
+
| typedframes | 0.3.1 | DataFrame column checker | 43ms ±668µs | 213ms ±4ms |
|
|
459
|
+
| ruff | 0.15.21 | Linter (no type checking) | 26ms ±1ms | 193ms ±2ms |
|
|
460
|
+
| ty | 0.0.58 | Type checker | 72ms ±3ms | 181ms ±9ms |
|
|
461
|
+
| pyrefly | 1.1.1 | Type checker | 107ms ±4ms | 557ms ±16ms |
|
|
462
|
+
| mypy | 2.2.0 | Type checker (no plugin) | 2.77s ±41ms | 4.21s ±71ms |
|
|
463
|
+
| mypy + typedframes | 2.2.0 | Type checker + column checker | 2.78s ±34ms | 4.48s ±63ms |
|
|
464
|
+
| pyright | 1.1.411 | Type checker | 743ms ±8ms | 3.34s ±43ms |
|
|
450
465
|
|
|
451
466
|
*Run `uv run python benchmarks/benchmark_checkers.py` to reproduce.*
|
|
452
467
|
|
|
@@ -659,7 +674,7 @@ Comprehensive comparison of pandas/DataFrame typing and validation tools. **type
|
|
|
659
674
|
|
|
660
675
|
| Feature | typedframes | Pandera | Great Expectations | strictly_typed_pandas | pandas-stubs | dataenforce | pandas-type-checks | StaticFrame | narwhals | dataframely | patito |
|
|
661
676
|
|---------------------------------|------------------------|-------------|--------------------|-----------------------|--------------|-------------|--------------------|------------------|----------|------------------|------------------|
|
|
662
|
-
| **Version tested** | 0.3.
|
|
677
|
+
| **Version tested** | 0.3.1 | 0.32.1 | 1.18.2 | 0.3.7 | 3.0.3 | 0.1.2 | 1.1.3 | 5.0.0 | 2.23.0 | 2.13.0 | 0.8.6 |
|
|
663
678
|
| **Analysis Type** |
|
|
664
679
|
| When errors are caught | **Static (lint-time)** | Runtime | Runtime | Runtime | Static | Runtime | Runtime | Runtime | Runtime | Runtime | Runtime |
|
|
665
680
|
| **Static Analysis (our focus)** |
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
> ⚠️ **Project Status: Proof of Concept**
|
|
10
10
|
>
|
|
11
|
-
> `typedframes` (v0.3.
|
|
11
|
+
> `typedframes` (v0.3.1) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
|
|
12
12
|
> integrations work, but expect rough edges. The codebase prioritizes demonstrating the viability of static DataFrame
|
|
13
13
|
> schema validation over production-grade stability.
|
|
14
14
|
>
|
|
@@ -289,6 +289,20 @@ The contract is resolved *transitively*: if a function only forwards its paramet
|
|
|
289
289
|
requirements, catching a missing column even when no single function in the chain touches it directly.
|
|
290
290
|
Column-list slices (`df[["a", "b"]]`) contribute to the contract too.
|
|
291
291
|
|
|
292
|
+
**Known limitations:**
|
|
293
|
+
- Cross-file delegate/schema resolution follows `from module import name`, plain `import module` +
|
|
294
|
+
`module.helper(df)` attribute access, and `from module import *` wildcard imports. A dotted import with no
|
|
295
|
+
alias (`import a.b.c`) only binds the first segment (`a`), matching Python's own binding rules, so a
|
|
296
|
+
deeply nested submodule accessed without an alias is not tracked.
|
|
297
|
+
- Contract inference is a single top-to-bottom pass over a function body, not full control-flow analysis.
|
|
298
|
+
Deeply nested control flow (nested `try`/`except`, `match`, comprehensions) may under-report a function's
|
|
299
|
+
true requirements.
|
|
300
|
+
- A cycle in the delegate graph (mutually- or self-delegating helpers) contributes only each function's own
|
|
301
|
+
direct requirements to the cycle, not the full transitive union — conservative rather than exhaustive.
|
|
302
|
+
- If two plainly-imported modules both define a same-named function, an attribute-style delegate call
|
|
303
|
+
(`module.helper(df)`) resolves to whichever one is found first — the checker doesn't disambiguate by which
|
|
304
|
+
module the call site actually used.
|
|
305
|
+
|
|
292
306
|
### See Also
|
|
293
307
|
|
|
294
308
|
- [`examples/inference_example.py`](examples/inference_example.py) — single-file walkthrough of all four inference
|
|
@@ -402,17 +416,18 @@ code reference.
|
|
|
402
416
|
|
|
403
417
|
Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
|
|
404
418
|
|
|
405
|
-
**Benchmark results** (
|
|
419
|
+
**Benchmark results** (20 runs, 3 warmup, caches cleared between runs):
|
|
420
|
+
*2026-07-13 · Darwin 25.5.0 · arm · CPython 3.14.4 · 64GiB RAM · Great Expectations pinned @ 1.9.3*
|
|
406
421
|
|
|
407
|
-
| Tool | Version | What it does | typedframes (13 files) | great_expectations (
|
|
422
|
+
| Tool | Version | What it does | typedframes (13 files) | great_expectations (482 files) |
|
|
408
423
|
|------|---------|--------------|------------------------|--------------------------------|
|
|
409
|
-
| typedframes | 0.3.
|
|
410
|
-
| ruff | 0.15.21 | Linter (no type checking) |
|
|
411
|
-
| ty | 0.0.58 | Type checker |
|
|
412
|
-
| pyrefly | 1.1.1 | Type checker |
|
|
413
|
-
| mypy | 2.2.0 | Type checker (no plugin) |
|
|
414
|
-
| mypy + typedframes | 2.2.0 | Type checker + column checker |
|
|
415
|
-
| pyright | 1.1.411 | Type checker |
|
|
424
|
+
| typedframes | 0.3.1 | DataFrame column checker | 43ms ±668µs | 213ms ±4ms |
|
|
425
|
+
| ruff | 0.15.21 | Linter (no type checking) | 26ms ±1ms | 193ms ±2ms |
|
|
426
|
+
| ty | 0.0.58 | Type checker | 72ms ±3ms | 181ms ±9ms |
|
|
427
|
+
| pyrefly | 1.1.1 | Type checker | 107ms ±4ms | 557ms ±16ms |
|
|
428
|
+
| mypy | 2.2.0 | Type checker (no plugin) | 2.77s ±41ms | 4.21s ±71ms |
|
|
429
|
+
| mypy + typedframes | 2.2.0 | Type checker + column checker | 2.78s ±34ms | 4.48s ±63ms |
|
|
430
|
+
| pyright | 1.1.411 | Type checker | 743ms ±8ms | 3.34s ±43ms |
|
|
416
431
|
|
|
417
432
|
*Run `uv run python benchmarks/benchmark_checkers.py` to reproduce.*
|
|
418
433
|
|
|
@@ -625,7 +640,7 @@ Comprehensive comparison of pandas/DataFrame typing and validation tools. **type
|
|
|
625
640
|
|
|
626
641
|
| Feature | typedframes | Pandera | Great Expectations | strictly_typed_pandas | pandas-stubs | dataenforce | pandas-type-checks | StaticFrame | narwhals | dataframely | patito |
|
|
627
642
|
|---------------------------------|------------------------|-------------|--------------------|-----------------------|--------------|-------------|--------------------|------------------|----------|------------------|------------------|
|
|
628
|
-
| **Version tested** | 0.3.
|
|
643
|
+
| **Version tested** | 0.3.1 | 0.32.1 | 1.18.2 | 0.3.7 | 3.0.3 | 0.1.2 | 1.1.3 | 5.0.0 | 2.23.0 | 2.13.0 | 0.8.6 |
|
|
629
644
|
| **Analysis Type** |
|
|
630
645
|
| When errors are caught | **Static (lint-time)** | Runtime | Runtime | Runtime | Static | Runtime | Runtime | Runtime | Runtime | Runtime | Runtime |
|
|
631
646
|
| **Static Analysis (our focus)** |
|
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "typedframes"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.1"
|
|
8
8
|
description = "Static analysis for pandas and polars DataFrames. Catch column errors at lint-time, not runtime."
|
|
9
9
|
keywords = ["pandas", "polars", "type-checking", "static-analysis", "dataframe", "linter", "mypy-plugin"]
|
|
10
10
|
readme = "README.md"
|
|
@@ -38,7 +38,9 @@ use ruff_text_size::Ranged;
|
|
|
38
38
|
use serde::{Deserialize, Serialize};
|
|
39
39
|
use std::collections::HashMap;
|
|
40
40
|
use std::fs;
|
|
41
|
+
use std::hash::{Hash, Hasher};
|
|
41
42
|
use std::path::{Path, PathBuf};
|
|
43
|
+
use std::sync::{Arc, Mutex};
|
|
42
44
|
/// Check a single Python file for DataFrame column errors.
|
|
43
45
|
///
|
|
44
46
|
/// Accepts an optional MessagePack-serialised [`ProjectIndex`] (produced by
|
|
@@ -62,7 +64,7 @@ fn check_file(file_path: String, index_bytes: Option<Vec<u8>>) -> PyResult<Strin
|
|
|
62
64
|
let mut linter = Linter::new();
|
|
63
65
|
|
|
64
66
|
if let Some(bytes) = index_bytes {
|
|
65
|
-
if let
|
|
67
|
+
if let Some(index) = get_cached_index(&bytes) {
|
|
66
68
|
linter.load_cross_file_symbols(&index, &source, path, &project_root);
|
|
67
69
|
}
|
|
68
70
|
}
|
|
@@ -79,6 +81,35 @@ fn check_file(file_path: String, index_bytes: Option<Vec<u8>>) -> PyResult<Strin
|
|
|
79
81
|
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(format!("{}", e)))
|
|
80
82
|
}
|
|
81
83
|
|
|
84
|
+
// The CLI calls check_file once per file in a project, passing the SAME serialised
|
|
85
|
+
// index_bytes on every call (see `_check_files` in cli.py, and mypy.py's per-file
|
|
86
|
+
// hook, which does the same across a single mypy run). Deserialising a project-wide
|
|
87
|
+
// index — which can be hundreds of KB for a large project, since it must carry every
|
|
88
|
+
// candidate delegate-target function, not just DataFrame-relevant ones — on every
|
|
89
|
+
// single file turns an O(1)-per-file operation into an O(project size)-per-file one,
|
|
90
|
+
// i.e. O(files^2) for a whole-project check. Cache the deserialised index keyed by a
|
|
91
|
+
// hash of its bytes (hashing is far cheaper than re-deserialising nested
|
|
92
|
+
// Strings/Vecs/HashMaps) so repeat calls with the same index are a cache hit; a
|
|
93
|
+
// different hash (e.g. a different project root between calls) correctly replaces
|
|
94
|
+
// the cached entry rather than silently reusing stale data.
|
|
95
|
+
static INDEX_CACHE: Mutex<Option<(u64, Arc<ProjectIndex>)>> = Mutex::new(None);
|
|
96
|
+
|
|
97
|
+
fn get_cached_index(bytes: &[u8]) -> Option<Arc<ProjectIndex>> {
|
|
98
|
+
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
|
99
|
+
bytes.hash(&mut hasher);
|
|
100
|
+
let hash = hasher.finish();
|
|
101
|
+
|
|
102
|
+
let mut cache = INDEX_CACHE.lock().ok()?;
|
|
103
|
+
if let Some((cached_hash, index)) = cache.as_ref() {
|
|
104
|
+
if *cached_hash == hash {
|
|
105
|
+
return Some(index.clone());
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
let index = Arc::new(rmp_serde::from_slice::<ProjectIndex>(bytes).ok()?);
|
|
109
|
+
*cache = Some((hash, index.clone()));
|
|
110
|
+
Some(index)
|
|
111
|
+
}
|
|
112
|
+
|
|
82
113
|
/// Build a cross-file symbol index for an entire project.
|
|
83
114
|
///
|
|
84
115
|
/// Walks all `.py` files under `project_root`, parses each one, and extracts
|
|
@@ -203,6 +234,15 @@ struct IndexFunction {
|
|
|
203
234
|
// Names of functions called with a tainted variable (the parameter itself, or a
|
|
204
235
|
// variable derived from it) as their first positional argument. Consumed only by
|
|
205
236
|
// resolve_transitive_requires; not needed once `requires` has been resolved.
|
|
237
|
+
// `#[serde(skip)]`: this is pure intermediate state, dead the moment
|
|
238
|
+
// build_index_internal finishes — it must never reach the serialised
|
|
239
|
+
// ProjectIndex that check_file deserialises once per file. On a large,
|
|
240
|
+
// delegate-heavy project (helper functions that just forward their first
|
|
241
|
+
// argument are extremely common Python, whether or not they touch a
|
|
242
|
+
// DataFrame) these lists dominate the index's serialised size, and paying
|
|
243
|
+
// that deserialisation cost once per file turns "more thorough analysis"
|
|
244
|
+
// into an O(files) cost repeated per file, i.e. O(files^2) overall.
|
|
245
|
+
#[serde(skip)]
|
|
206
246
|
delegates: Vec<String>,
|
|
207
247
|
// Schema name from the first parameter's annotation, e.g. "ReportSchema" from
|
|
208
248
|
// `Annotated[pl.DataFrame, ReportSchema]`; empty if the parameter isn't
|
|
@@ -210,6 +250,9 @@ struct IndexFunction {
|
|
|
210
250
|
// resolvable yet (it may be defined in a third file) — resolve_transitive_requires
|
|
211
251
|
// resolves it project-wide and overrides `requires` with the authoritative column
|
|
212
252
|
// list once every file's schemas are known, superseding the heuristic body-scan.
|
|
253
|
+
// `#[serde(skip)]`: same reasoning as `delegates` above — dead weight past
|
|
254
|
+
// resolve_param_schema_requires, must not be serialised into ProjectIndex.
|
|
255
|
+
#[serde(skip)]
|
|
213
256
|
param_schema_name: String,
|
|
214
257
|
}
|
|
215
258
|
|
|
@@ -220,6 +263,11 @@ struct IndexEntry {
|
|
|
220
263
|
functions: HashMap<String, IndexFunction>, // function name -> return type info
|
|
221
264
|
exports: Vec<String>, // names in __all__, for wildcard-import resolution
|
|
222
265
|
imports: HashMap<String, String>, // imported name -> dotted module it came from (`from X import Y`)
|
|
266
|
+
// Local alias -> dotted module name, from plain `import module [as alias]`
|
|
267
|
+
// statements. Lets resolve_delegate_target and load_cross_file_symbols follow
|
|
268
|
+
// attribute-style calls (`transforms.enrich(df)`) back to the module they came
|
|
269
|
+
// from, the same way `imports` does for `from transforms import enrich`.
|
|
270
|
+
module_aliases: HashMap<String, String>,
|
|
223
271
|
}
|
|
224
272
|
|
|
225
273
|
// In-memory cross-file symbol index.
|
|
@@ -229,6 +277,30 @@ struct IndexEntry {
|
|
|
229
277
|
struct ProjectIndex {
|
|
230
278
|
version: u32, // format version, currently always 1
|
|
231
279
|
files: HashMap<String, IndexEntry>, // absolute file path -> IndexEntry
|
|
280
|
+
// Project-wide schema name -> column list, unioned across every file's `schemas`
|
|
281
|
+
// map. Computed once here in build_index_internal so that per-file lookups (see
|
|
282
|
+
// Linter::load_cross_file_symbols, called once per file by `check_file`) are a
|
|
283
|
+
// single field read instead of a rebuild-by-iterating-every-file — the latter
|
|
284
|
+
// turned a per-file O(1) lookup into an O(files) one, i.e. O(files^2) for a whole
|
|
285
|
+
// project check. A schema may be defined in a THIRD file — neither the function's
|
|
286
|
+
// own file nor the one importing it — so this must stay project-wide rather than
|
|
287
|
+
// scoped to a single IndexEntry.
|
|
288
|
+
all_schemas: HashMap<String, Vec<String>>,
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Union schema name -> column list across every file's `schemas` map. First
|
|
292
|
+
// definition wins on name collision (matches the prior per-call behaviour in both
|
|
293
|
+
// resolve_param_schema_requires and load_cross_file_symbols).
|
|
294
|
+
fn compute_all_schemas(files: &HashMap<String, IndexEntry>) -> HashMap<String, Vec<String>> {
|
|
295
|
+
let mut all_schemas: HashMap<String, Vec<String>> = HashMap::new();
|
|
296
|
+
for entry in files.values() {
|
|
297
|
+
for (name, cols) in &entry.schemas {
|
|
298
|
+
all_schemas
|
|
299
|
+
.entry(name.clone())
|
|
300
|
+
.or_insert_with(|| cols.clone());
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
all_schemas
|
|
232
304
|
}
|
|
233
305
|
|
|
234
306
|
// ── Index helpers ──────────────────────────────────────────────────────────────
|
|
@@ -316,6 +388,7 @@ fn index_file(path: &Path) -> Option<IndexEntry> {
|
|
|
316
388
|
|
|
317
389
|
let mut exports = Vec::new();
|
|
318
390
|
let mut imports: HashMap<String, String> = HashMap::new();
|
|
391
|
+
let mut module_aliases: HashMap<String, String> = HashMap::new();
|
|
319
392
|
if let Ok(parsed) = parse_module(&source) {
|
|
320
393
|
let module = parsed.into_syntax();
|
|
321
394
|
for stmt in &module.body {
|
|
@@ -346,6 +419,23 @@ fn index_file(path: &Path) -> Option<IndexEntry> {
|
|
|
346
419
|
}
|
|
347
420
|
}
|
|
348
421
|
}
|
|
422
|
+
// Plain `import module [as alias]` / `import a.b.c [as alias]`. Without
|
|
423
|
+
// an alias, Python binds the FIRST segment of a dotted import (`import
|
|
424
|
+
// a.b.c` binds `a`), so a dotted single-segment module (the common case
|
|
425
|
+
// for a flat project: `import transforms`) binds cleanly; a genuinely
|
|
426
|
+
// nested `a.b.c` accessed as `a.b.c.func()` without an alias is not
|
|
427
|
+
// tracked here (`a.func()` wouldn't resolve to `a.b.c` anyway) — same
|
|
428
|
+
// conservative-not-exhaustive trade-off as the rest of this heuristic.
|
|
429
|
+
Stmt::Import(import_stmt) => {
|
|
430
|
+
for alias in &import_stmt.names {
|
|
431
|
+
let dotted = alias.name.id.to_string();
|
|
432
|
+
let local_name = match &alias.asname {
|
|
433
|
+
Some(asname) => asname.id.to_string(),
|
|
434
|
+
None => dotted.split('.').next().unwrap_or(&dotted).to_string(),
|
|
435
|
+
};
|
|
436
|
+
module_aliases.insert(local_name, dotted);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
349
439
|
_ => {}
|
|
350
440
|
}
|
|
351
441
|
}
|
|
@@ -356,6 +446,7 @@ fn index_file(path: &Path) -> Option<IndexEntry> {
|
|
|
356
446
|
functions,
|
|
357
447
|
exports,
|
|
358
448
|
imports,
|
|
449
|
+
module_aliases,
|
|
359
450
|
})
|
|
360
451
|
}
|
|
361
452
|
|
|
@@ -370,9 +461,14 @@ fn build_index_internal(project_root: &Path) -> ProjectIndex {
|
|
|
370
461
|
}
|
|
371
462
|
}
|
|
372
463
|
}
|
|
373
|
-
|
|
464
|
+
let all_schemas = compute_all_schemas(&files);
|
|
465
|
+
resolve_param_schema_requires(&mut files, &all_schemas);
|
|
374
466
|
resolve_transitive_requires(project_root, &mut files);
|
|
375
|
-
ProjectIndex {
|
|
467
|
+
ProjectIndex {
|
|
468
|
+
version: 1,
|
|
469
|
+
files,
|
|
470
|
+
all_schemas,
|
|
471
|
+
}
|
|
376
472
|
}
|
|
377
473
|
|
|
378
474
|
// Resolve each function's `param_schema_name` (the schema its first parameter is
|
|
@@ -384,16 +480,10 @@ fn build_index_internal(project_root: &Path) -> ProjectIndex {
|
|
|
384
480
|
// resolve it locally. Runs before resolve_transitive_requires so that functions
|
|
385
481
|
// delegating to a schema-annotated one inherit the authoritative set, not the
|
|
386
482
|
// heuristic fallback.
|
|
387
|
-
fn resolve_param_schema_requires(
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
all_schemas
|
|
392
|
-
.entry(name.clone())
|
|
393
|
-
.or_insert_with(|| cols.clone());
|
|
394
|
-
}
|
|
395
|
-
}
|
|
396
|
-
|
|
483
|
+
fn resolve_param_schema_requires(
|
|
484
|
+
files: &mut HashMap<String, IndexEntry>,
|
|
485
|
+
all_schemas: &HashMap<String, Vec<String>>,
|
|
486
|
+
) {
|
|
397
487
|
for entry in files.values_mut() {
|
|
398
488
|
for func in entry.functions.values_mut() {
|
|
399
489
|
if func.param_schema_name.is_empty() {
|
|
@@ -415,6 +505,35 @@ type FuncNode = (String, String);
|
|
|
415
505
|
// Resolve a delegate call target (a bare name, as written at the call site) to the
|
|
416
506
|
// node that actually defines it: a same-file function first, else a function
|
|
417
507
|
// reached by following `from_file`'s own `from X import name` statements.
|
|
508
|
+
// Resolve a dotted module name (e.g. "pkg.transforms") to the project file that
|
|
509
|
+
// defines it, trying both the project root and a `src/` layout — same candidate
|
|
510
|
+
// paths used throughout this module for cross-file resolution.
|
|
511
|
+
fn resolve_module_file(
|
|
512
|
+
module_name: &str,
|
|
513
|
+
project_root: &Path,
|
|
514
|
+
files: &HashMap<String, IndexEntry>,
|
|
515
|
+
) -> Option<String> {
|
|
516
|
+
let mod_path = module_name.replace('.', "/");
|
|
517
|
+
let candidates = [
|
|
518
|
+
project_root.join(format!("{mod_path}.py")),
|
|
519
|
+
project_root.join("src").join(format!("{mod_path}.py")),
|
|
520
|
+
];
|
|
521
|
+
candidates
|
|
522
|
+
.iter()
|
|
523
|
+
.filter_map(|p| p.to_str())
|
|
524
|
+
.find(|p| files.contains_key(*p))
|
|
525
|
+
.map(str::to_string)
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
// Resolve a delegate call target (a bare name, as written at the call site) to the
|
|
529
|
+
// node that actually defines it: a same-file function first, else a function
|
|
530
|
+
// reached by following `from_file`'s own `from X import name` statements, else —
|
|
531
|
+
// since attribute-style calls (`transforms.enrich(df)`) are recorded as the bare
|
|
532
|
+
// attribute name `enrich` with no module prefix (see scan_expr_for_contract) — a
|
|
533
|
+
// function by that name in any module `from_file` plainly `import`s. That last
|
|
534
|
+
// step can't distinguish which plainly-imported module a given call actually used
|
|
535
|
+
// if more than one defines a same-named function; same conservative trade-off as
|
|
536
|
+
// the rest of this heuristic (see the cycle-handling note on resolve_node_requires).
|
|
418
537
|
fn resolve_delegate_target(
|
|
419
538
|
from_file: &str,
|
|
420
539
|
callee: &str,
|
|
@@ -425,22 +544,21 @@ fn resolve_delegate_target(
|
|
|
425
544
|
if entry.functions.contains_key(callee) {
|
|
426
545
|
return Some((from_file.to_string(), callee.to_string()));
|
|
427
546
|
}
|
|
428
|
-
let module_name = entry.imports.get(callee)
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
.
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
} else {
|
|
442
|
-
None
|
|
547
|
+
if let Some(module_name) = entry.imports.get(callee) {
|
|
548
|
+
let target_file = resolve_module_file(module_name, project_root, files)?;
|
|
549
|
+
if files.get(&target_file)?.functions.contains_key(callee) {
|
|
550
|
+
return Some((target_file, callee.to_string()));
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
for module_name in entry.module_aliases.values() {
|
|
554
|
+
let Some(target_file) = resolve_module_file(module_name, project_root, files) else {
|
|
555
|
+
continue;
|
|
556
|
+
};
|
|
557
|
+
if files.get(&target_file)?.functions.contains_key(callee) {
|
|
558
|
+
return Some((target_file, callee.to_string()));
|
|
559
|
+
}
|
|
443
560
|
}
|
|
561
|
+
None
|
|
444
562
|
}
|
|
445
563
|
|
|
446
564
|
// Memoised DFS over the delegate graph: a function's fully resolved requirement set
|
|
@@ -970,15 +1088,12 @@ impl Linter {
|
|
|
970
1088
|
// pipeline.py imports only load_customers, not CustomerSchema directly. Looking
|
|
971
1089
|
// the schema up only in the function's own file's entry (as opposed to
|
|
972
1090
|
// project-wide) would silently fail in that case — no error, but no validation
|
|
973
|
-
// either, since self.schemas would never learn CustomerSchema's columns.
|
|
974
|
-
//
|
|
975
|
-
//
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
all_schemas.entry(name.as_str()).or_insert(cols);
|
|
980
|
-
}
|
|
981
|
-
}
|
|
1091
|
+
// either, since self.schemas would never learn CustomerSchema's columns.
|
|
1092
|
+
// `index.all_schemas` is this project-wide name -> columns registry — computed
|
|
1093
|
+
// ONCE in build_index_internal, not rebuilt here. This function runs once per
|
|
1094
|
+
// file checked (see `check_file`), so rebuilding a project-wide map in here
|
|
1095
|
+
// would cost O(files) per file, i.e. O(files^2) for a whole-project check.
|
|
1096
|
+
let all_schemas = &index.all_schemas;
|
|
982
1097
|
|
|
983
1098
|
let Ok(parsed) = parse_module(source) else {
|
|
984
1099
|
return;
|
|
@@ -1015,50 +1130,124 @@ impl Linter {
|
|
|
1015
1130
|
// Use the full resolved path (not just the basename) so error messages
|
|
1016
1131
|
// contain an openable `file:line` reference regardless of cwd.
|
|
1017
1132
|
let file_path_display = resolved_path.display().to_string();
|
|
1133
|
+
// `from X import *`: ruff represents the wildcard as a single alias named
|
|
1134
|
+
// "*". Expand to the module's declared __all__, or — matching real Python
|
|
1135
|
+
// semantics for a module with no __all__ — every public (non-`_`-prefixed)
|
|
1136
|
+
// function/schema name, since we can't tell which of those the file
|
|
1137
|
+
// actually goes on to use.
|
|
1138
|
+
let is_wildcard =
|
|
1139
|
+
import_from.names.len() == 1 && import_from.names[0].name.as_str() == "*";
|
|
1140
|
+
if is_wildcard {
|
|
1141
|
+
let names: Vec<String> = if !entry.exports.is_empty() {
|
|
1142
|
+
entry.exports.clone()
|
|
1143
|
+
} else {
|
|
1144
|
+
entry
|
|
1145
|
+
.functions
|
|
1146
|
+
.keys()
|
|
1147
|
+
.chain(entry.schemas.keys())
|
|
1148
|
+
.filter(|n| !n.starts_with('_'))
|
|
1149
|
+
.cloned()
|
|
1150
|
+
.collect()
|
|
1151
|
+
};
|
|
1152
|
+
for name in &names {
|
|
1153
|
+
self.import_name(entry, name, &file_path_display, all_schemas);
|
|
1154
|
+
}
|
|
1155
|
+
continue;
|
|
1156
|
+
}
|
|
1018
1157
|
for alias in &import_from.names {
|
|
1019
1158
|
let name = alias.name.id.as_str();
|
|
1020
|
-
|
|
1021
|
-
|
|
1159
|
+
self.import_name(entry, name, &file_path_display, all_schemas);
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
// Plain `import module [as alias]`: everything reachable via `module.name`
|
|
1164
|
+
// attribute access. Unlike `from X import name`, there's no explicit name
|
|
1165
|
+
// list to narrow to, so pull in every function this module defines — a call
|
|
1166
|
+
// site written as `module.func(df)` is resolved by check_call_requirements
|
|
1167
|
+
// looking up the bare name `func`, the same key space `from X import func`
|
|
1168
|
+
// would have populated (see visit_stmt's Expr::Attribute call-site handling).
|
|
1169
|
+
for stmt in &module.body {
|
|
1170
|
+
let Stmt::Import(import_stmt) = stmt else {
|
|
1171
|
+
continue;
|
|
1172
|
+
};
|
|
1173
|
+
for alias in &import_stmt.names {
|
|
1174
|
+
let dotted = alias.name.id.as_str();
|
|
1175
|
+
if dotted.starts_with("typedframes") {
|
|
1176
|
+
continue;
|
|
1022
1177
|
}
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
.strip_prefix("__inferred_")
|
|
1039
|
-
.and_then(|s| s.rsplit_once("_at_"))
|
|
1040
|
-
.and_then(|(_, l)| l.parse::<usize>().ok())
|
|
1041
|
-
.map(|l| format!(":{l}"))
|
|
1042
|
-
.unwrap_or_default();
|
|
1043
|
-
self.schema_origins.insert(
|
|
1044
|
-
func.returns_schema.clone(),
|
|
1045
|
-
format!("{name} ({file_path_display}{line_suffix})"),
|
|
1046
|
-
);
|
|
1047
|
-
}
|
|
1048
|
-
}
|
|
1049
|
-
// Record the parameter-column contract so calls to this function
|
|
1050
|
-
// (e.g. `trim(customers)`) can be validated at the call site,
|
|
1051
|
-
// where the actual argument's inferred schema is known.
|
|
1052
|
-
if !func.requires.is_empty() {
|
|
1053
|
-
let origin = format!("{name} ({file_path_display}:{})", func.def_line);
|
|
1054
|
-
self.param_requires
|
|
1055
|
-
.insert(name.to_string(), (func.requires.clone(), origin));
|
|
1056
|
-
}
|
|
1178
|
+
let Some(resolved_path) =
|
|
1179
|
+
resolve_module_file(dotted, project_root, &index.files).map(PathBuf::from)
|
|
1180
|
+
else {
|
|
1181
|
+
continue;
|
|
1182
|
+
};
|
|
1183
|
+
let Some(resolved_str) = resolved_path.to_str() else {
|
|
1184
|
+
continue;
|
|
1185
|
+
};
|
|
1186
|
+
let Some(entry) = index.files.get(resolved_str) else {
|
|
1187
|
+
continue;
|
|
1188
|
+
};
|
|
1189
|
+
let file_path_display = resolved_path.display().to_string();
|
|
1190
|
+
let names: Vec<String> = entry.functions.keys().cloned().collect();
|
|
1191
|
+
for name in &names {
|
|
1192
|
+
self.import_name(entry, name, &file_path_display, all_schemas);
|
|
1057
1193
|
}
|
|
1058
1194
|
}
|
|
1059
1195
|
}
|
|
1060
1196
|
}
|
|
1061
1197
|
|
|
1198
|
+
// Pull a single name's schema/function/param-contract info from a resolved
|
|
1199
|
+
// cross-file IndexEntry into this Linter's own local state. Shared by
|
|
1200
|
+
// `from X import name`, `from X import *`, and plain `import module` handling
|
|
1201
|
+
// in load_cross_file_symbols — all three ultimately need to do the same thing
|
|
1202
|
+
// for each name they bring into scope.
|
|
1203
|
+
fn import_name(
|
|
1204
|
+
&mut self,
|
|
1205
|
+
entry: &IndexEntry,
|
|
1206
|
+
name: &str,
|
|
1207
|
+
file_path_display: &str,
|
|
1208
|
+
all_schemas: &HashMap<String, Vec<String>>,
|
|
1209
|
+
) {
|
|
1210
|
+
if let Some(cols) = entry.schemas.get(name) {
|
|
1211
|
+
self.schemas.insert(name.to_string(), cols.clone());
|
|
1212
|
+
}
|
|
1213
|
+
let Some(func) = entry.functions.get(name) else {
|
|
1214
|
+
return;
|
|
1215
|
+
};
|
|
1216
|
+
if !func.returns_schema.is_empty() {
|
|
1217
|
+
self.functions
|
|
1218
|
+
.insert(name.to_string(), func.returns_schema.clone());
|
|
1219
|
+
}
|
|
1220
|
+
if let Some(schema_cols) = all_schemas.get(func.returns_schema.as_str()) {
|
|
1221
|
+
self.schemas
|
|
1222
|
+
.insert(func.returns_schema.clone(), schema_cols.clone());
|
|
1223
|
+
// Record origin so error messages point back to the source function/file.
|
|
1224
|
+
// The inferred schema name encodes the definition line as
|
|
1225
|
+
// `__inferred_<var>_at_<line>`, so we can surface the exact
|
|
1226
|
+
// location without storing extra metadata.
|
|
1227
|
+
if func.returns_schema.starts_with("__inferred_") {
|
|
1228
|
+
let line_suffix = func
|
|
1229
|
+
.returns_schema
|
|
1230
|
+
.strip_prefix("__inferred_")
|
|
1231
|
+
.and_then(|s| s.rsplit_once("_at_"))
|
|
1232
|
+
.and_then(|(_, l)| l.parse::<usize>().ok())
|
|
1233
|
+
.map(|l| format!(":{l}"))
|
|
1234
|
+
.unwrap_or_default();
|
|
1235
|
+
self.schema_origins.insert(
|
|
1236
|
+
func.returns_schema.clone(),
|
|
1237
|
+
format!("{name} ({file_path_display}{line_suffix})"),
|
|
1238
|
+
);
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
// Record the parameter-column contract so calls to this function
|
|
1242
|
+
// (e.g. `trim(customers)`) can be validated at the call site,
|
|
1243
|
+
// where the actual argument's inferred schema is known.
|
|
1244
|
+
if !func.requires.is_empty() {
|
|
1245
|
+
let origin = format!("{name} ({file_path_display}:{})", func.def_line);
|
|
1246
|
+
self.param_requires
|
|
1247
|
+
.insert(name.to_string(), (func.requires.clone(), origin));
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1062
1251
|
// Check if a base class name indicates a typedframes schema
|
|
1063
1252
|
fn is_schema_base(name: &str) -> bool {
|
|
1064
1253
|
matches!(
|
|
@@ -1397,12 +1586,28 @@ impl Linter {
|
|
|
1397
1586
|
Self::scan_expr_for_contract(tainted, &attr.value, direct, delegates);
|
|
1398
1587
|
}
|
|
1399
1588
|
Expr::Call(call) => {
|
|
1400
|
-
|
|
1401
|
-
|
|
1589
|
+
let first_arg_tainted =
|
|
1590
|
+
call.arguments.args.first().is_some_and(
|
|
1402
1591
|
|a| matches!(a, Expr::Name(n) if tainted.contains(n.id.as_str())),
|
|
1403
|
-
)
|
|
1592
|
+
);
|
|
1593
|
+
match &*call.func {
|
|
1594
|
+
Expr::Name(func_name) if first_arg_tainted => {
|
|
1404
1595
|
delegates.push(func_name.id.to_string());
|
|
1405
1596
|
}
|
|
1597
|
+
// `module.func(df)` via a plain `import module` — record the bare
|
|
1598
|
+
// attribute name as a delegate candidate the same way a bare-name
|
|
1599
|
+
// call would be (resolve_delegate_target follows it back to the
|
|
1600
|
+
// module). Guarded on `base` not itself being tainted so an actual
|
|
1601
|
+
// DataFrame method call (`df.merge(df)`) is never misread as a
|
|
1602
|
+
// delegate to a function literally named `merge`.
|
|
1603
|
+
Expr::Attribute(attr) if first_arg_tainted => {
|
|
1604
|
+
if let Expr::Name(base) = &*attr.value {
|
|
1605
|
+
if !tainted.contains(base.id.as_str()) {
|
|
1606
|
+
delegates.push(attr.attr.to_string());
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
_ => {}
|
|
1406
1611
|
}
|
|
1407
1612
|
Self::scan_expr_for_contract(tainted, &call.func, direct, delegates);
|
|
1408
1613
|
for arg in &call.arguments.args {
|
|
@@ -2701,6 +2906,24 @@ impl Linter {
|
|
|
2701
2906
|
}
|
|
2702
2907
|
}
|
|
2703
2908
|
}
|
|
2909
|
+
} else if let Expr::Name(base) = current_expr {
|
|
2910
|
+
// Handle result = module.trim_customers(customers) — an
|
|
2911
|
+
// attribute-style call to a delegate reached via a plain
|
|
2912
|
+
// `import module` (see load_cross_file_symbols, which
|
|
2913
|
+
// populates self.param_requires by bare function name
|
|
2914
|
+
// regardless of how it's called at the use site). Guarded
|
|
2915
|
+
// on `base` not being a tracked DataFrame variable so a
|
|
2916
|
+
// genuine method call (`df.merge(other)`) is never treated
|
|
2917
|
+
// as a call to a same-named cross-file delegate function.
|
|
2918
|
+
if !self.variables.contains_key(base.id.as_str()) {
|
|
2919
|
+
self.check_call_requirements(
|
|
2920
|
+
attr.attr.as_str(),
|
|
2921
|
+
call,
|
|
2922
|
+
current_line,
|
|
2923
|
+
current_col,
|
|
2924
|
+
errors,
|
|
2925
|
+
);
|
|
2926
|
+
}
|
|
2704
2927
|
}
|
|
2705
2928
|
} else if let Expr::Name(func_name) = &*call.func {
|
|
2706
2929
|
// Handle df = load_users() where load_users() -> PandasFrame[Schema]
|
|
@@ -2883,6 +3106,13 @@ impl Linter {
|
|
|
2883
3106
|
col,
|
|
2884
3107
|
errors,
|
|
2885
3108
|
);
|
|
3109
|
+
// Bare call statement via a plain `import module`, e.g.
|
|
3110
|
+
// `transforms.trim_customers(customers)` — guarded on `recv`
|
|
3111
|
+
// not being a tracked DataFrame variable so a genuine method
|
|
3112
|
+
// call is never treated as a call to a same-named delegate.
|
|
3113
|
+
if !self.variables.contains_key(recv.id.as_str()) {
|
|
3114
|
+
self.check_call_requirements(func_name, call, line, col, errors);
|
|
3115
|
+
}
|
|
2886
3116
|
}
|
|
2887
3117
|
} else if let Expr::Name(func_name) = &*call.func {
|
|
2888
3118
|
// Bare call statement, e.g. `trim_customers(customers)` with no
|
|
@@ -4282,4 +4512,158 @@ print(df["revenue"]) # typedframes: ignore[unknown-column, dropped-unknown-colu
|
|
|
4282
4512
|
// assert — unknown-column is in the comma-separated list, so suppressed
|
|
4283
4513
|
assert!(errors.is_empty(), "unexpected errors: {errors:?}");
|
|
4284
4514
|
}
|
|
4515
|
+
|
|
4516
|
+
#[test]
|
|
4517
|
+
fn test_should_detect_missing_column_through_plain_module_import_attribute_call() {
|
|
4518
|
+
// arrange: same shape as test_should_detect_missing_column_at_direct_call_site,
|
|
4519
|
+
// but pipeline.py calls `steps.postproc(df)` via a plain `import steps` rather
|
|
4520
|
+
// than `from steps import postproc` — the blind spot this test guards against.
|
|
4521
|
+
let dir = tempfile::tempdir().unwrap();
|
|
4522
|
+
let root = dir.path();
|
|
4523
|
+
fs::write(root.join("pyproject.toml"), "").unwrap();
|
|
4524
|
+
fs::write(
|
|
4525
|
+
root.join("loaders.py"),
|
|
4526
|
+
r#"
|
|
4527
|
+
import pandas as pd
|
|
4528
|
+
|
|
4529
|
+
def load(path: str) -> pd.DataFrame:
|
|
4530
|
+
df = pd.read_csv(path, usecols=["a", "b"])
|
|
4531
|
+
return df
|
|
4532
|
+
"#,
|
|
4533
|
+
)
|
|
4534
|
+
.unwrap();
|
|
4535
|
+
fs::write(
|
|
4536
|
+
root.join("steps.py"),
|
|
4537
|
+
r#"
|
|
4538
|
+
import pandas as pd
|
|
4539
|
+
|
|
4540
|
+
def postproc(df: pd.DataFrame) -> pd.DataFrame:
|
|
4541
|
+
y = df["c"]
|
|
4542
|
+
return df
|
|
4543
|
+
"#,
|
|
4544
|
+
)
|
|
4545
|
+
.unwrap();
|
|
4546
|
+
let pipeline_source = r#"
|
|
4547
|
+
import steps
|
|
4548
|
+
from loaders import load
|
|
4549
|
+
|
|
4550
|
+
def process(path: str) -> None:
|
|
4551
|
+
df = load(path)
|
|
4552
|
+
result = steps.postproc(df)
|
|
4553
|
+
print(result)
|
|
4554
|
+
"#;
|
|
4555
|
+
fs::write(root.join("pipeline.py"), pipeline_source).unwrap();
|
|
4556
|
+
|
|
4557
|
+
// act
|
|
4558
|
+
let index = build_index_internal(root);
|
|
4559
|
+
let mut linter = Linter::new();
|
|
4560
|
+
let pipeline_path = root.join("pipeline.py");
|
|
4561
|
+
linter.load_cross_file_symbols(&index, pipeline_source, &pipeline_path, root);
|
|
4562
|
+
let errors = linter
|
|
4563
|
+
.check_file_internal(pipeline_source, &pipeline_path)
|
|
4564
|
+
.unwrap();
|
|
4565
|
+
|
|
4566
|
+
// assert
|
|
4567
|
+
assert_eq!(errors.len(), 1, "expected one error, got: {errors:?}");
|
|
4568
|
+
assert_eq!(errors[0].code, "missing-column");
|
|
4569
|
+
assert!(errors[0].message.contains("missing column(s) {c}"));
|
|
4570
|
+
assert!(errors[0].message.contains("passed to postproc"));
|
|
4571
|
+
}
|
|
4572
|
+
|
|
4573
|
+
#[test]
|
|
4574
|
+
fn test_should_resolve_transitive_requires_through_plain_import_delegate() {
|
|
4575
|
+
// arrange: transform() only forwards df to steps.postproc(df) via a plain
|
|
4576
|
+
// `import steps` — the delegate-detection blind spot for attribute calls.
|
|
4577
|
+
let dir = tempfile::tempdir().unwrap();
|
|
4578
|
+
let root = dir.path();
|
|
4579
|
+
fs::write(root.join("pyproject.toml"), "").unwrap();
|
|
4580
|
+
fs::write(
|
|
4581
|
+
root.join("steps.py"),
|
|
4582
|
+
r#"
|
|
4583
|
+
import pandas as pd
|
|
4584
|
+
|
|
4585
|
+
def postproc(df: pd.DataFrame) -> pd.DataFrame:
|
|
4586
|
+
y = df["c"]
|
|
4587
|
+
return df
|
|
4588
|
+
"#,
|
|
4589
|
+
)
|
|
4590
|
+
.unwrap();
|
|
4591
|
+
fs::write(
|
|
4592
|
+
root.join("transforms.py"),
|
|
4593
|
+
r#"
|
|
4594
|
+
import steps
|
|
4595
|
+
|
|
4596
|
+
def transform(df):
|
|
4597
|
+
return steps.postproc(df)
|
|
4598
|
+
"#,
|
|
4599
|
+
)
|
|
4600
|
+
.unwrap();
|
|
4601
|
+
|
|
4602
|
+
// act
|
|
4603
|
+
let index = build_index_internal(root);
|
|
4604
|
+
let transforms_path = root.join("transforms.py").to_str().unwrap().to_string();
|
|
4605
|
+
let func = &index.files[&transforms_path].functions["transform"];
|
|
4606
|
+
|
|
4607
|
+
// assert: transform's own requires is the transitive union through the
|
|
4608
|
+
// attribute-style delegate call, i.e. {c} — not empty, which is what it
|
|
4609
|
+
// would be if the `steps.postproc(df)` call were never recognised as a
|
|
4610
|
+
// delegate at all.
|
|
4611
|
+
assert_eq!(func.requires, vec!["c".to_string()]);
|
|
4612
|
+
}
|
|
4613
|
+
|
|
4614
|
+
#[test]
|
|
4615
|
+
fn test_should_detect_missing_column_through_wildcard_import() {
|
|
4616
|
+
// arrange: pipeline.py brings postproc into scope via `from steps import *`
|
|
4617
|
+
// rather than naming it explicitly.
|
|
4618
|
+
let dir = tempfile::tempdir().unwrap();
|
|
4619
|
+
let root = dir.path();
|
|
4620
|
+
fs::write(root.join("pyproject.toml"), "").unwrap();
|
|
4621
|
+
fs::write(
|
|
4622
|
+
root.join("loaders.py"),
|
|
4623
|
+
r#"
|
|
4624
|
+
import pandas as pd
|
|
4625
|
+
|
|
4626
|
+
def load(path: str) -> pd.DataFrame:
|
|
4627
|
+
df = pd.read_csv(path, usecols=["a", "b"])
|
|
4628
|
+
return df
|
|
4629
|
+
"#,
|
|
4630
|
+
)
|
|
4631
|
+
.unwrap();
|
|
4632
|
+
fs::write(
|
|
4633
|
+
root.join("steps.py"),
|
|
4634
|
+
r#"
|
|
4635
|
+
import pandas as pd
|
|
4636
|
+
|
|
4637
|
+
def postproc(df: pd.DataFrame) -> pd.DataFrame:
|
|
4638
|
+
y = df["c"]
|
|
4639
|
+
return df
|
|
4640
|
+
"#,
|
|
4641
|
+
)
|
|
4642
|
+
.unwrap();
|
|
4643
|
+
let pipeline_source = r#"
|
|
4644
|
+
from steps import *
|
|
4645
|
+
from loaders import load
|
|
4646
|
+
|
|
4647
|
+
def process(path: str) -> None:
|
|
4648
|
+
df = load(path)
|
|
4649
|
+
result = postproc(df)
|
|
4650
|
+
print(result)
|
|
4651
|
+
"#;
|
|
4652
|
+
fs::write(root.join("pipeline.py"), pipeline_source).unwrap();
|
|
4653
|
+
|
|
4654
|
+
// act
|
|
4655
|
+
let index = build_index_internal(root);
|
|
4656
|
+
let mut linter = Linter::new();
|
|
4657
|
+
let pipeline_path = root.join("pipeline.py");
|
|
4658
|
+
linter.load_cross_file_symbols(&index, pipeline_source, &pipeline_path, root);
|
|
4659
|
+
let errors = linter
|
|
4660
|
+
.check_file_internal(pipeline_source, &pipeline_path)
|
|
4661
|
+
.unwrap();
|
|
4662
|
+
|
|
4663
|
+
// assert
|
|
4664
|
+
assert_eq!(errors.len(), 1, "expected one error, got: {errors:?}");
|
|
4665
|
+
assert_eq!(errors[0].code, "missing-column");
|
|
4666
|
+
assert!(errors[0].message.contains("missing column(s) {c}"));
|
|
4667
|
+
assert!(errors[0].message.contains("passed to postproc"));
|
|
4668
|
+
}
|
|
4285
4669
|
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|