typedframes 0.14.4__tar.gz → 0.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {typedframes-0.14.4 → typedframes-0.15.0}/PKG-INFO +73 -22
- {typedframes-0.14.4 → typedframes-0.15.0}/README.md +72 -21
- {typedframes-0.14.4 → typedframes-0.15.0}/pyproject.toml +16 -7
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/Cargo.lock +118 -119
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/Cargo.toml +2 -1
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/python/Cargo.toml +1 -1
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/apply.rs +355 -17
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/ast_extract.rs +184 -18
- typedframes-0.15.0/rust/src/call_contracts.rs +822 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/args.rs +19 -3
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/check.rs +247 -40
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/coverage.rs +56 -17
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/suggest.rs +54 -23
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/column_usage.rs +612 -8
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/config.rs +92 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/constants.rs +200 -2
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/contract.rs +88 -1
- typedframes-0.15.0/rust/src/dict_frames.rs +283 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/engine.rs +71 -17
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/errors.rs +18 -1
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/frame_ops.rs +346 -4
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/index.rs +598 -31
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/lib.rs +6 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/linter.rs +18385 -10004
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/notebook.rs +4 -1
- typedframes-0.15.0/rust/src/polars_names.rs +219 -0
- typedframes-0.15.0/rust/src/query_names.rs +147 -0
- typedframes-0.15.0/rust/src/returned_loads.rs +145 -0
- typedframes-0.15.0/rust/src/self_frames.rs +236 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/static_eval.rs +708 -20
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/suggest.rs +1367 -43
- typedframes-0.15.0/rust/src/typo.rs +337 -0
- typedframes-0.15.0/rust/tests/cli.rs +1455 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/__init__.py +1 -1
- typedframes-0.14.4/rust/src/typo.rs +0 -90
- typedframes-0.14.4/rust/tests/cli.rs +0 -745
- {typedframes-0.14.4 → typedframes-0.15.0}/.cargo/config.toml +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/LICENSE +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/README.md +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/benches/parser_bench.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/build.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/python/build.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/python/src/lib.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/python-data/scripts/typedframes-py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/call_usage.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/cache.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/collect.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/config.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/mod.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/pathutil.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/pyfmt.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/report.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/cli/style.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/column_consuming_methods.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/fast_hash.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/helpers.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/library_names.txt +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/main.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/parallel.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/phrase.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/report.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/report_minimize.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/report_names.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/sql.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/src/words.txt +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/rust/tests/integration_test.rs +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/__main__.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/_binary.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/_rust.pyi +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/base_schema.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/column.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/column_group.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/column_group_error.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/column_set.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/missing_dependency_error.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/mypy.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/pandera.py +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/py.typed +0 -0
- {typedframes-0.14.4 → typedframes-0.15.0}/src/typedframes/schema_algebra.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: typedframes
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.0
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -44,7 +44,7 @@ Project-URL: Repository, https://github.com/w-martin/typedframes
|
|
|
44
44
|
|
|
45
45
|
> ⚠️ **Project Status: Proof of Concept**
|
|
46
46
|
>
|
|
47
|
-
> `typedframes` (v0.
|
|
47
|
+
> `typedframes` (v0.15.0) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
|
|
48
48
|
> integrations work, but expect rough edges. The codebase prioritizes demonstrating the viability of static DataFrame
|
|
49
49
|
> column checking over production-grade stability.
|
|
50
50
|
>
|
|
@@ -70,7 +70,8 @@ typedframes check src/ --coverage-fail-under=90
|
|
|
70
70
|
`--coverage-fail-under=N` is the same idea as a test-coverage or mypy type-coverage gate, applied to how many DataFrames
|
|
71
71
|
the checker can resolve to a concrete column list — a DataFrame it merely recognizes but can't resolve columns for
|
|
72
72
|
(a Feast retrieval whose entity_df isn't itself concrete, a bare `-> pd.DataFrame` return with no attached Schema)
|
|
73
|
-
does not count, and
|
|
73
|
+
does not count, and the first column accessed on it is flagged with an `unverifiable-column` warning (once per frame,
|
|
74
|
+
with the fix) rather than passed through silently.
|
|
74
75
|
See [DataFrame Schema Coverage Thresholds](#dataframe-schema-coverage-thresholds-opt-in).
|
|
75
76
|
Add `BaseSchema` classes later for cross-file awareness and IDE autocomplete — see [Quick Start](#quick-start).
|
|
76
77
|
|
|
@@ -109,7 +110,7 @@ tracking and IDE autocomplete; they're a progressive enhancement, not a prerequi
|
|
|
109
110
|
|
|
110
111
|
**What you get:**
|
|
111
112
|
|
|
112
|
-
- ✅ **Works without schema annotations** - Column inference from `usecols=`, `
|
|
113
|
+
- ✅ **Works without schema annotations** - Column inference from `usecols=`, `columns=`, dict literals, and method chains catches errors on unannotated code
|
|
113
114
|
- ✅ **CI gate via coverage threshold** - `--coverage-fail-under=N` fails the build when too much of your codebase is invisible to the checker, the same way you'd gate on test coverage or a type checker's type-coverage number
|
|
114
115
|
- ✅ **Rust-fast** - Milliseconds, not seconds, even on hundreds of files; fast enough for pre-commit hooks and CI (see [benchmarks](#static-analysis-performance))
|
|
115
116
|
- ✅ **Cross-file awareness** - Add `BaseSchema` and typed return annotations to follow schemas across module boundaries
|
|
@@ -140,6 +141,23 @@ pip install typedframes[pandas] # includes pandas
|
|
|
140
141
|
pip install typedframes[polars] # includes polars
|
|
141
142
|
```
|
|
142
143
|
|
|
144
|
+
Without Python, for example in a minimal container, each release also carries a static Linux executable
|
|
145
|
+
(`x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl`) with no libc dependency, so it runs on any distribution,
|
|
146
|
+
including Alpine and an empty `scratch` image:
|
|
147
|
+
|
|
148
|
+
```shell
|
|
149
|
+
curl -fsSL https://github.com/w-martin/typedframes/releases/latest/download/typedframes-x86_64-unknown-linux-musl.tar.gz \
|
|
150
|
+
| tar -xz --strip-components=1 typedframes-x86_64-unknown-linux-musl/typedframes
|
|
151
|
+
./typedframes check src
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Each archive has a matching `.sha256` file next to it, and a build provenance attestation signed by the release
|
|
155
|
+
workflow. To verify an archive you have downloaded:
|
|
156
|
+
|
|
157
|
+
```shell
|
|
158
|
+
gh attestation verify typedframes-x86_64-unknown-linux-musl.tar.gz --repo w-martin/typedframes
|
|
159
|
+
```
|
|
160
|
+
|
|
143
161
|
Where the Python interpreter may run code but no other executable may be started, set `TYPEDFRAMES_INPROC=1`. Then
|
|
144
162
|
`python -m typedframes` and the mypy plugin run the same checker inside the interpreter, through an extension module in
|
|
145
163
|
the wheel, instead of starting the `typedframes` executable. Results and exit status are the same; when the checker is
|
|
@@ -251,7 +269,10 @@ checker catches real bugs from day one without it.
|
|
|
251
269
|
### Inferred Schemas
|
|
252
270
|
|
|
253
271
|
When you pass `usecols=` (pandas) or `schema=` / `columns=` (polars), the checker builds an inferred column set
|
|
254
|
-
and validates all subscript access against it — no schema annotation required
|
|
272
|
+
and validates all subscript access against it — no schema annotation required. A pandas `dtype=` mapping is not
|
|
273
|
+
a column list (it types some columns and says nothing about the rest); polars' `schema=` is complete. Writing a
|
|
274
|
+
new column (`df["new"] = ...`, `df[["a", "b"]] = ...`, `df.loc[mask, "flag"] = ...`) adds it to an inferred set;
|
|
275
|
+
only a declared `BaseSchema` rejects a column it does not list:
|
|
255
276
|
|
|
256
277
|
```python
|
|
257
278
|
# Checker infers {user_id, email} from usecols= — no annotation needed
|
|
@@ -284,6 +305,10 @@ trimmed = df.drop(columns=["age"])
|
|
|
284
305
|
# assign() — new column added to inferred set
|
|
285
306
|
augmented = df.assign(created_at="2024-01-01")
|
|
286
307
|
print(augmented["created_at"]) # ✓ OK
|
|
308
|
+
|
|
309
|
+
# query() / eval() strings — bare names are checked as columns
|
|
310
|
+
adults = df.query("age >= 18") # ✓ OK
|
|
311
|
+
# df.query("agee >= 18") # ✗ Error: 'agee' not in inferred column set
|
|
287
312
|
```
|
|
288
313
|
|
|
289
314
|
### Inference Gaps and Warnings
|
|
@@ -333,6 +358,17 @@ trimmed = df.drop(columns=["nonexistent"])
|
|
|
333
358
|
# ⚠ dropped-unknown-column: Dropped column 'nonexistent' does not exist in UserData
|
|
334
359
|
```
|
|
335
360
|
|
|
361
|
+
**Even without a column list, what a frame lost is known.** After
|
|
362
|
+
`items.drop(columns=["created_at"], inplace=True)`, `items = items.rename(columns={"old": "new"})` or
|
|
363
|
+
`pd.get_dummies(df, columns=["category"])`, reading `created_at`, `old` or `category` off that frame is an
|
|
364
|
+
`unknown-column` error, and reading a column ahead of the only statement that assigns it is the warning
|
|
365
|
+
`column-read-before-assignment` (the data may already have it). `a.merge(b)` follows pandas' `_x`/`_y`
|
|
366
|
+
(or `suffixes=`) renaming of the columns both sides carry. A column read once on such a frame, named nowhere
|
|
367
|
+
else in the file, and one slip from exactly one name the code does mention raises the warning
|
|
368
|
+
`likely-column-typo`. Details in
|
|
369
|
+
[the usage guide](docs/usage.md#frames-known-only-by-name) and the
|
|
370
|
+
[method matrix](docs/method-matrix.md).
|
|
371
|
+
|
|
336
372
|
### Function Parameter Contracts
|
|
337
373
|
|
|
338
374
|
Beyond validating access at the point it happens, the checker infers a *contract* for any function's
|
|
@@ -354,11 +390,25 @@ contact_label(customers)
|
|
|
354
390
|
# column(s) {email} — available: {customer_id, name, region}, required: {email, name}
|
|
355
391
|
```
|
|
356
392
|
|
|
393
|
+
The body counts only when every statement in it runs on each call: a read behind an `if`, a loop, a
|
|
394
|
+
conditional expression or `and`/`or` (`if "a" in df.columns: return df["a"]`) is no requirement.
|
|
395
|
+
|
|
357
396
|
The contract is resolved *transitively*: if a function only forwards its parameter to other functions
|
|
358
397
|
(`step1 = preprocess(df); step2 = enrich(step1)`), the checker follows the chain and unions their
|
|
359
398
|
requirements, catching a missing column even when no single function in the chain touches it directly.
|
|
360
399
|
Column-list slices (`df[["a", "b"]]`) contribute to the contract too.
|
|
361
400
|
|
|
401
|
+
Every other parameter has a contract too: the columns the body reads off it on every path, before
|
|
402
|
+
anything could have added them (reads after a branch, loop or `return`, or after the frame is written
|
|
403
|
+
to or handed to another function, don't count). Arguments are matched to parameters by position and by
|
|
404
|
+
keyword (`combine(left, right=orders)`), and a parameter passed straight on to another helper takes on
|
|
405
|
+
that helper's requirement, one level deep. A call is checked only when its callee is one known function:
|
|
406
|
+
a local function, an imported one, `module.helper(...)`, `self.helper(...)` (unless an instance
|
|
407
|
+
attribute of that name shadows it, a subclass in another file overrides it, or one in the same file
|
|
408
|
+
overrides it with a different requirement), or a method on a `Protocol`-typed receiver whose every
|
|
409
|
+
implementation in the project requires the same columns. Calls with `*args`/`**kwargs`, names bound to
|
|
410
|
+
two functions, and frames whose columns are unknown are left alone.
|
|
411
|
+
|
|
362
412
|
**Known limitations:**
|
|
363
413
|
- Cross-file delegate/schema resolution follows `from module import name`, plain `import module` +
|
|
364
414
|
`module.helper(df)` attribute access, and `from module import *` wildcard imports. A dotted import with no
|
|
@@ -369,9 +419,8 @@ Column-list slices (`df[["a", "b"]]`) contribute to the contract too.
|
|
|
369
419
|
true requirements.
|
|
370
420
|
- A cycle in the delegate graph (mutually- or self-delegating helpers) contributes only each function's own
|
|
371
421
|
direct requirements to the cycle, not the full transitive union — conservative rather than exhaustive.
|
|
372
|
-
-
|
|
373
|
-
|
|
374
|
-
module the call site actually used.
|
|
422
|
+
- The first parameter's transitive delegate chain, resolved at the index level, may still follow a bare
|
|
423
|
+
delegate name to whichever plainly-imported module defines it first.
|
|
375
424
|
|
|
376
425
|
### See Also
|
|
377
426
|
|
|
@@ -515,7 +564,7 @@ repos:
|
|
|
515
564
|
name: typedframes check
|
|
516
565
|
entry: typedframes check . --strict
|
|
517
566
|
language: python
|
|
518
|
-
additional_dependencies: ["typedframes==0.
|
|
567
|
+
additional_dependencies: ["typedframes==0.15.0"]
|
|
519
568
|
types_or: [python, jupyter]
|
|
520
569
|
pass_filenames: false
|
|
521
570
|
```
|
|
@@ -564,7 +613,7 @@ The action installs the PyPI wheel into a throwaway virtualenv and runs the chec
|
|
|
564
613
|
| Input | Default | |
|
|
565
614
|
|-------|---------|-|
|
|
566
615
|
| `path` | `.` | File or directory to check |
|
|
567
|
-
| `version` | `latest` | PyPI version to install, e.g. `"0.
|
|
616
|
+
| `version` | `latest` | PyPI version to install, e.g. `"0.15.0"` |
|
|
568
617
|
| `strict` | `true` | Fail the step on errors — `typedframes check` exits 0 without it |
|
|
569
618
|
| `coverage-fail-under` | *(unset)* | Minimum DataFrame schema coverage, e.g. `"90"` |
|
|
570
619
|
| `coverage-detail` | `summary` | Or `term-missing` for the per-file breakdown, `explain` to diagnose a lower-than-expected total |
|
|
@@ -582,6 +631,7 @@ typedframes suggest src/ # dry run: prints diffs, c
|
|
|
582
631
|
typedframes suggest src/ --apply # write the safe suggestions
|
|
583
632
|
typedframes suggest src/ --apply --only quiet-amber # act on just that site
|
|
584
633
|
typedframes suggest src/ --interactive # ask about each suggestion
|
|
634
|
+
typedframes suggest notebooks/ src/ # several paths, each looked at on its own
|
|
585
635
|
```
|
|
586
636
|
|
|
587
637
|
Edits need a clean git work tree and tracked files, each edited file is re-checked in memory before anything is written,
|
|
@@ -692,9 +742,10 @@ the
|
|
|
692
742
|
Some DataFrames are recognized but never resolve to a concrete column list — a Feast
|
|
693
743
|
retrieval whose `entity_df` isn't itself concretely typed, or a bare `-> pd.DataFrame`
|
|
694
744
|
return with no attached `Schema`. These do **not** count as covered, and accessing any
|
|
695
|
-
column on one is flagged `unverifiable-column`
|
|
696
|
-
the checker genuinely doesn't know whether the column exists, so it
|
|
697
|
-
guessing either way
|
|
745
|
+
column on one is flagged with an `unverifiable-column` warning (once per frame, not once per use) rather
|
|
746
|
+
than passed through silently — the checker genuinely doesn't know whether the column exists, so it
|
|
747
|
+
says so instead of guessing either way; a warning rather than an error, because not knowing is not the
|
|
748
|
+
same as being wrong (it fails the run under `--strict`). Inside a `test_*` function the warning is not raised: the test's own reads are the run-time check. A Feast retrieval whose `entity_df` argument IS a concretely typed
|
|
698
749
|
DataFrame is fully resolved instead: Feast's real output is `entity_df`'s own columns
|
|
699
750
|
plus the requested features, so both halves get checked normally. See
|
|
700
751
|
[Unresolved schemas](docs/usage.md#unresolved-schemas-recognized-but-not-column-checked).
|
|
@@ -704,17 +755,17 @@ plus the requested features, so both halves get checked normally. See
|
|
|
704
755
|
Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
|
|
705
756
|
|
|
706
757
|
**Benchmark results** (median of 5 runs ± one standard deviation, 3 warmup, caches cleared between runs):
|
|
707
|
-
*2026-
|
|
758
|
+
*2026-11-15 · Darwin 27.2.0 · arm · CPython 3.15.0 · 64GiB RAM · Great Expectations pinned @ 1.23.2*
|
|
708
759
|
|
|
709
760
|
| Tool | Version | What it does | typedframes (12 files) | great_expectations (494 files) |
|
|
710
761
|
|------|---------|--------------|------------------------|--------------------------------|
|
|
711
|
-
| typedframes | 0.
|
|
712
|
-
| ruff | 0.17.
|
|
713
|
-
| ty | 0.0.
|
|
714
|
-
| pyrefly | 1.4.0 | Type checker |
|
|
715
|
-
| mypy | 2.4.
|
|
716
|
-
| mypy + typedframes | 2.4.
|
|
717
|
-
| pyright | 1.1.414 | Type checker |
|
|
762
|
+
| typedframes | 0.15.0 | DataFrame column checker | 19.6 ± 0.3 ms | 115.8 ± 1.7 ms |
|
|
763
|
+
| ruff | 0.17.3 | Linter (no type checking) | 24.3 ± 0.6 ms | 220.7 ± 3.2 ms |
|
|
764
|
+
| ty | 0.0.95 | Type checker | 52.5 ± 2.3 ms | 768.4 ± 18.5 ms |
|
|
765
|
+
| pyrefly | 1.4.0 | Type checker | 109.3 ± 16.9 ms | 272.8 ± 11.4 ms |
|
|
766
|
+
| mypy | 2.4.1 | Type checker (no plugin) | 1.83 ± 0.14 s | 3.06 ± 0.03 s |
|
|
767
|
+
| mypy + typedframes | 2.4.1 | Type checker + column checker | 1.84 ± 0.02 s | 3.30 ± 0.01 s |
|
|
768
|
+
| pyright | 1.1.414 | Type checker | 674.0 ± 5.5 ms | 3.93 ± 0.04 s |
|
|
718
769
|
|
|
719
770
|
*Run `uv run python benchmarks/benchmark_checkers.py` to reproduce.*
|
|
720
771
|
|
|
@@ -1242,7 +1293,7 @@ A: The mypy plugin doesn't work with pyright. Use the standalone checker (`typed
|
|
|
1242
1293
|
validation. Schema descriptor access (`df[Schema.column]`) works natively in pyright without any plugin.
|
|
1243
1294
|
|
|
1244
1295
|
**Q: Do I need to write `BaseSchema` classes to get value?**
|
|
1245
|
-
A: No. The standalone checker works entirely from inference: `usecols=`/`columns
|
|
1296
|
+
A: No. The standalone checker works entirely from inference: `usecols=`/`columns=` arguments on read
|
|
1246
1297
|
calls give it enough information to validate column access and propagate that knowledge through method chains
|
|
1247
1298
|
(`rename`, `drop`, `assign`, `select`, …). `BaseSchema` is a progressive enhancement that unlocks cross-file
|
|
1248
1299
|
awareness (schemas travel with function return types across module boundaries) and IDE autocomplete via
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
> ⚠️ **Project Status: Proof of Concept**
|
|
10
10
|
>
|
|
11
|
-
> `typedframes` (v0.
|
|
11
|
+
> `typedframes` (v0.15.0) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
|
|
12
12
|
> integrations work, but expect rough edges. The codebase prioritizes demonstrating the viability of static DataFrame
|
|
13
13
|
> column checking over production-grade stability.
|
|
14
14
|
>
|
|
@@ -34,7 +34,8 @@ typedframes check src/ --coverage-fail-under=90
|
|
|
34
34
|
`--coverage-fail-under=N` is the same idea as a test-coverage or mypy type-coverage gate, applied to how many DataFrames
|
|
35
35
|
the checker can resolve to a concrete column list — a DataFrame it merely recognizes but can't resolve columns for
|
|
36
36
|
(a Feast retrieval whose entity_df isn't itself concrete, a bare `-> pd.DataFrame` return with no attached Schema)
|
|
37
|
-
does not count, and
|
|
37
|
+
does not count, and the first column accessed on it is flagged with an `unverifiable-column` warning (once per frame,
|
|
38
|
+
with the fix) rather than passed through silently.
|
|
38
39
|
See [DataFrame Schema Coverage Thresholds](#dataframe-schema-coverage-thresholds-opt-in).
|
|
39
40
|
Add `BaseSchema` classes later for cross-file awareness and IDE autocomplete — see [Quick Start](#quick-start).
|
|
40
41
|
|
|
@@ -73,7 +74,7 @@ tracking and IDE autocomplete; they're a progressive enhancement, not a prerequi
|
|
|
73
74
|
|
|
74
75
|
**What you get:**
|
|
75
76
|
|
|
76
|
-
- ✅ **Works without schema annotations** - Column inference from `usecols=`, `
|
|
77
|
+
- ✅ **Works without schema annotations** - Column inference from `usecols=`, `columns=`, dict literals, and method chains catches errors on unannotated code
|
|
77
78
|
- ✅ **CI gate via coverage threshold** - `--coverage-fail-under=N` fails the build when too much of your codebase is invisible to the checker, the same way you'd gate on test coverage or a type checker's type-coverage number
|
|
78
79
|
- ✅ **Rust-fast** - Milliseconds, not seconds, even on hundreds of files; fast enough for pre-commit hooks and CI (see [benchmarks](#static-analysis-performance))
|
|
79
80
|
- ✅ **Cross-file awareness** - Add `BaseSchema` and typed return annotations to follow schemas across module boundaries
|
|
@@ -104,6 +105,23 @@ pip install typedframes[pandas] # includes pandas
|
|
|
104
105
|
pip install typedframes[polars] # includes polars
|
|
105
106
|
```
|
|
106
107
|
|
|
108
|
+
Without Python, for example in a minimal container, each release also carries a static Linux executable
|
|
109
|
+
(`x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl`) with no libc dependency, so it runs on any distribution,
|
|
110
|
+
including Alpine and an empty `scratch` image:
|
|
111
|
+
|
|
112
|
+
```shell
|
|
113
|
+
curl -fsSL https://github.com/w-martin/typedframes/releases/latest/download/typedframes-x86_64-unknown-linux-musl.tar.gz \
|
|
114
|
+
| tar -xz --strip-components=1 typedframes-x86_64-unknown-linux-musl/typedframes
|
|
115
|
+
./typedframes check src
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Each archive has a matching `.sha256` file next to it, and a build provenance attestation signed by the release
|
|
119
|
+
workflow. To verify an archive you have downloaded:
|
|
120
|
+
|
|
121
|
+
```shell
|
|
122
|
+
gh attestation verify typedframes-x86_64-unknown-linux-musl.tar.gz --repo w-martin/typedframes
|
|
123
|
+
```
|
|
124
|
+
|
|
107
125
|
Where the Python interpreter may run code but no other executable may be started, set `TYPEDFRAMES_INPROC=1`. Then
|
|
108
126
|
`python -m typedframes` and the mypy plugin run the same checker inside the interpreter, through an extension module in
|
|
109
127
|
the wheel, instead of starting the `typedframes` executable. Results and exit status are the same; when the checker is
|
|
@@ -215,7 +233,10 @@ checker catches real bugs from day one without it.
|
|
|
215
233
|
### Inferred Schemas
|
|
216
234
|
|
|
217
235
|
When you pass `usecols=` (pandas) or `schema=` / `columns=` (polars), the checker builds an inferred column set
|
|
218
|
-
and validates all subscript access against it — no schema annotation required
|
|
236
|
+
and validates all subscript access against it — no schema annotation required. A pandas `dtype=` mapping is not
|
|
237
|
+
a column list (it types some columns and says nothing about the rest); polars' `schema=` is complete. Writing a
|
|
238
|
+
new column (`df["new"] = ...`, `df[["a", "b"]] = ...`, `df.loc[mask, "flag"] = ...`) adds it to an inferred set;
|
|
239
|
+
only a declared `BaseSchema` rejects a column it does not list:
|
|
219
240
|
|
|
220
241
|
```python
|
|
221
242
|
# Checker infers {user_id, email} from usecols= — no annotation needed
|
|
@@ -248,6 +269,10 @@ trimmed = df.drop(columns=["age"])
|
|
|
248
269
|
# assign() — new column added to inferred set
|
|
249
270
|
augmented = df.assign(created_at="2024-01-01")
|
|
250
271
|
print(augmented["created_at"]) # ✓ OK
|
|
272
|
+
|
|
273
|
+
# query() / eval() strings — bare names are checked as columns
|
|
274
|
+
adults = df.query("age >= 18") # ✓ OK
|
|
275
|
+
# df.query("agee >= 18") # ✗ Error: 'agee' not in inferred column set
|
|
251
276
|
```
|
|
252
277
|
|
|
253
278
|
### Inference Gaps and Warnings
|
|
@@ -297,6 +322,17 @@ trimmed = df.drop(columns=["nonexistent"])
|
|
|
297
322
|
# ⚠ dropped-unknown-column: Dropped column 'nonexistent' does not exist in UserData
|
|
298
323
|
```
|
|
299
324
|
|
|
325
|
+
**Even without a column list, what a frame lost is known.** After
|
|
326
|
+
`items.drop(columns=["created_at"], inplace=True)`, `items = items.rename(columns={"old": "new"})` or
|
|
327
|
+
`pd.get_dummies(df, columns=["category"])`, reading `created_at`, `old` or `category` off that frame is an
|
|
328
|
+
`unknown-column` error, and reading a column ahead of the only statement that assigns it is the warning
|
|
329
|
+
`column-read-before-assignment` (the data may already have it). `a.merge(b)` follows pandas' `_x`/`_y`
|
|
330
|
+
(or `suffixes=`) renaming of the columns both sides carry. A column read once on such a frame, named nowhere
|
|
331
|
+
else in the file, and one slip from exactly one name the code does mention raises the warning
|
|
332
|
+
`likely-column-typo`. Details in
|
|
333
|
+
[the usage guide](docs/usage.md#frames-known-only-by-name) and the
|
|
334
|
+
[method matrix](docs/method-matrix.md).
|
|
335
|
+
|
|
300
336
|
### Function Parameter Contracts
|
|
301
337
|
|
|
302
338
|
Beyond validating access at the point it happens, the checker infers a *contract* for any function's
|
|
@@ -318,11 +354,25 @@ contact_label(customers)
|
|
|
318
354
|
# column(s) {email} — available: {customer_id, name, region}, required: {email, name}
|
|
319
355
|
```
|
|
320
356
|
|
|
357
|
+
The body counts only when every statement in it runs on each call: a read behind an `if`, a loop, a
|
|
358
|
+
conditional expression or `and`/`or` (`if "a" in df.columns: return df["a"]`) is no requirement.
|
|
359
|
+
|
|
321
360
|
The contract is resolved *transitively*: if a function only forwards its parameter to other functions
|
|
322
361
|
(`step1 = preprocess(df); step2 = enrich(step1)`), the checker follows the chain and unions their
|
|
323
362
|
requirements, catching a missing column even when no single function in the chain touches it directly.
|
|
324
363
|
Column-list slices (`df[["a", "b"]]`) contribute to the contract too.
|
|
325
364
|
|
|
365
|
+
Every other parameter has a contract too: the columns the body reads off it on every path, before
|
|
366
|
+
anything could have added them (reads after a branch, loop or `return`, or after the frame is written
|
|
367
|
+
to or handed to another function, don't count). Arguments are matched to parameters by position and by
|
|
368
|
+
keyword (`combine(left, right=orders)`), and a parameter passed straight on to another helper takes on
|
|
369
|
+
that helper's requirement, one level deep. A call is checked only when its callee is one known function:
|
|
370
|
+
a local function, an imported one, `module.helper(...)`, `self.helper(...)` (unless an instance
|
|
371
|
+
attribute of that name shadows it, a subclass in another file overrides it, or one in the same file
|
|
372
|
+
overrides it with a different requirement), or a method on a `Protocol`-typed receiver whose every
|
|
373
|
+
implementation in the project requires the same columns. Calls with `*args`/`**kwargs`, names bound to
|
|
374
|
+
two functions, and frames whose columns are unknown are left alone.
|
|
375
|
+
|
|
326
376
|
**Known limitations:**
|
|
327
377
|
- Cross-file delegate/schema resolution follows `from module import name`, plain `import module` +
|
|
328
378
|
`module.helper(df)` attribute access, and `from module import *` wildcard imports. A dotted import with no
|
|
@@ -333,9 +383,8 @@ Column-list slices (`df[["a", "b"]]`) contribute to the contract too.
|
|
|
333
383
|
true requirements.
|
|
334
384
|
- A cycle in the delegate graph (mutually- or self-delegating helpers) contributes only each function's own
|
|
335
385
|
direct requirements to the cycle, not the full transitive union — conservative rather than exhaustive.
|
|
336
|
-
-
|
|
337
|
-
|
|
338
|
-
module the call site actually used.
|
|
386
|
+
- The first parameter's transitive delegate chain, resolved at the index level, may still follow a bare
|
|
387
|
+
delegate name to whichever plainly-imported module defines it first.
|
|
339
388
|
|
|
340
389
|
### See Also
|
|
341
390
|
|
|
@@ -479,7 +528,7 @@ repos:
|
|
|
479
528
|
name: typedframes check
|
|
480
529
|
entry: typedframes check . --strict
|
|
481
530
|
language: python
|
|
482
|
-
additional_dependencies: ["typedframes==0.
|
|
531
|
+
additional_dependencies: ["typedframes==0.15.0"]
|
|
483
532
|
types_or: [python, jupyter]
|
|
484
533
|
pass_filenames: false
|
|
485
534
|
```
|
|
@@ -528,7 +577,7 @@ The action installs the PyPI wheel into a throwaway virtualenv and runs the chec
|
|
|
528
577
|
| Input | Default | |
|
|
529
578
|
|-------|---------|-|
|
|
530
579
|
| `path` | `.` | File or directory to check |
|
|
531
|
-
| `version` | `latest` | PyPI version to install, e.g. `"0.
|
|
580
|
+
| `version` | `latest` | PyPI version to install, e.g. `"0.15.0"` |
|
|
532
581
|
| `strict` | `true` | Fail the step on errors — `typedframes check` exits 0 without it |
|
|
533
582
|
| `coverage-fail-under` | *(unset)* | Minimum DataFrame schema coverage, e.g. `"90"` |
|
|
534
583
|
| `coverage-detail` | `summary` | Or `term-missing` for the per-file breakdown, `explain` to diagnose a lower-than-expected total |
|
|
@@ -546,6 +595,7 @@ typedframes suggest src/ # dry run: prints diffs, c
|
|
|
546
595
|
typedframes suggest src/ --apply # write the safe suggestions
|
|
547
596
|
typedframes suggest src/ --apply --only quiet-amber # act on just that site
|
|
548
597
|
typedframes suggest src/ --interactive # ask about each suggestion
|
|
598
|
+
typedframes suggest notebooks/ src/ # several paths, each looked at on its own
|
|
549
599
|
```
|
|
550
600
|
|
|
551
601
|
Edits need a clean git work tree and tracked files, each edited file is re-checked in memory before anything is written,
|
|
@@ -656,9 +706,10 @@ the
|
|
|
656
706
|
Some DataFrames are recognized but never resolve to a concrete column list — a Feast
|
|
657
707
|
retrieval whose `entity_df` isn't itself concretely typed, or a bare `-> pd.DataFrame`
|
|
658
708
|
return with no attached `Schema`. These do **not** count as covered, and accessing any
|
|
659
|
-
column on one is flagged `unverifiable-column`
|
|
660
|
-
the checker genuinely doesn't know whether the column exists, so it
|
|
661
|
-
guessing either way
|
|
709
|
+
column on one is flagged with an `unverifiable-column` warning (once per frame, not once per use) rather
|
|
710
|
+
than passed through silently — the checker genuinely doesn't know whether the column exists, so it
|
|
711
|
+
says so instead of guessing either way; a warning rather than an error, because not knowing is not the
|
|
712
|
+
same as being wrong (it fails the run under `--strict`). Inside a `test_*` function the warning is not raised: the test's own reads are the run-time check. A Feast retrieval whose `entity_df` argument IS a concretely typed
|
|
662
713
|
DataFrame is fully resolved instead: Feast's real output is `entity_df`'s own columns
|
|
663
714
|
plus the requested features, so both halves get checked normally. See
|
|
664
715
|
[Unresolved schemas](docs/usage.md#unresolved-schemas-recognized-but-not-column-checked).
|
|
@@ -668,17 +719,17 @@ plus the requested features, so both halves get checked normally. See
|
|
|
668
719
|
Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
|
|
669
720
|
|
|
670
721
|
**Benchmark results** (median of 5 runs ± one standard deviation, 3 warmup, caches cleared between runs):
|
|
671
|
-
*2026-
|
|
722
|
+
*2026-11-15 · Darwin 27.2.0 · arm · CPython 3.15.0 · 64GiB RAM · Great Expectations pinned @ 1.23.2*
|
|
672
723
|
|
|
673
724
|
| Tool | Version | What it does | typedframes (12 files) | great_expectations (494 files) |
|
|
674
725
|
|------|---------|--------------|------------------------|--------------------------------|
|
|
675
|
-
| typedframes | 0.
|
|
676
|
-
| ruff | 0.17.
|
|
677
|
-
| ty | 0.0.
|
|
678
|
-
| pyrefly | 1.4.0 | Type checker |
|
|
679
|
-
| mypy | 2.4.
|
|
680
|
-
| mypy + typedframes | 2.4.
|
|
681
|
-
| pyright | 1.1.414 | Type checker |
|
|
726
|
+
| typedframes | 0.15.0 | DataFrame column checker | 19.6 ± 0.3 ms | 115.8 ± 1.7 ms |
|
|
727
|
+
| ruff | 0.17.3 | Linter (no type checking) | 24.3 ± 0.6 ms | 220.7 ± 3.2 ms |
|
|
728
|
+
| ty | 0.0.95 | Type checker | 52.5 ± 2.3 ms | 768.4 ± 18.5 ms |
|
|
729
|
+
| pyrefly | 1.4.0 | Type checker | 109.3 ± 16.9 ms | 272.8 ± 11.4 ms |
|
|
730
|
+
| mypy | 2.4.1 | Type checker (no plugin) | 1.83 ± 0.14 s | 3.06 ± 0.03 s |
|
|
731
|
+
| mypy + typedframes | 2.4.1 | Type checker + column checker | 1.84 ± 0.02 s | 3.30 ± 0.01 s |
|
|
732
|
+
| pyright | 1.1.414 | Type checker | 674.0 ± 5.5 ms | 3.93 ± 0.04 s |
|
|
682
733
|
|
|
683
734
|
*Run `uv run python benchmarks/benchmark_checkers.py` to reproduce.*
|
|
684
735
|
|
|
@@ -1206,7 +1257,7 @@ A: The mypy plugin doesn't work with pyright. Use the standalone checker (`typed
|
|
|
1206
1257
|
validation. Schema descriptor access (`df[Schema.column]`) works natively in pyright without any plugin.
|
|
1207
1258
|
|
|
1208
1259
|
**Q: Do I need to write `BaseSchema` classes to get value?**
|
|
1209
|
-
A: No. The standalone checker works entirely from inference: `usecols=`/`columns
|
|
1260
|
+
A: No. The standalone checker works entirely from inference: `usecols=`/`columns=` arguments on read
|
|
1210
1261
|
calls give it enough information to validate column access and propagate that knowledge through method chains
|
|
1211
1262
|
(`rename`, `drop`, `assign`, `select`, …). `BaseSchema` is a progressive enhancement that unlocks cross-file
|
|
1212
1263
|
awareness (schemas travel with function return types across module boundaries) and IDE autocomplete via
|
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "typedframes"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.15.0"
|
|
8
8
|
description = "Static analysis for pandas and polars DataFrames. Catch column errors at lint-time, not runtime."
|
|
9
9
|
keywords = ["pandas", "polars", "type-checking", "static-analysis", "dataframe", "linter", "mypy-plugin"]
|
|
10
10
|
readme = "README.md"
|
|
@@ -128,11 +128,11 @@ convention = "google"
|
|
|
128
128
|
|
|
129
129
|
[dependency-groups]
|
|
130
130
|
docs = [
|
|
131
|
-
"mkdocs-material>=9.
|
|
132
|
-
"mkdocstrings-python>=2.0.
|
|
131
|
+
"mkdocs-material>=9.8.0",
|
|
132
|
+
"mkdocstrings-python>=2.0.10",
|
|
133
133
|
]
|
|
134
134
|
dev = [
|
|
135
|
-
"pandas>=3.0
|
|
135
|
+
"pandas>=3.1.0",
|
|
136
136
|
"pandera>=0.34.1",
|
|
137
137
|
"pip-audit>=2.10.1",
|
|
138
138
|
"polars>=2.0.0",
|
|
@@ -147,11 +147,11 @@ dev = [
|
|
|
147
147
|
"pytest>=9.1.1",
|
|
148
148
|
"pytest-cov>=7.1.0",
|
|
149
149
|
"pytest-xdist>=3.8.0",
|
|
150
|
-
"ruff>=0.17.
|
|
151
|
-
"ty>=0.0.
|
|
150
|
+
"ruff>=0.17.2",
|
|
151
|
+
"ty>=0.0.91",
|
|
152
152
|
"openpyxl>=3.1.5",
|
|
153
153
|
"pyarrow>=26.0.0",
|
|
154
|
-
"xlsxwriter>=3.
|
|
154
|
+
"xlsxwriter>=3.3.0",
|
|
155
155
|
"fastexcel>=0.21.0",
|
|
156
156
|
"pandas-stubs>=3.0.5.260914",
|
|
157
157
|
"mypy>=2.4.0",
|
|
@@ -235,3 +235,12 @@ warnings = true
|
|
|
235
235
|
|
|
236
236
|
[tool.uv.workspace]
|
|
237
237
|
exclude = ["examples/*/*"]
|
|
238
|
+
|
|
239
|
+
[tool.uv]
|
|
240
|
+
# Security floors for transitive dependencies (Dependabot advisories).
|
|
241
|
+
constraint-dependencies = [
|
|
242
|
+
"certifi>=2026.10.18",
|
|
243
|
+
"filelock>=4.2.1",
|
|
244
|
+
"msgpack>=1.2.3",
|
|
245
|
+
"virtualenv>=21.14.9",
|
|
246
|
+
]
|