typedframes 0.5.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {typedframes-0.5.0 → typedframes-0.6.0}/PKG-INFO +157 -6
- {typedframes-0.5.0 → typedframes-0.6.0}/README.md +156 -5
- {typedframes-0.5.0 → typedframes-0.6.0}/pyproject.toml +5 -7
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/Cargo.lock +1 -1
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/Cargo.toml +1 -1
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/ast_extract.rs +3 -4
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/config.rs +55 -5
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/errors.rs +10 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/index.rs +609 -13
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/lib.rs +4 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/linter.rs +617 -144
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/notebook.rs +4 -4
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/pyapi.rs +25 -1
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/__init__.py +1 -1
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/_rust_checker.pyi +6 -2
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/cli.py +170 -90
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/.cargo/config.toml +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/README.md +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/benches/parser_bench.rs +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/constants.rs +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/contract.rs +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/main.rs +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/sql.rs +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/src/typo.rs +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/rust/tests/integration_test.rs +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/base_schema.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/column.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/column_group.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/column_group_error.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/column_set.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/missing_dependency_error.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/mypy.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/pandas.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/pandera.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/polars.py +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/py.typed +0 -0
- {typedframes-0.5.0 → typedframes-0.6.0}/src/typedframes/schema_algebra.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: typedframes
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -66,7 +66,10 @@ typedframes check src/ --coverage-fail-under=90
|
|
|
66
66
|
```
|
|
67
67
|
|
|
68
68
|
`--coverage-fail-under=N` is the same idea as a test-coverage or mypy type-coverage gate, applied to how many DataFrames
|
|
69
|
-
the checker can resolve
|
|
69
|
+
the checker can resolve to a concrete column list — a DataFrame it merely recognizes but can't resolve columns for
|
|
70
|
+
(a Feast retrieval whose entity_df isn't itself concrete, a bare `-> pd.DataFrame` return with no attached Schema)
|
|
71
|
+
does not count, and any column accessed on it is flagged `unverifiable-column` rather than passed through silently.
|
|
72
|
+
See [DataFrame Schema Coverage Thresholds](#dataframe-schema-coverage-thresholds-opt-in).
|
|
70
73
|
Add `BaseSchema` classes later for cross-file awareness and IDE autocomplete — see [Quick Start](#quick-start).
|
|
71
74
|
|
|
72
75
|
---
|
|
@@ -78,6 +81,7 @@ Add `BaseSchema` classes later for cross-file awareness and IDE autocomplete —
|
|
|
78
81
|
- [Quick Start](#quick-start)
|
|
79
82
|
- [Column Inference](#column-inference)
|
|
80
83
|
- [Static Analysis](#static-analysis)
|
|
84
|
+
- [Jupyter Notebooks](#jupyter-notebooks)
|
|
81
85
|
- [DataFrame Schema Coverage Thresholds](#dataframe-schema-coverage-thresholds-opt-in)
|
|
82
86
|
- [Static Analysis Performance](#static-analysis-performance)
|
|
83
87
|
- [Type Safety With Multiple Backends](#type-safety-with-multiple-backends)
|
|
@@ -428,7 +432,10 @@ typedframes check src/ --strict
|
|
|
428
432
|
# JSON output
|
|
429
433
|
typedframes check src/ --output-format=json
|
|
430
434
|
|
|
431
|
-
#
|
|
435
|
+
# Check a single file (follows that file's own imports, not the whole project)
|
|
436
|
+
typedframes check src/pipeline.py
|
|
437
|
+
|
|
438
|
+
# Skip the cross-file index entirely (every file in isolation, fastest)
|
|
432
439
|
typedframes check src/ --no-index
|
|
433
440
|
|
|
434
441
|
# Suppress all warnings (untracked-dataframe, dropped-unknown-column)
|
|
@@ -486,6 +493,135 @@ See the full [Method Matrix](https://typedframes.readthedocs.io/en/latest/method
|
|
|
486
493
|
for the complete list of tracked, passthrough, and untracked operations, plus the error
|
|
487
494
|
code reference.
|
|
488
495
|
|
|
496
|
+
### Pre-commit Hook
|
|
497
|
+
|
|
498
|
+
```yaml
|
|
499
|
+
# .pre-commit-config.yaml
|
|
500
|
+
repos:
|
|
501
|
+
- repo: local
|
|
502
|
+
hooks:
|
|
503
|
+
- id: typedframes
|
|
504
|
+
name: typedframes check
|
|
505
|
+
entry: typedframes check . --strict
|
|
506
|
+
language: python
|
|
507
|
+
additional_dependencies: ["typedframes==0.5.0"]
|
|
508
|
+
types_or: [python, jupyter]
|
|
509
|
+
pass_filenames: false
|
|
510
|
+
```
|
|
511
|
+
|
|
512
|
+
`pass_filenames: false` is deliberate: the checker takes a single path and builds its
|
|
513
|
+
cross-file index from it, so one run per commit still sees the modules the commit didn't
|
|
514
|
+
touch — a per-file invocation would lose that. `--strict` is what makes the hook block
|
|
515
|
+
the commit: `typedframes check` exits 0 by default even when it finds real errors. Add
|
|
516
|
+
the coverage gate to the same line:
|
|
517
|
+
|
|
518
|
+
```yaml
|
|
519
|
+
# with coverage gating
|
|
520
|
+
entry: typedframes check . --strict --coverage-fail-under=90
|
|
521
|
+
```
|
|
522
|
+
|
|
523
|
+
This repository also ships a `.pre-commit-hooks.yaml`, so once a tagged release contains
|
|
524
|
+
it you can point `repo:` straight at `https://github.com/w-martin/typedframes` with a
|
|
525
|
+
`rev:` and drop the boilerplate above. That form needs a Rust toolchain wherever the hook
|
|
526
|
+
runs — pre-commit's python installer builds the repository from source rather than
|
|
527
|
+
fetching a wheel — so the local hook stays the quicker one to install either way.
|
|
528
|
+
|
|
529
|
+
### GitHub Actions
|
|
530
|
+
|
|
531
|
+
```yaml
|
|
532
|
+
# .github/workflows/typedframes.yml
|
|
533
|
+
name: typedframes
|
|
534
|
+
on: [push, pull_request]
|
|
535
|
+
|
|
536
|
+
jobs:
|
|
537
|
+
typedframes:
|
|
538
|
+
runs-on: ubuntu-latest
|
|
539
|
+
steps:
|
|
540
|
+
- uses: actions/checkout@v4
|
|
541
|
+
# No tagged release contains action.yml yet. Repin this to that tag once one
|
|
542
|
+
# ships: an unpinned third-party action is a supply-chain risk, and @main is a
|
|
543
|
+
# stopgap rather than a recommendation.
|
|
544
|
+
- uses: w-martin/typedframes@main
|
|
545
|
+
with:
|
|
546
|
+
path: src/
|
|
547
|
+
coverage-fail-under: "90"
|
|
548
|
+
```
|
|
549
|
+
|
|
550
|
+
The action installs the PyPI wheel into a throwaway virtualenv and runs the checker with
|
|
551
|
+
`--output-format=github`, so errors arrive as annotations on the pull request diff.
|
|
552
|
+
|
|
553
|
+
| Input | Default | |
|
|
554
|
+
|-------|---------|-|
|
|
555
|
+
| `path` | `.` | File or directory to check |
|
|
556
|
+
| `version` | `latest` | PyPI version to install, e.g. `"0.5.0"` |
|
|
557
|
+
| `strict` | `true` | Fail the step on errors — `typedframes check` exits 0 without it |
|
|
558
|
+
| `coverage-fail-under` | *(unset)* | Minimum DataFrame schema coverage, e.g. `"90"` |
|
|
559
|
+
| `coverage-detail` | `summary` | Or `term-missing` for the per-file breakdown |
|
|
560
|
+
| `no-warnings` | `false` | Suppress warning-level diagnostics |
|
|
561
|
+
| `args` | *(empty)* | Escape hatch for `--no-index`, `--lenient-ingest`, `--no-info` |
|
|
562
|
+
|
|
563
|
+
---
|
|
564
|
+
|
|
565
|
+
## Jupyter Notebooks
|
|
566
|
+
|
|
567
|
+
The standalone checker reads `.ipynb` files directly — no `nbconvert`, no `jupytext`, no
|
|
568
|
+
flag, no conversion step. Notebooks are parsed with
|
|
569
|
+
[`ruff_notebook`](https://github.com/astral-sh/ruff/tree/main/crates/ruff_notebook), the
|
|
570
|
+
same crate Ruff uses for its own notebook support:
|
|
571
|
+
|
|
572
|
+
```shell
|
|
573
|
+
# Check one notebook
|
|
574
|
+
typedframes check analysis.ipynb
|
|
575
|
+
|
|
576
|
+
# Check a directory — .py and .ipynb files are both picked up, no flag needed
|
|
577
|
+
typedframes check notebooks/
|
|
578
|
+
|
|
579
|
+
# Output — `cell N` is the notebook's Nth cell; line:col are relative to that cell:
|
|
580
|
+
# analysis.ipynb:cell 6:2:1: warning[untracked-dataframe] columns unknown at lint time; specify
|
|
581
|
+
# `usecols`/`columns`, or annotate the variable's type
|
|
582
|
+
# analysis.ipynb:cell 10:1:1: error[unknown-column] Column 'revenue' does not exist in Orders
|
|
583
|
+
# {amount, customer_id, order_id} (defined at analysis.ipynb cell 2:8)
|
|
584
|
+
# ✗ Found 1 error, 2 warnings in 1 file (0.0s)
|
|
585
|
+
```
|
|
586
|
+
|
|
587
|
+
**Features:**
|
|
588
|
+
- ✅ Auto-detected — every `.ipynb` under a checked path, alongside `.py` files
|
|
589
|
+
- ✅ Diagnostics reported as `cell N:line:col`, mapped back to the cell that produced them
|
|
590
|
+
rather than a line number in the raw notebook JSON — including a same-notebook schema's
|
|
591
|
+
`(defined at ...)` cross-reference
|
|
592
|
+
- ✅ Same error codes, severities, and flags as `.py` files: `--strict`,
|
|
593
|
+
`--output-format`, `--no-warnings`, `--lenient-ingest`, coverage thresholds
|
|
594
|
+
- ✅ IPython magics and shell escapes parse natively (`%matplotlib inline`, `%%time`,
|
|
595
|
+
`!pip install ...`) — nothing has to be stripped first
|
|
596
|
+
- ✅ `# typedframes: ignore` suppresses a diagnostic on a line inside a cell, same as in a
|
|
597
|
+
`.py` file
|
|
598
|
+
- ✅ A schema imported from a project `.py` module resolves through the cross-file index,
|
|
599
|
+
so `orders = load_orders()` in a notebook is validated against the schema on
|
|
600
|
+
`load_orders`' return annotation
|
|
601
|
+
- ✅ `.ipynb_checkpoints/` is pruned by default, so stale checkpoint copies aren't
|
|
602
|
+
re-reported
|
|
603
|
+
|
|
604
|
+
**Caveats:**
|
|
605
|
+
- ❌ The mypy plugin ([Option 2](#option-2-mypy-plugin-comprehensive)) is `.py`-only —
|
|
606
|
+
mypy reads Python source files, so notebook checking is standalone-checker-only
|
|
607
|
+
- ❌ `cell N` counts *every* cell in the notebook, markdown and raw cells included — so
|
|
608
|
+
the number is a cell's position in the notebook, not "the Nth code cell"
|
|
609
|
+
- ❌ A notebook is never a *source* of schemas for other files (Python can't import an
|
|
610
|
+
`.ipynb`); a schema defined in a notebook is visible only inside that notebook
|
|
611
|
+
- ❌ A notebook that isn't valid notebook JSON, whose kernel isn't Python, or whose code
|
|
612
|
+
doesn't parse is skipped with a message on stderr and the run continues — a skip alone
|
|
613
|
+
never fails the run, `--strict` included
|
|
614
|
+
- ❌ Under `--output-format=github`, the annotation's `line`/`col` stay cell-relative and
|
|
615
|
+
the cell number moves into the annotation title, since GitHub annotations address raw
|
|
616
|
+
file lines and an `.ipynb` is JSON rather than source text
|
|
617
|
+
|
|
618
|
+
### See Also
|
|
619
|
+
|
|
620
|
+
- [`examples/features/ipynb_example.ipynb`](examples/features/ipynb_example.ipynb) — a
|
|
621
|
+
runnable notebook with one schema, an IPython magic, and three intentional bugs
|
|
622
|
+
(`untracked-dataframe`, `dropped-unknown-column`, `unknown-column`), all caught by
|
|
623
|
+
`typedframes check ipynb_example.ipynb` without running a cell.
|
|
624
|
+
|
|
489
625
|
---
|
|
490
626
|
|
|
491
627
|
## DataFrame Schema Coverage Thresholds (Opt-In)
|
|
@@ -516,6 +652,16 @@ breakdown of exactly which DataFrames cost you coverage (as text or, combined wi
|
|
|
516
652
|
the
|
|
517
653
|
[DataFrame schema coverage thresholds guide](docs/usage.md#dataframe-schema-coverage-thresholds).
|
|
518
654
|
|
|
655
|
+
Some DataFrames are recognized but never resolve to a concrete column list — a Feast
|
|
656
|
+
retrieval whose `entity_df` isn't itself concretely typed, or a bare `-> pd.DataFrame`
|
|
657
|
+
return with no attached `Schema`. These do **not** count as covered, and accessing any
|
|
658
|
+
column on one is flagged `unverifiable-column` rather than passed through silently —
|
|
659
|
+
the checker genuinely doesn't know whether the column exists, so it says so instead of
|
|
660
|
+
guessing either way. A Feast retrieval whose `entity_df` argument IS a concretely typed
|
|
661
|
+
DataFrame is fully resolved instead: Feast's real output is `entity_df`'s own columns
|
|
662
|
+
plus the requested features, so both halves get checked normally. See
|
|
663
|
+
[Unresolved schemas](docs/usage.md#unresolved-schemas-recognized-but-not-column-checked).
|
|
664
|
+
|
|
519
665
|
## Static Analysis Performance
|
|
520
666
|
|
|
521
667
|
Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
|
|
@@ -537,9 +683,11 @@ Fast feedback reduces development time. The typedframes Rust binary provides nea
|
|
|
537
683
|
|
|
538
684
|
The typedframes binary resolves column names within a file and, when a project index is present, across files too.
|
|
539
685
|
Run `typedframes check src/` to build the index automatically and catch errors like `df = load_users(); df["typo"]`
|
|
540
|
-
even when `load_users` is defined in another module.
|
|
541
|
-
|
|
542
|
-
|
|
686
|
+
even when `load_users` is defined in another module. Checking a single file
|
|
687
|
+
(`typedframes check src/pipeline.py`) builds a smaller index from that file's own imports instead of walking the whole
|
|
688
|
+
project, so it still resolves `load_users` without paying for files it never references. Pass `--no-index` to skip the
|
|
689
|
+
index entirely and check each file in isolation. Full type checkers (mypy, pyright, ty) analyze all Python types across
|
|
690
|
+
your entire codebase. Use both: the binary for fast iteration, mypy for comprehensive checking.
|
|
543
691
|
|
|
544
692
|
The standalone checker is built with [`ruff_python_parser`](https://github.com/astral-sh/ruff) for Python AST
|
|
545
693
|
parsing.
|
|
@@ -915,6 +1063,9 @@ Runnable versions of everything shown in [Quick Start](#quick-start) and
|
|
|
915
1063
|
[`multi_file_with_schema/`](examples/features/multi_file_with_schema/) — the same
|
|
916
1064
|
cross-file pipeline checked with and without schemas, see
|
|
917
1065
|
[`examples/features/README.md`](examples/features/README.md) for what each one catches
|
|
1066
|
+
- [`ipynb_example.ipynb`](examples/features/ipynb_example.ipynb) — the same checks in a
|
|
1067
|
+
Jupyter notebook, with `cell N:line:col` locations (see [Jupyter
|
|
1068
|
+
Notebooks](#jupyter-notebooks))
|
|
918
1069
|
|
|
919
1070
|
---
|
|
920
1071
|
|
|
@@ -32,7 +32,10 @@ typedframes check src/ --coverage-fail-under=90
|
|
|
32
32
|
```
|
|
33
33
|
|
|
34
34
|
`--coverage-fail-under=N` is the same idea as a test-coverage or mypy type-coverage gate, applied to how many DataFrames
|
|
35
|
-
the checker can resolve
|
|
35
|
+
the checker can resolve to a concrete column list — a DataFrame it merely recognizes but can't resolve columns for
|
|
36
|
+
(a Feast retrieval whose entity_df isn't itself concrete, a bare `-> pd.DataFrame` return with no attached Schema)
|
|
37
|
+
does not count, and any column accessed on it is flagged `unverifiable-column` rather than passed through silently.
|
|
38
|
+
See [DataFrame Schema Coverage Thresholds](#dataframe-schema-coverage-thresholds-opt-in).
|
|
36
39
|
Add `BaseSchema` classes later for cross-file awareness and IDE autocomplete — see [Quick Start](#quick-start).
|
|
37
40
|
|
|
38
41
|
---
|
|
@@ -44,6 +47,7 @@ Add `BaseSchema` classes later for cross-file awareness and IDE autocomplete —
|
|
|
44
47
|
- [Quick Start](#quick-start)
|
|
45
48
|
- [Column Inference](#column-inference)
|
|
46
49
|
- [Static Analysis](#static-analysis)
|
|
50
|
+
- [Jupyter Notebooks](#jupyter-notebooks)
|
|
47
51
|
- [DataFrame Schema Coverage Thresholds](#dataframe-schema-coverage-thresholds-opt-in)
|
|
48
52
|
- [Static Analysis Performance](#static-analysis-performance)
|
|
49
53
|
- [Type Safety With Multiple Backends](#type-safety-with-multiple-backends)
|
|
@@ -394,7 +398,10 @@ typedframes check src/ --strict
|
|
|
394
398
|
# JSON output
|
|
395
399
|
typedframes check src/ --output-format=json
|
|
396
400
|
|
|
397
|
-
#
|
|
401
|
+
# Check a single file (follows that file's own imports, not the whole project)
|
|
402
|
+
typedframes check src/pipeline.py
|
|
403
|
+
|
|
404
|
+
# Skip the cross-file index entirely (every file in isolation, fastest)
|
|
398
405
|
typedframes check src/ --no-index
|
|
399
406
|
|
|
400
407
|
# Suppress all warnings (untracked-dataframe, dropped-unknown-column)
|
|
@@ -452,6 +459,135 @@ See the full [Method Matrix](https://typedframes.readthedocs.io/en/latest/method
|
|
|
452
459
|
for the complete list of tracked, passthrough, and untracked operations, plus the error
|
|
453
460
|
code reference.
|
|
454
461
|
|
|
462
|
+
### Pre-commit Hook
|
|
463
|
+
|
|
464
|
+
```yaml
|
|
465
|
+
# .pre-commit-config.yaml
|
|
466
|
+
repos:
|
|
467
|
+
- repo: local
|
|
468
|
+
hooks:
|
|
469
|
+
- id: typedframes
|
|
470
|
+
name: typedframes check
|
|
471
|
+
entry: typedframes check . --strict
|
|
472
|
+
language: python
|
|
473
|
+
additional_dependencies: ["typedframes==0.5.0"]
|
|
474
|
+
types_or: [python, jupyter]
|
|
475
|
+
pass_filenames: false
|
|
476
|
+
```
|
|
477
|
+
|
|
478
|
+
`pass_filenames: false` is deliberate: the checker takes a single path and builds its
|
|
479
|
+
cross-file index from it, so one run per commit still sees the modules the commit didn't
|
|
480
|
+
touch — a per-file invocation would lose that. `--strict` is what makes the hook block
|
|
481
|
+
the commit: `typedframes check` exits 0 by default even when it finds real errors. Add
|
|
482
|
+
the coverage gate to the same line:
|
|
483
|
+
|
|
484
|
+
```yaml
|
|
485
|
+
# with coverage gating
|
|
486
|
+
entry: typedframes check . --strict --coverage-fail-under=90
|
|
487
|
+
```
|
|
488
|
+
|
|
489
|
+
This repository also ships a `.pre-commit-hooks.yaml`, so once a tagged release contains
|
|
490
|
+
it you can point `repo:` straight at `https://github.com/w-martin/typedframes` with a
|
|
491
|
+
`rev:` and drop the boilerplate above. That form needs a Rust toolchain wherever the hook
|
|
492
|
+
runs — pre-commit's python installer builds the repository from source rather than
|
|
493
|
+
fetching a wheel — so the local hook stays the quicker one to install either way.
|
|
494
|
+
|
|
495
|
+
### GitHub Actions
|
|
496
|
+
|
|
497
|
+
```yaml
|
|
498
|
+
# .github/workflows/typedframes.yml
|
|
499
|
+
name: typedframes
|
|
500
|
+
on: [push, pull_request]
|
|
501
|
+
|
|
502
|
+
jobs:
|
|
503
|
+
typedframes:
|
|
504
|
+
runs-on: ubuntu-latest
|
|
505
|
+
steps:
|
|
506
|
+
- uses: actions/checkout@v4
|
|
507
|
+
# No tagged release contains action.yml yet. Repin this to that tag once one
|
|
508
|
+
# ships: an unpinned third-party action is a supply-chain risk, and @main is a
|
|
509
|
+
# stopgap rather than a recommendation.
|
|
510
|
+
- uses: w-martin/typedframes@main
|
|
511
|
+
with:
|
|
512
|
+
path: src/
|
|
513
|
+
coverage-fail-under: "90"
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
The action installs the PyPI wheel into a throwaway virtualenv and runs the checker with
|
|
517
|
+
`--output-format=github`, so errors arrive as annotations on the pull request diff.
|
|
518
|
+
|
|
519
|
+
| Input | Default | |
|
|
520
|
+
|-------|---------|-|
|
|
521
|
+
| `path` | `.` | File or directory to check |
|
|
522
|
+
| `version` | `latest` | PyPI version to install, e.g. `"0.5.0"` |
|
|
523
|
+
| `strict` | `true` | Fail the step on errors — `typedframes check` exits 0 without it |
|
|
524
|
+
| `coverage-fail-under` | *(unset)* | Minimum DataFrame schema coverage, e.g. `"90"` |
|
|
525
|
+
| `coverage-detail` | `summary` | Or `term-missing` for the per-file breakdown |
|
|
526
|
+
| `no-warnings` | `false` | Suppress warning-level diagnostics |
|
|
527
|
+
| `args` | *(empty)* | Escape hatch for `--no-index`, `--lenient-ingest`, `--no-info` |
|
|
528
|
+
|
|
529
|
+
---
|
|
530
|
+
|
|
531
|
+
## Jupyter Notebooks
|
|
532
|
+
|
|
533
|
+
The standalone checker reads `.ipynb` files directly — no `nbconvert`, no `jupytext`, no
|
|
534
|
+
flag, no conversion step. Notebooks are parsed with
|
|
535
|
+
[`ruff_notebook`](https://github.com/astral-sh/ruff/tree/main/crates/ruff_notebook), the
|
|
536
|
+
same crate Ruff uses for its own notebook support:
|
|
537
|
+
|
|
538
|
+
```shell
|
|
539
|
+
# Check one notebook
|
|
540
|
+
typedframes check analysis.ipynb
|
|
541
|
+
|
|
542
|
+
# Check a directory — .py and .ipynb files are both picked up, no flag needed
|
|
543
|
+
typedframes check notebooks/
|
|
544
|
+
|
|
545
|
+
# Output — `cell N` is the notebook's Nth cell; line:col are relative to that cell:
|
|
546
|
+
# analysis.ipynb:cell 6:2:1: warning[untracked-dataframe] columns unknown at lint time; specify
|
|
547
|
+
# `usecols`/`columns`, or annotate the variable's type
|
|
548
|
+
# analysis.ipynb:cell 10:1:1: error[unknown-column] Column 'revenue' does not exist in Orders
|
|
549
|
+
# {amount, customer_id, order_id} (defined at analysis.ipynb cell 2:8)
|
|
550
|
+
# ✗ Found 1 error, 2 warnings in 1 file (0.0s)
|
|
551
|
+
```
|
|
552
|
+
|
|
553
|
+
**Features:**
|
|
554
|
+
- ✅ Auto-detected — every `.ipynb` under a checked path, alongside `.py` files
|
|
555
|
+
- ✅ Diagnostics reported as `cell N:line:col`, mapped back to the cell that produced them
|
|
556
|
+
rather than a line number in the raw notebook JSON — including a same-notebook schema's
|
|
557
|
+
`(defined at ...)` cross-reference
|
|
558
|
+
- ✅ Same error codes, severities, and flags as `.py` files: `--strict`,
|
|
559
|
+
`--output-format`, `--no-warnings`, `--lenient-ingest`, coverage thresholds
|
|
560
|
+
- ✅ IPython magics and shell escapes parse natively (`%matplotlib inline`, `%%time`,
|
|
561
|
+
`!pip install ...`) — nothing has to be stripped first
|
|
562
|
+
- ✅ `# typedframes: ignore` suppresses a diagnostic on a line inside a cell, same as in a
|
|
563
|
+
`.py` file
|
|
564
|
+
- ✅ A schema imported from a project `.py` module resolves through the cross-file index,
|
|
565
|
+
so `orders = load_orders()` in a notebook is validated against the schema on
|
|
566
|
+
`load_orders`' return annotation
|
|
567
|
+
- ✅ `.ipynb_checkpoints/` is pruned by default, so stale checkpoint copies aren't
|
|
568
|
+
re-reported
|
|
569
|
+
|
|
570
|
+
**Caveats:**
|
|
571
|
+
- ❌ The mypy plugin ([Option 2](#option-2-mypy-plugin-comprehensive)) is `.py`-only —
|
|
572
|
+
mypy reads Python source files, so notebook checking is standalone-checker-only
|
|
573
|
+
- ❌ `cell N` counts *every* cell in the notebook, markdown and raw cells included — so
|
|
574
|
+
the number is a cell's position in the notebook, not "the Nth code cell"
|
|
575
|
+
- ❌ A notebook is never a *source* of schemas for other files (Python can't import an
|
|
576
|
+
`.ipynb`); a schema defined in a notebook is visible only inside that notebook
|
|
577
|
+
- ❌ A notebook that isn't valid notebook JSON, whose kernel isn't Python, or whose code
|
|
578
|
+
doesn't parse is skipped with a message on stderr and the run continues — a skip alone
|
|
579
|
+
never fails the run, `--strict` included
|
|
580
|
+
- ❌ Under `--output-format=github`, the annotation's `line`/`col` stay cell-relative and
|
|
581
|
+
the cell number moves into the annotation title, since GitHub annotations address raw
|
|
582
|
+
file lines and an `.ipynb` is JSON rather than source text
|
|
583
|
+
|
|
584
|
+
### See Also
|
|
585
|
+
|
|
586
|
+
- [`examples/features/ipynb_example.ipynb`](examples/features/ipynb_example.ipynb) — a
|
|
587
|
+
runnable notebook with one schema, an IPython magic, and three intentional bugs
|
|
588
|
+
(`untracked-dataframe`, `dropped-unknown-column`, `unknown-column`), all caught by
|
|
589
|
+
`typedframes check ipynb_example.ipynb` without running a cell.
|
|
590
|
+
|
|
455
591
|
---
|
|
456
592
|
|
|
457
593
|
## DataFrame Schema Coverage Thresholds (Opt-In)
|
|
@@ -482,6 +618,16 @@ breakdown of exactly which DataFrames cost you coverage (as text or, combined wi
|
|
|
482
618
|
the
|
|
483
619
|
[DataFrame schema coverage thresholds guide](docs/usage.md#dataframe-schema-coverage-thresholds).
|
|
484
620
|
|
|
621
|
+
Some DataFrames are recognized but never resolve to a concrete column list — a Feast
|
|
622
|
+
retrieval whose `entity_df` isn't itself concretely typed, or a bare `-> pd.DataFrame`
|
|
623
|
+
return with no attached `Schema`. These do **not** count as covered, and accessing any
|
|
624
|
+
column on one is flagged `unverifiable-column` rather than passed through silently —
|
|
625
|
+
the checker genuinely doesn't know whether the column exists, so it says so instead of
|
|
626
|
+
guessing either way. A Feast retrieval whose `entity_df` argument IS a concretely typed
|
|
627
|
+
DataFrame is fully resolved instead: Feast's real output is `entity_df`'s own columns
|
|
628
|
+
plus the requested features, so both halves get checked normally. See
|
|
629
|
+
[Unresolved schemas](docs/usage.md#unresolved-schemas-recognized-but-not-column-checked).
|
|
630
|
+
|
|
485
631
|
## Static Analysis Performance
|
|
486
632
|
|
|
487
633
|
Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
|
|
@@ -503,9 +649,11 @@ Fast feedback reduces development time. The typedframes Rust binary provides nea
|
|
|
503
649
|
|
|
504
650
|
The typedframes binary resolves column names within a file and, when a project index is present, across files too.
|
|
505
651
|
Run `typedframes check src/` to build the index automatically and catch errors like `df = load_users(); df["typo"]`
|
|
506
|
-
even when `load_users` is defined in another module.
|
|
507
|
-
|
|
508
|
-
|
|
652
|
+
even when `load_users` is defined in another module. Checking a single file
|
|
653
|
+
(`typedframes check src/pipeline.py`) builds a smaller index from that file's own imports instead of walking the whole
|
|
654
|
+
project, so it still resolves `load_users` without paying for files it never references. Pass `--no-index` to skip the
|
|
655
|
+
index entirely and check each file in isolation. Full type checkers (mypy, pyright, ty) analyze all Python types across
|
|
656
|
+
your entire codebase. Use both: the binary for fast iteration, mypy for comprehensive checking.
|
|
509
657
|
|
|
510
658
|
The standalone checker is built with [`ruff_python_parser`](https://github.com/astral-sh/ruff) for Python AST
|
|
511
659
|
parsing.
|
|
@@ -881,6 +1029,9 @@ Runnable versions of everything shown in [Quick Start](#quick-start) and
|
|
|
881
1029
|
[`multi_file_with_schema/`](examples/features/multi_file_with_schema/) — the same
|
|
882
1030
|
cross-file pipeline checked with and without schemas, see
|
|
883
1031
|
[`examples/features/README.md`](examples/features/README.md) for what each one catches
|
|
1032
|
+
- [`ipynb_example.ipynb`](examples/features/ipynb_example.ipynb) — the same checks in a
|
|
1033
|
+
Jupyter notebook, with `cell N:line:col` locations (see [Jupyter
|
|
1034
|
+
Notebooks](#jupyter-notebooks))
|
|
884
1035
|
|
|
885
1036
|
---
|
|
886
1037
|
|
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "typedframes"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
description = "Static analysis for pandas and polars DataFrames. Catch column errors at lint-time, not runtime."
|
|
9
9
|
keywords = ["pandas", "polars", "type-checking", "static-analysis", "dataframe", "linter", "mypy-plugin"]
|
|
10
10
|
readme = "README.md"
|
|
@@ -135,7 +135,6 @@ dev = [
|
|
|
135
135
|
"complexipy>=6.2.0",
|
|
136
136
|
"coverage-threshold>=0.6.2",
|
|
137
137
|
"invoke>=3.0.3",
|
|
138
|
-
"licensecheck>=2025.1.0",
|
|
139
138
|
"maturin>=1.14.1",
|
|
140
139
|
"pre-commit>=4.6.1",
|
|
141
140
|
"pyrefly>=1.2.0",
|
|
@@ -151,11 +150,15 @@ dev = [
|
|
|
151
150
|
"fastexcel>=0.20.2",
|
|
152
151
|
"pandas-stubs>=3.0.5.260730",
|
|
153
152
|
"mypy>=2.3.1",
|
|
153
|
+
"trustedlicenses>=0.1.1",
|
|
154
154
|
]
|
|
155
155
|
|
|
156
156
|
[tool.bandit]
|
|
157
157
|
exclude_dirs = ["tests", "examples"]
|
|
158
158
|
|
|
159
|
+
[tool.trustedlicenses]
|
|
160
|
+
allowed-categories = ["Permissive", "Public Domain", "Copyleft Limited"]
|
|
161
|
+
|
|
159
162
|
[tool.coverage.report]
|
|
160
163
|
exclude_lines = [
|
|
161
164
|
"pragma: no cover",
|
|
@@ -237,10 +240,5 @@ invalid-return-type = "ignore"
|
|
|
237
240
|
enabled = true
|
|
238
241
|
warnings = true
|
|
239
242
|
|
|
240
|
-
[tool.licensecheck]
|
|
241
|
-
using = "PEP631"
|
|
242
|
-
groups = ["dev"]
|
|
243
|
-
ignore_packages = ["mypy-extensions", "typedframes", "pyrefly"]
|
|
244
|
-
|
|
245
243
|
[tool.uv.workspace]
|
|
246
244
|
exclude = ["examples/*/*"]
|
|
@@ -265,10 +265,9 @@ pub(crate) fn extract_schema_from_annotation(expr: &Expr) -> Option<&str> {
|
|
|
265
265
|
// `extract_schema_from_annotation` first; this is the fallback for the shape that
|
|
266
266
|
// leaves with no schema name to extract, which is what most third-party return
|
|
267
267
|
// annotations look like (a `py.typed` package has no reason to know about this
|
|
268
|
-
// project's `Schema` classes). The caller registers
|
|
269
|
-
// (empty column list, `
|
|
270
|
-
//
|
|
271
|
-
// than leaving the call untracked, and never manufactures a false unknown-column.
|
|
268
|
+
// project's `Schema` classes). The caller registers this as an unresolved schema
|
|
269
|
+
// (empty column list, `unresolved_schemas`) -- recognized as a DataFrame rather
|
|
270
|
+
// than untracked, with every column access raising `unverifiable-column`.
|
|
272
271
|
//
|
|
273
272
|
// Deliberately narrower than `extract_schema_from_annotation`: no quoted
|
|
274
273
|
// (forward-referenced) form, since a bare `pd.DataFrame`/`pl.DataFrame` return never
|
|
@@ -99,23 +99,49 @@ pub fn is_enabled(project_root: &Path) -> bool {
|
|
|
99
99
|
|
|
100
100
|
/// Walk up the directory tree from `start_path` until a `pyproject.toml` is found.
|
|
101
101
|
///
|
|
102
|
-
/// Returns
|
|
103
|
-
///
|
|
104
|
-
|
|
102
|
+
/// Returns `None` when no `pyproject.toml` exists anywhere in the ancestor chain --
|
|
103
|
+
/// i.e. there is no project root to speak of (a standalone script in a scratch
|
|
104
|
+
/// directory, a file under `/tmp`, ...). Callers that need a best-effort directory
|
|
105
|
+
/// regardless should use [`find_project_root`], which substitutes `start_path`.
|
|
106
|
+
///
|
|
107
|
+
/// `pyproject.toml` is the ONE project-root marker this checker recognises, and it is
|
|
108
|
+
/// deliberately the same marker used everywhere else: `load_linter_config` reads
|
|
109
|
+
/// `[tool.typedframes]` out of it, and `find_site_packages_dir` looks for `.venv`
|
|
110
|
+
/// beside it. Single-file and directory checks must agree on where the root is, or the
|
|
111
|
+
/// same file would resolve its imports differently depending on how it was checked.
|
|
112
|
+
pub fn find_project_root_opt(start_path: &Path) -> Option<PathBuf> {
|
|
105
113
|
let mut current = start_path.to_path_buf();
|
|
106
114
|
if current.is_file() {
|
|
107
115
|
current.pop();
|
|
108
116
|
}
|
|
109
117
|
loop {
|
|
110
118
|
if current.join("pyproject.toml").exists() {
|
|
111
|
-
|
|
119
|
+
// `current.pop()` can walk `current` all the way down to the empty
|
|
120
|
+
// `PathBuf` (when `start_path` was relative and its project root is the
|
|
121
|
+
// CWD itself). The empty PathBuf is a valid *join* target (joining onto
|
|
122
|
+
// it is a no-op) but `fs::canonicalize("")` errors -- unlike
|
|
123
|
+
// `fs::canonicalize(".")`, which resolves to the CWD -- so normalize here
|
|
124
|
+
// to keep that invariant for every caller that canonicalizes the root.
|
|
125
|
+
return Some(if current.as_os_str().is_empty() {
|
|
126
|
+
PathBuf::from(".")
|
|
127
|
+
} else {
|
|
128
|
+
current
|
|
129
|
+
});
|
|
112
130
|
}
|
|
113
131
|
if !current.pop() {
|
|
114
|
-
return
|
|
132
|
+
return None;
|
|
115
133
|
}
|
|
116
134
|
}
|
|
117
135
|
}
|
|
118
136
|
|
|
137
|
+
/// Walk up the directory tree from `start_path` until a `pyproject.toml` is found.
|
|
138
|
+
///
|
|
139
|
+
/// Returns the directory containing `pyproject.toml`, or `start_path` itself if no
|
|
140
|
+
/// `pyproject.toml` exists anywhere in the ancestor chain (e.g. standalone scripts).
|
|
141
|
+
pub fn find_project_root(start_path: &Path) -> PathBuf {
|
|
142
|
+
find_project_root_opt(start_path).unwrap_or_else(|| start_path.to_path_buf())
|
|
143
|
+
}
|
|
144
|
+
|
|
119
145
|
#[cfg(test)]
|
|
120
146
|
mod tests {
|
|
121
147
|
use super::*;
|
|
@@ -132,6 +158,30 @@ mod tests {
|
|
|
132
158
|
assert_eq!(find_project_root(root), root);
|
|
133
159
|
}
|
|
134
160
|
|
|
161
|
+
#[test]
|
|
162
|
+
fn test_find_project_root_opt_reports_when_there_is_no_root_at_all() {
|
|
163
|
+
// arrange: a directory tree with no pyproject.toml anywhere above it. The
|
|
164
|
+
// best-effort `find_project_root` still has to return *something*, which is
|
|
165
|
+
// exactly why the Option-returning variant exists -- a caller that needs to
|
|
166
|
+
// skip gracefully (single-file indexing) can't tell "root found" from "gave
|
|
167
|
+
// up and handed back the start path" through the infallible one.
|
|
168
|
+
let temp = tempfile::tempdir().unwrap();
|
|
169
|
+
let sub = temp.path().join("a/b/c");
|
|
170
|
+
fs::create_dir_all(&sub).unwrap();
|
|
171
|
+
|
|
172
|
+
// act / assert -- no pyproject.toml exists in the tempdir, but a real one may
|
|
173
|
+
// well exist further up (e.g. checked-out under a repo), so only assert the
|
|
174
|
+
// agreement between the two functions rather than an absolute None.
|
|
175
|
+
match find_project_root_opt(&sub) {
|
|
176
|
+
Some(found) => assert_eq!(find_project_root(&sub), found),
|
|
177
|
+
None => assert_eq!(find_project_root(&sub), sub),
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// A root that genuinely exists is reported as Some(that root).
|
|
181
|
+
fs::write(temp.path().join("pyproject.toml"), "").unwrap();
|
|
182
|
+
assert_eq!(find_project_root_opt(&sub), Some(temp.path().to_path_buf()));
|
|
183
|
+
}
|
|
184
|
+
|
|
135
185
|
#[test]
|
|
136
186
|
fn test_is_enabled() {
|
|
137
187
|
let temp = tempfile::tempdir().unwrap();
|
|
@@ -16,6 +16,12 @@ pub(crate) const CODE_RESERVED_NAME: &str = "reserved-name";
|
|
|
16
16
|
pub(crate) const CODE_UNTRACKED_DATAFRAME: &str = "untracked-dataframe";
|
|
17
17
|
pub(crate) const CODE_DROPPED_UNKNOWN_COLUMN: &str = "dropped-unknown-column";
|
|
18
18
|
pub(crate) const CODE_MISSING_COLUMN: &str = "missing-column";
|
|
19
|
+
// A recognized DataFrame with no resolvable column list (unresolved entity_df in
|
|
20
|
+
// a Feast retrieval, a bare `-> pd.DataFrame` return with no attached Schema).
|
|
21
|
+
// Unlike CODE_UNKNOWN_COLUMN, there's no schema to check against, so access is
|
|
22
|
+
// unverifiable rather than definitely wrong. Raised at every access site, since
|
|
23
|
+
// the unresolved state propagates through returns, reassignment, and merge/concat.
|
|
24
|
+
pub(crate) const CODE_UNVERIFIABLE_COLUMN: &str = "unverifiable-column";
|
|
19
25
|
|
|
20
26
|
// Return true if the source line at `line` (1-indexed) carries a
|
|
21
27
|
// `# typedframes: ignore` or `# typedframes: ignore[code]` comment.
|
|
@@ -48,6 +54,10 @@ pub(crate) fn is_line_ignored(source: &str, line: usize, code: &str) -> bool {
|
|
|
48
54
|
/// known column set (`dataframes_typed`). This is informational — a low ratio
|
|
49
55
|
/// means the check had little to validate, not that the file has fewer
|
|
50
56
|
/// problems. See [`crate::linter::Linter`] for exactly what is counted.
|
|
57
|
+
///
|
|
58
|
+
/// An origin with no resolvable column list (see `CODE_UNVERIFIABLE_COLUMN`) does
|
|
59
|
+
/// NOT count toward `dataframes_typed` — it falls into `untyped_sites` like any
|
|
60
|
+
/// other unresolved DataFrame.
|
|
51
61
|
#[derive(Debug, Serialize, Default)]
|
|
52
62
|
pub struct FileStats {
|
|
53
63
|
pub dataframes_total: usize,
|