typedframes 0.12.0__tar.gz → 0.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {typedframes-0.12.0 → typedframes-0.13.0}/PKG-INFO +14 -13
- {typedframes-0.12.0 → typedframes-0.13.0}/README.md +13 -12
- {typedframes-0.12.0 → typedframes-0.13.0}/pyproject.toml +1 -1
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/Cargo.lock +41 -1
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/Cargo.toml +10 -1
- typedframes-0.13.0/rust/src/call_usage.rs +425 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/column_usage.rs +36 -16
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/errors.rs +1 -0
- typedframes-0.13.0/rust/src/fast_hash.rs +106 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/helpers.rs +96 -5
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/index.rs +562 -249
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/lib.rs +29 -2
- typedframes-0.13.0/rust/src/library_names.txt +2852 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/linter.rs +533 -18
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/parallel.rs +76 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/pyapi.rs +327 -84
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/report.rs +233 -18
- typedframes-0.13.0/rust/src/report_minimize.rs +315 -0
- typedframes-0.13.0/rust/src/report_names.rs +311 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/static_eval.rs +372 -124
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/suggest.rs +45 -1
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/__init__.py +37 -9
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/_rust_checker.pyi +19 -7
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/cli.py +209 -43
- typedframes-0.12.0/rust/src/report_names.rs +0 -631
- {typedframes-0.12.0 → typedframes-0.13.0}/LICENSE +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/.cargo/config.toml +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/README.md +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/benches/parser_bench.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/build.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/apply.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/ast_extract.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/column_consuming_methods.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/config.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/constants.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/contract.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/frame_ops.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/main.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/notebook.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/phrase.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/sql.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/typo.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/words.txt +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/rust/tests/integration_test.rs +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/base_schema.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/cache.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/column.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/column_group.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/column_group_error.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/column_set.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/compat.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/missing_dependency_error.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/mypy.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/pandera.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/py.typed +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/python315.py +0 -0
- {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/schema_algebra.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: typedframes
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -44,7 +44,7 @@ Project-URL: Repository, https://github.com/w-martin/typedframes
|
|
|
44
44
|
|
|
45
45
|
> ⚠️ **Project Status: Proof of Concept**
|
|
46
46
|
>
|
|
47
|
-
> `typedframes` (v0.
|
|
47
|
+
> `typedframes` (v0.13.0) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
|
|
48
48
|
> integrations work, but expect rough edges. The codebase prioritizes demonstrating the viability of static DataFrame
|
|
49
49
|
> column checking over production-grade stability.
|
|
50
50
|
>
|
|
@@ -506,7 +506,7 @@ repos:
|
|
|
506
506
|
name: typedframes check
|
|
507
507
|
entry: typedframes check . --strict
|
|
508
508
|
language: python
|
|
509
|
-
additional_dependencies: ["typedframes==0.
|
|
509
|
+
additional_dependencies: ["typedframes==0.13.0"]
|
|
510
510
|
types_or: [python, jupyter]
|
|
511
511
|
pass_filenames: false
|
|
512
512
|
```
|
|
@@ -555,7 +555,7 @@ The action installs the PyPI wheel into a throwaway virtualenv and runs the chec
|
|
|
555
555
|
| Input | Default | |
|
|
556
556
|
|-------|---------|-|
|
|
557
557
|
| `path` | `.` | File or directory to check |
|
|
558
|
-
| `version` | `latest` | PyPI version to install, e.g. `"0.
|
|
558
|
+
| `version` | `latest` | PyPI version to install, e.g. `"0.13.0"` |
|
|
559
559
|
| `strict` | `true` | Fail the step on errors — `typedframes check` exits 0 without it |
|
|
560
560
|
| `coverage-fail-under` | *(unset)* | Minimum DataFrame schema coverage, e.g. `"90"` |
|
|
561
561
|
| `coverage-detail` | `summary` | Or `term-missing` for the per-file breakdown, `explain` to diagnose a lower-than-expected total |
|
|
@@ -583,7 +583,8 @@ and the files are written together or not at all. See the
|
|
|
583
583
|
|
|
584
584
|
If `check` or `suggest` gets a site wrong and the code can't be shared, `typedframes report quiet-amber src/` prints a
|
|
585
585
|
bug-report body with a redacted copy of the code around that site, checked against the checker so it reproduces the same
|
|
586
|
-
result.
|
|
586
|
+
result. `typedframes report --all src/` does this for every untyped site, grouped by why the checker decided as it did.
|
|
587
|
+
Redacted is not anonymous: read it before posting, and don't use it on code you aren't allowed to share. See
|
|
587
588
|
[Reporting a problem](docs/usage.md#reporting-a-problem-report), and
|
|
588
589
|
[Security and data handling](docs/security.md) for what the tool reads, writes and connects to.
|
|
589
590
|
|
|
@@ -694,17 +695,17 @@ plus the requested features, so both halves get checked normally. See
|
|
|
694
695
|
Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
|
|
695
696
|
|
|
696
697
|
**Benchmark results** (median of 5 runs ± one standard deviation, 3 warmup, caches cleared between runs):
|
|
697
|
-
*2026-10-
|
|
698
|
+
*2026-10-13 · Darwin 27.0.0 · arm · CPython 3.15.0 · 64GiB RAM · Great Expectations pinned @ 1.23.2*
|
|
698
699
|
|
|
699
700
|
| Tool | Version | What it does | typedframes (14 files) | great_expectations (494 files) |
|
|
700
701
|
|------|---------|--------------|------------------------|--------------------------------|
|
|
701
|
-
| typedframes | 0.
|
|
702
|
-
| ruff | 0.17.0 | Linter (no type checking) |
|
|
703
|
-
| ty | 0.0.84 | Type checker |
|
|
704
|
-
| pyrefly | 1.3.2 | Type checker |
|
|
705
|
-
| mypy | 2.4.0 | Type checker (no plugin) | 1.
|
|
706
|
-
| mypy + typedframes | 2.4.0 | Type checker + column checker | 1.84 ± 0.
|
|
707
|
-
| pyright | 1.1.414 | Type checker |
|
|
702
|
+
| typedframes | 0.13.0 | DataFrame column checker | 42.8 ± 0.3 ms | 72.4 ± 0.8 ms |
|
|
703
|
+
| ruff | 0.17.0 | Linter (no type checking) | 27.6 ± 1.0 ms | 222.1 ± 2.7 ms |
|
|
704
|
+
| ty | 0.0.84 | Type checker | 59.6 ± 1.4 ms | 761.4 ± 9.4 ms |
|
|
705
|
+
| pyrefly | 1.3.2 | Type checker | 98.0 ± 12.2 ms | 276.6 ± 7.3 ms |
|
|
706
|
+
| mypy | 2.4.0 | Type checker (no plugin) | 1.83 ± 0.13 s | 3.08 ± 0.03 s |
|
|
707
|
+
| mypy + typedframes | 2.4.0 | Type checker + column checker | 1.84 ± 0.03 s | 3.32 ± 0.09 s |
|
|
708
|
+
| pyright | 1.1.414 | Type checker | 812.0 ± 12.3 ms | 3.91 ± 0.04 s |
|
|
708
709
|
|
|
709
710
|
*Run `uv run python benchmarks/benchmark_checkers.py` to reproduce.*
|
|
710
711
|
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
> ⚠️ **Project Status: Proof of Concept**
|
|
10
10
|
>
|
|
11
|
-
> `typedframes` (v0.
|
|
11
|
+
> `typedframes` (v0.13.0) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
|
|
12
12
|
> integrations work, but expect rough edges. The codebase prioritizes demonstrating the viability of static DataFrame
|
|
13
13
|
> column checking over production-grade stability.
|
|
14
14
|
>
|
|
@@ -470,7 +470,7 @@ repos:
|
|
|
470
470
|
name: typedframes check
|
|
471
471
|
entry: typedframes check . --strict
|
|
472
472
|
language: python
|
|
473
|
-
additional_dependencies: ["typedframes==0.
|
|
473
|
+
additional_dependencies: ["typedframes==0.13.0"]
|
|
474
474
|
types_or: [python, jupyter]
|
|
475
475
|
pass_filenames: false
|
|
476
476
|
```
|
|
@@ -519,7 +519,7 @@ The action installs the PyPI wheel into a throwaway virtualenv and runs the chec
|
|
|
519
519
|
| Input | Default | |
|
|
520
520
|
|-------|---------|-|
|
|
521
521
|
| `path` | `.` | File or directory to check |
|
|
522
|
-
| `version` | `latest` | PyPI version to install, e.g. `"0.
|
|
522
|
+
| `version` | `latest` | PyPI version to install, e.g. `"0.13.0"` |
|
|
523
523
|
| `strict` | `true` | Fail the step on errors — `typedframes check` exits 0 without it |
|
|
524
524
|
| `coverage-fail-under` | *(unset)* | Minimum DataFrame schema coverage, e.g. `"90"` |
|
|
525
525
|
| `coverage-detail` | `summary` | Or `term-missing` for the per-file breakdown, `explain` to diagnose a lower-than-expected total |
|
|
@@ -547,7 +547,8 @@ and the files are written together or not at all. See the
|
|
|
547
547
|
|
|
548
548
|
If `check` or `suggest` gets a site wrong and the code can't be shared, `typedframes report quiet-amber src/` prints a
|
|
549
549
|
bug-report body with a redacted copy of the code around that site, checked against the checker so it reproduces the same
|
|
550
|
-
result.
|
|
550
|
+
result. `typedframes report --all src/` does this for every untyped site, grouped by why the checker decided as it did.
|
|
551
|
+
Redacted is not anonymous: read it before posting, and don't use it on code you aren't allowed to share. See
|
|
551
552
|
[Reporting a problem](docs/usage.md#reporting-a-problem-report), and
|
|
552
553
|
[Security and data handling](docs/security.md) for what the tool reads, writes and connects to.
|
|
553
554
|
|
|
@@ -658,17 +659,17 @@ plus the requested features, so both halves get checked normally. See
|
|
|
658
659
|
Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
|
|
659
660
|
|
|
660
661
|
**Benchmark results** (median of 5 runs ± one standard deviation, 3 warmup, caches cleared between runs):
|
|
661
|
-
*2026-10-
|
|
662
|
+
*2026-10-13 · Darwin 27.0.0 · arm · CPython 3.15.0 · 64GiB RAM · Great Expectations pinned @ 1.23.2*
|
|
662
663
|
|
|
663
664
|
| Tool | Version | What it does | typedframes (14 files) | great_expectations (494 files) |
|
|
664
665
|
|------|---------|--------------|------------------------|--------------------------------|
|
|
665
|
-
| typedframes | 0.
|
|
666
|
-
| ruff | 0.17.0 | Linter (no type checking) |
|
|
667
|
-
| ty | 0.0.84 | Type checker |
|
|
668
|
-
| pyrefly | 1.3.2 | Type checker |
|
|
669
|
-
| mypy | 2.4.0 | Type checker (no plugin) | 1.
|
|
670
|
-
| mypy + typedframes | 2.4.0 | Type checker + column checker | 1.84 ± 0.
|
|
671
|
-
| pyright | 1.1.414 | Type checker |
|
|
666
|
+
| typedframes | 0.13.0 | DataFrame column checker | 42.8 ± 0.3 ms | 72.4 ± 0.8 ms |
|
|
667
|
+
| ruff | 0.17.0 | Linter (no type checking) | 27.6 ± 1.0 ms | 222.1 ± 2.7 ms |
|
|
668
|
+
| ty | 0.0.84 | Type checker | 59.6 ± 1.4 ms | 761.4 ± 9.4 ms |
|
|
669
|
+
| pyrefly | 1.3.2 | Type checker | 98.0 ± 12.2 ms | 276.6 ± 7.3 ms |
|
|
670
|
+
| mypy | 2.4.0 | Type checker (no plugin) | 1.83 ± 0.13 s | 3.08 ± 0.03 s |
|
|
671
|
+
| mypy + typedframes | 2.4.0 | Type checker + column checker | 1.84 ± 0.03 s | 3.32 ± 0.09 s |
|
|
672
|
+
| pyright | 1.1.414 | Type checker | 812.0 ± 12.3 ms | 3.91 ± 0.04 s |
|
|
672
673
|
|
|
673
674
|
*Run `uv run python benchmarks/benchmark_checkers.py` to reproduce.*
|
|
674
675
|
|
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "typedframes"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.13.0"
|
|
8
8
|
description = "Static analysis for pandas and polars DataFrames. Catch column errors at lint-time, not runtime."
|
|
9
9
|
keywords = ["pandas", "polars", "type-checking", "static-analysis", "dataframe", "linter", "mypy-plugin"]
|
|
10
10
|
readme = "README.md"
|
|
@@ -535,6 +535,15 @@ version = "0.2.189"
|
|
|
535
535
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
536
536
|
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
|
537
537
|
|
|
538
|
+
[[package]]
|
|
539
|
+
name = "libmimalloc-sys"
|
|
540
|
+
version = "0.1.49"
|
|
541
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
542
|
+
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
|
|
543
|
+
dependencies = [
|
|
544
|
+
"cc",
|
|
545
|
+
]
|
|
546
|
+
|
|
538
547
|
[[package]]
|
|
539
548
|
name = "linux-raw-sys"
|
|
540
549
|
version = "0.12.1"
|
|
@@ -576,6 +585,15 @@ version = "2.8.3"
|
|
|
576
585
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
577
586
|
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
578
587
|
|
|
588
|
+
[[package]]
|
|
589
|
+
name = "mimalloc"
|
|
590
|
+
version = "0.1.52"
|
|
591
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
592
|
+
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
|
|
593
|
+
dependencies = [
|
|
594
|
+
"libmimalloc-sys",
|
|
595
|
+
]
|
|
596
|
+
|
|
579
597
|
[[package]]
|
|
580
598
|
name = "num-traits"
|
|
581
599
|
version = "0.2.19"
|
|
@@ -1281,6 +1299,26 @@ dependencies = [
|
|
|
1281
1299
|
"syn 3.0.3",
|
|
1282
1300
|
]
|
|
1283
1301
|
|
|
1302
|
+
[[package]]
|
|
1303
|
+
name = "tikv-jemalloc-sys"
|
|
1304
|
+
version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8"
|
|
1305
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1306
|
+
checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0"
|
|
1307
|
+
dependencies = [
|
|
1308
|
+
"cc",
|
|
1309
|
+
"libc",
|
|
1310
|
+
]
|
|
1311
|
+
|
|
1312
|
+
[[package]]
|
|
1313
|
+
name = "tikv-jemallocator"
|
|
1314
|
+
version = "0.7.0"
|
|
1315
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1316
|
+
checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd"
|
|
1317
|
+
dependencies = [
|
|
1318
|
+
"libc",
|
|
1319
|
+
"tikv-jemalloc-sys",
|
|
1320
|
+
]
|
|
1321
|
+
|
|
1284
1322
|
[[package]]
|
|
1285
1323
|
name = "tinytemplate"
|
|
1286
1324
|
version = "1.2.1"
|
|
@@ -1347,10 +1385,11 @@ checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2"
|
|
|
1347
1385
|
|
|
1348
1386
|
[[package]]
|
|
1349
1387
|
name = "typedframes_checker"
|
|
1350
|
-
version = "0.
|
|
1388
|
+
version = "0.13.0"
|
|
1351
1389
|
dependencies = [
|
|
1352
1390
|
"anyhow",
|
|
1353
1391
|
"criterion",
|
|
1392
|
+
"mimalloc",
|
|
1354
1393
|
"pyo3",
|
|
1355
1394
|
"rmp-serde",
|
|
1356
1395
|
"ruff_notebook",
|
|
@@ -1362,6 +1401,7 @@ dependencies = [
|
|
|
1362
1401
|
"serde_json",
|
|
1363
1402
|
"sqlparser",
|
|
1364
1403
|
"tempfile",
|
|
1404
|
+
"tikv-jemallocator",
|
|
1365
1405
|
"toml",
|
|
1366
1406
|
]
|
|
1367
1407
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "typedframes_checker"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.13.0"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
description = "A high-performance Rust-based checker for typedframes DataFrame column schemas"
|
|
6
6
|
license = "MIT"
|
|
@@ -52,6 +52,15 @@ sqlparser = "0.62"
|
|
|
52
52
|
# stayed green while local macOS runs did not.
|
|
53
53
|
pyo3 = { version = "0.29", features = ["abi3-py311"] }
|
|
54
54
|
|
|
55
|
+
# The allocator split ruff and ty use: jemalloc where it builds, mimalloc on Windows (jemalloc
|
|
56
|
+
# has no MSVC support), the platform allocator elsewhere. Keep the `cfg`s in `src/lib.rs` in
|
|
57
|
+
# step with these. `disable_initial_exec_tls` lets the library be loaded with `dlopen`.
|
|
58
|
+
[target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), not(target_os = "aix"), not(target_os = "android"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64", target_arch = "riscv64")))'.dependencies]
|
|
59
|
+
tikv-jemallocator = { version = "0.7", features = ["disable_initial_exec_tls"] }
|
|
60
|
+
|
|
61
|
+
[target.'cfg(target_os = "windows")'.dependencies]
|
|
62
|
+
mimalloc = { version = "0.1.52", default-features = false }
|
|
63
|
+
|
|
55
64
|
[dev-dependencies]
|
|
56
65
|
tempfile = "3.8"
|
|
57
66
|
criterion = { version = "0.8", features = ["html_reports"] }
|
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
//! What the callers of a function do with the frame it returns.
|
|
2
|
+
//!
|
|
3
|
+
//! A frame a function returns is only as wide as its callers need if every caller is seen
|
|
4
|
+
//! and each reads named columns and nothing else. This module observes one file: for each
|
|
5
|
+
//! callee name, the columns read from the result of every `x = name(...)` call, and whether
|
|
6
|
+
//! the name is also used in a way that hides a use (called other than as a plain assignment,
|
|
7
|
+
//! assigned inside a loop, handed on, passed around as a value, decorated). Names match by
|
|
8
|
+
//! bare name, so a call to an unrelated function of the same name only adds columns, never
|
|
9
|
+
//! removes them. The project-wide merge is `index::resolve_call_usage`.
|
|
10
|
+
|
|
11
|
+
use crate::column_usage::{
|
|
12
|
+
blocks_narrowing, declares_global_or_nonlocal, first_nested_scope_mention, scan_usage_with,
|
|
13
|
+
HelperResolver,
|
|
14
|
+
};
|
|
15
|
+
use crate::helpers::FileHelpers;
|
|
16
|
+
use crate::static_eval::{Analysis, ImportedModules};
|
|
17
|
+
use ruff_python_ast::visitor::{self as ast_visitor, Visitor};
|
|
18
|
+
use ruff_python_ast::{ExceptHandler, Expr, ModModule, Stmt};
|
|
19
|
+
use std::collections::{BTreeSet, HashMap, HashSet};
|
|
20
|
+
|
|
21
|
+
/// What one file shows about the calls to a name.
|
|
22
|
+
#[derive(Debug, Default, PartialEq)]
|
|
23
|
+
pub(crate) struct Observation {
|
|
24
|
+
/// Columns read from the result of the file's calls.
|
|
25
|
+
pub(crate) columns: BTreeSet<String>,
|
|
26
|
+
/// How many calls the file makes.
|
|
27
|
+
pub(crate) calls: usize,
|
|
28
|
+
/// Some use of the name or of a call's result isn't a read of named columns.
|
|
29
|
+
pub(crate) unknown: bool,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
#[derive(Debug, Default, PartialEq)]
|
|
33
|
+
pub(crate) struct FileCalls {
|
|
34
|
+
pub(crate) observations: HashMap<String, Observation>,
|
|
35
|
+
/// Names of the functions and methods the file defines.
|
|
36
|
+
pub(crate) defined: HashSet<String>,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
fn callee_name(func: &Expr) -> Option<&str> {
|
|
40
|
+
match func {
|
|
41
|
+
Expr::Name(n) => Some(n.id.as_str()),
|
|
42
|
+
Expr::Attribute(a) => Some(a.attr.as_str()),
|
|
43
|
+
_ => None,
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Every call by callee name, and every other mention of a name that could be a function
|
|
48
|
+
// being handed around (`callbacks = [load]`, `map(load, ...)`, `obj.load`).
|
|
49
|
+
#[derive(Default)]
|
|
50
|
+
struct Mentions {
|
|
51
|
+
calls: HashMap<String, usize>,
|
|
52
|
+
other: HashSet<String>,
|
|
53
|
+
defined: HashSet<String>,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
impl<'a> Visitor<'a> for Mentions {
|
|
57
|
+
fn visit_stmt(&mut self, stmt: &'a Stmt) {
|
|
58
|
+
// `from lib import load as fetch` calls `load` under another name.
|
|
59
|
+
let aliased = match stmt {
|
|
60
|
+
Stmt::Import(i) => Some(&i.names),
|
|
61
|
+
Stmt::ImportFrom(i) => Some(&i.names),
|
|
62
|
+
_ => None,
|
|
63
|
+
};
|
|
64
|
+
for alias in aliased.into_iter().flatten().filter(|a| a.asname.is_some()) {
|
|
65
|
+
self.other.insert(alias.name.to_string());
|
|
66
|
+
}
|
|
67
|
+
if let Stmt::FunctionDef(def) = stmt {
|
|
68
|
+
self.defined.insert(def.name.to_string());
|
|
69
|
+
// A decorator (a route, a task, a fixture) may call it where this can't see.
|
|
70
|
+
if !def.decorator_list.is_empty() {
|
|
71
|
+
self.other.insert(def.name.to_string());
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
ast_visitor::walk_stmt(self, stmt);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
fn visit_expr(&mut self, expr: &'a Expr) {
|
|
78
|
+
match expr {
|
|
79
|
+
Expr::Call(call) => {
|
|
80
|
+
if let Some(name) = callee_name(&call.func) {
|
|
81
|
+
*self.calls.entry(name.to_string()).or_default() += 1;
|
|
82
|
+
}
|
|
83
|
+
match &*call.func {
|
|
84
|
+
Expr::Name(_) => {}
|
|
85
|
+
Expr::Attribute(a) => self.visit_expr(&a.value),
|
|
86
|
+
other => self.visit_expr(other),
|
|
87
|
+
}
|
|
88
|
+
for arg in &call.arguments.args {
|
|
89
|
+
self.visit_expr(arg);
|
|
90
|
+
}
|
|
91
|
+
for keyword in &call.arguments.keywords {
|
|
92
|
+
self.visit_expr(&keyword.value);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
Expr::Name(n) => {
|
|
96
|
+
self.other.insert(n.id.to_string());
|
|
97
|
+
}
|
|
98
|
+
Expr::Attribute(a) => {
|
|
99
|
+
self.other.insert(a.attr.to_string());
|
|
100
|
+
self.visit_expr(&a.value);
|
|
101
|
+
}
|
|
102
|
+
_ => ast_visitor::walk_expr(self, expr),
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// The remainder of each enclosing block, innermost first, after `extra`.
|
|
108
|
+
fn following<'a>(extra: &[&'a [Stmt]], rest: &'a [Stmt], tails: &[&'a [Stmt]]) -> Vec<&'a [Stmt]> {
|
|
109
|
+
let mut chain: Vec<&[Stmt]> = extra.to_vec();
|
|
110
|
+
chain.push(rest);
|
|
111
|
+
chain.extend_from_slice(tails);
|
|
112
|
+
chain
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Finds each `x = name(...)` and reads off what the rest of its scope does with `x`.
|
|
116
|
+
struct Scanner<'h> {
|
|
117
|
+
helpers: &'h dyn HelperResolver,
|
|
118
|
+
// The module's own statements: a name assigned there can be imported and read by other
|
|
119
|
+
// files, which this scan doesn't follow.
|
|
120
|
+
module: &'h [Stmt],
|
|
121
|
+
// Only calls to these names are worth following.
|
|
122
|
+
defined: &'h HashSet<String>,
|
|
123
|
+
// By callee name: the calls whose result was fully accounted for, and their columns.
|
|
124
|
+
known: HashMap<String, Observation>,
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
impl Scanner<'_> {
|
|
128
|
+
fn scope(&mut self, body: &[Stmt]) {
|
|
129
|
+
self.block(body, &[], false, body);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
fn class_body(&mut self, body: &[Stmt]) {
|
|
133
|
+
for stmt in body {
|
|
134
|
+
match stmt {
|
|
135
|
+
Stmt::FunctionDef(def) => self.scope(&def.body),
|
|
136
|
+
Stmt::ClassDef(inner) => self.class_body(&inner.body),
|
|
137
|
+
_ => {}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
fn block<'a>(
|
|
143
|
+
&mut self,
|
|
144
|
+
block: &'a [Stmt],
|
|
145
|
+
tails: &[&'a [Stmt]],
|
|
146
|
+
in_loop: bool,
|
|
147
|
+
scope_body: &'a [Stmt],
|
|
148
|
+
) {
|
|
149
|
+
for (i, stmt) in block.iter().enumerate() {
|
|
150
|
+
let rest = &block[i + 1..];
|
|
151
|
+
self.assignment(stmt, rest, tails, in_loop, scope_body);
|
|
152
|
+
let after = |extra: &[&'a [Stmt]]| following(extra, rest, tails);
|
|
153
|
+
match stmt {
|
|
154
|
+
Stmt::FunctionDef(def) => self.scope(&def.body),
|
|
155
|
+
Stmt::ClassDef(class) => self.class_body(&class.body),
|
|
156
|
+
Stmt::If(s) => {
|
|
157
|
+
let chain = after(&[]);
|
|
158
|
+
self.block(&s.body, &chain, in_loop, scope_body);
|
|
159
|
+
for clause in &s.elif_else_clauses {
|
|
160
|
+
self.block(&clause.body, &chain, in_loop, scope_body);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
Stmt::With(s) => self.block(&s.body, &after(&[]), in_loop, scope_body),
|
|
164
|
+
Stmt::Match(s) => {
|
|
165
|
+
let chain = after(&[]);
|
|
166
|
+
for case in &s.cases {
|
|
167
|
+
self.block(&case.body, &chain, in_loop, scope_body);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
Stmt::Try(s) => {
|
|
171
|
+
let handlers: Vec<&[Stmt]> = s
|
|
172
|
+
.handlers
|
|
173
|
+
.iter()
|
|
174
|
+
.map(|h| {
|
|
175
|
+
let ExceptHandler::ExceptHandler(h) = h;
|
|
176
|
+
h.body.as_slice()
|
|
177
|
+
})
|
|
178
|
+
.collect();
|
|
179
|
+
let finalbody = s.finalbody.as_slice();
|
|
180
|
+
let mut body_extra: Vec<&[Stmt]> = handlers.clone();
|
|
181
|
+
body_extra.push(s.orelse.as_slice());
|
|
182
|
+
body_extra.push(finalbody);
|
|
183
|
+
self.block(&s.body, &after(&body_extra), in_loop, scope_body);
|
|
184
|
+
let tail = after(&[finalbody]);
|
|
185
|
+
for handler in &handlers {
|
|
186
|
+
self.block(handler, &tail, in_loop, scope_body);
|
|
187
|
+
}
|
|
188
|
+
self.block(&s.orelse, &tail, in_loop, scope_body);
|
|
189
|
+
self.block(finalbody, &after(&[]), in_loop, scope_body);
|
|
190
|
+
}
|
|
191
|
+
Stmt::For(s) => {
|
|
192
|
+
let chain = after(&[]);
|
|
193
|
+
self.block(&s.body, &chain, true, scope_body);
|
|
194
|
+
self.block(&s.orelse, &chain, in_loop, scope_body);
|
|
195
|
+
}
|
|
196
|
+
Stmt::While(s) => {
|
|
197
|
+
let chain = after(&[]);
|
|
198
|
+
self.block(&s.body, &chain, true, scope_body);
|
|
199
|
+
self.block(&s.orelse, &chain, in_loop, scope_body);
|
|
200
|
+
}
|
|
201
|
+
_ => {}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
fn assignment<'a>(
|
|
207
|
+
&mut self,
|
|
208
|
+
stmt: &'a Stmt,
|
|
209
|
+
rest: &'a [Stmt],
|
|
210
|
+
tails: &[&'a [Stmt]],
|
|
211
|
+
in_loop: bool,
|
|
212
|
+
scope_body: &'a [Stmt],
|
|
213
|
+
) {
|
|
214
|
+
let (target, value) = match stmt {
|
|
215
|
+
Stmt::Assign(a) => match (a.targets.as_slice(), &*a.value) {
|
|
216
|
+
([Expr::Name(target)], value) => (target, value),
|
|
217
|
+
_ => return,
|
|
218
|
+
},
|
|
219
|
+
Stmt::AnnAssign(a) => match (&*a.target, a.value.as_deref()) {
|
|
220
|
+
(Expr::Name(target), Some(value)) => (target, value),
|
|
221
|
+
_ => return,
|
|
222
|
+
},
|
|
223
|
+
_ => return,
|
|
224
|
+
};
|
|
225
|
+
let Expr::Call(call) = value else {
|
|
226
|
+
return;
|
|
227
|
+
};
|
|
228
|
+
// In a loop the result can be read before the next pass reassigns it.
|
|
229
|
+
let Some(callee) =
|
|
230
|
+
callee_name(&call.func).filter(|name| !in_loop && self.defined.contains(*name))
|
|
231
|
+
else {
|
|
232
|
+
return;
|
|
233
|
+
};
|
|
234
|
+
if std::ptr::eq(scope_body, self.module) {
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
let downstream = std::iter::once(rest)
|
|
238
|
+
.chain(tails.iter().copied())
|
|
239
|
+
.flat_map(|stmts| stmts.iter());
|
|
240
|
+
let usage = scan_usage_with(target.id.as_str(), downstream, &|_| 0, self.helpers);
|
|
241
|
+
if usage.other_uses.iter().any(blocks_narrowing)
|
|
242
|
+
|| first_nested_scope_mention(scope_body, &usage.ever_tainted).is_some()
|
|
243
|
+
|| usage
|
|
244
|
+
.ever_tainted
|
|
245
|
+
.iter()
|
|
246
|
+
.any(|alias| declares_global_or_nonlocal(scope_body, alias))
|
|
247
|
+
{
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
let seen = self.known.entry(callee.to_string()).or_default();
|
|
251
|
+
seen.calls += 1;
|
|
252
|
+
seen.columns
|
|
253
|
+
.extend(usage.accesses.iter().map(|a| a.column.clone()));
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/// The names of the functions and methods `module` defines.
|
|
258
|
+
pub(crate) fn defined_names(module: &ModModule) -> HashSet<String> {
|
|
259
|
+
let mut mentions = Mentions::default();
|
|
260
|
+
for stmt in &module.body {
|
|
261
|
+
mentions.visit_stmt(stmt);
|
|
262
|
+
}
|
|
263
|
+
mentions.defined
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/// What `module` shows about the calls to each name it calls or mentions. Only calls to a
|
|
267
|
+
/// name in `defined` (what the project defines) have their results followed.
|
|
268
|
+
pub(crate) fn module_call_usage(module: &ModModule, defined: &HashSet<String>) -> FileCalls {
|
|
269
|
+
let mut mentions = Mentions::default();
|
|
270
|
+
for stmt in &module.body {
|
|
271
|
+
mentions.visit_stmt(stmt);
|
|
272
|
+
}
|
|
273
|
+
let none = ImportedModules::new();
|
|
274
|
+
let analysis = Analysis::build(module, &none);
|
|
275
|
+
let imported = HashMap::new();
|
|
276
|
+
let helpers = FileHelpers::new(&analysis, &imported).with_methods(module);
|
|
277
|
+
let mut scanner = Scanner {
|
|
278
|
+
helpers: &helpers,
|
|
279
|
+
module: &module.body,
|
|
280
|
+
defined,
|
|
281
|
+
known: HashMap::new(),
|
|
282
|
+
};
|
|
283
|
+
scanner.scope(&module.body);
|
|
284
|
+
|
|
285
|
+
let mut observations: HashMap<String, Observation> = HashMap::new();
|
|
286
|
+
for (name, calls) in &mentions.calls {
|
|
287
|
+
let known = scanner.known.remove(name);
|
|
288
|
+
let accounted = known.as_ref().map_or(0, |k| k.calls);
|
|
289
|
+
observations.insert(
|
|
290
|
+
name.clone(),
|
|
291
|
+
Observation {
|
|
292
|
+
columns: known.map(|k| k.columns).unwrap_or_default(),
|
|
293
|
+
calls: *calls,
|
|
294
|
+
unknown: *calls != accounted || mentions.other.contains(name),
|
|
295
|
+
},
|
|
296
|
+
);
|
|
297
|
+
}
|
|
298
|
+
for name in &mentions.other {
|
|
299
|
+
observations.entry(name.clone()).or_default().unknown = true;
|
|
300
|
+
}
|
|
301
|
+
FileCalls {
|
|
302
|
+
observations,
|
|
303
|
+
defined: mentions.defined,
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
#[cfg(test)]
|
|
308
|
+
mod tests {
|
|
309
|
+
use super::*;
|
|
310
|
+
use ruff_python_parser::parse_module;
|
|
311
|
+
|
|
312
|
+
fn observe(source: &str) -> FileCalls {
|
|
313
|
+
let module = parse_module(source).unwrap().into_syntax();
|
|
314
|
+
let mut defined = defined_names(&module);
|
|
315
|
+
defined.extend(["load".to_string()]);
|
|
316
|
+
module_call_usage(&module, &defined)
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
fn columns(calls: &FileCalls, name: &str) -> Vec<String> {
|
|
320
|
+
calls.observations[name].columns.iter().cloned().collect()
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
#[test]
|
|
324
|
+
fn test_should_collect_the_columns_read_from_each_assigned_call() {
|
|
325
|
+
let calls = observe(
|
|
326
|
+
"def f():\n a = load()\n print(a[\"x\"])\n b = load()\n return b[\"y\"]\n",
|
|
327
|
+
);
|
|
328
|
+
|
|
329
|
+
let seen = &calls.observations["load"];
|
|
330
|
+
assert!(!seen.unknown);
|
|
331
|
+
assert_eq!(seen.calls, 2);
|
|
332
|
+
assert_eq!(columns(&calls, "load"), vec!["x", "y"]);
|
|
333
|
+
assert!(calls.defined.contains("f"));
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
#[test]
|
|
337
|
+
fn test_should_see_calls_through_methods_and_nested_blocks() {
|
|
338
|
+
let calls = observe(
|
|
339
|
+
"class S:\n def f(self):\n with ctx():\n a = self.load()\n return a[\"x\"]\n",
|
|
340
|
+
);
|
|
341
|
+
|
|
342
|
+
assert!(!calls.observations["load"].unknown);
|
|
343
|
+
assert_eq!(columns(&calls, "load"), vec!["x"]);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
#[test]
|
|
347
|
+
fn test_should_not_know_a_call_whose_result_is_handed_on() {
|
|
348
|
+
let calls = observe("def f():\n a = load()\n return a\n");
|
|
349
|
+
|
|
350
|
+
assert!(calls.observations["load"].unknown);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
#[test]
|
|
354
|
+
fn test_should_not_know_a_call_that_is_not_a_plain_assignment() {
|
|
355
|
+
let calls = observe("def f():\n a = load()\n return load()[\"x\"], a[\"y\"]\n");
|
|
356
|
+
|
|
357
|
+
assert!(calls.observations["load"].unknown);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
#[test]
|
|
361
|
+
fn test_should_not_know_a_call_inside_a_loop() {
|
|
362
|
+
let calls =
|
|
363
|
+
observe("def f(xs):\n for x in xs:\n a = load(x)\n print(a[\"y\"])\n");
|
|
364
|
+
|
|
365
|
+
assert!(calls.observations["load"].unknown);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
#[test]
|
|
369
|
+
fn test_should_not_know_a_function_passed_around_as_a_value() {
|
|
370
|
+
let calls = observe("def f():\n a = load()\n print(a[\"x\"])\n register(load)\n");
|
|
371
|
+
|
|
372
|
+
assert!(calls.observations["load"].unknown);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
#[test]
|
|
376
|
+
fn test_should_not_know_a_decorated_function() {
|
|
377
|
+
let calls = observe("@route\ndef load():\n return 1\n\n\ndef f():\n a = load()\n print(a[\"x\"])\n");
|
|
378
|
+
|
|
379
|
+
assert!(calls.observations["load"].unknown);
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
#[test]
|
|
383
|
+
fn test_should_read_through_a_helper_that_only_reads_columns() {
|
|
384
|
+
let calls = observe(
|
|
385
|
+
"def total(frame):\n return frame[\"z\"]\n\n\ndef f():\n a = load()\n return total(a)\n",
|
|
386
|
+
);
|
|
387
|
+
|
|
388
|
+
assert!(!calls.observations["load"].unknown);
|
|
389
|
+
assert_eq!(columns(&calls, "load"), vec!["z"]);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
#[test]
|
|
393
|
+
fn test_should_not_count_an_unrelated_assignment_of_something_else() {
|
|
394
|
+
let calls = observe("def f():\n a = 1\n return a\n");
|
|
395
|
+
|
|
396
|
+
assert!(!calls.observations.contains_key("load"));
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
#[test]
|
|
400
|
+
fn test_should_not_know_a_function_another_file_imports_under_an_alias() {
|
|
401
|
+
let calls = observe(
|
|
402
|
+
"from lib import load as fetch\n\n\ndef f():\n a = fetch()\n return a[\"x\"]\n",
|
|
403
|
+
);
|
|
404
|
+
|
|
405
|
+
assert!(calls.observations["load"].unknown);
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
#[test]
|
|
409
|
+
fn test_should_not_know_a_call_assigned_at_module_level() {
|
|
410
|
+
let calls = observe("a = load()\nprint(a[\"x\"])\n");
|
|
411
|
+
|
|
412
|
+
assert!(calls.observations["load"].unknown);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
#[test]
|
|
416
|
+
fn test_should_not_follow_calls_to_names_the_project_does_not_define() {
|
|
417
|
+
let module = parse_module("def f():\n a = other()\n return a[\"x\"]\n")
|
|
418
|
+
.unwrap()
|
|
419
|
+
.into_syntax();
|
|
420
|
+
|
|
421
|
+
let calls = module_call_usage(&module, &defined_names(&module));
|
|
422
|
+
|
|
423
|
+
assert!(calls.observations["other"].unknown);
|
|
424
|
+
}
|
|
425
|
+
}
|