typedframes 0.12.0__tar.gz → 0.13.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {typedframes-0.12.0 → typedframes-0.13.0}/PKG-INFO +14 -13
  2. {typedframes-0.12.0 → typedframes-0.13.0}/README.md +13 -12
  3. {typedframes-0.12.0 → typedframes-0.13.0}/pyproject.toml +1 -1
  4. {typedframes-0.12.0 → typedframes-0.13.0}/rust/Cargo.lock +41 -1
  5. {typedframes-0.12.0 → typedframes-0.13.0}/rust/Cargo.toml +10 -1
  6. typedframes-0.13.0/rust/src/call_usage.rs +425 -0
  7. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/column_usage.rs +36 -16
  8. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/errors.rs +1 -0
  9. typedframes-0.13.0/rust/src/fast_hash.rs +106 -0
  10. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/helpers.rs +96 -5
  11. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/index.rs +562 -249
  12. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/lib.rs +29 -2
  13. typedframes-0.13.0/rust/src/library_names.txt +2852 -0
  14. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/linter.rs +533 -18
  15. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/parallel.rs +76 -0
  16. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/pyapi.rs +327 -84
  17. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/report.rs +233 -18
  18. typedframes-0.13.0/rust/src/report_minimize.rs +315 -0
  19. typedframes-0.13.0/rust/src/report_names.rs +311 -0
  20. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/static_eval.rs +372 -124
  21. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/suggest.rs +45 -1
  22. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/__init__.py +37 -9
  23. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/_rust_checker.pyi +19 -7
  24. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/cli.py +209 -43
  25. typedframes-0.12.0/rust/src/report_names.rs +0 -631
  26. {typedframes-0.12.0 → typedframes-0.13.0}/LICENSE +0 -0
  27. {typedframes-0.12.0 → typedframes-0.13.0}/rust/.cargo/config.toml +0 -0
  28. {typedframes-0.12.0 → typedframes-0.13.0}/rust/README.md +0 -0
  29. {typedframes-0.12.0 → typedframes-0.13.0}/rust/benches/parser_bench.rs +0 -0
  30. {typedframes-0.12.0 → typedframes-0.13.0}/rust/build.rs +0 -0
  31. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/apply.rs +0 -0
  32. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/ast_extract.rs +0 -0
  33. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/column_consuming_methods.rs +0 -0
  34. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/config.rs +0 -0
  35. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/constants.rs +0 -0
  36. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/contract.rs +0 -0
  37. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/frame_ops.rs +0 -0
  38. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/main.rs +0 -0
  39. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/notebook.rs +0 -0
  40. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/phrase.rs +0 -0
  41. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/sql.rs +0 -0
  42. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/typo.rs +0 -0
  43. {typedframes-0.12.0 → typedframes-0.13.0}/rust/src/words.txt +0 -0
  44. {typedframes-0.12.0 → typedframes-0.13.0}/rust/tests/integration_test.rs +0 -0
  45. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/base_schema.py +0 -0
  46. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/cache.py +0 -0
  47. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/column.py +0 -0
  48. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/column_group.py +0 -0
  49. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/column_group_error.py +0 -0
  50. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/column_set.py +0 -0
  51. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/compat.py +0 -0
  52. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/missing_dependency_error.py +0 -0
  53. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/mypy.py +0 -0
  54. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/pandera.py +0 -0
  55. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/py.typed +0 -0
  56. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/python315.py +0 -0
  57. {typedframes-0.12.0 → typedframes-0.13.0}/src/typedframes/schema_algebra.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: typedframes
3
- Version: 0.12.0
3
+ Version: 0.13.0
4
4
  Classifier: Development Status :: 3 - Alpha
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: License :: OSI Approved :: MIT License
@@ -44,7 +44,7 @@ Project-URL: Repository, https://github.com/w-martin/typedframes
44
44
 
45
45
  > ⚠️ **Project Status: Proof of Concept**
46
46
  >
47
- > `typedframes` (v0.12.0) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
47
+ > `typedframes` (v0.13.0) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
48
48
  > integrations work, but expect rough edges. The codebase prioritizes demonstrating the viability of static DataFrame
49
49
  > column checking over production-grade stability.
50
50
  >
@@ -506,7 +506,7 @@ repos:
506
506
  name: typedframes check
507
507
  entry: typedframes check . --strict
508
508
  language: python
509
- additional_dependencies: ["typedframes==0.12.0"]
509
+ additional_dependencies: ["typedframes==0.13.0"]
510
510
  types_or: [python, jupyter]
511
511
  pass_filenames: false
512
512
  ```
@@ -555,7 +555,7 @@ The action installs the PyPI wheel into a throwaway virtualenv and runs the chec
555
555
  | Input | Default | |
556
556
  |-------|---------|-|
557
557
  | `path` | `.` | File or directory to check |
558
- | `version` | `latest` | PyPI version to install, e.g. `"0.12.0"` |
558
+ | `version` | `latest` | PyPI version to install, e.g. `"0.13.0"` |
559
559
  | `strict` | `true` | Fail the step on errors — `typedframes check` exits 0 without it |
560
560
  | `coverage-fail-under` | *(unset)* | Minimum DataFrame schema coverage, e.g. `"90"` |
561
561
  | `coverage-detail` | `summary` | Or `term-missing` for the per-file breakdown, `explain` to diagnose a lower-than-expected total |
@@ -583,7 +583,8 @@ and the files are written together or not at all. See the
583
583
 
584
584
  If `check` or `suggest` gets a site wrong and the code can't be shared, `typedframes report quiet-amber src/` prints a
585
585
  bug-report body with a redacted copy of the code around that site, checked against the checker so it reproduces the same
586
- result. Redacted is not anonymous: read it before posting, and don't use it on code you aren't allowed to share. See
586
+ result. `typedframes report --all src/` does this for every untyped site, grouped by why the checker decided as it did.
587
+ Redacted is not anonymous: read it before posting, and don't use it on code you aren't allowed to share. See
587
588
  [Reporting a problem](docs/usage.md#reporting-a-problem-report), and
588
589
  [Security and data handling](docs/security.md) for what the tool reads, writes and connects to.
589
590
 
@@ -694,17 +695,17 @@ plus the requested features, so both halves get checked normally. See
694
695
  Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
695
696
 
696
697
  **Benchmark results** (median of 5 runs ± one standard deviation, 3 warmup, caches cleared between runs):
697
- *2026-10-11 · Darwin 27.0.0 · arm · CPython 3.15.0 · 64GiB RAM · Great Expectations pinned @ 1.23.2*
698
+ *2026-10-13 · Darwin 27.0.0 · arm · CPython 3.15.0 · 64GiB RAM · Great Expectations pinned @ 1.23.2*
698
699
 
699
700
  | Tool | Version | What it does | typedframes (14 files) | great_expectations (494 files) |
700
701
  |------|---------|--------------|------------------------|--------------------------------|
701
- | typedframes | 0.12.0 | DataFrame column checker | 41.1 ± 0.7 ms | 122.1 ± 0.6 ms |
702
- | ruff | 0.17.0 | Linter (no type checking) | 21.5 ± 0.6 ms | 212.9 ± 1.5 ms |
703
- | ty | 0.0.84 | Type checker | 50.2 ± 0.3 ms | 751.7 ± 7.6 ms |
704
- | pyrefly | 1.3.2 | Type checker | 91.5 ± 2.5 ms | 268.9 ± 3.8 ms |
705
- | mypy | 2.4.0 | Type checker (no plugin) | 1.82 ± 0.03 s | 3.04 ± 0.02 s |
706
- | mypy + typedframes | 2.4.0 | Type checker + column checker | 1.84 ± 0.02 s | 3.46 ± 0.02 s |
707
- | pyright | 1.1.414 | Type checker | 794.2 ± 13.8 ms | 3.90 ± 0.03 s |
702
+ | typedframes | 0.13.0 | DataFrame column checker | 42.8 ± 0.3 ms | 72.4 ± 0.8 ms |
703
+ | ruff | 0.17.0 | Linter (no type checking) | 27.6 ± 1.0 ms | 222.1 ± 2.7 ms |
704
+ | ty | 0.0.84 | Type checker | 59.6 ± 1.4 ms | 761.4 ± 9.4 ms |
705
+ | pyrefly | 1.3.2 | Type checker | 98.0 ± 12.2 ms | 276.6 ± 7.3 ms |
706
+ | mypy | 2.4.0 | Type checker (no plugin) | 1.83 ± 0.13 s | 3.08 ± 0.03 s |
707
+ | mypy + typedframes | 2.4.0 | Type checker + column checker | 1.84 ± 0.03 s | 3.32 ± 0.09 s |
708
+ | pyright | 1.1.414 | Type checker | 812.0 ± 12.3 ms | 3.91 ± 0.04 s |
708
709
 
709
710
  *Run `uv run python benchmarks/benchmark_checkers.py` to reproduce.*
710
711
 
@@ -8,7 +8,7 @@
8
8
 
9
9
  > ⚠️ **Project Status: Proof of Concept**
10
10
  >
11
- > `typedframes` (v0.12.0) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
11
+ > `typedframes` (v0.13.0) is currently an experimental proof-of-concept. The core static analysis and mypy/Rust
12
12
  > integrations work, but expect rough edges. The codebase prioritizes demonstrating the viability of static DataFrame
13
13
  > column checking over production-grade stability.
14
14
  >
@@ -470,7 +470,7 @@ repos:
470
470
  name: typedframes check
471
471
  entry: typedframes check . --strict
472
472
  language: python
473
- additional_dependencies: ["typedframes==0.12.0"]
473
+ additional_dependencies: ["typedframes==0.13.0"]
474
474
  types_or: [python, jupyter]
475
475
  pass_filenames: false
476
476
  ```
@@ -519,7 +519,7 @@ The action installs the PyPI wheel into a throwaway virtualenv and runs the chec
519
519
  | Input | Default | |
520
520
  |-------|---------|-|
521
521
  | `path` | `.` | File or directory to check |
522
- | `version` | `latest` | PyPI version to install, e.g. `"0.12.0"` |
522
+ | `version` | `latest` | PyPI version to install, e.g. `"0.13.0"` |
523
523
  | `strict` | `true` | Fail the step on errors — `typedframes check` exits 0 without it |
524
524
  | `coverage-fail-under` | *(unset)* | Minimum DataFrame schema coverage, e.g. `"90"` |
525
525
  | `coverage-detail` | `summary` | Or `term-missing` for the per-file breakdown, `explain` to diagnose a lower-than-expected total |
@@ -547,7 +547,8 @@ and the files are written together or not at all. See the
547
547
 
548
548
  If `check` or `suggest` gets a site wrong and the code can't be shared, `typedframes report quiet-amber src/` prints a
549
549
  bug-report body with a redacted copy of the code around that site, checked against the checker so it reproduces the same
550
- result. Redacted is not anonymous: read it before posting, and don't use it on code you aren't allowed to share. See
550
+ result. `typedframes report --all src/` does this for every untyped site, grouped by why the checker decided as it did.
551
+ Redacted is not anonymous: read it before posting, and don't use it on code you aren't allowed to share. See
551
552
  [Reporting a problem](docs/usage.md#reporting-a-problem-report), and
552
553
  [Security and data handling](docs/security.md) for what the tool reads, writes and connects to.
553
554
 
@@ -658,17 +659,17 @@ plus the requested features, so both halves get checked normally. See
658
659
  Fast feedback reduces development time. The typedframes Rust binary provides near-instant column checking.
659
660
 
660
661
  **Benchmark results** (median of 5 runs ± one standard deviation, 3 warmup, caches cleared between runs):
661
- *2026-10-11 · Darwin 27.0.0 · arm · CPython 3.15.0 · 64GiB RAM · Great Expectations pinned @ 1.23.2*
662
+ *2026-10-13 · Darwin 27.0.0 · arm · CPython 3.15.0 · 64GiB RAM · Great Expectations pinned @ 1.23.2*
662
663
 
663
664
  | Tool | Version | What it does | typedframes (14 files) | great_expectations (494 files) |
664
665
  |------|---------|--------------|------------------------|--------------------------------|
665
- | typedframes | 0.12.0 | DataFrame column checker | 41.1 ± 0.7 ms | 122.1 ± 0.6 ms |
666
- | ruff | 0.17.0 | Linter (no type checking) | 21.5 ± 0.6 ms | 212.9 ± 1.5 ms |
667
- | ty | 0.0.84 | Type checker | 50.2 ± 0.3 ms | 751.7 ± 7.6 ms |
668
- | pyrefly | 1.3.2 | Type checker | 91.5 ± 2.5 ms | 268.9 ± 3.8 ms |
669
- | mypy | 2.4.0 | Type checker (no plugin) | 1.82 ± 0.03 s | 3.04 ± 0.02 s |
670
- | mypy + typedframes | 2.4.0 | Type checker + column checker | 1.84 ± 0.02 s | 3.46 ± 0.02 s |
671
- | pyright | 1.1.414 | Type checker | 794.2 ± 13.8 ms | 3.90 ± 0.03 s |
666
+ | typedframes | 0.13.0 | DataFrame column checker | 42.8 ± 0.3 ms | 72.4 ± 0.8 ms |
667
+ | ruff | 0.17.0 | Linter (no type checking) | 27.6 ± 1.0 ms | 222.1 ± 2.7 ms |
668
+ | ty | 0.0.84 | Type checker | 59.6 ± 1.4 ms | 761.4 ± 9.4 ms |
669
+ | pyrefly | 1.3.2 | Type checker | 98.0 ± 12.2 ms | 276.6 ± 7.3 ms |
670
+ | mypy | 2.4.0 | Type checker (no plugin) | 1.83 ± 0.13 s | 3.08 ± 0.03 s |
671
+ | mypy + typedframes | 2.4.0 | Type checker + column checker | 1.84 ± 0.03 s | 3.32 ± 0.09 s |
672
+ | pyright | 1.1.414 | Type checker | 812.0 ± 12.3 ms | 3.91 ± 0.04 s |
672
673
 
673
674
  *Run `uv run python benchmarks/benchmark_checkers.py` to reproduce.*
674
675
 
@@ -4,7 +4,7 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "typedframes"
7
- version = "0.12.0"
7
+ version = "0.13.0"
8
8
  description = "Static analysis for pandas and polars DataFrames. Catch column errors at lint-time, not runtime."
9
9
  keywords = ["pandas", "polars", "type-checking", "static-analysis", "dataframe", "linter", "mypy-plugin"]
10
10
  readme = "README.md"
@@ -535,6 +535,15 @@ version = "0.2.189"
535
535
  source = "registry+https://github.com/rust-lang/crates.io-index"
536
536
  checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
537
537
 
538
+ [[package]]
539
+ name = "libmimalloc-sys"
540
+ version = "0.1.49"
541
+ source = "registry+https://github.com/rust-lang/crates.io-index"
542
+ checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
543
+ dependencies = [
544
+ "cc",
545
+ ]
546
+
538
547
  [[package]]
539
548
  name = "linux-raw-sys"
540
549
  version = "0.12.1"
@@ -576,6 +585,15 @@ version = "2.8.3"
576
585
  source = "registry+https://github.com/rust-lang/crates.io-index"
577
586
  checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
578
587
 
588
+ [[package]]
589
+ name = "mimalloc"
590
+ version = "0.1.52"
591
+ source = "registry+https://github.com/rust-lang/crates.io-index"
592
+ checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
593
+ dependencies = [
594
+ "libmimalloc-sys",
595
+ ]
596
+
579
597
  [[package]]
580
598
  name = "num-traits"
581
599
  version = "0.2.19"
@@ -1281,6 +1299,26 @@ dependencies = [
1281
1299
  "syn 3.0.3",
1282
1300
  ]
1283
1301
 
1302
+ [[package]]
1303
+ name = "tikv-jemalloc-sys"
1304
+ version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8"
1305
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1306
+ checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0"
1307
+ dependencies = [
1308
+ "cc",
1309
+ "libc",
1310
+ ]
1311
+
1312
+ [[package]]
1313
+ name = "tikv-jemallocator"
1314
+ version = "0.7.0"
1315
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1316
+ checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd"
1317
+ dependencies = [
1318
+ "libc",
1319
+ "tikv-jemalloc-sys",
1320
+ ]
1321
+
1284
1322
  [[package]]
1285
1323
  name = "tinytemplate"
1286
1324
  version = "1.2.1"
@@ -1347,10 +1385,11 @@ checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2"
1347
1385
 
1348
1386
  [[package]]
1349
1387
  name = "typedframes_checker"
1350
- version = "0.12.0"
1388
+ version = "0.13.0"
1351
1389
  dependencies = [
1352
1390
  "anyhow",
1353
1391
  "criterion",
1392
+ "mimalloc",
1354
1393
  "pyo3",
1355
1394
  "rmp-serde",
1356
1395
  "ruff_notebook",
@@ -1362,6 +1401,7 @@ dependencies = [
1362
1401
  "serde_json",
1363
1402
  "sqlparser",
1364
1403
  "tempfile",
1404
+ "tikv-jemallocator",
1365
1405
  "toml",
1366
1406
  ]
1367
1407
 
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "typedframes_checker"
3
- version = "0.12.0"
3
+ version = "0.13.0"
4
4
  edition = "2021"
5
5
  description = "A high-performance Rust-based checker for typedframes DataFrame column schemas"
6
6
  license = "MIT"
@@ -52,6 +52,15 @@ sqlparser = "0.62"
52
52
  # stayed green while local macOS runs did not.
53
53
  pyo3 = { version = "0.29", features = ["abi3-py311"] }
54
54
 
55
+ # The allocator split ruff and ty use: jemalloc where it builds, mimalloc on Windows (jemalloc
56
+ # has no MSVC support), the platform allocator elsewhere. Keep the `cfg`s in `src/lib.rs` in
57
+ # step with these. `disable_initial_exec_tls` lets the library be loaded with `dlopen`.
58
+ [target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), not(target_os = "aix"), not(target_os = "android"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64", target_arch = "riscv64")))'.dependencies]
59
+ tikv-jemallocator = { version = "0.7", features = ["disable_initial_exec_tls"] }
60
+
61
+ [target.'cfg(target_os = "windows")'.dependencies]
62
+ mimalloc = { version = "0.1.52", default-features = false }
63
+
55
64
  [dev-dependencies]
56
65
  tempfile = "3.8"
57
66
  criterion = { version = "0.8", features = ["html_reports"] }
@@ -0,0 +1,425 @@
1
+ //! What the callers of a function do with the frame it returns.
2
+ //!
3
+ //! A frame a function returns is only as wide as its callers need if every caller is seen
4
+ //! and each reads named columns and nothing else. This module observes one file: for each
5
+ //! callee name, the columns read from the result of every `x = name(...)` call, and whether
6
+ //! the name is also used in a way that hides a use (called other than as a plain assignment,
7
+ //! assigned inside a loop, handed on, passed around as a value, decorated). Names match by
8
+ //! bare name, so a call to an unrelated function of the same name only adds columns, never
9
+ //! removes them. The project-wide merge is `index::resolve_call_usage`.
10
+
11
+ use crate::column_usage::{
12
+ blocks_narrowing, declares_global_or_nonlocal, first_nested_scope_mention, scan_usage_with,
13
+ HelperResolver,
14
+ };
15
+ use crate::helpers::FileHelpers;
16
+ use crate::static_eval::{Analysis, ImportedModules};
17
+ use ruff_python_ast::visitor::{self as ast_visitor, Visitor};
18
+ use ruff_python_ast::{ExceptHandler, Expr, ModModule, Stmt};
19
+ use std::collections::{BTreeSet, HashMap, HashSet};
20
+
21
+ /// What one file shows about the calls to a name.
22
+ #[derive(Debug, Default, PartialEq)]
23
+ pub(crate) struct Observation {
24
+ /// Columns read from the result of the file's calls.
25
+ pub(crate) columns: BTreeSet<String>,
26
+ /// How many calls the file makes.
27
+ pub(crate) calls: usize,
28
+ /// Some use of the name or of a call's result isn't a read of named columns.
29
+ pub(crate) unknown: bool,
30
+ }
31
+
32
+ #[derive(Debug, Default, PartialEq)]
33
+ pub(crate) struct FileCalls {
34
+ pub(crate) observations: HashMap<String, Observation>,
35
+ /// Names of the functions and methods the file defines.
36
+ pub(crate) defined: HashSet<String>,
37
+ }
38
+
39
+ fn callee_name(func: &Expr) -> Option<&str> {
40
+ match func {
41
+ Expr::Name(n) => Some(n.id.as_str()),
42
+ Expr::Attribute(a) => Some(a.attr.as_str()),
43
+ _ => None,
44
+ }
45
+ }
46
+
47
+ // Every call by callee name, and every other mention of a name that could be a function
48
+ // being handed around (`callbacks = [load]`, `map(load, ...)`, `obj.load`).
49
+ #[derive(Default)]
50
+ struct Mentions {
51
+ calls: HashMap<String, usize>,
52
+ other: HashSet<String>,
53
+ defined: HashSet<String>,
54
+ }
55
+
56
+ impl<'a> Visitor<'a> for Mentions {
57
+ fn visit_stmt(&mut self, stmt: &'a Stmt) {
58
+ // `from lib import load as fetch` calls `load` under another name.
59
+ let aliased = match stmt {
60
+ Stmt::Import(i) => Some(&i.names),
61
+ Stmt::ImportFrom(i) => Some(&i.names),
62
+ _ => None,
63
+ };
64
+ for alias in aliased.into_iter().flatten().filter(|a| a.asname.is_some()) {
65
+ self.other.insert(alias.name.to_string());
66
+ }
67
+ if let Stmt::FunctionDef(def) = stmt {
68
+ self.defined.insert(def.name.to_string());
69
+ // A decorator (a route, a task, a fixture) may call it where this can't see.
70
+ if !def.decorator_list.is_empty() {
71
+ self.other.insert(def.name.to_string());
72
+ }
73
+ }
74
+ ast_visitor::walk_stmt(self, stmt);
75
+ }
76
+
77
+ fn visit_expr(&mut self, expr: &'a Expr) {
78
+ match expr {
79
+ Expr::Call(call) => {
80
+ if let Some(name) = callee_name(&call.func) {
81
+ *self.calls.entry(name.to_string()).or_default() += 1;
82
+ }
83
+ match &*call.func {
84
+ Expr::Name(_) => {}
85
+ Expr::Attribute(a) => self.visit_expr(&a.value),
86
+ other => self.visit_expr(other),
87
+ }
88
+ for arg in &call.arguments.args {
89
+ self.visit_expr(arg);
90
+ }
91
+ for keyword in &call.arguments.keywords {
92
+ self.visit_expr(&keyword.value);
93
+ }
94
+ }
95
+ Expr::Name(n) => {
96
+ self.other.insert(n.id.to_string());
97
+ }
98
+ Expr::Attribute(a) => {
99
+ self.other.insert(a.attr.to_string());
100
+ self.visit_expr(&a.value);
101
+ }
102
+ _ => ast_visitor::walk_expr(self, expr),
103
+ }
104
+ }
105
+ }
106
+
107
+ // The remainder of each enclosing block, innermost first, after `extra`.
108
+ fn following<'a>(extra: &[&'a [Stmt]], rest: &'a [Stmt], tails: &[&'a [Stmt]]) -> Vec<&'a [Stmt]> {
109
+ let mut chain: Vec<&[Stmt]> = extra.to_vec();
110
+ chain.push(rest);
111
+ chain.extend_from_slice(tails);
112
+ chain
113
+ }
114
+
115
+ // Finds each `x = name(...)` and reads off what the rest of its scope does with `x`.
116
+ struct Scanner<'h> {
117
+ helpers: &'h dyn HelperResolver,
118
+ // The module's own statements: a name assigned there can be imported and read by other
119
+ // files, which this scan doesn't follow.
120
+ module: &'h [Stmt],
121
+ // Only calls to these names are worth following.
122
+ defined: &'h HashSet<String>,
123
+ // By callee name: the calls whose result was fully accounted for, and their columns.
124
+ known: HashMap<String, Observation>,
125
+ }
126
+
127
+ impl Scanner<'_> {
128
+ fn scope(&mut self, body: &[Stmt]) {
129
+ self.block(body, &[], false, body);
130
+ }
131
+
132
+ fn class_body(&mut self, body: &[Stmt]) {
133
+ for stmt in body {
134
+ match stmt {
135
+ Stmt::FunctionDef(def) => self.scope(&def.body),
136
+ Stmt::ClassDef(inner) => self.class_body(&inner.body),
137
+ _ => {}
138
+ }
139
+ }
140
+ }
141
+
142
+ fn block<'a>(
143
+ &mut self,
144
+ block: &'a [Stmt],
145
+ tails: &[&'a [Stmt]],
146
+ in_loop: bool,
147
+ scope_body: &'a [Stmt],
148
+ ) {
149
+ for (i, stmt) in block.iter().enumerate() {
150
+ let rest = &block[i + 1..];
151
+ self.assignment(stmt, rest, tails, in_loop, scope_body);
152
+ let after = |extra: &[&'a [Stmt]]| following(extra, rest, tails);
153
+ match stmt {
154
+ Stmt::FunctionDef(def) => self.scope(&def.body),
155
+ Stmt::ClassDef(class) => self.class_body(&class.body),
156
+ Stmt::If(s) => {
157
+ let chain = after(&[]);
158
+ self.block(&s.body, &chain, in_loop, scope_body);
159
+ for clause in &s.elif_else_clauses {
160
+ self.block(&clause.body, &chain, in_loop, scope_body);
161
+ }
162
+ }
163
+ Stmt::With(s) => self.block(&s.body, &after(&[]), in_loop, scope_body),
164
+ Stmt::Match(s) => {
165
+ let chain = after(&[]);
166
+ for case in &s.cases {
167
+ self.block(&case.body, &chain, in_loop, scope_body);
168
+ }
169
+ }
170
+ Stmt::Try(s) => {
171
+ let handlers: Vec<&[Stmt]> = s
172
+ .handlers
173
+ .iter()
174
+ .map(|h| {
175
+ let ExceptHandler::ExceptHandler(h) = h;
176
+ h.body.as_slice()
177
+ })
178
+ .collect();
179
+ let finalbody = s.finalbody.as_slice();
180
+ let mut body_extra: Vec<&[Stmt]> = handlers.clone();
181
+ body_extra.push(s.orelse.as_slice());
182
+ body_extra.push(finalbody);
183
+ self.block(&s.body, &after(&body_extra), in_loop, scope_body);
184
+ let tail = after(&[finalbody]);
185
+ for handler in &handlers {
186
+ self.block(handler, &tail, in_loop, scope_body);
187
+ }
188
+ self.block(&s.orelse, &tail, in_loop, scope_body);
189
+ self.block(finalbody, &after(&[]), in_loop, scope_body);
190
+ }
191
+ Stmt::For(s) => {
192
+ let chain = after(&[]);
193
+ self.block(&s.body, &chain, true, scope_body);
194
+ self.block(&s.orelse, &chain, in_loop, scope_body);
195
+ }
196
+ Stmt::While(s) => {
197
+ let chain = after(&[]);
198
+ self.block(&s.body, &chain, true, scope_body);
199
+ self.block(&s.orelse, &chain, in_loop, scope_body);
200
+ }
201
+ _ => {}
202
+ }
203
+ }
204
+ }
205
+
206
+ fn assignment<'a>(
207
+ &mut self,
208
+ stmt: &'a Stmt,
209
+ rest: &'a [Stmt],
210
+ tails: &[&'a [Stmt]],
211
+ in_loop: bool,
212
+ scope_body: &'a [Stmt],
213
+ ) {
214
+ let (target, value) = match stmt {
215
+ Stmt::Assign(a) => match (a.targets.as_slice(), &*a.value) {
216
+ ([Expr::Name(target)], value) => (target, value),
217
+ _ => return,
218
+ },
219
+ Stmt::AnnAssign(a) => match (&*a.target, a.value.as_deref()) {
220
+ (Expr::Name(target), Some(value)) => (target, value),
221
+ _ => return,
222
+ },
223
+ _ => return,
224
+ };
225
+ let Expr::Call(call) = value else {
226
+ return;
227
+ };
228
+ // In a loop the result can be read before the next pass reassigns it.
229
+ let Some(callee) =
230
+ callee_name(&call.func).filter(|name| !in_loop && self.defined.contains(*name))
231
+ else {
232
+ return;
233
+ };
234
+ if std::ptr::eq(scope_body, self.module) {
235
+ return;
236
+ }
237
+ let downstream = std::iter::once(rest)
238
+ .chain(tails.iter().copied())
239
+ .flat_map(|stmts| stmts.iter());
240
+ let usage = scan_usage_with(target.id.as_str(), downstream, &|_| 0, self.helpers);
241
+ if usage.other_uses.iter().any(blocks_narrowing)
242
+ || first_nested_scope_mention(scope_body, &usage.ever_tainted).is_some()
243
+ || usage
244
+ .ever_tainted
245
+ .iter()
246
+ .any(|alias| declares_global_or_nonlocal(scope_body, alias))
247
+ {
248
+ return;
249
+ }
250
+ let seen = self.known.entry(callee.to_string()).or_default();
251
+ seen.calls += 1;
252
+ seen.columns
253
+ .extend(usage.accesses.iter().map(|a| a.column.clone()));
254
+ }
255
+ }
256
+
257
+ /// The names of the functions and methods `module` defines.
258
+ pub(crate) fn defined_names(module: &ModModule) -> HashSet<String> {
259
+ let mut mentions = Mentions::default();
260
+ for stmt in &module.body {
261
+ mentions.visit_stmt(stmt);
262
+ }
263
+ mentions.defined
264
+ }
265
+
266
+ /// What `module` shows about the calls to each name it calls or mentions. Only calls to a
267
+ /// name in `defined` (what the project defines) have their results followed.
268
+ pub(crate) fn module_call_usage(module: &ModModule, defined: &HashSet<String>) -> FileCalls {
269
+ let mut mentions = Mentions::default();
270
+ for stmt in &module.body {
271
+ mentions.visit_stmt(stmt);
272
+ }
273
+ let none = ImportedModules::new();
274
+ let analysis = Analysis::build(module, &none);
275
+ let imported = HashMap::new();
276
+ let helpers = FileHelpers::new(&analysis, &imported).with_methods(module);
277
+ let mut scanner = Scanner {
278
+ helpers: &helpers,
279
+ module: &module.body,
280
+ defined,
281
+ known: HashMap::new(),
282
+ };
283
+ scanner.scope(&module.body);
284
+
285
+ let mut observations: HashMap<String, Observation> = HashMap::new();
286
+ for (name, calls) in &mentions.calls {
287
+ let known = scanner.known.remove(name);
288
+ let accounted = known.as_ref().map_or(0, |k| k.calls);
289
+ observations.insert(
290
+ name.clone(),
291
+ Observation {
292
+ columns: known.map(|k| k.columns).unwrap_or_default(),
293
+ calls: *calls,
294
+ unknown: *calls != accounted || mentions.other.contains(name),
295
+ },
296
+ );
297
+ }
298
+ for name in &mentions.other {
299
+ observations.entry(name.clone()).or_default().unknown = true;
300
+ }
301
+ FileCalls {
302
+ observations,
303
+ defined: mentions.defined,
304
+ }
305
+ }
306
+
307
+ #[cfg(test)]
308
+ mod tests {
309
+ use super::*;
310
+ use ruff_python_parser::parse_module;
311
+
312
+ fn observe(source: &str) -> FileCalls {
313
+ let module = parse_module(source).unwrap().into_syntax();
314
+ let mut defined = defined_names(&module);
315
+ defined.extend(["load".to_string()]);
316
+ module_call_usage(&module, &defined)
317
+ }
318
+
319
+ fn columns(calls: &FileCalls, name: &str) -> Vec<String> {
320
+ calls.observations[name].columns.iter().cloned().collect()
321
+ }
322
+
323
+ #[test]
324
+ fn test_should_collect_the_columns_read_from_each_assigned_call() {
325
+ let calls = observe(
326
+ "def f():\n a = load()\n print(a[\"x\"])\n b = load()\n return b[\"y\"]\n",
327
+ );
328
+
329
+ let seen = &calls.observations["load"];
330
+ assert!(!seen.unknown);
331
+ assert_eq!(seen.calls, 2);
332
+ assert_eq!(columns(&calls, "load"), vec!["x", "y"]);
333
+ assert!(calls.defined.contains("f"));
334
+ }
335
+
336
+ #[test]
337
+ fn test_should_see_calls_through_methods_and_nested_blocks() {
338
+ let calls = observe(
339
+ "class S:\n def f(self):\n with ctx():\n a = self.load()\n return a[\"x\"]\n",
340
+ );
341
+
342
+ assert!(!calls.observations["load"].unknown);
343
+ assert_eq!(columns(&calls, "load"), vec!["x"]);
344
+ }
345
+
346
+ #[test]
347
+ fn test_should_not_know_a_call_whose_result_is_handed_on() {
348
+ let calls = observe("def f():\n a = load()\n return a\n");
349
+
350
+ assert!(calls.observations["load"].unknown);
351
+ }
352
+
353
+ #[test]
354
+ fn test_should_not_know_a_call_that_is_not_a_plain_assignment() {
355
+ let calls = observe("def f():\n a = load()\n return load()[\"x\"], a[\"y\"]\n");
356
+
357
+ assert!(calls.observations["load"].unknown);
358
+ }
359
+
360
+ #[test]
361
+ fn test_should_not_know_a_call_inside_a_loop() {
362
+ let calls =
363
+ observe("def f(xs):\n for x in xs:\n a = load(x)\n print(a[\"y\"])\n");
364
+
365
+ assert!(calls.observations["load"].unknown);
366
+ }
367
+
368
+ #[test]
369
+ fn test_should_not_know_a_function_passed_around_as_a_value() {
370
+ let calls = observe("def f():\n a = load()\n print(a[\"x\"])\n register(load)\n");
371
+
372
+ assert!(calls.observations["load"].unknown);
373
+ }
374
+
375
+ #[test]
376
+ fn test_should_not_know_a_decorated_function() {
377
+ let calls = observe("@route\ndef load():\n return 1\n\n\ndef f():\n a = load()\n print(a[\"x\"])\n");
378
+
379
+ assert!(calls.observations["load"].unknown);
380
+ }
381
+
382
+ #[test]
383
+ fn test_should_read_through_a_helper_that_only_reads_columns() {
384
+ let calls = observe(
385
+ "def total(frame):\n return frame[\"z\"]\n\n\ndef f():\n a = load()\n return total(a)\n",
386
+ );
387
+
388
+ assert!(!calls.observations["load"].unknown);
389
+ assert_eq!(columns(&calls, "load"), vec!["z"]);
390
+ }
391
+
392
+ #[test]
393
+ fn test_should_not_count_an_unrelated_assignment_of_something_else() {
394
+ let calls = observe("def f():\n a = 1\n return a\n");
395
+
396
+ assert!(!calls.observations.contains_key("load"));
397
+ }
398
+
399
+ #[test]
400
+ fn test_should_not_know_a_function_another_file_imports_under_an_alias() {
401
+ let calls = observe(
402
+ "from lib import load as fetch\n\n\ndef f():\n a = fetch()\n return a[\"x\"]\n",
403
+ );
404
+
405
+ assert!(calls.observations["load"].unknown);
406
+ }
407
+
408
+ #[test]
409
+ fn test_should_not_know_a_call_assigned_at_module_level() {
410
+ let calls = observe("a = load()\nprint(a[\"x\"])\n");
411
+
412
+ assert!(calls.observations["load"].unknown);
413
+ }
414
+
415
+ #[test]
416
+ fn test_should_not_follow_calls_to_names_the_project_does_not_define() {
417
+ let module = parse_module("def f():\n a = other()\n return a[\"x\"]\n")
418
+ .unwrap()
419
+ .into_syntax();
420
+
421
+ let calls = module_call_usage(&module, &defined_names(&module));
422
+
423
+ assert!(calls.observations["other"].unknown);
424
+ }
425
+ }