pypreflight 1.0.0__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. pypreflight-2.0.0/.gitignore +76 -0
  2. pypreflight-2.0.0/CHANGELOG.md +12 -0
  3. pypreflight-2.0.0/PKG-INFO +480 -0
  4. pypreflight-2.0.0/README.md +431 -0
  5. pypreflight-2.0.0/docs/fit_hygiene.md +89 -0
  6. pypreflight-2.0.0/examples/clean.csv +61 -0
  7. pypreflight-2.0.0/examples/disguised_missing_units.csv +61 -0
  8. pypreflight-2.0.0/examples/messy_noheader_semicolon.csv +60 -0
  9. pypreflight-2.0.0/pyproject.toml +134 -0
  10. pypreflight-2.0.0/src/preflight/__init__.py +21 -0
  11. pypreflight-2.0.0/src/preflight/assemble.py +597 -0
  12. pypreflight-2.0.0/src/preflight/clean/__init__.py +1 -0
  13. pypreflight-2.0.0/src/preflight/clean/_common.py +31 -0
  14. pypreflight-2.0.0/src/preflight/clean/cleaner.py +273 -0
  15. pypreflight-2.0.0/src/preflight/clean/config.py +40 -0
  16. pypreflight-2.0.0/src/preflight/clean/dedupe.py +88 -0
  17. pypreflight-2.0.0/src/preflight/clean/drops.py +206 -0
  18. pypreflight-2.0.0/src/preflight/clean/impute.py +214 -0
  19. pypreflight-2.0.0/src/preflight/clean/markers.py +90 -0
  20. pypreflight-2.0.0/src/preflight/clean/normalize.py +161 -0
  21. pypreflight-2.0.0/src/preflight/clean/outliers.py +170 -0
  22. pypreflight-2.0.0/src/preflight/clean/rows.py +173 -0
  23. pypreflight-2.0.0/src/preflight/cli.py +1 -0
  24. pypreflight-2.0.0/src/preflight/decisions.py +188 -0
  25. pypreflight-2.0.0/src/preflight/engineer/__init__.py +1 -0
  26. pypreflight-2.0.0/src/preflight/engineer/config.py +83 -0
  27. pypreflight-2.0.0/src/preflight/engineer/datetime.py +102 -0
  28. pypreflight-2.0.0/src/preflight/engineer/encoding.py +90 -0
  29. pypreflight-2.0.0/src/preflight/engineer/engineer.py +352 -0
  30. pypreflight-2.0.0/src/preflight/engineer/features.py +259 -0
  31. pypreflight-2.0.0/src/preflight/engineer/numeric.py +46 -0
  32. pypreflight-2.0.0/src/preflight/engineer/vif.py +153 -0
  33. pypreflight-2.0.0/src/preflight/infer/__init__.py +32 -0
  34. pypreflight-2.0.0/src/preflight/infer/config.py +37 -0
  35. pypreflight-2.0.0/src/preflight/infer/missing.py +243 -0
  36. pypreflight-2.0.0/src/preflight/infer/types.py +800 -0
  37. pypreflight-2.0.0/src/preflight/ingest/__init__.py +185 -0
  38. pypreflight-2.0.0/src/preflight/ingest/dialect.py +369 -0
  39. pypreflight-2.0.0/src/preflight/ingest/encoding.py +199 -0
  40. pypreflight-2.0.0/src/preflight/ingest/header.py +317 -0
  41. pypreflight-2.0.0/src/preflight/ingest/loader.py +208 -0
  42. pypreflight-2.0.0/src/preflight/interact/__init__.py +1 -0
  43. pypreflight-2.0.0/src/preflight/interact/channels/__init__.py +17 -0
  44. pypreflight-2.0.0/src/preflight/interact/channels/base.py +35 -0
  45. pypreflight-2.0.0/src/preflight/interact/channels/callback.py +34 -0
  46. pypreflight-2.0.0/src/preflight/interact/channels/notebook.py +108 -0
  47. pypreflight-2.0.0/src/preflight/interact/channels/review_file.py +128 -0
  48. pypreflight-2.0.0/src/preflight/interact/channels/terminal.py +185 -0
  49. pypreflight-2.0.0/src/preflight/interact/modes.py +138 -0
  50. pypreflight-2.0.0/src/preflight/interact/preview.py +174 -0
  51. pypreflight-2.0.0/src/preflight/interact/question.py +391 -0
  52. pypreflight-2.0.0/src/preflight/interact/resolve.py +232 -0
  53. pypreflight-2.0.0/src/preflight/interact/store.py +108 -0
  54. pypreflight-2.0.0/src/preflight/interact/triggers.py +306 -0
  55. pypreflight-2.0.0/src/preflight/plan.py +1281 -0
  56. pypreflight-2.0.0/src/preflight/prepare.py +122 -0
  57. pypreflight-2.0.0/src/preflight/py.typed +0 -0
  58. pypreflight-2.0.0/src/preflight/report/__init__.py +13 -0
  59. pypreflight-2.0.0/src/preflight/report/plan_pdf.py +518 -0
  60. {pypreflight-1.0.0/src/preflight → pypreflight-2.0.0/src/preflight/report}/report.py +77 -170
  61. pypreflight-2.0.0/src/preflight/report/report_pdf.py +599 -0
  62. pypreflight-2.0.0/src/preflight/report/templates.py +896 -0
  63. pypreflight-2.0.0/src/preflight/schema.py +182 -0
  64. pypreflight-2.0.0/src/preflight/signals.py +317 -0
  65. pypreflight-2.0.0/src/preflight/types.py +89 -0
  66. {pypreflight-1.0.0 → pypreflight-2.0.0}/src/preflight/validation.py +4 -6
  67. pypreflight-2.0.0/tests/conftest.py +88 -0
  68. pypreflight-2.0.0/tests/property/.gitkeep +0 -0
  69. pypreflight-2.0.0/tests/property/test_ingest_roundtrip.py +79 -0
  70. pypreflight-2.0.0/tests/property/test_performance_smoke.py +75 -0
  71. pypreflight-2.0.0/tests/property/test_plan_determinism_and_purity.py +115 -0
  72. pypreflight-2.0.0/tests/property/test_schema_roundtrip.py +93 -0
  73. pypreflight-2.0.0/tests/regression/.gitkeep +0 -0
  74. pypreflight-2.0.0/tests/regression/adapter.py +468 -0
  75. pypreflight-2.0.0/tests/regression/test_mvp_acceptance.py +640 -0
  76. pypreflight-2.0.0/tests/unit/.gitkeep +0 -0
  77. pypreflight-2.0.0/tests/unit/conftest.py +61 -0
  78. pypreflight-2.0.0/tests/unit/data/infer_golden_pre_sampling.json +1 -0
  79. pypreflight-2.0.0/tests/unit/test_assemble.py +136 -0
  80. pypreflight-2.0.0/tests/unit/test_benchmark_regressions.py +268 -0
  81. pypreflight-2.0.0/tests/unit/test_channels.py +383 -0
  82. pypreflight-2.0.0/tests/unit/test_clean.py +175 -0
  83. pypreflight-2.0.0/tests/unit/test_dry_run.py +81 -0
  84. pypreflight-2.0.0/tests/unit/test_encoding_evidence.py +104 -0
  85. pypreflight-2.0.0/tests/unit/test_engineer.py +153 -0
  86. pypreflight-2.0.0/tests/unit/test_fit_hygiene.py +167 -0
  87. pypreflight-2.0.0/tests/unit/test_infer.py +280 -0
  88. pypreflight-2.0.0/tests/unit/test_infer_sampling.py +165 -0
  89. pypreflight-2.0.0/tests/unit/test_ingest.py +408 -0
  90. pypreflight-2.0.0/tests/unit/test_ingest_edges.py +202 -0
  91. pypreflight-2.0.0/tests/unit/test_jargon_lint.py +240 -0
  92. pypreflight-2.0.0/tests/unit/test_modes.py +224 -0
  93. pypreflight-2.0.0/tests/unit/test_no_extras.py +83 -0
  94. pypreflight-2.0.0/tests/unit/test_nonblocking.py +94 -0
  95. pypreflight-2.0.0/tests/unit/test_normalize_categorical.py +50 -0
  96. pypreflight-2.0.0/tests/unit/test_output_location.py +71 -0
  97. pypreflight-2.0.0/tests/unit/test_plan.py +168 -0
  98. pypreflight-2.0.0/tests/unit/test_plan_apply_equivalence.py +157 -0
  99. pypreflight-2.0.0/tests/unit/test_prep_integration.py +198 -0
  100. pypreflight-2.0.0/tests/unit/test_prep_steps.py +224 -0
  101. pypreflight-2.0.0/tests/unit/test_question.py +257 -0
  102. pypreflight-2.0.0/tests/unit/test_report_pdf.py +81 -0
  103. pypreflight-2.0.0/tests/unit/test_store.py +183 -0
  104. pypreflight-2.0.0/tests/unit/test_units_generic.py +112 -0
  105. pypreflight-1.0.0/.claude/agents/claude-md-updater.md +0 -40
  106. pypreflight-1.0.0/.claude/agents/debugger.md +0 -26
  107. pypreflight-1.0.0/.claude/agents/documenter.md +0 -30
  108. pypreflight-1.0.0/.claude/agents/implementer.md +0 -27
  109. pypreflight-1.0.0/.claude/agents/planner.md +0 -25
  110. pypreflight-1.0.0/.claude/agents/refactor.md +0 -25
  111. pypreflight-1.0.0/.claude/agents/reviewer.md +0 -38
  112. pypreflight-1.0.0/.claude/agents/test-writer.md +0 -33
  113. pypreflight-1.0.0/.claude/commands/debug.md +0 -8
  114. pypreflight-1.0.0/.claude/commands/done.md +0 -15
  115. pypreflight-1.0.0/.claude/commands/review.md +0 -12
  116. pypreflight-1.0.0/.claude/commands/ship.md +0 -27
  117. pypreflight-1.0.0/.gitignore +0 -41
  118. pypreflight-1.0.0/CHANGELOG.md +0 -49
  119. pypreflight-1.0.0/CLAUDE.md +0 -132
  120. pypreflight-1.0.0/PKG-INFO +0 -366
  121. pypreflight-1.0.0/Plan.md +0 -169
  122. pypreflight-1.0.0/README.md +0 -333
  123. pypreflight-1.0.0/prompt.txt +0 -36
  124. pypreflight-1.0.0/pyproject.toml +0 -72
  125. pypreflight-1.0.0/requirements.txt +0 -9
  126. pypreflight-1.0.0/src/preflight/__init__.py +0 -322
  127. pypreflight-1.0.0/src/preflight/assembler.py +0 -386
  128. pypreflight-1.0.0/src/preflight/cleaner.py +0 -359
  129. pypreflight-1.0.0/src/preflight/cli.py +0 -257
  130. pypreflight-1.0.0/src/preflight/compare_report.py +0 -219
  131. pypreflight-1.0.0/src/preflight/engineer.py +0 -717
  132. pypreflight-1.0.0/src/preflight/profiler.py +0 -406
  133. pypreflight-1.0.0/src/preflight/types.py +0 -129
  134. pypreflight-1.0.0/tests/1.0.0/phase1.py +0 -212
  135. pypreflight-1.0.0/tests/1.0.0/phase2.py +0 -283
  136. pypreflight-1.0.0/tests/1.0.0/phase3.py +0 -281
  137. pypreflight-1.0.0/tests/1.0.0/phase4.py +0 -214
  138. pypreflight-1.0.0/tests/1.0.0/phase5.py +0 -191
  139. pypreflight-1.0.0/tests/1.0.0/phase6.py +0 -257
  140. pypreflight-1.0.0/tests/1.0.0/phase7.py +0 -237
  141. pypreflight-1.0.0/tests/1.0.0/phase8.py +0 -229
  142. pypreflight-1.0.0/tests/1.0.0/phase8_1.py +0 -230
  143. pypreflight-1.0.0/tests/1.0.0/test_full_v1_regression.py +0 -460
  144. pypreflight-1.0.0/tests/edge_cases/test_cardinality_extremes.py +0 -73
  145. pypreflight-1.0.0/tests/edge_cases/test_degenerate_columns.py +0 -87
  146. pypreflight-1.0.0/tests/edge_cases/test_degenerate_shapes.py +0 -55
  147. pypreflight-1.0.0/tests/integration/test_adult_income.py +0 -85
  148. pypreflight-1.0.0/tests/integration/test_house_prices.py +0 -64
  149. pypreflight-1.0.0/tests/integration/test_titanic.py +0 -81
  150. pypreflight-1.0.0/tests/phase2.py +0 -305
  151. pypreflight-1.0.0/tests/phase3.py +0 -184
  152. pypreflight-1.0.0/tests/repo_hygiene.py +0 -32
  153. pypreflight-1.0.0/tests/test_assembler.py +0 -90
  154. pypreflight-1.0.0/tests/test_cleaner.py +0 -106
  155. pypreflight-1.0.0/tests/test_cleaner_coverage.py +0 -24
  156. pypreflight-1.0.0/tests/test_cli.py +0 -103
  157. pypreflight-1.0.0/tests/test_cli_coverage.py +0 -50
  158. pypreflight-1.0.0/tests/test_cli_features.py +0 -212
  159. pypreflight-1.0.0/tests/test_coverage_gate.py +0 -11
  160. pypreflight-1.0.0/tests/test_engineer.py +0 -109
  161. pypreflight-1.0.0/tests/test_full.py +0 -812
  162. pypreflight-1.0.0/tests/test_init.py +0 -86
  163. pypreflight-1.0.0/tests/test_profiler.py +0 -95
  164. pypreflight-1.0.0/tests/test_report.py +0 -89
  165. pypreflight-1.0.0/tests/test_report_readability.py +0 -197
  166. pypreflight-1.0.0/tests/test_scaffold.py +0 -53
  167. pypreflight-1.0.0/tests/test_types.py +0 -75
  168. pypreflight-1.0.0/tests/test_validation.py +0 -289
  169. {pypreflight-1.0.0 → pypreflight-2.0.0}/LICENSE +0 -0
@@ -0,0 +1,76 @@
1
+ # ---------------------------------------------------------------------------
2
+ # Python bytecode and build output
3
+ # ---------------------------------------------------------------------------
4
+ __pycache__/
5
+ *.py[cod]
6
+ *$py.class
7
+ *.so
8
+ build/
9
+ dist/
10
+ wheels/
11
+ *.egg-info/
12
+ *.egg
13
+ .eggs/
14
+ pip-wheel-metadata/
15
+
16
+ # ---------------------------------------------------------------------------
17
+ # Virtual environments
18
+ # ---------------------------------------------------------------------------
19
+ .venv/
20
+ venv/
21
+ env/
22
+ .python-version
23
+
24
+ # ---------------------------------------------------------------------------
25
+ # Test, coverage, lint and type-check caches
26
+ # ---------------------------------------------------------------------------
27
+ .pytest_cache/
28
+ .hypothesis/
29
+ .nox/
30
+ .tox/
31
+ .coverage
32
+ .coverage.*
33
+ coverage.xml
34
+ htmlcov/
35
+ junit*.xml
36
+ .ruff_cache/
37
+ .mypy_cache/
38
+ .dmypy.json
39
+ .pyre/
40
+
41
+ # ---------------------------------------------------------------------------
42
+ # Notebooks
43
+ # ---------------------------------------------------------------------------
44
+ .ipynb_checkpoints/
45
+
46
+ # ---------------------------------------------------------------------------
47
+ # Editors, OS files, local tooling
48
+ # ---------------------------------------------------------------------------
49
+ .vscode/
50
+ .idea/
51
+ *.swp
52
+ *~
53
+ .DS_Store
54
+ Thumbs.db
55
+ .claude/
56
+ CLAUDE.md
57
+
58
+ # ---------------------------------------------------------------------------
59
+ # Project-specific
60
+ # ---------------------------------------------------------------------------
61
+ # Raw corpus files are downloaded by scripts/acquire_corpus.py (labels are tracked).
62
+ corpus/raw/
63
+ # Generated measurement output from scripts/corpus_accuracy.py and friends.
64
+ scripts/_*.json
65
+ # PreFlight's default output folder (created wherever it is run).
66
+ preflight_out/
67
+ # Local review logs and internal working notes (never published).
68
+ .review/
69
+ dev-docs/
70
+ prompt
71
+
72
+ # Personal sandbox for trying the package locally (not published).
73
+ playground/
74
+
75
+ # personal docs
76
+ deployment.md
@@ -0,0 +1,12 @@
1
+ # CHANGELOG
2
+
3
+ ## 2.0.0 (2026-11-16)
4
+
5
+ ### Changed
6
+ - A column whose type is clear but is flagged only because some values are placeholders for
7
+ missing data (`?`, `N/A`, `-999`) now recommends that type instead of "Keep as plain text",
8
+ and `mode="defaults"` converts it. Close calls, units, leading zeros and ambiguous dates
9
+ still default to plain text. Stored answers for affected columns are asked again once,
10
+ because the decision's fingerprint includes its default.
11
+ - Type questions always offer "Keep as plain text", and explain when a column is flagged only
12
+ for missing-value placeholders.
@@ -0,0 +1,480 @@
1
+ Metadata-Version: 2.5
2
+ Name: pypreflight
3
+ Version: 2.0.0
4
+ Summary: A pip-installable Python library that takes a raw pandas DataFrame and returns a cleaned DataFrame, a reusable sklearn Pipeline, and a structured Report — automatically.
5
+ Project-URL: Homepage, https://github.com/VAIyerAmogha/PreFlight
6
+ Project-URL: Demo, https://pre-flight-tan.vercel.app
7
+ Project-URL: Repository, https://github.com/VAIyerAmogha/PreFlight
8
+ Project-URL: Documentation, https://github.com/VAIyerAmogha/PreFlight#readme
9
+ Project-URL: Issues, https://github.com/VAIyerAmogha/PreFlight/issues
10
+ Project-URL: Changelog, https://github.com/VAIyerAmogha/PreFlight/blob/main/CHANGELOG.md
11
+ Author: VAIyerAmogha
12
+ License: MIT License
13
+ License-File: LICENSE
14
+ Keywords: feature-engineering,machine-learning,pandas,preprocessing,scikit-learn
15
+ Classifier: Development Status :: 3 - Alpha
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: License :: OSI Approved :: MIT License
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
22
+ Requires-Python: >=3.10
23
+ Requires-Dist: charset-normalizer
24
+ Requires-Dist: joblib
25
+ Requires-Dist: matplotlib
26
+ Requires-Dist: numpy
27
+ Requires-Dist: pandas
28
+ Requires-Dist: python-dateutil
29
+ Requires-Dist: reportlab>=4.0.0
30
+ Requires-Dist: scikit-learn
31
+ Requires-Dist: scipy
32
+ Requires-Dist: typer
33
+ Provides-Extra: dev
34
+ Requires-Dist: hypothesis; extra == 'dev'
35
+ Requires-Dist: mypy; extra == 'dev'
36
+ Requires-Dist: nox; extra == 'dev'
37
+ Requires-Dist: pre-commit; extra == 'dev'
38
+ Requires-Dist: pypdf>=5.0.0; extra == 'dev'
39
+ Requires-Dist: pytest; extra == 'dev'
40
+ Requires-Dist: pytest-cov; extra == 'dev'
41
+ Requires-Dist: pytest-timeout; extra == 'dev'
42
+ Requires-Dist: pytest-xdist; extra == 'dev'
43
+ Requires-Dist: ruff; extra == 'dev'
44
+ Provides-Extra: interactive
45
+ Requires-Dist: rich>=13; extra == 'interactive'
46
+ Provides-Extra: notebook
47
+ Requires-Dist: ipywidgets>=8; extra == 'notebook'
48
+ Description-Content-Type: text/markdown
49
+
50
+ <div align="center">
51
+
52
+ # PreFlight
53
+
54
+ **Turn a messy CSV into model-ready data, with every choice written down.**
55
+
56
+ [![PyPI](https://img.shields.io/pypi/v/pypreflight?include_prereleases&label=PyPI)](https://pypi.org/project/pypreflight/)
57
+ [![Python](https://img.shields.io/pypi/pyversions/pypreflight)](https://pypi.org/project/pypreflight/)
58
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/VAIyerAmogha/PreFlight/blob/main/LICENSE)
59
+ [![Typed](https://img.shields.io/badge/typing-mypy%20strict-informational)](https://github.com/VAIyerAmogha/PreFlight/tree/main/typings)
60
+
61
+ **[Try it in your browser](https://pre-flight-tan.vercel.app)**
62
+
63
+ [Installation](#installation) ·
64
+ [Quickstart](#quickstart) ·
65
+ [How it works](#how-it-works) ·
66
+ [API](#api-reference) ·
67
+ [Accuracy](#accuracy) ·
68
+ [Limitations](#known-limitations) ·
69
+ [Contributing](#contributing)
70
+
71
+ </div>
72
+
73
+ ---
74
+
75
+ PreFlight is a Python library that prepares tabular data for machine learning. You give it the
76
+ path to a CSV file and the name of your target column. It works out how the file is encoded,
77
+ what separates the cells and where the header is. It then infers a type for every column,
78
+ plans the cleaning and feature engineering, and gives you back:
79
+
80
+ - a typed, cleaned **pandas DataFrame**;
81
+ - an **unfitted scikit-learn `Pipeline`** holding every step that learns from data, ready to be
82
+ cloned and fitted inside each cross-validation fold;
83
+ - a **Plan PDF** before anything runs and a **Report PDF** after, both readable by someone who
84
+ has never opened the code;
85
+ - **JSON records** of the schema, every decision and every answer a person gave.
86
+
87
+ Each choice PreFlight makes is a `Decision` carrying its evidence, a confidence score, a safe
88
+ default and a one-sentence explanation. When the evidence is too close to call, PreFlight
89
+ leaves the data as text and asks a person. It does not guess, and it never modifies the source file.
90
+
91
+ > **Status:** PreFlight 2.0 is in **alpha** (`2.0.0.dev0`). The public API listed below is stable in shape,
92
+ > but details may change before 2.0.0. Please [open an issue](https://github.com/VAIyerAmogha/PreFlight/issues)
93
+ > if something surprises you.
94
+
95
+ > [!WARNING]
96
+ > **PreFlight can make mistakes.** It infers types, file formats and cleaning steps from the data,
97
+ > and inference can be wrong. Review the Plan PDF and `decisions.json` before you rely on the output,
98
+ > especially for high-stakes work. See [Known limitations](#known-limitations).
99
+
100
+ ## Why PreFlight
101
+
102
+ `pandas.read_csv(path)` assumes UTF-8, commas and a header on the first line. Real files break
103
+ those assumptions all the time: European exports separated by semicolons, Latin-1 text,
104
+ headerless dumps, title rows above the table, `-999` or `?` standing in for "missing",
105
+ `69 kg` in a numeric column. Most preprocessing tools either fail on these files or, worse,
106
+ handle them silently in a way nobody checks.
107
+
108
+ PreFlight is built on five rules:
109
+
110
+ | Rule | What it means in practice |
111
+ |---|---|
112
+ | **Everything is a Decision** | Encoding, separator, header row, every column type, every fill value and encoder is a record with evidence and confidence. |
113
+ | **Plan before apply** | `plan()` only reads and proposes. Nothing changes until `apply()`, so a dry run and a Plan PDF come for free. |
114
+ | **Abstain, don't guess** | If two readings of a column score too closely, or a conversion would lose information, the column stays as text and becomes a question. |
115
+ | **Ask once, never hang** | All questions are batched into one review. With no person available, the run stops with a clear list instead of waiting. Answers are stored by fingerprint and reused. |
116
+ | **Fit hygiene** | Steps that learn nothing run on the table. Steps that learn from data (medians, encoders, clipping limits) go into an unfitted pipeline you fit per fold, so CV scores stay honest. |
117
+
118
+ ## Installation
119
+
120
+ ```bash
121
+ pip install --pre pypreflight
122
+ ```
123
+
124
+ The distribution is named `pypreflight`. The import name is `preflight`.
125
+
126
+ Optional extras:
127
+
128
+ ```bash
129
+ pip install --pre "pypreflight[interactive]" # rich-formatted terminal questions
130
+ pip install --pre "pypreflight[notebook]" # ipywidgets questions inside Jupyter
131
+ ```
132
+
133
+ PreFlight supports Python **3.10, 3.11 and 3.12**. It builds on pandas, NumPy, SciPy,
134
+ scikit-learn, charset-normalizer and ReportLab.
135
+
136
+ ## Quickstart
137
+
138
+ ### One call
139
+
140
+ ```python
141
+ import preflight
142
+
143
+ out = preflight.prepare("patients.csv", target="outcome")
144
+ ```
145
+
146
+ In a terminal or notebook, PreFlight shows any open questions on one screen and waits for your
147
+ answers. In a script, CI job or other non-interactive session, it stops and lists the questions
148
+ instead of hanging (see [Resolution modes](#resolution-modes)).
149
+
150
+ `out` is a dictionary:
151
+
152
+ | Key | Contents |
153
+ |---|---|
154
+ | `apply_result` | `ApplyResult` with `.data` (typed table), `.prepared` (typed and cleaned), `.pipeline`, `.decisions`, `.anomalies` |
155
+ | `pipeline` | **Unfitted** scikit-learn `Pipeline`. Fit it on training rows only. |
156
+ | `pipeline_fitted_on_all_rows` | The same pipeline fitted on every row, for transforming new data once model selection is done. Not for evaluation. |
157
+ | `plan` | The final `Plan` |
158
+ | `paths` | Paths of every file written |
159
+
160
+ All files are written to `./preflight_out/` unless you pass `out_dir`:
161
+
162
+ ```
163
+ preflight_out/
164
+ ├── plan.json # the full plan, machine-readable
165
+ ├── plan.pdf # what PreFlight intends to do, and what it asked
166
+ ├── cleaned_data.csv # the prepared table
167
+ ├── schema.json # encoding, dialect, header, per-column types and anomalies
168
+ ├── decisions.json # every final Decision with evidence and who decided it
169
+ ├── pipeline.joblib # unfitted pipeline
170
+ ├── pipeline_fitted_on_all_rows.joblib
171
+ └── report.pdf # what was done, by whom, and what looked unusual
172
+ ```
173
+
174
+ ### Look before you leap
175
+
176
+ `plan()` is pure. It reads the file, proposes, and writes nothing.
177
+
178
+ ```python
179
+ import preflight
180
+
181
+ p = preflight.plan("examples/disguised_missing_units.csv", target="outcome")
182
+
183
+ for d in p.decisions:
184
+ flag = "ASK" if d.needs_attention else ""
185
+ print(f"{d.stage:<9}{d.kind:<22}{str(d.columns):<15}{d.action:<26}{d.confidence:.2f} {flag}")
186
+ ```
187
+
188
+ ```
189
+ reading encoding [] use_encoding_utf-8 1.00
190
+ reading dialect [] separate_by_, 0.95
191
+ reading header [] use_header_row_0 0.85
192
+ infer type_inference ['patient_id'] keep_as_id 0.98 ASK
193
+ infer type_inference ['weight'] cast_to_int 0.90 ASK
194
+ infer type_inference ['temp'] cast_to_float 1.00 ASK
195
+ infer type_inference ['smoker'] cast_to_boolean 1.00 ASK
196
+ infer type_inference ['outcome'] keep_as_categorical 0.95
197
+ infer ambiguous_task ['outcome'] classification 0.50 ASK
198
+ clean normalize_categories ['patient_id'] normalize_categories 1.00
199
+ clean add_missing_indicator ['weight'] add_missing_indicator 1.00
200
+ clean fill_median ['weight'] fill_median 0.95
201
+ clean add_missing_indicator ['temp'] add_missing_indicator 1.00
202
+ clean fill_median ['temp'] fill_median 0.95
203
+ clean clip_outliers ['weight'] clip_outliers 0.95
204
+ clean clip_outliers ['temp'] clip_outliers 0.95
205
+ engineer target_encode ['patient_id'] target_encode_cross_fit 0.95
206
+ ```
207
+
208
+ This file contains four traps, and each one is caught and explained:
209
+
210
+ - `weight` holds values like `69 kg` and some `N/A`. The Decision reads: *"Column 'weight' holds
211
+ numbers with the unit 'kg'. Read as numbers; the unit 'kg' is kept in this record."* It asks
212
+ first, because stripping a unit loses information.
213
+ - `temp` uses `-999` for "no reading". PreFlight flags it because every other value is at least
214
+ 36.1, not because `-999` is on a list.
215
+ - `smoker` is a `Y`/`N` column where `?` means unknown.
216
+ - `patient_id` scores almost equally as an identifier and as a category, so it is marked for review.
217
+
218
+ `preflight.dry_run()` does the same and also resolves the questions and writes `plan.json`
219
+ and `plan.pdf`, without producing any cleaned data.
220
+
221
+ ### Files `pandas.read_csv` gets wrong
222
+
223
+ ```python
224
+ p = preflight.plan("examples/messy_noheader_semicolon.csv") # 5.5;2.9;2.2;versicolor ...
225
+
226
+ for d in p.decisions[:3]:
227
+ print(d.action, d.confidence, "|", d.plain_language)
228
+ ```
229
+
230
+ ```
231
+ use_encoding_utf-8 1.0 | Read the file using standard text format.
232
+ separate_by_; 0.9 | Separate columns using semicolon separators.
233
+ no_header 0.98 | No column names row found; assign automatic column names.
234
+ ```
235
+
236
+ `pandas.read_csv` returns a single column named `5.5;2.9;2.2;versicolor` for this file. PreFlight
237
+ splits it into `col_0` … `col_3` and types them as float, float, float and categorical.
238
+
239
+ ### Cross-validation without leakage
240
+
241
+ The returned pipeline is unfitted and works with `sklearn.base.clone`, so every statistic
242
+ (fill values, clipping limits, target encodings) is learned from the training fold alone:
243
+
244
+ ```python
245
+ from sklearn.base import clone
246
+ from sklearn.model_selection import KFold
247
+
248
+ df = out["apply_result"].prepared
249
+ X, y = df.drop(columns=["outcome"]), df["outcome"]
250
+
251
+ for train, test in KFold(5, shuffle=True, random_state=0).split(X):
252
+ pipe = clone(out["pipeline"])
253
+ X_train = pipe.fit_transform(X.iloc[train], y.iloc[train])
254
+ X_test = pipe.transform(X.iloc[test])
255
+ # fit and score your model here
256
+ ```
257
+
258
+ Which steps run on the table and which go into the pipeline is documented step by step in
259
+ [`docs/fit_hygiene.md`](https://github.com/VAIyerAmogha/PreFlight/blob/main/docs/fit_hygiene.md).
260
+
261
+ ## How it works
262
+
263
+ ```
264
+ your_file.csv (never modified)
265
+ │
266
+ ▼
267
+ 1. READ encoding → separator → header row → raw table, all text
268
+ ▼
269
+ 2. TYPE per column: find disguised missing values, → TableSchema
270
+ score candidate types, pick a clear winner
271
+ or abstain (keep as text and ask)
272
+ ▼
273
+ 3. PLAN simulated on a private copy: → Plan (ordered Decisions)
274
+ markers → units → duplicates → column drops plan.json · plan.pdf
275
+ → text tidying → fills → clipping → leakage
276
+ → encoding
277
+ ▼
278
+ 4. RESOLVE open questions batched into one review;
279
+ stored answers reused by fingerprint
280
+ ▼
281
+ 5. APPLY learn-nothing steps → prepared DataFrame
282
+ learn-from-data steps → unfitted sklearn Pipeline
283
+ ▼
284
+ report.pdf · decisions.json · schema.json · pipeline.joblib
285
+ ```
286
+
287
+ ### Decisions
288
+
289
+ Every step above produces `Decision` objects. The fields you will use most:
290
+
291
+ | Field | Meaning |
292
+ |---|---|
293
+ | `stage`, `kind` | Where it came from (`reading`, `infer`, `clean`, `engineer`) and what it is |
294
+ | `columns` | Columns it applies to |
295
+ | `action` | What will be done, e.g. `cast_to_float`, `fill_median` |
296
+ | `evidence` | The measurements behind it: scores, counts, learned values |
297
+ | `confidence` | 0 to 1 |
298
+ | `default` | The safe choice used when nobody answers |
299
+ | `needs_attention` | `True` if a person should confirm it |
300
+ | `plain_language` | One sentence a non-specialist can read |
301
+ | `status`, `history` | Who decided it (automatic, default or user) and how it changed |
302
+
303
+ ### Resolution modes
304
+
305
+ Pass `mode=` to `prepare()`, `dry_run()`, `resolve()` or `apply()`:
306
+
307
+ | Mode | Behaviour |
308
+ |---|---|
309
+ | `"auto"` *(default)* | `"ask"` when the session is interactive (terminal or notebook), `"strict"` otherwise |
310
+ | `"ask"` | Show all open questions through a channel and use the answers |
311
+ | `"defaults"` | Accept the safe default for every question. Uncertain types stay as text. |
312
+ | `"strict"` | Raise `StrictResolutionError` listing every open question. Never waits. |
313
+
314
+ A strict run on the example file stops like this:
315
+
316
+ ```
317
+ Strict mode: 5 unresolved item(s), and this session cannot ask (no terminal or notebook).
318
+ To proceed: rerun with mode='defaults' to accept the safe default for each item; or write a
319
+ review file with mode='ask', channel='review_file', fill in its answers, and run again (stored
320
+ answers are reused); or run interactively with mode='ask'.
321
+ - id: dec_infer_patient_id_1e3f2400 | column: patient_id | How should column 'patient_id' be read?
322
+ - id: dec_infer_weight_b4b5b4ac | column: weight | How should column 'weight' be read?
323
+ ...
324
+ ```
325
+
326
+ > **Tip:** `"defaults"` is deliberately cautious: the safe default for an uncertain column type is to keep
327
+ > the original text, because keeping text never destroys information. For the best result on a
328
+ > new dataset, answer the questions once. The answers are stored and reused on later runs.
329
+
330
+ ### Channels
331
+
332
+ With `mode="ask"`, the `channel=` argument decides how questions reach a person:
333
+
334
+ | Channel | Use it for |
335
+ |---|---|
336
+ | `"terminal"` | Interactive prompt. Formatted with `rich` if the `interactive` extra is installed. |
337
+ | `"notebook"` | ipywidgets form inside Jupyter (`notebook` extra) |
338
+ | `"review_file"` | Asynchronous review. Writes `review.json`, raises `ReviewPending`, and continues when you call `preflight.resolve("…/review.json")` after filling in the answers. |
339
+ | a callable | Your own UI or automation. Receives the list of `Question` objects and returns `{question_id: option}`. |
340
+
341
+ Each question carries its text, an explanation, a summary of the evidence, the options, the
342
+ recommended default and a preview of real values before and after each option. A callable
343
+ channel looks like this:
344
+
345
+ ```python
346
+ def answer(questions):
347
+ choices = {}
348
+ for q in questions:
349
+ print(q.column, q.text, q.options)
350
+ choices[q.question_id] = q.recommended_default
351
+ return choices
352
+
353
+ out = preflight.prepare("patients.csv", target="outcome", mode="ask", channel=answer)
354
+ ```
355
+
356
+ ## API reference
357
+
358
+ Everything below is importable from the top-level `preflight` package.
359
+
360
+ | Function | Purpose | Writes files |
361
+ |---|---|---|
362
+ | `plan(path, target=None, task=None, model_hint="tree", feature_config=None)` | Read, infer and propose. Returns a `Plan`. | No |
363
+ | `dry_run(path, target=None, task=None, out_dir=None, mode="auto", channel=None, ...)` | `plan` plus question resolution. Returns `(Plan, plan_pdf_path)`. | `plan.json`, `plan.pdf`, `answers.json` |
364
+ | `resolve(plan_or_path, mode=None, channel=None, out_dir=None, target=None, task=None)` | Answer every open question. Accepts a `Plan`, a data file, or a filled-in `review.json`. | `plan.json`, `plan.pdf`, `answers.json`, maybe `review.json` |
365
+ | `apply(plan, decisions=None, mode=None, channel=None, out_dir=None)` | Run the plan. Returns an `ApplyResult`. | Only when `mode` is given (resolved plan) |
366
+ | `prepare(path, target, task=None, mode="auto", channel=None, out_dir=None, model_hint="tree", feature_config=None)` | Everything: plan → resolve → apply → save → Report PDF | All outputs |
367
+
368
+ Classes: `Plan` (`to_json`, `save_json`, `from_json`, `from_dict`), `Decision`, `TableSchema`.
369
+
370
+ Common arguments:
371
+
372
+ - **`task`**: `"classification"`, `"regression"` or `None` to infer it from the target.
373
+ - **`model_hint`**: `"tree"` or `"linear"`. Shapes the engineering step: `"tree"` ordinal-encodes
374
+ low-variety categories, `"linear"` one-hot encodes them and standard-scales numeric features.
375
+ - **`out_dir`**: defaults to `./preflight_out/`. PreFlight never writes next to the source file,
376
+ which may be read-only or shared.
377
+
378
+ ## Accuracy
379
+
380
+ PreFlight is measured against a corpus of **254 real public CSV files** (Rdatasets, TidyTuesday,
381
+ FiveThirtyEight, UCI, seaborn, pandas test fixtures and the Frictionless messy-CSV tests) with
382
+ **2,850 hand-labelled columns**. The labels and their sources live in
383
+ [`corpus/labels/`](https://github.com/VAIyerAmogha/PreFlight/tree/main/corpus/labels).
384
+
385
+ | Metric | v1.0.1 | v2 |
386
+ |---|---:|---:|
387
+ | Files that crash | 10 / 254 | **0 / 254** |
388
+ | Column type exactly right | 80.77% | **91.23%** |
389
+ | Column type right *or* flagged for review | n/a | **99.54%** |
390
+ | Wrong type with no flag (silent failures) | 9.40% | **0.46%** |
391
+ | File format right (encoding + separator + header) | 91.73% | 91.73% |
392
+
393
+ In plain terms, fewer than 1 column in 100 gets a wrong type without a warning.
394
+
395
+ These numbers come with caveats. The held-out part of the corpus was examined during
396
+ development, so it is not a fully independent test. Most remaining type misses are row-number
397
+ columns read as integers, and nearly all of those are flagged for review.
398
+
399
+ ## Known limitations
400
+
401
+ **PreFlight can make mistakes.** Every type, format and cleaning choice is inferred from the data
402
+ by heuristics, not known for certain, and some of them will be wrong for your file:
403
+
404
+ - On the benchmark corpus, about **1 column in 11** gets the wrong type (most are flagged for review,
405
+ but **0.46% are wrong with no flag**), and the file format is misread for about **1 file in 12**.
406
+ - Your data may look nothing like the corpus, so your error rate may be higher.
407
+ - A confident decision is not a guaranteed one. Confidence scores measure how clearly the evidence
408
+ points one way, not whether the answer is right.
409
+ - Recommended answers to questions are a safe default, not domain knowledge. Only you know what a
410
+ column means.
411
+
412
+ Treat the output as a well-documented first draft. Read the Plan PDF before running `prepare()`,
413
+ check the Report PDF and the cleaned data afterwards, and override any decision you disagree with.
414
+ PreFlight is provided as is, without warranty (see the [license](https://github.com/VAIyerAmogha/PreFlight/blob/main/LICENSE)).
415
+
416
+ Specific known gaps:
417
+
418
+ - **CSV-like files only.** No Excel, JSON or Parquet, and no DataFrame input in 2.0.
419
+ - **No command-line interface yet.** Use the Python API.
420
+ - **The leakage check is numeric-only.** A text or yes/no copy of the target is not detected.
421
+ - **Text identifier columns** with many unique values are target-encoded rather than dropped.
422
+ - **Dialect detection** tries four separators and looks at the first 128 KB of the file.
423
+ - **Type conversion** runs per value in Python, so very large files are slower than they need to be.
424
+ - **Numeric columns that are 30–60% blank** keep their blanks. Add an imputer if your model cannot
425
+ handle missing values.
426
+
427
+ Bug reports and pull requests for any of these are welcome.
428
+
429
+ ## Contributing
430
+
431
+ ```bash
432
+ git clone https://github.com/VAIyerAmogha/PreFlight.git
433
+ cd PreFlight
434
+ python -m venv .venv && source .venv/bin/activate
435
+ pip install -e ".[dev,interactive,notebook]"
436
+ ```
437
+
438
+ Run the checks CI runs:
439
+
440
+ ```bash
441
+ pytest -q # fast suite: unit, property-based and regression tests
442
+ ruff check src/ tests/ # lint
443
+ mypy --strict src/preflight # type check
444
+ nox # all of the above on Python 3.10, 3.11 and 3.12
445
+ ```
446
+
447
+ The corpus suites run every test against all 254 files. They are slow and deselected by
448
+ default. The raw files are not stored in the repository, so download them first:
449
+
450
+ ```bash
451
+ python scripts/acquire_corpus.py
452
+ pytest -m corpus -n 4 --dist loadgroup
453
+ ```
454
+
455
+ Repository layout:
456
+
457
+ ```
458
+ src/preflight/
459
+ ├── ingest/ encoding, dialect and header detection
460
+ ├── infer/ per-column type inference and disguised-missing detection
461
+ ├── clean/ markers, duplicates, drops, normalisation, imputation, outliers
462
+ ├── engineer/ encoding, scaling, datetime and numeric features, VIF
463
+ ├── interact/ questions, modes, channels and the answer store
464
+ ├── report/ Plan PDF and Report PDF
465
+ ├── plan.py the orchestrator: plan(), apply(), dry_run()
466
+ ├── prepare.py the one-call entry point
467
+ └── assemble.py builds the scikit-learn pipeline
468
+ tests/ unit/ · property/ · regression/ · corpus/
469
+ corpus/labels/ ground-truth labels for the 254-file corpus
470
+ examples/ small CSVs used in this README
471
+ ```
472
+
473
+ Before opening a pull request, please make sure the fast suite, ruff and mypy all pass, and add
474
+ a test for any behaviour you change. Changes to type inference or ingestion should also be checked
475
+ against the corpus suite.
476
+
477
+ ## License
478
+
479
+ PreFlight is released under the [MIT License](https://github.com/VAIyerAmogha/PreFlight/blob/main/LICENSE).
480
+ All runtime dependencies are under permissive licences.