statskeptic 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. statskeptic-0.1.0/.env.example +13 -0
  2. statskeptic-0.1.0/.github/workflows/ci.yml +50 -0
  3. statskeptic-0.1.0/.github/workflows/publish.yml +22 -0
  4. statskeptic-0.1.0/.gitignore +53 -0
  5. statskeptic-0.1.0/CHANGELOG.md +29 -0
  6. statskeptic-0.1.0/LICENSE +21 -0
  7. statskeptic-0.1.0/PKG-INFO +201 -0
  8. statskeptic-0.1.0/README.md +160 -0
  9. statskeptic-0.1.0/RELEASING.md +67 -0
  10. statskeptic-0.1.0/examples/biomarker_screen.csv +141 -0
  11. statskeptic-0.1.0/examples/clean_ab_test.csv +401 -0
  12. statskeptic-0.1.0/examples/exercise_health.csv +301 -0
  13. statskeptic-0.1.0/examples/make_demo_data.py +108 -0
  14. statskeptic-0.1.0/examples/skewed_trial.csv +91 -0
  15. statskeptic-0.1.0/examples/small_trial.csv +19 -0
  16. statskeptic-0.1.0/pyproject.toml +126 -0
  17. statskeptic-0.1.0/src/statskeptic/__init__.py +43 -0
  18. statskeptic-0.1.0/src/statskeptic/agent.py +208 -0
  19. statskeptic-0.1.0/src/statskeptic/cli.py +82 -0
  20. statskeptic-0.1.0/src/statskeptic/critique/__init__.py +25 -0
  21. statskeptic-0.1.0/src/statskeptic/critique/engine.py +418 -0
  22. statskeptic-0.1.0/src/statskeptic/critique/models.py +66 -0
  23. statskeptic-0.1.0/src/statskeptic/critique/revise.py +168 -0
  24. statskeptic-0.1.0/src/statskeptic/errors.py +20 -0
  25. statskeptic-0.1.0/src/statskeptic/execution.py +137 -0
  26. statskeptic-0.1.0/src/statskeptic/loader.py +33 -0
  27. statskeptic-0.1.0/src/statskeptic/plan/__init__.py +11 -0
  28. statskeptic-0.1.0/src/statskeptic/plan/models.py +65 -0
  29. statskeptic-0.1.0/src/statskeptic/plan/planner.py +512 -0
  30. statskeptic-0.1.0/src/statskeptic/profile/__init__.py +10 -0
  31. statskeptic-0.1.0/src/statskeptic/profile/models.py +83 -0
  32. statskeptic-0.1.0/src/statskeptic/profile/profiler.py +152 -0
  33. statskeptic-0.1.0/src/statskeptic/py.typed +0 -0
  34. statskeptic-0.1.0/src/statskeptic/report/__init__.py +3 -0
  35. statskeptic-0.1.0/src/statskeptic/report/models.py +209 -0
  36. statskeptic-0.1.0/src/statskeptic/stats/__init__.py +45 -0
  37. statskeptic-0.1.0/src/statskeptic/stats/_support.py +78 -0
  38. statskeptic-0.1.0/src/statskeptic/stats/association.py +261 -0
  39. statskeptic-0.1.0/src/statskeptic/stats/assumptions.py +260 -0
  40. statskeptic-0.1.0/src/statskeptic/stats/k_group.py +163 -0
  41. statskeptic-0.1.0/src/statskeptic/stats/regression.py +231 -0
  42. statskeptic-0.1.0/src/statskeptic/stats/results.py +185 -0
  43. statskeptic-0.1.0/src/statskeptic/stats/two_group.py +252 -0
  44. statskeptic-0.1.0/tests/conftest.py +72 -0
  45. statskeptic-0.1.0/tests/test_agent.py +75 -0
  46. statskeptic-0.1.0/tests/test_assumptions.py +67 -0
  47. statskeptic-0.1.0/tests/test_corpus_csv_wrangling.py +69 -0
  48. statskeptic-0.1.0/tests/test_corpus_pmlb.py +63 -0
  49. statskeptic-0.1.0/tests/test_critique_revise.py +142 -0
  50. statskeptic-0.1.0/tests/test_edge_cases.py +226 -0
  51. statskeptic-0.1.0/tests/test_execution_paths.py +103 -0
  52. statskeptic-0.1.0/tests/test_loader.py +96 -0
  53. statskeptic-0.1.0/tests/test_planner.py +92 -0
  54. statskeptic-0.1.0/tests/test_profiler.py +50 -0
  55. statskeptic-0.1.0/tests/test_properties.py +88 -0
  56. statskeptic-0.1.0/tests/test_render_planner_extra.py +82 -0
  57. statskeptic-0.1.0/tests/test_report_cli.py +93 -0
  58. statskeptic-0.1.0/tests/test_stats_association.py +63 -0
  59. statskeptic-0.1.0/tests/test_stats_kgroup_regression.py +88 -0
  60. statskeptic-0.1.0/tests/test_stats_two_group.py +74 -0
@@ -0,0 +1,13 @@
1
+ # Copy to .env (gitignored) and fill in. Never commit real keys.
2
+
3
+ # Optional: enables the LLM planner/interpreter/critic. Without it, statskeptic runs
4
+ # in rule-only mode (fixed analysis plans + the deterministic critique rubric). Use
5
+ # the cheapest capable model and keep call counts small.
6
+ ANTHROPIC_API_KEY=
7
+
8
+ # Optional model override. Defaults to the cheapest capable Claude.
9
+ STATSKEPTIC_MODEL=
10
+
11
+ # Where statskeptic writes analyses, reports, and cached LLM responses.
12
+ # Defaults to ./.statskeptic (gitignored).
13
+ STATSKEPTIC_DATA_DIR=
@@ -0,0 +1,50 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ lint:
10
+ name: lint and type-check
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - uses: actions/checkout@v4
14
+ - uses: actions/setup-python@v5
15
+ with:
16
+ python-version: "3.12"
17
+ - run: pip install -e ".[dev]"
18
+ - run: black --check src tests
19
+ - run: ruff check src tests
20
+ - run: mypy src
21
+
22
+ test:
23
+ name: test (${{ matrix.os }}, py${{ matrix.python-version }})
24
+ runs-on: ${{ matrix.os }}
25
+ strategy:
26
+ fail-fast: false
27
+ matrix:
28
+ os: [ubuntu-latest, windows-latest, macos-latest]
29
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
30
+ steps:
31
+ - uses: actions/checkout@v4
32
+ - uses: actions/setup-python@v5
33
+ with:
34
+ python-version: ${{ matrix.python-version }}
35
+ - run: python -m pip install --upgrade pip
36
+ - run: pip install -e ".[dev]"
37
+ # The default gate excludes the network corpus sweeps (-m 'not slow').
38
+ - run: pytest -q
39
+
40
+ package:
41
+ name: build and check the distribution
42
+ runs-on: ubuntu-latest
43
+ steps:
44
+ - uses: actions/checkout@v4
45
+ - uses: actions/setup-python@v5
46
+ with:
47
+ python-version: "3.12"
48
+ - run: pip install build twine
49
+ - run: python -m build
50
+ - run: twine check dist/*
@@ -0,0 +1,22 @@
1
+ name: publish
2
+
3
+ # Publishes to PyPI when you cut a GitHub Release. Uses Trusted Publishing (OIDC),
4
+ # so no API token or password is stored anywhere.
5
+ on:
6
+ release:
7
+ types: [published]
8
+
9
+ jobs:
10
+ publish:
11
+ runs-on: ubuntu-latest
12
+ environment: pypi
13
+ permissions:
14
+ id-token: write # required for Trusted Publishing
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - uses: actions/setup-python@v5
18
+ with:
19
+ python-version: "3.12"
20
+ - run: pip install build
21
+ - run: python -m build
22
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,53 @@
1
+ # Byte-compiled / optimized / cache
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ .python-version
6
+
7
+ # Distribution / packaging / build
8
+ build/
9
+ dist/
10
+ *.egg-info/
11
+ *.egg
12
+ .eggs/
13
+ wheels/
14
+ sdist/
15
+ MANIFEST
16
+
17
+ # Virtual environments
18
+ .venv/
19
+ venv/
20
+ env/
21
+ ENV/
22
+ .tox/
23
+
24
+ # Test / coverage / type-check / lint caches
25
+ .pytest_cache/
26
+ .benchmarks/
27
+ .coverage
28
+ .coverage.*
29
+ coverage.xml
30
+ htmlcov/
31
+ .cache/
32
+ .mypy_cache/
33
+ .dmypy.json
34
+ .ruff_cache/
35
+ .hypothesis/
36
+
37
+ # Secrets - never commit keys
38
+ .env
39
+ .env.*
40
+ !.env.example
41
+
42
+ # Local run artifacts (analyses, reports, cached LLM responses)
43
+ .statskeptic/
44
+ *.statskeptic.json
45
+
46
+ # Documentation builds
47
+ site/
48
+ docs/_build/
49
+
50
+ # OS / editor cruft
51
+ .DS_Store
52
+ *.swp
53
+ *~
@@ -0,0 +1,29 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project aims to
5
+ follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
+
7
+ ## [0.1.0] - 2026-06-01
8
+
9
+ First public release: the deterministic core, with no LLM required.
10
+
11
+ ### Added
12
+ - `analyze(data, question)` and a `statskeptic analyze` CLI that profile a dataset,
13
+ plan a vetted analysis, run it, critique it against a methodological rubric, revise
14
+ what can be fixed, and report what the data shows and what it cannot conclude.
15
+ - Vetted statistical routines, each validated against hand-computed answers: Student's
16
+ t, Welch's t, Mann-Whitney U, Pearson, Spearman, chi-square, Fisher's exact, one-way
17
+ ANOVA, Kruskal-Wallis, OLS, and logistic regression. Every result carries an effect
18
+ size, a confidence interval where one is defined, and the computation that produced it.
19
+ - A deterministic critique engine and revision loop covering assumption violations,
20
+ multiple comparisons, confounding, low power, data leakage, and outlier sensitivity.
21
+ - Robust CSV reading via CleverCSV (dialect, quoting, and encoding detection), with
22
+ infinities treated as missing data.
23
+ - Meaningful CLI exit codes (defensible / cannot-conclude / declined / usage / internal).
24
+
25
+ ### Notes
26
+ - The model never produces a number; every statistic originates in executed, re-runnable
27
+ code. "The data cannot support a reliable answer" is a first-class success state.
28
+ - An optional LLM critic and clinical/financial domain packs are planned; the check
29
+ registry and planner are built as the extension points for them.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 David Burton
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,201 @@
1
+ Metadata-Version: 2.4
2
+ Name: statskeptic
3
+ Version: 0.1.0
4
+ Summary: An autonomous data-analysis agent that red-teams its own conclusions and reports what it cannot prove.
5
+ Project-URL: Homepage, https://github.com/Burton-David/statskeptic
6
+ Project-URL: Repository, https://github.com/Burton-David/statskeptic
7
+ Project-URL: Issues, https://github.com/Burton-David/statskeptic/issues
8
+ Author-email: David Burton <42814680+Burton-David@users.noreply.github.com>
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: agents,ai-safety,data-analysis,llm,reproducibility,statistics
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
21
+ Classifier: Topic :: Scientific/Engineering :: Mathematics
22
+ Requires-Python: >=3.10
23
+ Requires-Dist: clevercsv>=0.8
24
+ Requires-Dist: numpy>=1.24
25
+ Requires-Dist: pandas>=2.0
26
+ Requires-Dist: pydantic>=2.6
27
+ Requires-Dist: scipy>=1.11
28
+ Requires-Dist: statsmodels>=0.14
29
+ Provides-Extra: dev
30
+ Requires-Dist: black==25.9.0; extra == 'dev'
31
+ Requires-Dist: hypothesis>=6.100; extra == 'dev'
32
+ Requires-Dist: mypy>=1.8; extra == 'dev'
33
+ Requires-Dist: pandas-stubs; extra == 'dev'
34
+ Requires-Dist: pmlb>=1.0; extra == 'dev'
35
+ Requires-Dist: pytest-cov>=4.1; extra == 'dev'
36
+ Requires-Dist: pytest>=7.4; extra == 'dev'
37
+ Requires-Dist: ruff>=0.4; extra == 'dev'
38
+ Provides-Extra: llm
39
+ Requires-Dist: anthropic>=0.39; extra == 'llm'
40
+ Description-Content-Type: text/markdown
41
+
42
+ # statskeptic
43
+
44
+ **A data-analysis agent that red-teams its own conclusions.**
45
+
46
+ Give it a dataset and a question. statskeptic profiles the data, picks a vetted
47
+ statistical method, runs it, and then turns on the result: it attacks its own analysis
48
+ against a methodological rubric (assumption violations, multiple comparisons,
49
+ confounding, underpowered samples, data leakage, outlier sensitivity), revises what it
50
+ can, and reports what the data shows **and what it cannot conclude**.
51
+
52
+ Two rules make it different from the fluent-but-wrong tools it competes with:
53
+
54
+ 1. **The model never produces a number.** Every statistic comes from real, tested code
55
+ (scipy / statsmodels) and ships with the exact call that produced it, so any figure
56
+ can be re-run and checked. statskeptic selects methods and interprets them; it does
57
+ not invent them.
58
+ 2. **"Cannot conclude" is a success state.** Over-claiming is the cardinal sin here.
59
+ When the data does not support a reliable answer, statskeptic says so plainly, and a
60
+ non-zero exit code lets a pipeline act on it.
61
+
62
+ ## A trap a naive tool walks into
63
+
64
+ `examples/skewed_trial.csv` is a two-arm trial where recovery time is heavily
65
+ right-skewed and there is no real difference between the arms. Point a tool that reaches
66
+ straight for a t-test at it and you get a confident false positive: `p = 0.014`,
67
+ "significant," ship it.
68
+
69
+ ```
70
+ $ statskeptic analyze examples/skewed_trial.csv -q "Does the drug reduce recovery hours?"
71
+
72
+ ## Mann-Whitney U
73
+ comparing 'recovery_hours' across 'arm': two groups, so a t-test is the usual first pass
74
+
75
+ - Result: U = 814, p = 0.110 (not significant at alpha=0.05)
76
+ - Effect: rank_biserial_r = -0.196
77
+ - location shift (drug - placebo): 95% CI [-17.5, 1.1]
78
+ - n = 90
79
+
80
+ ### Revisions
81
+ - Switched from Student's t-test to Mann-Whitney U (assumption.normality): data is
82
+ non-normal; the rank-based test is valid here. p 0.014 -> 0.110.
83
+
84
+ ### Objections raised
85
+ - None outstanding.
86
+
87
+ ## What this cannot conclude
88
+ - Nothing beyond the assumptions and caveats noted above.
89
+ ```
90
+
91
+ statskeptic planned the same t-test a careful analyst would reach for first, then its
92
+ normality check fired, the revision loop switched to the rank-based test, and the
93
+ "significant" result evaporated. The audit trail shows the switch and the p-value before
94
+ and after. The false positive never leaves the building.
95
+
96
+ ## What it catches
97
+
98
+ Each objection is grounded in the actual numbers and carries a concrete remedy. Some are
99
+ fixed automatically by re-running; others can only be flagged, and those push the verdict
100
+ toward "cannot conclude."
101
+
102
+ | Objection | What fires it | What statskeptic does |
103
+ | --- | --- | --- |
104
+ | Non-normality | Shapiro plus a real skew magnitude, not a trivial deviation | switch to the rank test (Mann-Whitney, Kruskal-Wallis, Spearman) |
105
+ | Unequal variance | Levene on a pooled-variance t-test | switch to Welch's t-test |
106
+ | Sparse contingency cells | expected counts below Cochran's threshold | switch a 2x2 to Fisher's exact test |
107
+ | Multiple comparisons | many tests run against one outcome | apply a Holm correction and re-read significance |
108
+ | Confounding | a causal question on observational data | name a candidate confounder; refuse the causal claim |
109
+ | Low power | a non-significant result where only a large effect was detectable | report the minimum detectable effect; refuse to read "no effect" |
110
+ | Data leakage | an identifier used as a predictor | drop it and re-fit |
111
+ | Outlier sensitivity | dropping extreme points flips significance | switch to a rank-based test |
112
+
113
+ The vetted toolset covers two-group comparisons (Student's t, Welch, Mann-Whitney),
114
+ k-group comparisons (one-way ANOVA, Kruskal-Wallis), association (Pearson, Spearman,
115
+ chi-square, Fisher's exact), and regression (OLS, logistic). Each routine reports an
116
+ effect size and, where one is defined, a confidence interval, and lists the assumptions
117
+ it checked against your data.
118
+
119
+ ## Install
120
+
121
+ ```
122
+ git clone https://github.com/Burton-David/statskeptic
123
+ cd statskeptic
124
+ pip install -e .
125
+ ```
126
+
127
+ Python 3.10 or newer. The core needs no API key and makes no network calls.
128
+
129
+ ## Usage
130
+
131
+ ```
132
+ statskeptic analyze data.csv --question "Does the treatment change recovery?"
133
+ ```
134
+
135
+ The reader detects the file's dialect (delimiter, quoting, encoding) with CleverCSV, so
136
+ semicolon-delimited, tab-delimited, or non-UTF-8 files load as the table they actually
137
+ are rather than a single mangled column, and infinities are treated as missing data.
138
+
139
+ Options:
140
+
141
+ - `--json` emits the full typed report, every number traceable to its computation.
142
+ - `--outcome`, `--group` / `--by`, `--predictors` name columns when the question is
143
+ ambiguous (the planner declines rather than guess).
144
+ - `--alpha` sets the significance level (default 0.05).
145
+ - `--quiet` suppresses the report body and returns only the exit code.
146
+
147
+ Exit codes make it scriptable as a gate:
148
+
149
+ | code | meaning |
150
+ | --- | --- |
151
+ | 0 | a defensible result (with caveats counts as defensible) |
152
+ | 2 | the data cannot support a reliable answer |
153
+ | 3 | the question does not map to a vetted method |
154
+ | 64 | usage error (bad flags, missing file, unknown column) |
155
+ | 70 | a statistical routine failed and the cause is reported, not hidden |
156
+
157
+ As a library:
158
+
159
+ ```python
160
+ from statskeptic import analyze
161
+
162
+ report = analyze("data.csv", "Does exercise cause better health?")
163
+ print(report.explain()) # markdown
164
+ report.to_json() # the full typed report
165
+ report.verdict # defensible / defensible_with_caveats / cannot_conclude / declined
166
+ ```
167
+
168
+ ## Try the planted-trap corpus
169
+
170
+ `examples/` ships five datasets, each with one planted flaw, generated by a seeded script
171
+ so the numbers above are reproducible (`python examples/make_demo_data.py`):
172
+
173
+ ```
174
+ statskeptic analyze examples/biomarker_screen.csv -q "Which markers are associated with the outcome?"
175
+ statskeptic analyze examples/exercise_health.csv -q "Does more exercise cause a better health score?"
176
+ statskeptic analyze examples/small_trial.csv -q "Does the treatment change the test score?"
177
+ statskeptic analyze examples/clean_ab_test.csv -q "Does the variant change order value?"
178
+ ```
179
+
180
+ The biomarker screen finds 4 markers significant at `p<0.05`, then a Holm correction
181
+ across the 24 tests leaves only the one real signal standing. The exercise question
182
+ reports a strong correlation and still refuses to call it causal, naming age as the
183
+ likely confounder. The small trial returns "cannot conclude": at nine per arm, only a
184
+ large effect was ever detectable. The clean A/B test returns a plain, defensible yes.
185
+
186
+ ## Honest limits
187
+
188
+ - Causal critique is a flag, not an engine. statskeptic names a candidate confounder and
189
+ declines the causal claim; it does not estimate causal effects.
190
+ - The rule-based planner maps a question to a method by keywords and column structure. It
191
+ declines ambiguous questions rather than guess, so you may need `--outcome` / `--group`
192
+ to point it at the right columns.
193
+ - Independence is assumed and stated, not tested. It is a property of the study design,
194
+ which the data alone cannot reveal.
195
+ - An optional LLM critic (for context-specific objections the static rubric cannot
196
+ encode) and clinical / financial domain packs are planned extensions, not yet shipped.
197
+ The check registry and the planner are built as the seams for them.
198
+
199
+ ## License
200
+
201
+ MIT.
@@ -0,0 +1,160 @@
1
+ # statskeptic
2
+
3
+ **A data-analysis agent that red-teams its own conclusions.**
4
+
5
+ Give it a dataset and a question. statskeptic profiles the data, picks a vetted
6
+ statistical method, runs it, and then turns on the result: it attacks its own analysis
7
+ against a methodological rubric (assumption violations, multiple comparisons,
8
+ confounding, underpowered samples, data leakage, outlier sensitivity), revises what it
9
+ can, and reports what the data shows **and what it cannot conclude**.
10
+
11
+ Two rules make it different from the fluent-but-wrong tools it competes with:
12
+
13
+ 1. **The model never produces a number.** Every statistic comes from real, tested code
14
+ (scipy / statsmodels) and ships with the exact call that produced it, so any figure
15
+ can be re-run and checked. statskeptic selects methods and interprets them; it does
16
+ not invent them.
17
+ 2. **"Cannot conclude" is a success state.** Over-claiming is the cardinal sin here.
18
+ When the data does not support a reliable answer, statskeptic says so plainly, and a
19
+ non-zero exit code lets a pipeline act on it.
20
+
21
+ ## A trap a naive tool walks into
22
+
23
+ `examples/skewed_trial.csv` is a two-arm trial where recovery time is heavily
24
+ right-skewed and there is no real difference between the arms. Point a tool that reaches
25
+ straight for a t-test at it and you get a confident false positive: `p = 0.014`,
26
+ "significant," ship it.
27
+
28
+ ```
29
+ $ statskeptic analyze examples/skewed_trial.csv -q "Does the drug reduce recovery hours?"
30
+
31
+ ## Mann-Whitney U
32
+ comparing 'recovery_hours' across 'arm': two groups, so a t-test is the usual first pass
33
+
34
+ - Result: U = 814, p = 0.110 (not significant at alpha=0.05)
35
+ - Effect: rank_biserial_r = -0.196
36
+ - location shift (drug - placebo): 95% CI [-17.5, 1.1]
37
+ - n = 90
38
+
39
+ ### Revisions
40
+ - Switched from Student's t-test to Mann-Whitney U (assumption.normality): data is
41
+ non-normal; the rank-based test is valid here. p 0.014 -> 0.110.
42
+
43
+ ### Objections raised
44
+ - None outstanding.
45
+
46
+ ## What this cannot conclude
47
+ - Nothing beyond the assumptions and caveats noted above.
48
+ ```
49
+
50
+ statskeptic planned the same t-test a careful analyst would reach for first, then its
51
+ normality check fired, the revision loop switched to the rank-based test, and the
52
+ "significant" result evaporated. The audit trail shows the switch and the p-value before
53
+ and after. The false positive never leaves the building.
54
+
55
+ ## What it catches
56
+
57
+ Each objection is grounded in the actual numbers and carries a concrete remedy. Some are
58
+ fixed automatically by re-running; others can only be flagged, and those push the verdict
59
+ toward "cannot conclude."
60
+
61
+ | Objection | What fires it | What statskeptic does |
62
+ | --- | --- | --- |
63
+ | Non-normality | Shapiro plus a real skew magnitude, not a trivial deviation | switch to the rank test (Mann-Whitney, Kruskal-Wallis, Spearman) |
64
+ | Unequal variance | Levene on a pooled-variance t-test | switch to Welch's t-test |
65
+ | Sparse contingency cells | expected counts below Cochran's threshold | switch a 2x2 to Fisher's exact test |
66
+ | Multiple comparisons | many tests run against one outcome | apply a Holm correction and re-read significance |
67
+ | Confounding | a causal question on observational data | name a candidate confounder; refuse the causal claim |
68
+ | Low power | a non-significant result where only a large effect was detectable | report the minimum detectable effect; refuse to read "no effect" |
69
+ | Data leakage | an identifier used as a predictor | drop it and re-fit |
70
+ | Outlier sensitivity | dropping extreme points flips significance | switch to a rank-based test |
71
+
72
+ The vetted toolset covers two-group comparisons (Student's t, Welch, Mann-Whitney),
73
+ k-group comparisons (one-way ANOVA, Kruskal-Wallis), association (Pearson, Spearman,
74
+ chi-square, Fisher's exact), and regression (OLS, logistic). Each routine reports an
75
+ effect size and, where one is defined, a confidence interval, and lists the assumptions
76
+ it checked against your data.
77
+
78
+ ## Install
79
+
80
+ ```
81
+ git clone https://github.com/Burton-David/statskeptic
82
+ cd statskeptic
83
+ pip install -e .
84
+ ```
85
+
86
+ Python 3.10 or newer. The core needs no API key and makes no network calls.
87
+
88
+ ## Usage
89
+
90
+ ```
91
+ statskeptic analyze data.csv --question "Does the treatment change recovery?"
92
+ ```
93
+
94
+ The reader detects the file's dialect (delimiter, quoting, encoding) with CleverCSV, so
95
+ semicolon-delimited, tab-delimited, or non-UTF-8 files load as the table they actually
96
+ are rather than a single mangled column, and infinities are treated as missing data.
97
+
98
+ Options:
99
+
100
+ - `--json` emits the full typed report, every number traceable to its computation.
101
+ - `--outcome`, `--group` / `--by`, `--predictors` name columns when the question is
102
+ ambiguous (the planner declines rather than guess).
103
+ - `--alpha` sets the significance level (default 0.05).
104
+ - `--quiet` suppresses the report body and returns only the exit code.
105
+
106
+ Exit codes make it scriptable as a gate:
107
+
108
+ | code | meaning |
109
+ | --- | --- |
110
+ | 0 | a defensible result (with caveats counts as defensible) |
111
+ | 2 | the data cannot support a reliable answer |
112
+ | 3 | the question does not map to a vetted method |
113
+ | 64 | usage error (bad flags, missing file, unknown column) |
114
+ | 70 | a statistical routine failed and the cause is reported, not hidden |
115
+
116
+ As a library:
117
+
118
+ ```python
119
+ from statskeptic import analyze
120
+
121
+ report = analyze("data.csv", "Does exercise cause better health?")
122
+ print(report.explain()) # markdown
123
+ report.to_json() # the full typed report
124
+ report.verdict # defensible / defensible_with_caveats / cannot_conclude / declined
125
+ ```
126
+
127
+ ## Try the planted-trap corpus
128
+
129
+ `examples/` ships five datasets, each with one planted flaw, generated by a seeded script
130
+ so the numbers above are reproducible (`python examples/make_demo_data.py`):
131
+
132
+ ```
133
+ statskeptic analyze examples/biomarker_screen.csv -q "Which markers are associated with the outcome?"
134
+ statskeptic analyze examples/exercise_health.csv -q "Does more exercise cause a better health score?"
135
+ statskeptic analyze examples/small_trial.csv -q "Does the treatment change the test score?"
136
+ statskeptic analyze examples/clean_ab_test.csv -q "Does the variant change order value?"
137
+ ```
138
+
139
+ The biomarker screen finds 4 markers significant at `p<0.05`, then a Holm correction
140
+ across the 24 tests leaves only the one real signal standing. The exercise question
141
+ reports a strong correlation and still refuses to call it causal, naming age as the
142
+ likely confounder. The small trial returns "cannot conclude": at nine per arm, only a
143
+ large effect was ever detectable. The clean A/B test returns a plain, defensible yes.
144
+
145
+ ## Honest limits
146
+
147
+ - Causal critique is a flag, not an engine. statskeptic names a candidate confounder and
148
+ declines the causal claim; it does not estimate causal effects.
149
+ - The rule-based planner maps a question to a method by keywords and column structure. It
150
+ declines ambiguous questions rather than guess, so you may need `--outcome` / `--group`
151
+ to point it at the right columns.
152
+ - Independence is assumed and stated, not tested. It is a property of the study design,
153
+ which the data alone cannot reveal.
154
+ - An optional LLM critic (for context-specific objections the static rubric cannot
155
+ encode) and clinical / financial domain packs are planned extensions, not yet shipped.
156
+ The check registry and the planner are built as the seams for them.
157
+
158
+ ## License
159
+
160
+ MIT.
@@ -0,0 +1,67 @@
1
+ # Releasing statskeptic
2
+
3
+ This project publishes to PyPI from GitHub Actions using **Trusted Publishing**
4
+ (OIDC), so no API token or password is stored. The release workflow is
5
+ `.github/workflows/publish.yml`; it runs when a GitHub Release is published.
6
+
7
+ ## One-time setup
8
+
9
+ 1. Push the repository to GitHub as `Burton-David/statskeptic` (or update the
10
+ `[project.urls]` in `pyproject.toml` and the values below to match the real path).
11
+ 2. On GitHub: **Settings -> Environments -> New environment**, named `pypi`.
12
+ 3. On PyPI: **Your account -> Publishing -> Add a new pending publisher**, with:
13
+
14
+ | Field | Value |
15
+ | --- | --- |
16
+ | PyPI Project Name | `statskeptic` |
17
+ | Owner | `Burton-David` |
18
+ | Repository name | `statskeptic` |
19
+ | Workflow name | `publish.yml` |
20
+ | Environment name | `pypi` |
21
+
22
+ "Pending publisher" is the right choice for the first release, before the project
23
+ exists on PyPI. After the first publish it becomes a normal Trusted Publisher.
24
+
25
+ ## Cutting a release
26
+
27
+ 1. Bump `version` in `pyproject.toml` (Semantic Versioning).
28
+ 2. Add a section to `CHANGELOG.md`.
29
+ 3. Commit, then tag and push:
30
+ ```
31
+ git commit -am "release: vX.Y.Z"
32
+ git tag vX.Y.Z
33
+ git push && git push --tags
34
+ ```
35
+ 4. On GitHub: **Releases -> Draft a new release**, choose the tag, publish it. The
36
+ `publish` workflow builds the sdist + wheel and uploads them to PyPI.
37
+
38
+ ## Dry run on TestPyPI (recommended for the first time)
39
+
40
+ Configure a second pending publisher on https://test.pypi.org with the same values,
41
+ then either publish a pre-release tag or run a one-off upload locally:
42
+
43
+ ```
44
+ python -m build
45
+ twine upload --repository testpypi dist/*
46
+ pip install --index-url https://test.pypi.org/simple/ statskeptic
47
+ ```
48
+
49
+ ## Manual fallback (API token)
50
+
51
+ If Trusted Publishing is not set up yet and you need to publish now, create a PyPI
52
+ API token (Account settings -> API tokens) and upload the built artifacts directly:
53
+
54
+ ```
55
+ python -m build
56
+ twine check dist/*
57
+ twine upload dist/* # username: __token__, password: the API token
58
+ ```
59
+
60
+ ## Before any publish
61
+
62
+ ```
63
+ python -m build
64
+ twine check dist/* # metadata and README render
65
+ ```
66
+
67
+ A version can only be uploaded to PyPI once; bump the version for every release.