statskeptic 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- statskeptic-0.1.0/.env.example +13 -0
- statskeptic-0.1.0/.github/workflows/ci.yml +50 -0
- statskeptic-0.1.0/.github/workflows/publish.yml +22 -0
- statskeptic-0.1.0/.gitignore +53 -0
- statskeptic-0.1.0/CHANGELOG.md +29 -0
- statskeptic-0.1.0/LICENSE +21 -0
- statskeptic-0.1.0/PKG-INFO +201 -0
- statskeptic-0.1.0/README.md +160 -0
- statskeptic-0.1.0/RELEASING.md +67 -0
- statskeptic-0.1.0/examples/biomarker_screen.csv +141 -0
- statskeptic-0.1.0/examples/clean_ab_test.csv +401 -0
- statskeptic-0.1.0/examples/exercise_health.csv +301 -0
- statskeptic-0.1.0/examples/make_demo_data.py +108 -0
- statskeptic-0.1.0/examples/skewed_trial.csv +91 -0
- statskeptic-0.1.0/examples/small_trial.csv +19 -0
- statskeptic-0.1.0/pyproject.toml +126 -0
- statskeptic-0.1.0/src/statskeptic/__init__.py +43 -0
- statskeptic-0.1.0/src/statskeptic/agent.py +208 -0
- statskeptic-0.1.0/src/statskeptic/cli.py +82 -0
- statskeptic-0.1.0/src/statskeptic/critique/__init__.py +25 -0
- statskeptic-0.1.0/src/statskeptic/critique/engine.py +418 -0
- statskeptic-0.1.0/src/statskeptic/critique/models.py +66 -0
- statskeptic-0.1.0/src/statskeptic/critique/revise.py +168 -0
- statskeptic-0.1.0/src/statskeptic/errors.py +20 -0
- statskeptic-0.1.0/src/statskeptic/execution.py +137 -0
- statskeptic-0.1.0/src/statskeptic/loader.py +33 -0
- statskeptic-0.1.0/src/statskeptic/plan/__init__.py +11 -0
- statskeptic-0.1.0/src/statskeptic/plan/models.py +65 -0
- statskeptic-0.1.0/src/statskeptic/plan/planner.py +512 -0
- statskeptic-0.1.0/src/statskeptic/profile/__init__.py +10 -0
- statskeptic-0.1.0/src/statskeptic/profile/models.py +83 -0
- statskeptic-0.1.0/src/statskeptic/profile/profiler.py +152 -0
- statskeptic-0.1.0/src/statskeptic/py.typed +0 -0
- statskeptic-0.1.0/src/statskeptic/report/__init__.py +3 -0
- statskeptic-0.1.0/src/statskeptic/report/models.py +209 -0
- statskeptic-0.1.0/src/statskeptic/stats/__init__.py +45 -0
- statskeptic-0.1.0/src/statskeptic/stats/_support.py +78 -0
- statskeptic-0.1.0/src/statskeptic/stats/association.py +261 -0
- statskeptic-0.1.0/src/statskeptic/stats/assumptions.py +260 -0
- statskeptic-0.1.0/src/statskeptic/stats/k_group.py +163 -0
- statskeptic-0.1.0/src/statskeptic/stats/regression.py +231 -0
- statskeptic-0.1.0/src/statskeptic/stats/results.py +185 -0
- statskeptic-0.1.0/src/statskeptic/stats/two_group.py +252 -0
- statskeptic-0.1.0/tests/conftest.py +72 -0
- statskeptic-0.1.0/tests/test_agent.py +75 -0
- statskeptic-0.1.0/tests/test_assumptions.py +67 -0
- statskeptic-0.1.0/tests/test_corpus_csv_wrangling.py +69 -0
- statskeptic-0.1.0/tests/test_corpus_pmlb.py +63 -0
- statskeptic-0.1.0/tests/test_critique_revise.py +142 -0
- statskeptic-0.1.0/tests/test_edge_cases.py +226 -0
- statskeptic-0.1.0/tests/test_execution_paths.py +103 -0
- statskeptic-0.1.0/tests/test_loader.py +96 -0
- statskeptic-0.1.0/tests/test_planner.py +92 -0
- statskeptic-0.1.0/tests/test_profiler.py +50 -0
- statskeptic-0.1.0/tests/test_properties.py +88 -0
- statskeptic-0.1.0/tests/test_render_planner_extra.py +82 -0
- statskeptic-0.1.0/tests/test_report_cli.py +93 -0
- statskeptic-0.1.0/tests/test_stats_association.py +63 -0
- statskeptic-0.1.0/tests/test_stats_kgroup_regression.py +88 -0
- statskeptic-0.1.0/tests/test_stats_two_group.py +74 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Copy to .env (gitignored) and fill in. Never commit real keys.
|
|
2
|
+
|
|
3
|
+
# Optional: enables the LLM planner/interpreter/critic. Without it, statskeptic runs
|
|
4
|
+
# in rule-only mode (fixed analysis plans + the deterministic critique rubric). Use
|
|
5
|
+
# the cheapest capable model and keep call counts small.
|
|
6
|
+
ANTHROPIC_API_KEY=
|
|
7
|
+
|
|
8
|
+
# Optional model override. Defaults to the cheapest capable Claude.
|
|
9
|
+
STATSKEPTIC_MODEL=
|
|
10
|
+
|
|
11
|
+
# Where statskeptic writes analyses, reports, and cached LLM responses.
|
|
12
|
+
# Defaults to ./.statskeptic (gitignored).
|
|
13
|
+
STATSKEPTIC_DATA_DIR=
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
lint:
|
|
10
|
+
name: lint and type-check
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v4
|
|
14
|
+
- uses: actions/setup-python@v5
|
|
15
|
+
with:
|
|
16
|
+
python-version: "3.12"
|
|
17
|
+
- run: pip install -e ".[dev]"
|
|
18
|
+
- run: black --check src tests
|
|
19
|
+
- run: ruff check src tests
|
|
20
|
+
- run: mypy src
|
|
21
|
+
|
|
22
|
+
test:
|
|
23
|
+
name: test (${{ matrix.os }}, py${{ matrix.python-version }})
|
|
24
|
+
runs-on: ${{ matrix.os }}
|
|
25
|
+
strategy:
|
|
26
|
+
fail-fast: false
|
|
27
|
+
matrix:
|
|
28
|
+
os: [ubuntu-latest, windows-latest, macos-latest]
|
|
29
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
30
|
+
steps:
|
|
31
|
+
- uses: actions/checkout@v4
|
|
32
|
+
- uses: actions/setup-python@v5
|
|
33
|
+
with:
|
|
34
|
+
python-version: ${{ matrix.python-version }}
|
|
35
|
+
- run: python -m pip install --upgrade pip
|
|
36
|
+
- run: pip install -e ".[dev]"
|
|
37
|
+
# The default gate excludes the network corpus sweeps (-m 'not slow').
|
|
38
|
+
- run: pytest -q
|
|
39
|
+
|
|
40
|
+
package:
|
|
41
|
+
name: build and check the distribution
|
|
42
|
+
runs-on: ubuntu-latest
|
|
43
|
+
steps:
|
|
44
|
+
- uses: actions/checkout@v4
|
|
45
|
+
- uses: actions/setup-python@v5
|
|
46
|
+
with:
|
|
47
|
+
python-version: "3.12"
|
|
48
|
+
- run: pip install build twine
|
|
49
|
+
- run: python -m build
|
|
50
|
+
- run: twine check dist/*
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
# Publishes to PyPI when you cut a GitHub Release. Uses Trusted Publishing (OIDC),
|
|
4
|
+
# so no API token or password is stored anywhere.
|
|
5
|
+
on:
|
|
6
|
+
release:
|
|
7
|
+
types: [published]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
publish:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
environment: pypi
|
|
13
|
+
permissions:
|
|
14
|
+
id-token: write # required for Trusted Publishing
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: "3.12"
|
|
20
|
+
- run: pip install build
|
|
21
|
+
- run: python -m build
|
|
22
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Byte-compiled / optimized / cache
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
.python-version
|
|
6
|
+
|
|
7
|
+
# Distribution / packaging / build
|
|
8
|
+
build/
|
|
9
|
+
dist/
|
|
10
|
+
*.egg-info/
|
|
11
|
+
*.egg
|
|
12
|
+
.eggs/
|
|
13
|
+
wheels/
|
|
14
|
+
sdist/
|
|
15
|
+
MANIFEST
|
|
16
|
+
|
|
17
|
+
# Virtual environments
|
|
18
|
+
.venv/
|
|
19
|
+
venv/
|
|
20
|
+
env/
|
|
21
|
+
ENV/
|
|
22
|
+
.tox/
|
|
23
|
+
|
|
24
|
+
# Test / coverage / type-check / lint caches
|
|
25
|
+
.pytest_cache/
|
|
26
|
+
.benchmarks/
|
|
27
|
+
.coverage
|
|
28
|
+
.coverage.*
|
|
29
|
+
coverage.xml
|
|
30
|
+
htmlcov/
|
|
31
|
+
.cache/
|
|
32
|
+
.mypy_cache/
|
|
33
|
+
.dmypy.json
|
|
34
|
+
.ruff_cache/
|
|
35
|
+
.hypothesis/
|
|
36
|
+
|
|
37
|
+
# Secrets - never commit keys
|
|
38
|
+
.env
|
|
39
|
+
.env.*
|
|
40
|
+
!.env.example
|
|
41
|
+
|
|
42
|
+
# Local run artifacts (analyses, reports, cached LLM responses)
|
|
43
|
+
.statskeptic/
|
|
44
|
+
*.statskeptic.json
|
|
45
|
+
|
|
46
|
+
# Documentation builds
|
|
47
|
+
site/
|
|
48
|
+
docs/_build/
|
|
49
|
+
|
|
50
|
+
# OS / editor cruft
|
|
51
|
+
.DS_Store
|
|
52
|
+
*.swp
|
|
53
|
+
*~
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project aims to
|
|
5
|
+
follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - 2026-06-01
|
|
8
|
+
|
|
9
|
+
First public release: the deterministic core, with no LLM required.
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
- `analyze(data, question)` and a `statskeptic analyze` CLI that profile a dataset,
|
|
13
|
+
plan a vetted analysis, run it, critique it against a methodological rubric, revise
|
|
14
|
+
what can be fixed, and report what the data shows and what it cannot conclude.
|
|
15
|
+
- Vetted statistical routines, each validated against hand-computed answers: Student's
|
|
16
|
+
t, Welch's t, Mann-Whitney U, Pearson, Spearman, chi-square, Fisher's exact, one-way
|
|
17
|
+
ANOVA, Kruskal-Wallis, OLS, and logistic regression. Every result carries an effect
|
|
18
|
+
size, a confidence interval where one is defined, and the computation that produced it.
|
|
19
|
+
- A deterministic critique engine and revision loop covering assumption violations,
|
|
20
|
+
multiple comparisons, confounding, low power, data leakage, and outlier sensitivity.
|
|
21
|
+
- Robust CSV reading via CleverCSV (dialect, quoting, and encoding detection), with
|
|
22
|
+
infinities treated as missing data.
|
|
23
|
+
- Meaningful CLI exit codes (defensible / cannot-conclude / declined / usage / internal).
|
|
24
|
+
|
|
25
|
+
### Notes
|
|
26
|
+
- The model never produces a number; every statistic originates in executed, re-runnable
|
|
27
|
+
code. "The data cannot support a reliable answer" is a first-class success state.
|
|
28
|
+
- An optional LLM critic and clinical/financial domain packs are planned; the check
|
|
29
|
+
registry and planner are built as the extension points for them.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 David Burton
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: statskeptic
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An autonomous data-analysis agent that red-teams its own conclusions and reports what it cannot prove.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Burton-David/statskeptic
|
|
6
|
+
Project-URL: Repository, https://github.com/Burton-David/statskeptic
|
|
7
|
+
Project-URL: Issues, https://github.com/Burton-David/statskeptic/issues
|
|
8
|
+
Author-email: David Burton <42814680+Burton-David@users.noreply.github.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agents,ai-safety,data-analysis,llm,reproducibility,statistics
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Mathematics
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: clevercsv>=0.8
|
|
24
|
+
Requires-Dist: numpy>=1.24
|
|
25
|
+
Requires-Dist: pandas>=2.0
|
|
26
|
+
Requires-Dist: pydantic>=2.6
|
|
27
|
+
Requires-Dist: scipy>=1.11
|
|
28
|
+
Requires-Dist: statsmodels>=0.14
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: black==25.9.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: hypothesis>=6.100; extra == 'dev'
|
|
32
|
+
Requires-Dist: mypy>=1.8; extra == 'dev'
|
|
33
|
+
Requires-Dist: pandas-stubs; extra == 'dev'
|
|
34
|
+
Requires-Dist: pmlb>=1.0; extra == 'dev'
|
|
35
|
+
Requires-Dist: pytest-cov>=4.1; extra == 'dev'
|
|
36
|
+
Requires-Dist: pytest>=7.4; extra == 'dev'
|
|
37
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
38
|
+
Provides-Extra: llm
|
|
39
|
+
Requires-Dist: anthropic>=0.39; extra == 'llm'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# statskeptic
|
|
43
|
+
|
|
44
|
+
**A data-analysis agent that red-teams its own conclusions.**
|
|
45
|
+
|
|
46
|
+
Give it a dataset and a question. statskeptic profiles the data, picks a vetted
|
|
47
|
+
statistical method, runs it, and then turns on the result: it attacks its own analysis
|
|
48
|
+
against a methodological rubric (assumption violations, multiple comparisons,
|
|
49
|
+
confounding, underpowered samples, data leakage, outlier sensitivity), revises what it
|
|
50
|
+
can, and reports what the data shows **and what it cannot conclude**.
|
|
51
|
+
|
|
52
|
+
Two rules make it different from the fluent-but-wrong tools it competes with:
|
|
53
|
+
|
|
54
|
+
1. **The model never produces a number.** Every statistic comes from real, tested code
|
|
55
|
+
(scipy / statsmodels) and ships with the exact call that produced it, so any figure
|
|
56
|
+
can be re-run and checked. statskeptic selects methods and interprets them; it does
|
|
57
|
+
not invent them.
|
|
58
|
+
2. **"Cannot conclude" is a success state.** Over-claiming is the cardinal sin here.
|
|
59
|
+
When the data does not support a reliable answer, statskeptic says so plainly, and a
|
|
60
|
+
non-zero exit code lets a pipeline act on it.
|
|
61
|
+
|
|
62
|
+
## A trap a naive tool walks into
|
|
63
|
+
|
|
64
|
+
`examples/skewed_trial.csv` is a two-arm trial where recovery time is heavily
|
|
65
|
+
right-skewed and there is no real difference between the arms. Point a tool that reaches
|
|
66
|
+
straight for a t-test at it and you get a confident false positive: `p = 0.014`,
|
|
67
|
+
"significant," ship it.
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
$ statskeptic analyze examples/skewed_trial.csv -q "Does the drug reduce recovery hours?"
|
|
71
|
+
|
|
72
|
+
## Mann-Whitney U
|
|
73
|
+
comparing 'recovery_hours' across 'arm': two groups, so a t-test is the usual first pass
|
|
74
|
+
|
|
75
|
+
- Result: U = 814, p = 0.110 (not significant at alpha=0.05)
|
|
76
|
+
- Effect: rank_biserial_r = -0.196
|
|
77
|
+
- location shift (drug - placebo): 95% CI [-17.5, 1.1]
|
|
78
|
+
- n = 90
|
|
79
|
+
|
|
80
|
+
### Revisions
|
|
81
|
+
- Switched from Student's t-test to Mann-Whitney U (assumption.normality): data is
|
|
82
|
+
non-normal; the rank-based test is valid here. p 0.014 -> 0.110.
|
|
83
|
+
|
|
84
|
+
### Objections raised
|
|
85
|
+
- None outstanding.
|
|
86
|
+
|
|
87
|
+
## What this cannot conclude
|
|
88
|
+
- Nothing beyond the assumptions and caveats noted above.
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
statskeptic planned the same t-test a careful analyst would reach for first, then its
|
|
92
|
+
normality check fired, the revision loop switched to the rank-based test, and the
|
|
93
|
+
"significant" result evaporated. The audit trail shows the switch and the p-value before
|
|
94
|
+
and after. The false positive never leaves the building.
|
|
95
|
+
|
|
96
|
+
## What it catches
|
|
97
|
+
|
|
98
|
+
Each objection is grounded in the actual numbers and carries a concrete remedy. Some are
|
|
99
|
+
fixed automatically by re-running; others can only be flagged, and those push the verdict
|
|
100
|
+
toward "cannot conclude."
|
|
101
|
+
|
|
102
|
+
| Objection | What fires it | What statskeptic does |
|
|
103
|
+
| --- | --- | --- |
|
|
104
|
+
| Non-normality | Shapiro plus a real skew magnitude, not a trivial deviation | switch to the rank test (Mann-Whitney, Kruskal-Wallis, Spearman) |
|
|
105
|
+
| Unequal variance | Levene on a pooled-variance t-test | switch to Welch's t-test |
|
|
106
|
+
| Sparse contingency cells | expected counts below Cochran's threshold | switch a 2x2 to Fisher's exact test |
|
|
107
|
+
| Multiple comparisons | many tests run against one outcome | apply a Holm correction and re-read significance |
|
|
108
|
+
| Confounding | a causal question on observational data | name a candidate confounder; refuse the causal claim |
|
|
109
|
+
| Low power | a non-significant result where only a large effect was detectable | report the minimum detectable effect; refuse to read "no effect" |
|
|
110
|
+
| Data leakage | an identifier used as a predictor | drop it and re-fit |
|
|
111
|
+
| Outlier sensitivity | dropping extreme points flips significance | switch to a rank-based test |
|
|
112
|
+
|
|
113
|
+
The vetted toolset covers two-group comparisons (Student's t, Welch, Mann-Whitney),
|
|
114
|
+
k-group comparisons (one-way ANOVA, Kruskal-Wallis), association (Pearson, Spearman,
|
|
115
|
+
chi-square, Fisher's exact), and regression (OLS, logistic). Each routine reports an
|
|
116
|
+
effect size and, where one is defined, a confidence interval, and lists the assumptions
|
|
117
|
+
it checked against your data.
|
|
118
|
+
|
|
119
|
+
## Install
|
|
120
|
+
|
|
121
|
+
```
|
|
122
|
+
git clone https://github.com/Burton-David/statskeptic
|
|
123
|
+
cd statskeptic
|
|
124
|
+
pip install -e .
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Python 3.10 or newer. The core needs no API key and makes no network calls.
|
|
128
|
+
|
|
129
|
+
## Usage
|
|
130
|
+
|
|
131
|
+
```
|
|
132
|
+
statskeptic analyze data.csv --question "Does the treatment change recovery?"
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
The reader detects the file's dialect (delimiter, quoting, encoding) with CleverCSV, so
|
|
136
|
+
semicolon-delimited, tab-delimited, or non-UTF-8 files load as the table they actually
|
|
137
|
+
are rather than a single mangled column, and infinities are treated as missing data.
|
|
138
|
+
|
|
139
|
+
Options:
|
|
140
|
+
|
|
141
|
+
- `--json` emits the full typed report, every number traceable to its computation.
|
|
142
|
+
- `--outcome`, `--group` / `--by`, `--predictors` name columns when the question is
|
|
143
|
+
ambiguous (the planner declines rather than guess).
|
|
144
|
+
- `--alpha` sets the significance level (default 0.05).
|
|
145
|
+
- `--quiet` suppresses the report body and returns only the exit code.
|
|
146
|
+
|
|
147
|
+
Exit codes make it scriptable as a gate:
|
|
148
|
+
|
|
149
|
+
| code | meaning |
|
|
150
|
+
| --- | --- |
|
|
151
|
+
| 0 | a defensible result (with caveats counts as defensible) |
|
|
152
|
+
| 2 | the data cannot support a reliable answer |
|
|
153
|
+
| 3 | the question does not map to a vetted method |
|
|
154
|
+
| 64 | usage error (bad flags, missing file, unknown column) |
|
|
155
|
+
| 70 | a statistical routine failed and the cause is reported, not hidden |
|
|
156
|
+
|
|
157
|
+
As a library:
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
from statskeptic import analyze
|
|
161
|
+
|
|
162
|
+
report = analyze("data.csv", "Does exercise cause better health?")
|
|
163
|
+
print(report.explain()) # markdown
|
|
164
|
+
report.to_json() # the full typed report
|
|
165
|
+
report.verdict # defensible / defensible_with_caveats / cannot_conclude / declined
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
## Try the planted-trap corpus
|
|
169
|
+
|
|
170
|
+
`examples/` ships five datasets, each with one planted flaw, generated by a seeded script
|
|
171
|
+
so the numbers above are reproducible (`python examples/make_demo_data.py`):
|
|
172
|
+
|
|
173
|
+
```
|
|
174
|
+
statskeptic analyze examples/biomarker_screen.csv -q "Which markers are associated with the outcome?"
|
|
175
|
+
statskeptic analyze examples/exercise_health.csv -q "Does more exercise cause a better health score?"
|
|
176
|
+
statskeptic analyze examples/small_trial.csv -q "Does the treatment change the test score?"
|
|
177
|
+
statskeptic analyze examples/clean_ab_test.csv -q "Does the variant change order value?"
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
The biomarker screen finds 4 markers significant at `p<0.05`, then a Holm correction
|
|
181
|
+
across the 24 tests leaves only the one real signal standing. The exercise question
|
|
182
|
+
reports a strong correlation and still refuses to call it causal, naming age as the
|
|
183
|
+
likely confounder. The small trial returns "cannot conclude": at nine per arm, only a
|
|
184
|
+
large effect was ever detectable. The clean A/B test returns a plain, defensible yes.
|
|
185
|
+
|
|
186
|
+
## Honest limits
|
|
187
|
+
|
|
188
|
+
- Causal critique is a flag, not an engine. statskeptic names a candidate confounder and
|
|
189
|
+
declines the causal claim; it does not estimate causal effects.
|
|
190
|
+
- The rule-based planner maps a question to a method by keywords and column structure. It
|
|
191
|
+
declines ambiguous questions rather than guess, so you may need `--outcome` / `--group`
|
|
192
|
+
to point it at the right columns.
|
|
193
|
+
- Independence is assumed and stated, not tested. It is a property of the study design,
|
|
194
|
+
which the data alone cannot reveal.
|
|
195
|
+
- An optional LLM critic (for context-specific objections the static rubric cannot
|
|
196
|
+
encode) and clinical / financial domain packs are planned extensions, not yet shipped.
|
|
197
|
+
The check registry and the planner are built as the seams for them.
|
|
198
|
+
|
|
199
|
+
## License
|
|
200
|
+
|
|
201
|
+
MIT.
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# statskeptic
|
|
2
|
+
|
|
3
|
+
**A data-analysis agent that red-teams its own conclusions.**
|
|
4
|
+
|
|
5
|
+
Give it a dataset and a question. statskeptic profiles the data, picks a vetted
|
|
6
|
+
statistical method, runs it, and then turns on the result: it attacks its own analysis
|
|
7
|
+
against a methodological rubric (assumption violations, multiple comparisons,
|
|
8
|
+
confounding, underpowered samples, data leakage, outlier sensitivity), revises what it
|
|
9
|
+
can, and reports what the data shows **and what it cannot conclude**.
|
|
10
|
+
|
|
11
|
+
Two rules make it different from the fluent-but-wrong tools it competes with:
|
|
12
|
+
|
|
13
|
+
1. **The model never produces a number.** Every statistic comes from real, tested code
|
|
14
|
+
(scipy / statsmodels) and ships with the exact call that produced it, so any figure
|
|
15
|
+
can be re-run and checked. statskeptic selects methods and interprets them; it does
|
|
16
|
+
not invent them.
|
|
17
|
+
2. **"Cannot conclude" is a success state.** Over-claiming is the cardinal sin here.
|
|
18
|
+
When the data does not support a reliable answer, statskeptic says so plainly, and a
|
|
19
|
+
non-zero exit code lets a pipeline act on it.
|
|
20
|
+
|
|
21
|
+
## A trap a naive tool walks into
|
|
22
|
+
|
|
23
|
+
`examples/skewed_trial.csv` is a two-arm trial where recovery time is heavily
|
|
24
|
+
right-skewed and there is no real difference between the arms. Point a tool that reaches
|
|
25
|
+
straight for a t-test at it and you get a confident false positive: `p = 0.014`,
|
|
26
|
+
"significant," ship it.
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
$ statskeptic analyze examples/skewed_trial.csv -q "Does the drug reduce recovery hours?"
|
|
30
|
+
|
|
31
|
+
## Mann-Whitney U
|
|
32
|
+
comparing 'recovery_hours' across 'arm': two groups, so a t-test is the usual first pass
|
|
33
|
+
|
|
34
|
+
- Result: U = 814, p = 0.110 (not significant at alpha=0.05)
|
|
35
|
+
- Effect: rank_biserial_r = -0.196
|
|
36
|
+
- location shift (drug - placebo): 95% CI [-17.5, 1.1]
|
|
37
|
+
- n = 90
|
|
38
|
+
|
|
39
|
+
### Revisions
|
|
40
|
+
- Switched from Student's t-test to Mann-Whitney U (assumption.normality): data is
|
|
41
|
+
non-normal; the rank-based test is valid here. p 0.014 -> 0.110.
|
|
42
|
+
|
|
43
|
+
### Objections raised
|
|
44
|
+
- None outstanding.
|
|
45
|
+
|
|
46
|
+
## What this cannot conclude
|
|
47
|
+
- Nothing beyond the assumptions and caveats noted above.
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
statskeptic planned the same t-test a careful analyst would reach for first, then its
|
|
51
|
+
normality check fired, the revision loop switched to the rank-based test, and the
|
|
52
|
+
"significant" result evaporated. The audit trail shows the switch and the p-value before
|
|
53
|
+
and after. The false positive never leaves the building.
|
|
54
|
+
|
|
55
|
+
## What it catches
|
|
56
|
+
|
|
57
|
+
Each objection is grounded in the actual numbers and carries a concrete remedy. Some are
|
|
58
|
+
fixed automatically by re-running; others can only be flagged, and those push the verdict
|
|
59
|
+
toward "cannot conclude."
|
|
60
|
+
|
|
61
|
+
| Objection | What fires it | What statskeptic does |
|
|
62
|
+
| --- | --- | --- |
|
|
63
|
+
| Non-normality | Shapiro plus a real skew magnitude, not a trivial deviation | switch to the rank test (Mann-Whitney, Kruskal-Wallis, Spearman) |
|
|
64
|
+
| Unequal variance | Levene on a pooled-variance t-test | switch to Welch's t-test |
|
|
65
|
+
| Sparse contingency cells | expected counts below Cochran's threshold | switch a 2x2 to Fisher's exact test |
|
|
66
|
+
| Multiple comparisons | many tests run against one outcome | apply a Holm correction and re-read significance |
|
|
67
|
+
| Confounding | a causal question on observational data | name a candidate confounder; refuse the causal claim |
|
|
68
|
+
| Low power | a non-significant result where only a large effect was detectable | report the minimum detectable effect; refuse to read "no effect" |
|
|
69
|
+
| Data leakage | an identifier used as a predictor | drop it and re-fit |
|
|
70
|
+
| Outlier sensitivity | dropping extreme points flips significance | switch to a rank-based test |
|
|
71
|
+
|
|
72
|
+
The vetted toolset covers two-group comparisons (Student's t, Welch, Mann-Whitney),
|
|
73
|
+
k-group comparisons (one-way ANOVA, Kruskal-Wallis), association (Pearson, Spearman,
|
|
74
|
+
chi-square, Fisher's exact), and regression (OLS, logistic). Each routine reports an
|
|
75
|
+
effect size and, where one is defined, a confidence interval, and lists the assumptions
|
|
76
|
+
it checked against your data.
|
|
77
|
+
|
|
78
|
+
## Install
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
git clone https://github.com/Burton-David/statskeptic
|
|
82
|
+
cd statskeptic
|
|
83
|
+
pip install -e .
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Python 3.10 or newer. The core needs no API key and makes no network calls.
|
|
87
|
+
|
|
88
|
+
## Usage
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
statskeptic analyze data.csv --question "Does the treatment change recovery?"
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
The reader detects the file's dialect (delimiter, quoting, encoding) with CleverCSV, so
|
|
95
|
+
semicolon-delimited, tab-delimited, or non-UTF-8 files load as the table they actually
|
|
96
|
+
are rather than a single mangled column, and infinities are treated as missing data.
|
|
97
|
+
|
|
98
|
+
Options:
|
|
99
|
+
|
|
100
|
+
- `--json` emits the full typed report, every number traceable to its computation.
|
|
101
|
+
- `--outcome`, `--group` / `--by`, `--predictors` name columns when the question is
|
|
102
|
+
ambiguous (the planner declines rather than guess).
|
|
103
|
+
- `--alpha` sets the significance level (default 0.05).
|
|
104
|
+
- `--quiet` suppresses the report body and returns only the exit code.
|
|
105
|
+
|
|
106
|
+
Exit codes make it scriptable as a gate:
|
|
107
|
+
|
|
108
|
+
| code | meaning |
|
|
109
|
+
| --- | --- |
|
|
110
|
+
| 0 | a defensible result (with caveats counts as defensible) |
|
|
111
|
+
| 2 | the data cannot support a reliable answer |
|
|
112
|
+
| 3 | the question does not map to a vetted method |
|
|
113
|
+
| 64 | usage error (bad flags, missing file, unknown column) |
|
|
114
|
+
| 70 | a statistical routine failed and the cause is reported, not hidden |
|
|
115
|
+
|
|
116
|
+
As a library:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from statskeptic import analyze
|
|
120
|
+
|
|
121
|
+
report = analyze("data.csv", "Does exercise cause better health?")
|
|
122
|
+
print(report.explain()) # markdown
|
|
123
|
+
report.to_json() # the full typed report
|
|
124
|
+
report.verdict # defensible / defensible_with_caveats / cannot_conclude / declined
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Try the planted-trap corpus
|
|
128
|
+
|
|
129
|
+
`examples/` ships five datasets, each with one planted flaw, generated by a seeded script
|
|
130
|
+
so the numbers above are reproducible (`python examples/make_demo_data.py`):
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
statskeptic analyze examples/biomarker_screen.csv -q "Which markers are associated with the outcome?"
|
|
134
|
+
statskeptic analyze examples/exercise_health.csv -q "Does more exercise cause a better health score?"
|
|
135
|
+
statskeptic analyze examples/small_trial.csv -q "Does the treatment change the test score?"
|
|
136
|
+
statskeptic analyze examples/clean_ab_test.csv -q "Does the variant change order value?"
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The biomarker screen finds 4 markers significant at `p<0.05`, then a Holm correction
|
|
140
|
+
across the 24 tests leaves only the one real signal standing. The exercise question
|
|
141
|
+
reports a strong correlation and still refuses to call it causal, naming age as the
|
|
142
|
+
likely confounder. The small trial returns "cannot conclude": at nine per arm, only a
|
|
143
|
+
large effect was ever detectable. The clean A/B test returns a plain, defensible yes.
|
|
144
|
+
|
|
145
|
+
## Honest limits
|
|
146
|
+
|
|
147
|
+
- Causal critique is a flag, not an engine. statskeptic names a candidate confounder and
|
|
148
|
+
declines the causal claim; it does not estimate causal effects.
|
|
149
|
+
- The rule-based planner maps a question to a method by keywords and column structure. It
|
|
150
|
+
declines ambiguous questions rather than guess, so you may need `--outcome` / `--group`
|
|
151
|
+
to point it at the right columns.
|
|
152
|
+
- Independence is assumed and stated, not tested. It is a property of the study design,
|
|
153
|
+
which the data alone cannot reveal.
|
|
154
|
+
- An optional LLM critic (for context-specific objections the static rubric cannot
|
|
155
|
+
encode) and clinical / financial domain packs are planned extensions, not yet shipped.
|
|
156
|
+
The check registry and the planner are built as the seams for them.
|
|
157
|
+
|
|
158
|
+
## License
|
|
159
|
+
|
|
160
|
+
MIT.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Releasing statskeptic
|
|
2
|
+
|
|
3
|
+
This project publishes to PyPI from GitHub Actions using **Trusted Publishing**
|
|
4
|
+
(OIDC), so no API token or password is stored. The release workflow is
|
|
5
|
+
`.github/workflows/publish.yml`; it runs when a GitHub Release is published.
|
|
6
|
+
|
|
7
|
+
## One-time setup
|
|
8
|
+
|
|
9
|
+
1. Push the repository to GitHub as `Burton-David/statskeptic` (or update the
|
|
10
|
+
`[project.urls]` in `pyproject.toml` and the values below to match the real path).
|
|
11
|
+
2. On GitHub: **Settings -> Environments -> New environment**, named `pypi`.
|
|
12
|
+
3. On PyPI: **Your account -> Publishing -> Add a new pending publisher**, with:
|
|
13
|
+
|
|
14
|
+
| Field | Value |
|
|
15
|
+
| --- | --- |
|
|
16
|
+
| PyPI Project Name | `statskeptic` |
|
|
17
|
+
| Owner | `Burton-David` |
|
|
18
|
+
| Repository name | `statskeptic` |
|
|
19
|
+
| Workflow name | `publish.yml` |
|
|
20
|
+
| Environment name | `pypi` |
|
|
21
|
+
|
|
22
|
+
"Pending publisher" is the right choice for the first release, before the project
|
|
23
|
+
exists on PyPI. After the first publish it becomes a normal Trusted Publisher.
|
|
24
|
+
|
|
25
|
+
## Cutting a release
|
|
26
|
+
|
|
27
|
+
1. Bump `version` in `pyproject.toml` (Semantic Versioning).
|
|
28
|
+
2. Add a section to `CHANGELOG.md`.
|
|
29
|
+
3. Commit, then tag and push:
|
|
30
|
+
```
|
|
31
|
+
git commit -am "release: vX.Y.Z"
|
|
32
|
+
git tag vX.Y.Z
|
|
33
|
+
git push && git push --tags
|
|
34
|
+
```
|
|
35
|
+
4. On GitHub: **Releases -> Draft a new release**, choose the tag, publish it. The
|
|
36
|
+
`publish` workflow builds the sdist + wheel and uploads them to PyPI.
|
|
37
|
+
|
|
38
|
+
## Dry run on TestPyPI (recommended for the first time)
|
|
39
|
+
|
|
40
|
+
Configure a second pending publisher on https://test.pypi.org with the same values,
|
|
41
|
+
then either publish a pre-release tag or run a one-off upload locally:
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
python -m build
|
|
45
|
+
twine upload --repository testpypi dist/*
|
|
46
|
+
pip install --index-url https://test.pypi.org/simple/ statskeptic
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Manual fallback (API token)
|
|
50
|
+
|
|
51
|
+
If Trusted Publishing is not set up yet and you need to publish now, create a PyPI
|
|
52
|
+
API token (Account settings -> API tokens) and upload the built artifacts directly:
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
python -m build
|
|
56
|
+
twine check dist/*
|
|
57
|
+
twine upload dist/* # username: __token__, password: the API token
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Before any publish
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
python -m build
|
|
64
|
+
twine check dist/* # metadata and README render
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
A version can only be uploaded to PyPI once; bump the version for every release.
|