stats3 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stats3-0.1.0/LICENSE +21 -0
- stats3-0.1.0/PKG-INFO +232 -0
- stats3-0.1.0/README.md +200 -0
- stats3-0.1.0/pyproject.toml +61 -0
- stats3-0.1.0/setup.cfg +4 -0
- stats3-0.1.0/src/stats3/__init__.py +88 -0
- stats3-0.1.0/src/stats3/__version__.py +1 -0
- stats3-0.1.0/src/stats3/_data.py +269 -0
- stats3-0.1.0/src/stats3/checks.py +461 -0
- stats3-0.1.0/src/stats3/core.py +530 -0
- stats3-0.1.0/src/stats3/counts.py +643 -0
- stats3-0.1.0/src/stats3/effects.py +481 -0
- stats3-0.1.0/src/stats3/formula.py +433 -0
- stats3-0.1.0/src/stats3/jupyter.py +77 -0
- stats3-0.1.0/src/stats3/masking.py +213 -0
- stats3-0.1.0/src/stats3/models.py +1497 -0
- stats3-0.1.0/src/stats3/py.typed +0 -0
- stats3-0.1.0/src/stats3/ranks.py +750 -0
- stats3-0.1.0/src/stats3.egg-info/PKG-INFO +232 -0
- stats3-0.1.0/src/stats3.egg-info/SOURCES.txt +27 -0
- stats3-0.1.0/src/stats3.egg-info/dependency_links.txt +1 -0
- stats3-0.1.0/src/stats3.egg-info/requires.txt +13 -0
- stats3-0.1.0/src/stats3.egg-info/top_level.txt +1 -0
- stats3-0.1.0/tests/test_backends.py +153 -0
- stats3-0.1.0/tests/test_masking.py +116 -0
- stats3-0.1.0/tests/test_models.py +266 -0
- stats3-0.1.0/tests/test_t_test.py +226 -0
- stats3-0.1.0/tests/test_tier1.py +309 -0
- stats3-0.1.0/tests/test_tier2.py +259 -0
stats3-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 rleyvasal
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
stats3-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stats3
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: R's statistical tests on Polars, for Polars, pandas and tidy3 data, with R's results
|
|
5
|
+
Author: Rigoberto Leyva Salmeron
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/rleyvasal/stats3
|
|
8
|
+
Project-URL: Changelog, https://github.com/rleyvasal/stats3/blob/main/CHANGELOG.md
|
|
9
|
+
Keywords: statistics,hypothesis tests,polars,pandas,tidyverse,R
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Mathematics
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: polars>=1.39
|
|
21
|
+
Requires-Dist: scipy>=1.18
|
|
22
|
+
Provides-Extra: pandas
|
|
23
|
+
Requires-Dist: pandas>=2.2.2; extra == "pandas"
|
|
24
|
+
Requires-Dist: pyarrow>=16; extra == "pandas"
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
27
|
+
Requires-Dist: pandas>=2.2.2; extra == "dev"
|
|
28
|
+
Requires-Dist: pyarrow>=16; extra == "dev"
|
|
29
|
+
Requires-Dist: tidy3>=0.8.1; extra == "dev"
|
|
30
|
+
Requires-Dist: ipython>=8.0; extra == "dev"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# stats3
|
|
34
|
+
|
|
35
|
+
**R's statistical tests on Polars, for Polars, pandas and tidy3 data.**
|
|
36
|
+
Write a test the way R users do, run it once per group, and get R's numbers
|
|
37
|
+
back as a table in the same kind of frame you passed in.
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install stats3 # Polars data
|
|
41
|
+
pip install "stats3[pandas]" # pandas data too
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
stats3 needs Python 3.12+, Polars and SciPy 1.18+ (the first SciPy whose
|
|
45
|
+
distribution functions match R to full precision).
|
|
46
|
+
|
|
47
|
+
Every test takes R's arguments and defaults, drops missing values as R does,
|
|
48
|
+
raises R's errors and warnings, and is checked against R 4.6.
|
|
49
|
+
|
|
50
|
+
## One test per group
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
import polars as pl
|
|
54
|
+
from stats3 import *
|
|
55
|
+
|
|
56
|
+
penguins = pl.read_csv(
|
|
57
|
+
"https://raw.githubusercontent.com/allisonhorst/palmerpenguins/main/inst/extdata/penguins.csv",
|
|
58
|
+
null_values="NA",
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
t_test(penguins, "body_mass_g ~ sex", by="species")
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
```text
|
|
65
|
+
┌───────────┬─────────────┬────────────┬────────────┬────────────┬─────────────┬─────────────┐
|
|
66
|
+
│ species ┆ estimate ┆ statistic ┆ df ┆ p_value ┆ conf_low ┆ conf_high │
|
|
67
|
+
│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │
|
|
68
|
+
│ str ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │
|
|
69
|
+
╞═══════════╪═════════════╪════════════╪════════════╪════════════╪═════════════╪═════════════╡
|
|
70
|
+
│ Adelie ┆ -674.657534 ┆ -13.126286 ┆ 135.693579 ┆ 6.4023e-26 ┆ -776.301162 ┆ -573.013906 │
|
|
71
|
+
│ Chinstrap ┆ -411.764706 ┆ -5.207734 ┆ 62.575105 ┆ 0.000002 ┆ -569.790344 ┆ -253.739068 │
|
|
72
|
+
│ Gentoo ┆ -805.094686 ┆ -14.761368 ┆ 116.643504 ┆ 1.8678e-28 ┆ -913.113043 ┆ -697.07633 │
|
|
73
|
+
└───────────┴─────────────┴────────────┴────────────┴────────────┴─────────────┴─────────────┘
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
(The table also has `estimate1`, `estimate2`, `method` and `alternative`.)
|
|
77
|
+
`by=` is dplyr's `.by`: one test per group, one row per test, groups in order.
|
|
78
|
+
|
|
79
|
+
## Any frame in, the same frame out
|
|
80
|
+
|
|
81
|
+
| You pass | Groups | You get back |
|
|
82
|
+
|---|---|---|
|
|
83
|
+
| Polars `DataFrame` / `LazyFrame` | `by=` | Polars `DataFrame` / `LazyFrame` |
|
|
84
|
+
| pandas `DataFrame` | `by=` | pandas `DataFrame` |
|
|
85
|
+
| tidy3 frame | `group_by()` | tidy3 frame |
|
|
86
|
+
| dict of columns | `by=` | Polars `DataFrame` |
|
|
87
|
+
|
|
88
|
+
`as_="polars"`, `"pandas"` or `"tidy3"` asks for another type. Factor levels
|
|
89
|
+
come from the data: pandas `category` and Polars `Enum` columns order the
|
|
90
|
+
levels of `y ~ g` (the first level is `estimate1`) and of the groups.
|
|
91
|
+
|
|
92
|
+
Each test can be called with the data first, or piped:
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
t_test(penguins, "body_mass_g ~ sex", by="species") # data first
|
|
96
|
+
penguins >> t_test("body_mass_g ~ sex", by="species") # pipe
|
|
97
|
+
penguins_pd.pipe(t_test("body_mass_g ~ sex", by="species")) # pandas .pipe
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
With [tidy3](https://github.com/rleyvasal/tidy3) the tests are pipe steps
|
|
101
|
+
like any verb, and its groups and factor levels carry through:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from tidy3 import *
|
|
105
|
+
|
|
106
|
+
tidy(penguins)
|
|
107
|
+
>> group_by(species, island)
|
|
108
|
+
>> wilcox_test(body_mass_g ~ sex)
|
|
109
|
+
>> group_by(species)
|
|
110
|
+
>> mutate(p_adj = p_adjust(p_value, "BH"))
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## In a notebook: R's shorthand
|
|
114
|
+
|
|
115
|
+
In Jupyter or IPython, after `from stats3 import *` (or `%load_ext
|
|
116
|
+
stats3.jupyter`), from the next cell on formulas need no quotes and columns
|
|
117
|
+
are bare names, inside stats3's calls:
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
penguins >> t_test(body_mass_g ~ sex, by=species)
|
|
121
|
+
penguins >> binom_test(sex == "male")
|
|
122
|
+
cor_test(penguins, bill_length_mm, bill_depth_mm, method="kendall")
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
A name the notebook defines keeps its value; any other bare name in a stats3
|
|
126
|
+
call is a column. With tidy3 loaded the two packages share the shorthand. In
|
|
127
|
+
a `.py` script write `"body_mass_g ~ sex"` and `pl.col("sex") == "male"`.
|
|
128
|
+
|
|
129
|
+
## Models
|
|
130
|
+
|
|
131
|
+
`lm()` and `glm()` take R's formulas and print a summary you can read at a
|
|
132
|
+
glance: the coefficients with their 95% intervals, then the fit.
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
fit = lm(penguins, "body_mass_g ~ flipper_length_mm + species")
|
|
136
|
+
fit
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
```text
|
|
140
|
+
lm(body_mass_g ~ flipper_length_mm + species) n = 342
|
|
141
|
+
┌───────────────────┬──────────────┬──────────────┬──────────────┬───────────┬────────────┐
|
|
142
|
+
│ term ┆ estimate ┆ conf_low ┆ conf_high ┆ std_error ┆ p_value │
|
|
143
|
+
╞═══════════════════╪══════════════╪══════════════╪══════════════╪═══════════╪════════════╡
|
|
144
|
+
│ (Intercept) ┆ -4031.476891 ┆ -5180.506847 ┆ -2882.446935 ┆ 584.15134 ┆ 2.5515e-11 │
|
|
145
|
+
│ flipper_length_mm ┆ 40.705401 ┆ 34.664683 ┆ 46.746119 ┆ 3.07102 ┆ 1.4016e-32 │
|
|
146
|
+
│ speciesChinstrap ┆ -206.51012 ┆ -320.066724 ┆ -92.953517 ┆ 57.730646 ┆ 0.000398 │
|
|
147
|
+
│ speciesGentoo ┆ 266.809603 ┆ 79.425129 ┆ 454.194077 ┆ 95.263741 ┆ 0.005392 │
|
|
148
|
+
└───────────────────┴──────────────┴──────────────┴──────────────┴───────────┴────────────┘
|
|
149
|
+
residual SE 375.5 on 338 df · R² 0.7826 · adjusted R² 0.7807
|
|
150
|
+
F(3, 338) = 405.7, p < 2.2e-16
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
broom's verbs give tables (in the data's frame type), and `by=` fits one
|
|
154
|
+
model per group:
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
tidy(fit, conf_int=True) # term, estimate, std_error, statistic, p_value, conf_low, conf_high
|
|
158
|
+
glance(fit) # r_squared, adj_r_squared, sigma, statistic, p_value, AIC, BIC, ...
|
|
159
|
+
augment(fit) # the rows with .fitted, .resid, .hat, .sigma, .cooksd, .std_resid
|
|
160
|
+
anova(fit) # sequential sums of squares; anova(fit1, fit2) compares models
|
|
161
|
+
anova(fit, type=2) # car's type II tests; type=3 with contrasts="sum"
|
|
162
|
+
predict(fit, new_rows, interval="prediction")
|
|
163
|
+
|
|
164
|
+
glance(lm(penguins, "body_mass_g ~ flipper_length_mm", by="species")) # one row per species
|
|
165
|
+
|
|
166
|
+
glm(penguins, "sex ~ body_mass_g + species", family=binomial())
|
|
167
|
+
glm(mtcars, "carb ~ hp + offset(log(gear))", family=poisson())
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Formulas follow R: `a * b`, `a:b`, `(a + b)^2`, `0 +` / `- 1`, `.`, `log(x)`,
|
|
171
|
+
`I(x^2)`, `factor(cyl)`, `offset(...)`, and `cbind(successes, failures)` for
|
|
172
|
+
binomial models. Factors use treatment contrasts with the first level as the
|
|
173
|
+
reference (a pandas category or Polars `Enum` sets the order), with R's
|
|
174
|
+
column names (`contrasts="sum"` gives R's `contr.sum`, as type III tests
|
|
175
|
+
want), and redundant columns are reported as aliased, as in R. For a `glm`,
|
|
176
|
+
`tidy(conf_int=True)` gives R's profile-likelihood intervals (`confint`);
|
|
177
|
+
`conf_method="wald"` gives `confint.default`.
|
|
178
|
+
Families: `gaussian`, `binomial`, `poisson`, `Gamma`, `inverse_gaussian`,
|
|
179
|
+
`quasibinomial`, `quasipoisson`, with R's links. `fit >> tidy()` works too,
|
|
180
|
+
and with tidy3 loaded, `tidy(data)` still makes a tidy3 frame.
|
|
181
|
+
|
|
182
|
+
The engine is stats3's own: weighted least squares by QR for `lm`, and a
|
|
183
|
+
port of R's `glm.fit` (same starting values, convergence rule and step
|
|
184
|
+
halving) for `glm`. On 1,000,000 rows with 13 coefficients `lm` takes about
|
|
185
|
+
0.16 s and a Poisson `glm` about 0.6 s, against about 1.1 s and 1.9 s for
|
|
186
|
+
statsmodels' formula interface.
|
|
187
|
+
|
|
188
|
+
## What's in it
|
|
189
|
+
|
|
190
|
+
| R | stats3 |
|
|
191
|
+
|---|---|
|
|
192
|
+
| `t.test` | `t_test("y ~ g")`, `t_test(x, mu=)`, `t_test(x, y, paired=True)` |
|
|
193
|
+
| `wilcox.test` | `wilcox_test(...)`, same forms; R 4.6's exact tests with ties |
|
|
194
|
+
| `kruskal.test` | `kruskal_test("y ~ g")` |
|
|
195
|
+
| `aov`, `TukeyHSD` | `aov("y ~ g")`, `tukey_hsd("y ~ g")` (one grouping column) |
|
|
196
|
+
| `cor.test` | `cor_test(x, y, method="pearson" / "spearman" / "kendall")` |
|
|
197
|
+
| `chisq.test` | `chisq_test(x, y)`, `chisq_test(x, p=[...])` |
|
|
198
|
+
| `fisher.test` | `fisher_test(x, y)`: odds ratio for 2 x 2, exact p-value for larger tables |
|
|
199
|
+
| `prop.test`, `binom.test` | `prop_test(x, n)`, `binom_test(sex == "male")` |
|
|
200
|
+
| `shapiro.test`, `ks.test` | `shapiro_test(x)`, `ks_test(x, "pnorm", mean=, sd=)`, `ks_test(x, y)` |
|
|
201
|
+
| `bartlett.test`, `var.test`, `car::leveneTest` | `bartlett_test`, `var_test`, `levene_test` |
|
|
202
|
+
| `p.adjust` | `p_adjust(p, "BH")`: on a vector or Series, or a column in `with_columns` / `mutate` |
|
|
203
|
+
| effectsize | `cohens_d`, `hedges_g`, `eta_squared` |
|
|
204
|
+
| `power.t.test`, `power.prop.test` | `power_t_test`, `power_prop_test` |
|
|
205
|
+
| `lm`, `glm` | `lm`, `glm`, with broom's `tidy`, `glance`, `augment`, and `anova`, `predict` |
|
|
206
|
+
| `car::Anova` | `anova(fit, type=2)`, `anova(fit, type=3)` |
|
|
207
|
+
|
|
208
|
+
Next: estimated marginal means, negative binomial, robust and quantile
|
|
209
|
+
regression, ordinal models, and mixed models.
|
|
210
|
+
|
|
211
|
+
## How it works, and how it is checked
|
|
212
|
+
|
|
213
|
+
Most tests need only a few numbers per group: counts, means, variances,
|
|
214
|
+
rank sums, cross-tabulated counts. Polars computes them for every group in
|
|
215
|
+
one lazy query, and SciPy's distribution functions (Polars has none) turn
|
|
216
|
+
them into p-values for all groups at once. Over 1,000 groups of 1,000 rows
|
|
217
|
+
most tests run 4-30 times faster than calling SciPy once per group. Tests
|
|
218
|
+
that need every value (Shapiro-Wilk, Kendall's tau) run per group. stats3
|
|
219
|
+
needs only Polars and SciPy.
|
|
220
|
+
|
|
221
|
+
`tests/r_oracle_cases.R` runs the same tests in R; `pytest` compares every
|
|
222
|
+
column, to 1e-9 relative, on Polars, lazy Polars, pandas and tidy3 input.
|
|
223
|
+
Where R's own numerics are approximate (Tukey's distribution, intervals
|
|
224
|
+
found with `uniroot`, the noncentral t above ncp 37.62, the asymptotic
|
|
225
|
+
Kolmogorov series), stats3 computes to full precision and the test compares
|
|
226
|
+
at R's accuracy, with the reason written next to the tolerance. Without R,
|
|
227
|
+
the SciPy comparisons still run.
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
pip install -e ".[dev]"
|
|
231
|
+
pytest
|
|
232
|
+
```
|
stats3-0.1.0/README.md
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# stats3
|
|
2
|
+
|
|
3
|
+
**R's statistical tests on Polars, for Polars, pandas and tidy3 data.**
|
|
4
|
+
Write a test the way R users do, run it once per group, and get R's numbers
|
|
5
|
+
back as a table in the same kind of frame you passed in.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install stats3 # Polars data
|
|
9
|
+
pip install "stats3[pandas]" # pandas data too
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
stats3 needs Python 3.12+, Polars and SciPy 1.18+ (the first SciPy whose
|
|
13
|
+
distribution functions match R to full precision).
|
|
14
|
+
|
|
15
|
+
Every test takes R's arguments and defaults, drops missing values as R does,
|
|
16
|
+
raises R's errors and warnings, and is checked against R 4.6.
|
|
17
|
+
|
|
18
|
+
## One test per group
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
import polars as pl
|
|
22
|
+
from stats3 import *
|
|
23
|
+
|
|
24
|
+
penguins = pl.read_csv(
|
|
25
|
+
"https://raw.githubusercontent.com/allisonhorst/palmerpenguins/main/inst/extdata/penguins.csv",
|
|
26
|
+
null_values="NA",
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
t_test(penguins, "body_mass_g ~ sex", by="species")
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
```text
|
|
33
|
+
┌───────────┬─────────────┬────────────┬────────────┬────────────┬─────────────┬─────────────┐
|
|
34
|
+
│ species ┆ estimate ┆ statistic ┆ df ┆ p_value ┆ conf_low ┆ conf_high │
|
|
35
|
+
│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │
|
|
36
|
+
│ str ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │
|
|
37
|
+
╞═══════════╪═════════════╪════════════╪════════════╪════════════╪═════════════╪═════════════╡
|
|
38
|
+
│ Adelie ┆ -674.657534 ┆ -13.126286 ┆ 135.693579 ┆ 6.4023e-26 ┆ -776.301162 ┆ -573.013906 │
|
|
39
|
+
│ Chinstrap ┆ -411.764706 ┆ -5.207734 ┆ 62.575105 ┆ 0.000002 ┆ -569.790344 ┆ -253.739068 │
|
|
40
|
+
│ Gentoo ┆ -805.094686 ┆ -14.761368 ┆ 116.643504 ┆ 1.8678e-28 ┆ -913.113043 ┆ -697.07633 │
|
|
41
|
+
└───────────┴─────────────┴────────────┴────────────┴────────────┴─────────────┴─────────────┘
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
(The table also has `estimate1`, `estimate2`, `method` and `alternative`.)
|
|
45
|
+
`by=` is dplyr's `.by`: one test per group, one row per test, groups in order.
|
|
46
|
+
|
|
47
|
+
## Any frame in, the same frame out
|
|
48
|
+
|
|
49
|
+
| You pass | Groups | You get back |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| Polars `DataFrame` / `LazyFrame` | `by=` | Polars `DataFrame` / `LazyFrame` |
|
|
52
|
+
| pandas `DataFrame` | `by=` | pandas `DataFrame` |
|
|
53
|
+
| tidy3 frame | `group_by()` | tidy3 frame |
|
|
54
|
+
| dict of columns | `by=` | Polars `DataFrame` |
|
|
55
|
+
|
|
56
|
+
`as_="polars"`, `"pandas"` or `"tidy3"` asks for another type. Factor levels
|
|
57
|
+
come from the data: pandas `category` and Polars `Enum` columns order the
|
|
58
|
+
levels of `y ~ g` (the first level is `estimate1`) and of the groups.
|
|
59
|
+
|
|
60
|
+
Each test can be called with the data first, or piped:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
t_test(penguins, "body_mass_g ~ sex", by="species") # data first
|
|
64
|
+
penguins >> t_test("body_mass_g ~ sex", by="species") # pipe
|
|
65
|
+
penguins_pd.pipe(t_test("body_mass_g ~ sex", by="species")) # pandas .pipe
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
With [tidy3](https://github.com/rleyvasal/tidy3) the tests are pipe steps
|
|
69
|
+
like any verb, and its groups and factor levels carry through:
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from tidy3 import *
|
|
73
|
+
|
|
74
|
+
tidy(penguins)
|
|
75
|
+
>> group_by(species, island)
|
|
76
|
+
>> wilcox_test(body_mass_g ~ sex)
|
|
77
|
+
>> group_by(species)
|
|
78
|
+
>> mutate(p_adj = p_adjust(p_value, "BH"))
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## In a notebook: R's shorthand
|
|
82
|
+
|
|
83
|
+
In Jupyter or IPython, after `from stats3 import *` (or `%load_ext
|
|
84
|
+
stats3.jupyter`), from the next cell on formulas need no quotes and columns
|
|
85
|
+
are bare names, inside stats3's calls:
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
penguins >> t_test(body_mass_g ~ sex, by=species)
|
|
89
|
+
penguins >> binom_test(sex == "male")
|
|
90
|
+
cor_test(penguins, bill_length_mm, bill_depth_mm, method="kendall")
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
A name the notebook defines keeps its value; any other bare name in a stats3
|
|
94
|
+
call is a column. With tidy3 loaded the two packages share the shorthand. In
|
|
95
|
+
a `.py` script write `"body_mass_g ~ sex"` and `pl.col("sex") == "male"`.
|
|
96
|
+
|
|
97
|
+
## Models
|
|
98
|
+
|
|
99
|
+
`lm()` and `glm()` take R's formulas and print a summary you can read at a
|
|
100
|
+
glance: the coefficients with their 95% intervals, then the fit.
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
fit = lm(penguins, "body_mass_g ~ flipper_length_mm + species")
|
|
104
|
+
fit
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
```text
|
|
108
|
+
lm(body_mass_g ~ flipper_length_mm + species) n = 342
|
|
109
|
+
┌───────────────────┬──────────────┬──────────────┬──────────────┬───────────┬────────────┐
|
|
110
|
+
│ term ┆ estimate ┆ conf_low ┆ conf_high ┆ std_error ┆ p_value │
|
|
111
|
+
╞═══════════════════╪══════════════╪══════════════╪══════════════╪═══════════╪════════════╡
|
|
112
|
+
│ (Intercept) ┆ -4031.476891 ┆ -5180.506847 ┆ -2882.446935 ┆ 584.15134 ┆ 2.5515e-11 │
|
|
113
|
+
│ flipper_length_mm ┆ 40.705401 ┆ 34.664683 ┆ 46.746119 ┆ 3.07102 ┆ 1.4016e-32 │
|
|
114
|
+
│ speciesChinstrap ┆ -206.51012 ┆ -320.066724 ┆ -92.953517 ┆ 57.730646 ┆ 0.000398 │
|
|
115
|
+
│ speciesGentoo ┆ 266.809603 ┆ 79.425129 ┆ 454.194077 ┆ 95.263741 ┆ 0.005392 │
|
|
116
|
+
└───────────────────┴──────────────┴──────────────┴──────────────┴───────────┴────────────┘
|
|
117
|
+
residual SE 375.5 on 338 df · R² 0.7826 · adjusted R² 0.7807
|
|
118
|
+
F(3, 338) = 405.7, p < 2.2e-16
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
broom's verbs give tables (in the data's frame type), and `by=` fits one
|
|
122
|
+
model per group:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
tidy(fit, conf_int=True) # term, estimate, std_error, statistic, p_value, conf_low, conf_high
|
|
126
|
+
glance(fit) # r_squared, adj_r_squared, sigma, statistic, p_value, AIC, BIC, ...
|
|
127
|
+
augment(fit) # the rows with .fitted, .resid, .hat, .sigma, .cooksd, .std_resid
|
|
128
|
+
anova(fit) # sequential sums of squares; anova(fit1, fit2) compares models
|
|
129
|
+
anova(fit, type=2) # car's type II tests; type=3 with contrasts="sum"
|
|
130
|
+
predict(fit, new_rows, interval="prediction")
|
|
131
|
+
|
|
132
|
+
glance(lm(penguins, "body_mass_g ~ flipper_length_mm", by="species")) # one row per species
|
|
133
|
+
|
|
134
|
+
glm(penguins, "sex ~ body_mass_g + species", family=binomial())
|
|
135
|
+
glm(mtcars, "carb ~ hp + offset(log(gear))", family=poisson())
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Formulas follow R: `a * b`, `a:b`, `(a + b)^2`, `0 +` / `- 1`, `.`, `log(x)`,
|
|
139
|
+
`I(x^2)`, `factor(cyl)`, `offset(...)`, and `cbind(successes, failures)` for
|
|
140
|
+
binomial models. Factors use treatment contrasts with the first level as the
|
|
141
|
+
reference (a pandas category or Polars `Enum` sets the order), with R's
|
|
142
|
+
column names (`contrasts="sum"` gives R's `contr.sum`, as type III tests
|
|
143
|
+
want), and redundant columns are reported as aliased, as in R. For a `glm`,
|
|
144
|
+
`tidy(conf_int=True)` gives R's profile-likelihood intervals (`confint`);
|
|
145
|
+
`conf_method="wald"` gives `confint.default`.
|
|
146
|
+
Families: `gaussian`, `binomial`, `poisson`, `Gamma`, `inverse_gaussian`,
|
|
147
|
+
`quasibinomial`, `quasipoisson`, with R's links. `fit >> tidy()` works too,
|
|
148
|
+
and with tidy3 loaded, `tidy(data)` still makes a tidy3 frame.
|
|
149
|
+
|
|
150
|
+
The engine is stats3's own: weighted least squares by QR for `lm`, and a
|
|
151
|
+
port of R's `glm.fit` (same starting values, convergence rule and step
|
|
152
|
+
halving) for `glm`. On 1,000,000 rows with 13 coefficients `lm` takes about
|
|
153
|
+
0.16 s and a Poisson `glm` about 0.6 s, against about 1.1 s and 1.9 s for
|
|
154
|
+
statsmodels' formula interface.
|
|
155
|
+
|
|
156
|
+
## What's in it
|
|
157
|
+
|
|
158
|
+
| R | stats3 |
|
|
159
|
+
|---|---|
|
|
160
|
+
| `t.test` | `t_test("y ~ g")`, `t_test(x, mu=)`, `t_test(x, y, paired=True)` |
|
|
161
|
+
| `wilcox.test` | `wilcox_test(...)`, same forms; R 4.6's exact tests with ties |
|
|
162
|
+
| `kruskal.test` | `kruskal_test("y ~ g")` |
|
|
163
|
+
| `aov`, `TukeyHSD` | `aov("y ~ g")`, `tukey_hsd("y ~ g")` (one grouping column) |
|
|
164
|
+
| `cor.test` | `cor_test(x, y, method="pearson" / "spearman" / "kendall")` |
|
|
165
|
+
| `chisq.test` | `chisq_test(x, y)`, `chisq_test(x, p=[...])` |
|
|
166
|
+
| `fisher.test` | `fisher_test(x, y)`: odds ratio for 2 x 2, exact p-value for larger tables |
|
|
167
|
+
| `prop.test`, `binom.test` | `prop_test(x, n)`, `binom_test(sex == "male")` |
|
|
168
|
+
| `shapiro.test`, `ks.test` | `shapiro_test(x)`, `ks_test(x, "pnorm", mean=, sd=)`, `ks_test(x, y)` |
|
|
169
|
+
| `bartlett.test`, `var.test`, `car::leveneTest` | `bartlett_test`, `var_test`, `levene_test` |
|
|
170
|
+
| `p.adjust` | `p_adjust(p, "BH")`: on a vector or Series, or a column in `with_columns` / `mutate` |
|
|
171
|
+
| effectsize | `cohens_d`, `hedges_g`, `eta_squared` |
|
|
172
|
+
| `power.t.test`, `power.prop.test` | `power_t_test`, `power_prop_test` |
|
|
173
|
+
| `lm`, `glm` | `lm`, `glm`, with broom's `tidy`, `glance`, `augment`, and `anova`, `predict` |
|
|
174
|
+
| `car::Anova` | `anova(fit, type=2)`, `anova(fit, type=3)` |
|
|
175
|
+
|
|
176
|
+
Next: estimated marginal means, negative binomial, robust and quantile
|
|
177
|
+
regression, ordinal models, and mixed models.
|
|
178
|
+
|
|
179
|
+
## How it works, and how it is checked
|
|
180
|
+
|
|
181
|
+
Most tests need only a few numbers per group: counts, means, variances,
|
|
182
|
+
rank sums, cross-tabulated counts. Polars computes them for every group in
|
|
183
|
+
one lazy query, and SciPy's distribution functions (Polars has none) turn
|
|
184
|
+
them into p-values for all groups at once. Over 1,000 groups of 1,000 rows
|
|
185
|
+
most tests run 4-30 times faster than calling SciPy once per group. Tests
|
|
186
|
+
that need every value (Shapiro-Wilk, Kendall's tau) run per group. stats3
|
|
187
|
+
needs only Polars and SciPy.
|
|
188
|
+
|
|
189
|
+
`tests/r_oracle_cases.R` runs the same tests in R; `pytest` compares every
|
|
190
|
+
column, to 1e-9 relative, on Polars, lazy Polars, pandas and tidy3 input.
|
|
191
|
+
Where R's own numerics are approximate (Tukey's distribution, intervals
|
|
192
|
+
found with `uniroot`, the noncentral t above ncp 37.62, the asymptotic
|
|
193
|
+
Kolmogorov series), stats3 computes to full precision and the test compares
|
|
194
|
+
at R's accuracy, with the reason written next to the tolerance. Without R,
|
|
195
|
+
the SciPy comparisons still run.
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
pip install -e ".[dev]"
|
|
199
|
+
pytest
|
|
200
|
+
```
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "stats3"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "R's statistical tests on Polars, for Polars, pandas and tidy3 data, with R's results"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Rigoberto Leyva Salmeron" }]
|
|
14
|
+
keywords = ["statistics", "hypothesis tests", "polars", "pandas", "tidyverse", "R"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Programming Language :: Python :: 3.14",
|
|
22
|
+
"Topic :: Scientific/Engineering :: Mathematics",
|
|
23
|
+
]
|
|
24
|
+
dependencies = [
|
|
25
|
+
# Summary statistics for every group come from one Polars query.
|
|
26
|
+
"polars>=1.39",
|
|
27
|
+
# p-values and intervals come from scipy's distribution functions
|
|
28
|
+
# (Polars has none); NumPy arrives with scipy. 1.18 is the first whose
|
|
29
|
+
# inverse t and Shapiro-Wilk match R to full precision.
|
|
30
|
+
"scipy>=1.18",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
# pandas frames in and out: Polars converts them through pyarrow (the
|
|
35
|
+
# first releases built for NumPy 2, which scipy 1.18 needs).
|
|
36
|
+
pandas = ["pandas>=2.2.2", "pyarrow>=16"]
|
|
37
|
+
dev = [
|
|
38
|
+
"pytest>=7.0",
|
|
39
|
+
"pandas>=2.2.2",
|
|
40
|
+
"pyarrow>=16",
|
|
41
|
+
# Only to test that tidy3 frames, groups and expressions work.
|
|
42
|
+
"tidy3>=0.8.1",
|
|
43
|
+
"ipython>=8.0",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[project.urls]
|
|
47
|
+
Homepage = "https://github.com/rleyvasal/stats3"
|
|
48
|
+
Changelog = "https://github.com/rleyvasal/stats3/blob/main/CHANGELOG.md"
|
|
49
|
+
|
|
50
|
+
[tool.setuptools.dynamic]
|
|
51
|
+
version = { attr = "stats3.__version__.__version__" }
|
|
52
|
+
|
|
53
|
+
[tool.setuptools.packages.find]
|
|
54
|
+
where = ["src"]
|
|
55
|
+
|
|
56
|
+
[tool.setuptools.package-data]
|
|
57
|
+
stats3 = ["py.typed"]
|
|
58
|
+
|
|
59
|
+
[tool.pytest.ini_options]
|
|
60
|
+
testpaths = ["tests"]
|
|
61
|
+
pythonpath = ["src", "tests"]
|
stats3-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""R's statistical tests on Polars, for Polars, pandas and tidy3 data.
|
|
2
|
+
|
|
3
|
+
from stats3 import *
|
|
4
|
+
|
|
5
|
+
penguins >> t_test("body_mass_g ~ sex", by="species") # Polars or pandas
|
|
6
|
+
t_test(penguins, "body_mass_g ~ sex", by="species") # data first
|
|
7
|
+
penguins >> group_by(species) >> t_test(body_mass_g ~ sex) # tidy3, notebook
|
|
8
|
+
|
|
9
|
+
Each test returns one row per group, with R's defaults and R's results, in
|
|
10
|
+
the kind of frame it was given.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from stats3.__version__ import __version__
|
|
14
|
+
from stats3.checks import bartlett_test, ks_test, levene_test, shapiro_test, var_test
|
|
15
|
+
from stats3.core import aov, t_test, tukey_hsd
|
|
16
|
+
from stats3.counts import binom_test, chisq_test, fisher_test, prop_test
|
|
17
|
+
from stats3.effects import (
|
|
18
|
+
cohens_d,
|
|
19
|
+
eta_squared,
|
|
20
|
+
hedges_g,
|
|
21
|
+
p_adjust,
|
|
22
|
+
power_prop_test,
|
|
23
|
+
power_t_test,
|
|
24
|
+
)
|
|
25
|
+
from stats3.models import (
|
|
26
|
+
Gamma,
|
|
27
|
+
anova,
|
|
28
|
+
augment,
|
|
29
|
+
binomial,
|
|
30
|
+
gaussian,
|
|
31
|
+
glance,
|
|
32
|
+
glm,
|
|
33
|
+
inverse_gaussian,
|
|
34
|
+
lm,
|
|
35
|
+
poisson,
|
|
36
|
+
predict,
|
|
37
|
+
quasibinomial,
|
|
38
|
+
quasipoisson,
|
|
39
|
+
tidy,
|
|
40
|
+
)
|
|
41
|
+
from stats3.ranks import cor_test, kruskal_test, wilcox_test
|
|
42
|
+
|
|
43
|
+
# In IPython / Jupyter, turn on unquoted formulas and bare column names in
|
|
44
|
+
# stats3 calls (from the next cell on), as tidy3 does for its verbs.
|
|
45
|
+
try:
|
|
46
|
+
from stats3.jupyter import enable as _enable_notebook
|
|
47
|
+
|
|
48
|
+
_enable_notebook()
|
|
49
|
+
except Exception: # pragma: no cover - never let the shell block an import
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
__all__ = [
|
|
53
|
+
"anova",
|
|
54
|
+
"aov",
|
|
55
|
+
"augment",
|
|
56
|
+
"bartlett_test",
|
|
57
|
+
"binom_test",
|
|
58
|
+
"binomial",
|
|
59
|
+
"chisq_test",
|
|
60
|
+
"cohens_d",
|
|
61
|
+
"cor_test",
|
|
62
|
+
"eta_squared",
|
|
63
|
+
"fisher_test",
|
|
64
|
+
"Gamma",
|
|
65
|
+
"gaussian",
|
|
66
|
+
"glance",
|
|
67
|
+
"glm",
|
|
68
|
+
"hedges_g",
|
|
69
|
+
"inverse_gaussian",
|
|
70
|
+
"kruskal_test",
|
|
71
|
+
"ks_test",
|
|
72
|
+
"levene_test",
|
|
73
|
+
"lm",
|
|
74
|
+
"p_adjust",
|
|
75
|
+
"poisson",
|
|
76
|
+
"power_prop_test",
|
|
77
|
+
"power_t_test",
|
|
78
|
+
"predict",
|
|
79
|
+
"prop_test",
|
|
80
|
+
"quasibinomial",
|
|
81
|
+
"quasipoisson",
|
|
82
|
+
"shapiro_test",
|
|
83
|
+
"t_test",
|
|
84
|
+
"tidy",
|
|
85
|
+
"tukey_hsd",
|
|
86
|
+
"var_test",
|
|
87
|
+
"wilcox_test",
|
|
88
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|