scistackstats 0.1.41__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scistackstats/__init__.py +59 -0
- scistackstats/catalog.py +186 -0
- scistackstats/codegen.py +148 -0
- scistackstats/describe.py +212 -0
- scistackstats/design.py +181 -0
- scistackstats/effects.py +112 -0
- scistackstats/engine.py +255 -0
- scistackstats/export.py +221 -0
- scistackstats/format.py +653 -0
- scistackstats/overlays.py +118 -0
- scistackstats/restore.py +169 -0
- scistackstats/run.py +422 -0
- scistackstats/seed.py +85 -0
- scistackstats/spec.py +316 -0
- scistackstats/tables.py +267 -0
- scistackstats-0.1.41.dist-info/METADATA +35 -0
- scistackstats-0.1.41.dist-info/RECORD +18 -0
- scistackstats-0.1.41.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""
|
|
2
|
+
scistackstats — statistics from the same design a figure draws.
|
|
3
|
+
|
|
4
|
+
StatStudio's engine, usable from code (the GUI is a thin wrapper). The design
|
|
5
|
+
is a plot-layer ``PlotSpec`` (grouping layers = factors, separate figures and
|
|
6
|
+
panels = strata, collapse chain = unit of analysis), so a table and a figure
|
|
7
|
+
built from the same design summarise the same rows. Every number is computed
|
|
8
|
+
by csv-stats (``csvstats``), through :mod:`scistackstats.engine`.
|
|
9
|
+
|
|
10
|
+
::
|
|
11
|
+
|
|
12
|
+
from scistackplot import PlotSpec, Role, PlotKind
|
|
13
|
+
from scistackstats import seed_from_plot, load_tables, describe_table, run
|
|
14
|
+
|
|
15
|
+
plot = PlotSpec(measures=["Ratings"], roles={"brand": Role.GROUP, "subject": Role.COLLAPSE},
|
|
16
|
+
groups=["brand"], kind=PlotKind.BAR)
|
|
17
|
+
spec = seed_from_plot(plot).spec
|
|
18
|
+
tables = load_tables(source, spec) # any scistackplot DataSource
|
|
19
|
+
descriptives = describe_table(spec, tables) # mean (SD) per brand, across subjects
|
|
20
|
+
report = run(spec, tables) # paired comparisons of the brands, Holm-adjusted
|
|
21
|
+
print(export_comparisons(report, spec, "md", tables=tables)) # txt md csv latex html json
|
|
22
|
+
|
|
23
|
+
Design: ``docs/claude/statistics-design.md``; plan ``.claude/plan-statistics.md``.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
from .codegen import NotExportable, generate_stats_function, why_not_exportable
|
|
29
|
+
from .catalog import TESTS, available_tests, select_tests, why_unavailable
|
|
30
|
+
from .describe import CELL_LAYOUTS, STATISTICS, DescriptivesTable, describe_table
|
|
31
|
+
from .design import Design, Factor, Tiers, comparison_tiers, derive_design
|
|
32
|
+
from .effects import EFFECT_SIZES, DifferenceSpec, default_effect_size, difference_spec
|
|
33
|
+
from .engine import CsvStatsEngine, EngineRefusal, FakeEngine, get_engine, use_engine
|
|
34
|
+
from .export import FORMATS, export_comparisons, export_descriptives, render
|
|
35
|
+
from .format import Grid, Labels, apa_sentence, comparisons_grid, descriptives_grids, labels_for, omnibus_grid, sweep_grid
|
|
36
|
+
from .run import Comparison, StatsReport, run
|
|
37
|
+
from .seed import SeedError, Seeded, seed_from_plot
|
|
38
|
+
from .spec import DV, Analysis, DescriptiveSpec, FormatSpec, StatsSpec, StatsSpecError
|
|
39
|
+
from .tables import ANALYSIS_KIND, UNIT, AnalysisFrame, AnalysisFrameError, analysis_frame, load_tables
|
|
40
|
+
|
|
41
|
+
__all__ = [
|
|
42
|
+
"ANALYSIS_KIND", "Analysis", "AnalysisFrame", "AnalysisFrameError", "CELL_LAYOUTS", "Comparison",
|
|
43
|
+
"CsvStatsEngine", "DV", "DescriptiveSpec", "DescriptivesTable", "Design", "DifferenceSpec",
|
|
44
|
+
"EFFECT_SIZES", "EngineRefusal", "Factor", "FakeEngine", "STATISTICS", "SeedError", "Seeded",
|
|
45
|
+
"StatsReport", "StatsSpec", "StatsSpecError", "TESTS", "Tiers", "UNIT", "analysis_frame",
|
|
46
|
+
"available_tests", "comparison_tiers", "default_effect_size", "derive_design", "describe_table",
|
|
47
|
+
"difference_spec", "get_engine", "load_tables", "run", "seed_from_plot", "select_tests",
|
|
48
|
+
"use_engine", "why_unavailable",
|
|
49
|
+
"FORMATS", "FormatSpec", "Grid", "Labels", "apa_sentence", "comparisons_grid", "descriptives_grids",
|
|
50
|
+
"export_comparisons", "export_descriptives", "labels_for", "omnibus_grid", "render", "sweep_grid",
|
|
51
|
+
"NotExportable", "generate_stats_function", "why_not_exportable",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
from importlib import metadata as _metadata
|
|
56
|
+
|
|
57
|
+
__version__ = _metadata.version("scistackstats")
|
|
58
|
+
except Exception: # pragma: no cover - running from a source tree
|
|
59
|
+
__version__ = "0.0.0"
|
scistackstats/catalog.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Which test runs where, and why not — the one refusal owner.
|
|
2
|
+
|
|
3
|
+
:func:`why_unavailable` answers "can test T run on this design?" with ``None``
|
|
4
|
+
or a reason. The GUI greys a menu entry with that reason, and the runner
|
|
5
|
+
records it as a refusal. :func:`select_tests` picks the tests for an analysis
|
|
6
|
+
from the design, and always says why (S12: never switch silently).
|
|
7
|
+
|
|
8
|
+
This module knows the *statistical* requirements: a paired test needs a
|
|
9
|
+
within factor and a unit, a two-sample test needs two levels, and so on.
|
|
10
|
+
What csv-stats does not support *yet* is csv-stats' to say: those come back
|
|
11
|
+
from the engine at run time as ``not yet in csv-stats: …`` refusals. Copying
|
|
12
|
+
csv-stats' support matrix here would give it a second owner.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from .design import BETWEEN, MIXED, WITHIN, Design
|
|
21
|
+
from .spec import Analysis
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class StatTest:
|
|
26
|
+
id: str
|
|
27
|
+
label: str
|
|
28
|
+
#: "pairwise" (two levels of one factor) or "omnibus" (every factor).
|
|
29
|
+
role: str
|
|
30
|
+
family: str
|
|
31
|
+
#: For pairwise tests: the factor kind it needs.
|
|
32
|
+
kind: str | None = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
TESTS: dict[str, StatTest] = {
|
|
36
|
+
t.id: t
|
|
37
|
+
for t in (
|
|
38
|
+
StatTest("ttest_ind_welch", "Welch's t-test", "pairwise", "parametric", BETWEEN),
|
|
39
|
+
StatTest("ttest_ind_student", "Student's t-test", "pairwise", "parametric", BETWEEN),
|
|
40
|
+
StatTest("ttest_paired", "Paired t-test", "pairwise", "parametric", WITHIN),
|
|
41
|
+
StatTest("mann_whitney", "Mann–Whitney U", "pairwise", "nonparametric", BETWEEN),
|
|
42
|
+
StatTest("wilcoxon", "Wilcoxon signed-rank", "pairwise", "nonparametric", WITHIN),
|
|
43
|
+
StatTest("spm_ttest2", "SPM two-sample t (1-D curves)", "pairwise", "spm", BETWEEN),
|
|
44
|
+
StatTest("spm_ttest_paired", "SPM paired t (1-D curves)", "pairwise", "spm", WITHIN),
|
|
45
|
+
StatTest("anova", "ANOVA", "omnibus", "parametric"),
|
|
46
|
+
StatTest("kruskal", "Kruskal–Wallis", "omnibus", "nonparametric"),
|
|
47
|
+
StatTest("friedman", "Friedman", "omnibus", "nonparametric"),
|
|
48
|
+
)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _design_problem(design: Design) -> str | None:
|
|
53
|
+
if not design.factors:
|
|
54
|
+
return "nothing is grouped: add a grouping layer to compare its levels"
|
|
55
|
+
if design.mixed:
|
|
56
|
+
names = ", ".join(design.mixed)
|
|
57
|
+
return f"{names}: some units at one level, some at several; set within or between first"
|
|
58
|
+
if design.within and design.unit is None:
|
|
59
|
+
return "a within-subject factor needs an experimental unit (collapse to one, without weight by N)"
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def why_unavailable(test_id: str, design: Design) -> str | None:
|
|
64
|
+
"""``None`` if ``test_id`` can run on ``design``, else the reason."""
|
|
65
|
+
info = TESTS.get(test_id)
|
|
66
|
+
if info is None:
|
|
67
|
+
return f"unknown test {test_id!r}"
|
|
68
|
+
problem = _design_problem(design)
|
|
69
|
+
if problem:
|
|
70
|
+
return problem
|
|
71
|
+
if info.role == "pairwise":
|
|
72
|
+
primary = design.factors[0]
|
|
73
|
+
if len(primary.levels) < 2:
|
|
74
|
+
return f"{primary.name} has fewer than 2 levels here"
|
|
75
|
+
if primary.kind != info.kind:
|
|
76
|
+
return f"{info.label} needs a {info.kind}-subject factor; {primary.name} is {primary.kind}"
|
|
77
|
+
return None
|
|
78
|
+
# Omnibus
|
|
79
|
+
if test_id == "kruskal":
|
|
80
|
+
if len(design.factors) != 1 or design.factors[0].kind != BETWEEN:
|
|
81
|
+
return "Kruskal–Wallis takes one between-subject factor"
|
|
82
|
+
if len(design.factors[0].levels) < 2:
|
|
83
|
+
return f"{design.factors[0].name} has fewer than 2 levels here"
|
|
84
|
+
if test_id == "friedman":
|
|
85
|
+
if len(design.factors) != 1 or design.factors[0].kind != WITHIN:
|
|
86
|
+
return "Friedman takes one within-subject factor"
|
|
87
|
+
if len(design.factors[0].levels) < 3:
|
|
88
|
+
return "Friedman needs 3+ levels; with 2, the Wilcoxon test is the test"
|
|
89
|
+
if test_id == "anova":
|
|
90
|
+
thin = [f.name for f in design.factors if len(f.levels) < 2]
|
|
91
|
+
if thin:
|
|
92
|
+
return f"{', '.join(thin)} has fewer than 2 levels here"
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class Selection:
|
|
98
|
+
omnibus: str | None
|
|
99
|
+
omnibus_reason: str | None
|
|
100
|
+
pairwise: str | None
|
|
101
|
+
pairwise_reason: str | None
|
|
102
|
+
#: Why these tests (shown in the GUI and the report).
|
|
103
|
+
why: str
|
|
104
|
+
|
|
105
|
+
def to_dict(self) -> dict[str, Any]:
|
|
106
|
+
return {"omnibus": self.omnibus, "omnibus_reason": self.omnibus_reason,
|
|
107
|
+
"pairwise": self.pairwise, "pairwise_reason": self.pairwise_reason, "why": self.why}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _pairwise_default(analysis: Analysis, design: Design) -> tuple[str | None, str]:
|
|
111
|
+
primary = design.factors[0] if design.factors else None
|
|
112
|
+
if primary is None:
|
|
113
|
+
return None, "no grouping layer"
|
|
114
|
+
within = primary.kind == WITHIN
|
|
115
|
+
if analysis.family == "spm":
|
|
116
|
+
test = "spm_ttest_paired" if within else "spm_ttest2"
|
|
117
|
+
elif analysis.family == "nonparametric":
|
|
118
|
+
test = "wilcoxon" if within else "mann_whitney"
|
|
119
|
+
elif within:
|
|
120
|
+
test = "ttest_paired"
|
|
121
|
+
else:
|
|
122
|
+
test = "ttest_ind_student" if analysis.equal_var else "ttest_ind_welch"
|
|
123
|
+
because = f"{primary.name} is {primary.kind}-subject" + (" (overridden)" if primary.overridden else "")
|
|
124
|
+
if test.startswith("ttest_ind"):
|
|
125
|
+
because += ", equal variances assumed" if analysis.equal_var else ", equal variances not assumed"
|
|
126
|
+
return test, because
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _omnibus_default(analysis: Analysis, design: Design) -> tuple[str | None, str]:
|
|
130
|
+
if not design.factors:
|
|
131
|
+
return None, "no grouping layer"
|
|
132
|
+
if analysis.family == "spm":
|
|
133
|
+
return None, "SPM on curves runs the pairwise tests; an SPM ANOVA is not yet in csv-stats"
|
|
134
|
+
if analysis.family == "parametric":
|
|
135
|
+
return "anova", "parametric family"
|
|
136
|
+
if len(design.factors) == 1:
|
|
137
|
+
f = design.factors[0]
|
|
138
|
+
if f.kind == WITHIN:
|
|
139
|
+
return "friedman", f"{f.name} is within-subject"
|
|
140
|
+
return "kruskal", f"{f.name} is between-subject"
|
|
141
|
+
return None, "no rank-based test for a factorial design; the pairwise rank tests still run"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def select_tests(analysis: Analysis, design: Design) -> Selection:
|
|
145
|
+
"""The omnibus and pairwise tests for ``analysis`` on ``design``.
|
|
146
|
+
|
|
147
|
+
An override (``omnibus_test`` / ``pairwise_test``) wins, and is still
|
|
148
|
+
checked by :func:`why_unavailable`.
|
|
149
|
+
"""
|
|
150
|
+
omnibus = omnibus_reason = pairwise = pairwise_reason = None
|
|
151
|
+
whys: list[str] = []
|
|
152
|
+
if analysis.omnibus:
|
|
153
|
+
if analysis.omnibus_test:
|
|
154
|
+
omnibus, because = analysis.omnibus_test, "chosen"
|
|
155
|
+
if TESTS.get(omnibus) and TESTS[omnibus].role != "omnibus":
|
|
156
|
+
omnibus_reason = f"{TESTS[omnibus].label} is not an omnibus test"
|
|
157
|
+
else:
|
|
158
|
+
omnibus, because = _omnibus_default(analysis, design)
|
|
159
|
+
if omnibus is None:
|
|
160
|
+
omnibus_reason = because
|
|
161
|
+
elif omnibus_reason is None:
|
|
162
|
+
omnibus_reason = why_unavailable(omnibus, design)
|
|
163
|
+
if omnibus:
|
|
164
|
+
whys.append(f"omnibus: {TESTS[omnibus].label if omnibus in TESTS else omnibus} ({because})")
|
|
165
|
+
if analysis.pairwise:
|
|
166
|
+
if analysis.pairwise_test:
|
|
167
|
+
pairwise, because = analysis.pairwise_test, "chosen"
|
|
168
|
+
if TESTS.get(pairwise) and TESTS[pairwise].role != "pairwise":
|
|
169
|
+
pairwise_reason = f"{TESTS[pairwise].label} is not a pairwise test"
|
|
170
|
+
else:
|
|
171
|
+
pairwise, because = _pairwise_default(analysis, design)
|
|
172
|
+
if pairwise is None:
|
|
173
|
+
pairwise_reason = because
|
|
174
|
+
elif pairwise_reason is None:
|
|
175
|
+
pairwise_reason = why_unavailable(pairwise, design)
|
|
176
|
+
if pairwise:
|
|
177
|
+
whys.append(f"pairwise: {TESTS[pairwise].label if pairwise in TESTS else pairwise} ({because})")
|
|
178
|
+
return Selection(omnibus, omnibus_reason, pairwise, pairwise_reason, "; ".join(whys) or "nothing selected")
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def available_tests(design: Design) -> list[dict[str, Any]]:
|
|
182
|
+
"""Every test with its availability, for the GUI's "+ Add test" menu."""
|
|
183
|
+
return [
|
|
184
|
+
{"id": t.id, "label": t.label, "role": t.role, "family": t.family, "reason": why_unavailable(t.id, design)}
|
|
185
|
+
for t in TESTS.values()
|
|
186
|
+
]
|
scistackstats/codegen.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Code export: a ``stat_`` function that reproduces a StatStudio table (Stage 11).
|
|
2
|
+
|
|
3
|
+
The generated function **calls scistackstats** with the spec embedded as a
|
|
4
|
+
literal. It does not restate the statistics: design derivation, test
|
|
5
|
+
selection, correction families and every csv-stats call stay in their one
|
|
6
|
+
owner, so the exported code cannot drift from what StatStudio showed. It
|
|
7
|
+
receives the long-format table ``for_each`` hands a ``stat_`` function
|
|
8
|
+
(schema keys as columns, ``as_table`` on), rebuilds the ``LongTable`` with
|
|
9
|
+
the level order and schema the preview used, and returns the report dict
|
|
10
|
+
(stored by scidb's ``stat_`` machinery: lineage, ``finalized``, ``scidb
|
|
11
|
+
report``).
|
|
12
|
+
|
|
13
|
+
First pass: a DV must be a single-column Variable whose factors are its
|
|
14
|
+
schema keys. :func:`why_not_exportable` refuses, with the reason, for chosen
|
|
15
|
+
``ColName`` columns, joined grouping variables, variant rows or variant
|
|
16
|
+
axes, and an x variable. Rebuilding those tables from ``for_each`` inputs
|
|
17
|
+
would be a second copy of the source's join and melt logic.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import keyword
|
|
23
|
+
import pprint
|
|
24
|
+
import re
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
from scistacklog import Log
|
|
29
|
+
|
|
30
|
+
from scistackplot import LongTable
|
|
31
|
+
|
|
32
|
+
from .spec import StatsSpec
|
|
33
|
+
|
|
34
|
+
LAYER = "scistackstats"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class NotExportable(ValueError):
|
|
38
|
+
"""This stats spec cannot be exported as code yet; the message says why."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _ident(text: str) -> str:
|
|
42
|
+
name = re.sub(r"\W+", "_", str(text)).strip("_").lower() or "stats"
|
|
43
|
+
if name[0].isdigit() or keyword.iskeyword(name):
|
|
44
|
+
name = f"v_{name}"
|
|
45
|
+
return name
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def default_function_name(stats_spec: StatsSpec) -> str:
|
|
49
|
+
return f"stat_{_ident(stats_spec.targets[0].variable)}"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def input_params(stats_spec: StatsSpec) -> dict[str, str]:
|
|
53
|
+
"""``{parameter: variable}``: ``df`` for one variable, ``df_<name>`` for several."""
|
|
54
|
+
variables = list(dict.fromkeys(dv.variable for dv in stats_spec.targets))
|
|
55
|
+
if len(variables) == 1:
|
|
56
|
+
return {"df": variables[0]}
|
|
57
|
+
return {f"df_{_ident(v)}": v for v in variables}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def why_not_exportable(stats_spec: StatsSpec, tables: dict[str, LongTable]) -> str | None:
|
|
61
|
+
design = stats_spec.design
|
|
62
|
+
if any(dv.columns is not None for dv in stats_spec.targets):
|
|
63
|
+
return "chosen columns of a multi-column variable are not exported as code yet"
|
|
64
|
+
if design.factor_variables:
|
|
65
|
+
return "joined grouping variables are not exported as code yet"
|
|
66
|
+
if design.x_measure:
|
|
67
|
+
return "an x variable is not exported as code yet"
|
|
68
|
+
if design.variant_sets:
|
|
69
|
+
return "variant rows are not exported as code yet"
|
|
70
|
+
for dv in stats_spec.targets:
|
|
71
|
+
table = tables.get(dv.variable)
|
|
72
|
+
if table is None:
|
|
73
|
+
return f"no table loaded for {dv.variable}"
|
|
74
|
+
if table.variant_factors:
|
|
75
|
+
names = [f.name for f in table.variant_factors]
|
|
76
|
+
return f"{dv.variable} has variant axes ({', '.join(names)}); they are not exported as code yet"
|
|
77
|
+
if table.field_factors:
|
|
78
|
+
return f"{dv.variable} is a multi-column variable; it is not exported as code yet"
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass
|
|
83
|
+
class StatsFunction:
|
|
84
|
+
name: str
|
|
85
|
+
source: str
|
|
86
|
+
#: ``{parameter: variable}`` the function takes.
|
|
87
|
+
inputs: dict[str, str] = field(default_factory=dict)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _table_literal(variable: str, param: str, table: LongTable) -> list[str]:
|
|
91
|
+
factors = list(table.factor_names)
|
|
92
|
+
levels = {f: [str(v) for v in table.factor(f).levels] for f in factors}
|
|
93
|
+
return [
|
|
94
|
+
f" {variable!r}: LongTable.from_frame(",
|
|
95
|
+
f" {param},",
|
|
96
|
+
f" factors=[c for c in {factors!r} if c in {param}.columns],",
|
|
97
|
+
f" measures=[{variable!r}],",
|
|
98
|
+
f" name={variable!r},",
|
|
99
|
+
f" schema_levels={list(table.schema_levels)!r},",
|
|
100
|
+
f" level_order={levels!r},",
|
|
101
|
+
" ),",
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def generate_stats_function(stats_spec: StatsSpec, tables: dict[str, LongTable], *,
|
|
106
|
+
function_name: str | None = None) -> StatsFunction:
|
|
107
|
+
"""The ``stat_`` function source (module docstring). Raises
|
|
108
|
+
:class:`NotExportable` with the reason when the spec is not exportable."""
|
|
109
|
+
reason = why_not_exportable(stats_spec, tables)
|
|
110
|
+
if reason:
|
|
111
|
+
Log.info("[stats codegen] not exportable: %s", reason, layer=LAYER)
|
|
112
|
+
raise NotExportable(reason)
|
|
113
|
+
name = function_name or default_function_name(stats_spec)
|
|
114
|
+
if not name.isidentifier() or keyword.iskeyword(name):
|
|
115
|
+
raise NotExportable(f"{name!r} is not a valid function name")
|
|
116
|
+
params = input_params(stats_spec)
|
|
117
|
+
spec_literal = pprint.pformat(stats_spec.to_dict(), indent=1, width=96, sort_dicts=True)
|
|
118
|
+
dvs = ", ".join(dv.key for dv in stats_spec.targets)
|
|
119
|
+
analyses = ", ".join(f"{a.id} ({a.family})" for a in stats_spec.comparisons) or "descriptives only"
|
|
120
|
+
lines = [
|
|
121
|
+
f"def {name}({', '.join(params)}):",
|
|
122
|
+
f' """StatStudio statistics for {dvs}: {analyses}.',
|
|
123
|
+
"",
|
|
124
|
+
" Generated by scistackstats. The statistics are scistackstats' and",
|
|
125
|
+
" csv-stats' own (this function calls them with the saved settings), so",
|
|
126
|
+
" the result is what StatStudio showed for the same rows.",
|
|
127
|
+
' """',
|
|
128
|
+
" from scistackplot import LongTable",
|
|
129
|
+
" from scistackstats import StatsSpec, describe_table, run",
|
|
130
|
+
"",
|
|
131
|
+
" spec = StatsSpec.from_dict(",
|
|
132
|
+
*(" " + line for line in spec_literal.splitlines()),
|
|
133
|
+
" )",
|
|
134
|
+
" tables = {",
|
|
135
|
+
]
|
|
136
|
+
for param, variable in params.items():
|
|
137
|
+
lines += [" " + line for line in _table_literal(variable, param, tables[variable])]
|
|
138
|
+
lines += [
|
|
139
|
+
" }",
|
|
140
|
+
" report = run(spec, tables).to_dict()",
|
|
141
|
+
' report["descriptives"] = describe_table(spec, tables).to_dict()',
|
|
142
|
+
" return report",
|
|
143
|
+
"",
|
|
144
|
+
]
|
|
145
|
+
source = "\n".join(lines)
|
|
146
|
+
Log.info("[stats codegen] %s: %d input(s), %d analysis(es), %d line(s)", name, len(params),
|
|
147
|
+
len(stats_spec.comparisons), len(lines), layer=LAYER)
|
|
148
|
+
return StatsFunction(name=name, source=source, inputs=params)
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""The Descriptives tab: summary statistics per DV × stratum × cell (S18).
|
|
2
|
+
|
|
3
|
+
Every number is ``csvstats.descriptive.summarize`` (through the engine): the
|
|
4
|
+
one owner of each formula, the same one the plot's error bars use. This
|
|
5
|
+
module only decides *which rows* make a cell, from the :class:`AnalysisFrame`
|
|
6
|
+
the comparisons read too.
|
|
7
|
+
|
|
8
|
+
Layout:
|
|
9
|
+
- one **block per DV** (S16);
|
|
10
|
+
- one **section per stratum** (separate figures × panels);
|
|
11
|
+
- **columns** = every combination of the grouping layers, outermost first
|
|
12
|
+
(nested headers, as the figure's nested ticks read);
|
|
13
|
+
- each cell holds every statistic; :class:`~scistackstats.spec.DescriptiveSpec`
|
|
14
|
+
picks what is shown. A combination with no rows in a stratum is a cell with
|
|
15
|
+
``n = 0``, so every section has the same columns.
|
|
16
|
+
|
|
17
|
+
``n`` counts units (the plot's sample), because the rows are already
|
|
18
|
+
collapsed to one per unit per cell.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import itertools
|
|
24
|
+
import math
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
from scistacklog import Log
|
|
29
|
+
|
|
30
|
+
from scistackplot import LongTable
|
|
31
|
+
|
|
32
|
+
from .engine import get_engine
|
|
33
|
+
from .spec import StatsSpec
|
|
34
|
+
from .tables import UNIT, AnalysisFrame, AnalysisFrameError, analysis_frame
|
|
35
|
+
|
|
36
|
+
LAYER = "scistackstats"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class StatisticInfo:
|
|
41
|
+
id: str
|
|
42
|
+
label: str
|
|
43
|
+
#: Keys of ``csvstats.descriptive.summarize`` this statistic shows.
|
|
44
|
+
keys: tuple[str, ...]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
#: The built-in statistics (S18), in picker order.
|
|
48
|
+
STATISTICS: dict[str, StatisticInfo] = {
|
|
49
|
+
s.id: s
|
|
50
|
+
for s in (
|
|
51
|
+
StatisticInfo("n", "n", ("n",)),
|
|
52
|
+
StatisticInfo("n_missing", "n missing", ("n_missing",)),
|
|
53
|
+
StatisticInfo("mean", "Mean", ("mean",)),
|
|
54
|
+
StatisticInfo("sd", "SD", ("sd",)),
|
|
55
|
+
StatisticInfo("sem", "SEM", ("sem",)),
|
|
56
|
+
StatisticInfo("ci95", "95% CI (t-based)", ("ci_low", "ci_high")),
|
|
57
|
+
StatisticInfo("median", "Median", ("median",)),
|
|
58
|
+
StatisticInfo("quartiles", "Q1, Q3", ("q1", "q3")),
|
|
59
|
+
StatisticInfo("iqr", "IQR", ("iqr",)),
|
|
60
|
+
StatisticInfo("min", "Min", ("min",)),
|
|
61
|
+
StatisticInfo("max", "Max", ("max",)),
|
|
62
|
+
StatisticInfo("range", "Range", ("range",)),
|
|
63
|
+
StatisticInfo("variance", "Variance", ("variance",)),
|
|
64
|
+
StatisticInfo("cv", "CV (SD / mean)", ("cv",)),
|
|
65
|
+
StatisticInfo("skewness", "Skewness (G1)", ("skewness",)),
|
|
66
|
+
StatisticInfo("kurtosis", "Excess kurtosis (G2)", ("kurtosis",)),
|
|
67
|
+
StatisticInfo("mode", "Mode", ("mode", "mode_count")),
|
|
68
|
+
)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
#: What each combined cell (``DescriptiveSpec.cell``) shows, and its header.
|
|
72
|
+
CELL_LAYOUTS: dict[str, tuple[str, tuple[str, ...]]] = {
|
|
73
|
+
"mean_sd": ("Mean (SD)", ("mean", "sd")),
|
|
74
|
+
"mean_sem": ("Mean (SEM)", ("mean", "sem")),
|
|
75
|
+
"mean_ci95": ("Mean [95% CI]", ("mean", "ci_low", "ci_high")),
|
|
76
|
+
"median_iqr": ("Median [Q1, Q3]", ("median", "q1", "q3")),
|
|
77
|
+
"none": ("", ()),
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _clean(stats: dict[str, Any]) -> dict[str, Any]:
|
|
82
|
+
out = {}
|
|
83
|
+
for key, value in stats.items():
|
|
84
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
85
|
+
value = None
|
|
86
|
+
out[key] = value
|
|
87
|
+
return out
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass
|
|
91
|
+
class Cell:
|
|
92
|
+
#: ``{factor: level}``, outermost first; ``{}`` when there are no layers.
|
|
93
|
+
column: dict[str, str]
|
|
94
|
+
stats: dict[str, Any]
|
|
95
|
+
|
|
96
|
+
def to_dict(self) -> dict[str, Any]:
|
|
97
|
+
return {"column": dict(self.column), "stats": _clean(self.stats)}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class Section:
|
|
102
|
+
stratum: dict[str, str]
|
|
103
|
+
cells: list[Cell]
|
|
104
|
+
#: Every unit in the section, across cells.
|
|
105
|
+
n_units: int | None
|
|
106
|
+
|
|
107
|
+
def to_dict(self) -> dict[str, Any]:
|
|
108
|
+
return {"stratum": dict(self.stratum), "n_units": self.n_units, "cells": [c.to_dict() for c in self.cells]}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@dataclass
|
|
112
|
+
class DVBlock:
|
|
113
|
+
dv: str
|
|
114
|
+
#: Grouping layers, outermost first (the header rows).
|
|
115
|
+
layers: list[str]
|
|
116
|
+
#: Every column combination, outermost first, in level order.
|
|
117
|
+
columns: list[dict[str, str]]
|
|
118
|
+
unit: str | None
|
|
119
|
+
sections: list[Section] = field(default_factory=list)
|
|
120
|
+
notes: list[str] = field(default_factory=list)
|
|
121
|
+
refused: str | None = None
|
|
122
|
+
#: ``AnalysisFrame.summary()``: unit, averaged keys, strata (for captions).
|
|
123
|
+
rows: dict[str, Any] = field(default_factory=dict)
|
|
124
|
+
|
|
125
|
+
def to_dict(self) -> dict[str, Any]:
|
|
126
|
+
return {
|
|
127
|
+
"dv": self.dv, "layers": self.layers, "columns": self.columns, "unit": self.unit,
|
|
128
|
+
"sections": [s.to_dict() for s in self.sections], "notes": self.notes, "refused": self.refused,
|
|
129
|
+
"rows": dict(self.rows),
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass
|
|
134
|
+
class DescriptivesTable:
|
|
135
|
+
blocks: list[DVBlock]
|
|
136
|
+
cell: str
|
|
137
|
+
statistics: list[str]
|
|
138
|
+
engine: dict[str, str]
|
|
139
|
+
|
|
140
|
+
def to_dict(self) -> dict[str, Any]:
|
|
141
|
+
header, keys = CELL_LAYOUTS[self.cell]
|
|
142
|
+
return {
|
|
143
|
+
"cell": self.cell, "cell_header": header, "cell_keys": list(keys),
|
|
144
|
+
"statistics": [{"id": s, "label": STATISTICS[s].label, "keys": list(STATISTICS[s].keys)}
|
|
145
|
+
for s in self.statistics],
|
|
146
|
+
"blocks": [b.to_dict() for b in self.blocks],
|
|
147
|
+
"engine": dict(self.engine),
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _columns(frame: AnalysisFrame) -> list[dict[str, Any]]:
|
|
152
|
+
layers = frame.outer_first
|
|
153
|
+
if not layers:
|
|
154
|
+
return [{}]
|
|
155
|
+
return [dict(zip(layers, combo)) for combo in itertools.product(*(frame.levels[n] for n in layers))]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def describe_frame(frame: AnalysisFrame) -> DVBlock:
|
|
159
|
+
engine = get_engine()
|
|
160
|
+
layers = frame.outer_first
|
|
161
|
+
columns = _columns(frame)
|
|
162
|
+
block = DVBlock(
|
|
163
|
+
dv=frame.dv.key, layers=layers,
|
|
164
|
+
columns=[{k: str(v) for k, v in c.items()} for c in columns],
|
|
165
|
+
unit=frame.unit, notes=list(frame.notes), rows=frame.summary(),
|
|
166
|
+
)
|
|
167
|
+
for stratum, rows in frame.stratum_groups():
|
|
168
|
+
cells = []
|
|
169
|
+
for column in columns:
|
|
170
|
+
part = rows
|
|
171
|
+
for name, level in column.items():
|
|
172
|
+
part = part[part[name] == level]
|
|
173
|
+
cells.append(Cell({k: str(v) for k, v in column.items()}, engine.summarize(part[frame.y].to_numpy())))
|
|
174
|
+
n_units = int(rows[UNIT].nunique()) if frame.unit is not None else None
|
|
175
|
+
block.sections.append(Section({k: str(v) for k, v in stratum.items()}, cells, n_units))
|
|
176
|
+
return block
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def describe_table(stats_spec: StatsSpec, tables: dict[str, LongTable]) -> DescriptivesTable:
|
|
180
|
+
"""The Descriptives tab for every DV of ``stats_spec``.
|
|
181
|
+
|
|
182
|
+
``tables`` maps each DV's variable to its LongTable (``tables.load_tables``).
|
|
183
|
+
A DV whose rows cannot be built is a block with ``refused`` set. It is
|
|
184
|
+
reported, not raised, so one bad DV never blanks the table.
|
|
185
|
+
"""
|
|
186
|
+
unknown = [s for s in stats_spec.descriptives.statistics if s not in STATISTICS]
|
|
187
|
+
if unknown:
|
|
188
|
+
raise ValueError(f"unknown descriptive statistic(s) {unknown}; choose from {list(STATISTICS)}")
|
|
189
|
+
engine = get_engine()
|
|
190
|
+
blocks: list[DVBlock] = []
|
|
191
|
+
with Log.timer("describe", layer=LAYER, extra=f"{len(stats_spec.targets)} DV(s)") as timing:
|
|
192
|
+
for dv in stats_spec.targets:
|
|
193
|
+
with timing.phase(f"dv {dv.key}"):
|
|
194
|
+
try:
|
|
195
|
+
frame = analysis_frame(stats_spec, dv, tables[dv.variable])
|
|
196
|
+
except (AnalysisFrameError, KeyError) as exc:
|
|
197
|
+
reason = str(exc) if isinstance(exc, AnalysisFrameError) else f"no table loaded for {dv.variable}"
|
|
198
|
+
Log.warn("[describe] %s refused: %s", dv.key, reason, layer=LAYER)
|
|
199
|
+
blocks.append(DVBlock(dv=dv.key, layers=[], columns=[], unit=None, refused=reason))
|
|
200
|
+
continue
|
|
201
|
+
block = describe_frame(frame)
|
|
202
|
+
ns = [c.stats.get("n", 0) for s in block.sections for c in s.cells]
|
|
203
|
+
Log.info(
|
|
204
|
+
"[describe] %s: %d section(s) x %d column(s), n per cell %s..%s",
|
|
205
|
+
dv.key, len(block.sections), len(block.columns),
|
|
206
|
+
min(ns) if ns else 0, max(ns) if ns else 0, layer=LAYER,
|
|
207
|
+
)
|
|
208
|
+
blocks.append(block)
|
|
209
|
+
return DescriptivesTable(
|
|
210
|
+
blocks=blocks, cell=stats_spec.descriptives.cell,
|
|
211
|
+
statistics=list(stats_spec.descriptives.statistics), engine=engine.versions(),
|
|
212
|
+
)
|