scistackstats 0.1.41__tar.gz

scistackstats-0.1.41/.gitignore

@@ -0,0 +1,37 @@
+.venv
+.DS_Store
+*/*/__pycache__
+others_projects/sciforge
+scistack-gui/extension/node_modules/
+scistack-gui/frontend/node_modules/__pycache__/ 
+*.pyc 
+__pycache__/ 
+*.pyc
+*.pyo
+*.egg-info/
+# Python package build output (rebuilt fresh by CI on every publish; a
+# committed dist/ gets deleted by `rm -rf dist` mid-build and dirties the
+# tree for hatch-vcs). Excludes scistack-gui/extension/dist/, which is an
+# intentionally committed compiled bundle, not a Python build artifact.
+/dist/
+/*/dist/
+# Generated database artifacts (DuckDB data/lineage + write-ahead logs)
+*.duckdb
+*.duckdb.wal
+*.wal
+scistack-gui/frontend/node_modules/
+# Compiled output of the frontend's React-free unit tests (npm test in
+# scistack-gui/frontend). Regenerated by `tsc -p tsconfig.test.json`; unlike
+# extension/dist/, nothing loads it at runtime.
+scistack-gui/frontend/dist/
+# mkdocs build output (regenerated by `mkdocs build`)
+/site/
+# Runtime output of code_export_service (pipeline-to-code export) — timestamped
+# per-run files, not source.
+/exports/
+*.log
+output.txt
+scimatlab/tests/matlab/test-failures.txt
+
+# csv-stats is its own repo (github.com/mtillman14/csv-stats), cloned here for editable dev
+/csv-stats/

scistackstats-0.1.41/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.5
+Name: scistackstats
+Version: 0.1.41
+Summary: Statistics for scistack: StatStudio designs, descriptives and tests from Plot Studio roles (math by csv-stats)
+Author: SciStack Contributors
+License-Expression: MIT
+Keywords: hypothesis-testing,provenance,scidb,scistack,statistics
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: csv-stats[full]>=0.2
+Requires-Dist: numpy>=1.21
+Requires-Dist: pandas>=1.5
+Requires-Dist: scistacklog==0.1.41
+Requires-Dist: scistackplot==0.1.41
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# scistackstats
+
+Statistics from the same design a figure draws: StatStudio's engine,
+usable from code. Every number is computed by [csv-stats](https://github.com/mtillman14/csv-stats).
+
+See `docs/claude/statistics-design.md` for the design and the module docstrings for the API.

scistackstats-0.1.41/README.md

@@ -0,0 +1,6 @@
+# scistackstats
+
+Statistics from the same design a figure draws: StatStudio's engine,
+usable from code. Every number is computed by [csv-stats](https://github.com/mtillman14/csv-stats).
+
+See `docs/claude/statistics-design.md` for the design and the module docstrings for the API.

scistackstats-0.1.41/pyproject.toml

@@ -0,0 +1,72 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs", "hatch-dependency-coversion==0.0.1a4"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+source = "vcs"
+raw-options = { search_parent_directories = true, local_scheme = "no-local-version" }
+
+# Sibling SciStack packages are pinned to EXACTLY this build's version at build
+# time (lockstep releases), so [project] dependencies lists them unversioned. The
+# extra "dynamic" entry is what makes hatchling run the hook. Every sibling in
+# dependencies must be named here: CI's build job fails a wheel whose internal
+# dependency is not "==<its own version>".
+[tool.hatch.metadata.hooks.dependency-coversion]
+override-versions-of = ["scistackplot", "scistacklog"]
+
+[project]
+name = "scistackstats"
+dynamic = ["version", "dependency-coversioning"]
+description = "Statistics for scistack: StatStudio designs, descriptives and tests from Plot Studio roles (math by csv-stats)"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+    { name = "SciStack Contributors" }
+]
+keywords = [
+    "statistics",
+    "hypothesis-testing",
+    "scidb",
+    "scistack",
+    "provenance",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Typing :: Typed",
+]
+dependencies = [
+    "pandas>=1.5",
+    "numpy>=1.21",
+    # Every statistic is computed by csv-stats (S6); [full] has the tests.
+    "csv-stats[full]>=0.2",
+    "scistackplot",
+    "scistacklog",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/src",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scistackstats"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

scistackstats-0.1.41/src/scistackstats/__init__.py

@@ -0,0 +1,59 @@
+"""
+scistackstats — statistics from the same design a figure draws.
+
+StatStudio's engine, usable from code (the GUI is a thin wrapper). The design
+is a plot-layer ``PlotSpec`` (grouping layers = factors, separate figures and
+panels = strata, collapse chain = unit of analysis), so a table and a figure
+built from the same design summarise the same rows. Every number is computed
+by csv-stats (``csvstats``), through :mod:`scistackstats.engine`.
+
+::
+
+    from scistackplot import PlotSpec, Role, PlotKind
+    from scistackstats import seed_from_plot, load_tables, describe_table, run
+
+    plot = PlotSpec(measures=["Ratings"], roles={"brand": Role.GROUP, "subject": Role.COLLAPSE},
+                    groups=["brand"], kind=PlotKind.BAR)
+    spec = seed_from_plot(plot).spec
+    tables = load_tables(source, spec)          # any scistackplot DataSource
+    descriptives = describe_table(spec, tables)  # mean (SD) per brand, across subjects
+    report = run(spec, tables)                   # paired comparisons of the brands, Holm-adjusted
+    print(export_comparisons(report, spec, "md", tables=tables))  # txt md csv latex html json
+
+Design: ``docs/claude/statistics-design.md``; plan ``.claude/plan-statistics.md``.
+"""
+
+from __future__ import annotations
+
+from .codegen import NotExportable, generate_stats_function, why_not_exportable
+from .catalog import TESTS, available_tests, select_tests, why_unavailable
+from .describe import CELL_LAYOUTS, STATISTICS, DescriptivesTable, describe_table
+from .design import Design, Factor, Tiers, comparison_tiers, derive_design
+from .effects import EFFECT_SIZES, DifferenceSpec, default_effect_size, difference_spec
+from .engine import CsvStatsEngine, EngineRefusal, FakeEngine, get_engine, use_engine
+from .export import FORMATS, export_comparisons, export_descriptives, render
+from .format import Grid, Labels, apa_sentence, comparisons_grid, descriptives_grids, labels_for, omnibus_grid, sweep_grid
+from .run import Comparison, StatsReport, run
+from .seed import SeedError, Seeded, seed_from_plot
+from .spec import DV, Analysis, DescriptiveSpec, FormatSpec, StatsSpec, StatsSpecError
+from .tables import ANALYSIS_KIND, UNIT, AnalysisFrame, AnalysisFrameError, analysis_frame, load_tables
+
+__all__ = [
+    "ANALYSIS_KIND", "Analysis", "AnalysisFrame", "AnalysisFrameError", "CELL_LAYOUTS", "Comparison",
+    "CsvStatsEngine", "DV", "DescriptiveSpec", "DescriptivesTable", "Design", "DifferenceSpec",
+    "EFFECT_SIZES", "EngineRefusal", "Factor", "FakeEngine", "STATISTICS", "SeedError", "Seeded",
+    "StatsReport", "StatsSpec", "StatsSpecError", "TESTS", "Tiers", "UNIT", "analysis_frame",
+    "available_tests", "comparison_tiers", "default_effect_size", "derive_design", "describe_table",
+    "difference_spec", "get_engine", "load_tables", "run", "seed_from_plot", "select_tests",
+    "use_engine", "why_unavailable",
+    "FORMATS", "FormatSpec", "Grid", "Labels", "apa_sentence", "comparisons_grid", "descriptives_grids",
+    "export_comparisons", "export_descriptives", "labels_for", "omnibus_grid", "render", "sweep_grid",
+    "NotExportable", "generate_stats_function", "why_not_exportable",
+]
+
+try:
+    from importlib import metadata as _metadata
+
+    __version__ = _metadata.version("scistackstats")
+except Exception:  # pragma: no cover - running from a source tree
+    __version__ = "0.0.0"

scistackstats-0.1.41/src/scistackstats/catalog.py

@@ -0,0 +1,186 @@
+"""Which test runs where, and why not — the one refusal owner.
+
+:func:`why_unavailable` answers "can test T run on this design?" with ``None``
+or a reason. The GUI greys a menu entry with that reason, and the runner
+records it as a refusal. :func:`select_tests` picks the tests for an analysis
+from the design, and always says why (S12: never switch silently).
+
+This module knows the *statistical* requirements: a paired test needs a
+within factor and a unit, a two-sample test needs two levels, and so on.
+What csv-stats does not support *yet* is csv-stats' to say: those come back
+from the engine at run time as ``not yet in csv-stats: …`` refusals. Copying
+csv-stats' support matrix here would give it a second owner.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from .design import BETWEEN, MIXED, WITHIN, Design
+from .spec import Analysis
+
+
+@dataclass(frozen=True)
+class StatTest:
+    id: str
+    label: str
+    #: "pairwise" (two levels of one factor) or "omnibus" (every factor).
+    role: str
+    family: str
+    #: For pairwise tests: the factor kind it needs.
+    kind: str | None = None
+
+
+TESTS: dict[str, StatTest] = {
+    t.id: t
+    for t in (
+        StatTest("ttest_ind_welch", "Welch's t-test", "pairwise", "parametric", BETWEEN),
+        StatTest("ttest_ind_student", "Student's t-test", "pairwise", "parametric", BETWEEN),
+        StatTest("ttest_paired", "Paired t-test", "pairwise", "parametric", WITHIN),
+        StatTest("mann_whitney", "Mann–Whitney U", "pairwise", "nonparametric", BETWEEN),
+        StatTest("wilcoxon", "Wilcoxon signed-rank", "pairwise", "nonparametric", WITHIN),
+        StatTest("spm_ttest2", "SPM two-sample t (1-D curves)", "pairwise", "spm", BETWEEN),
+        StatTest("spm_ttest_paired", "SPM paired t (1-D curves)", "pairwise", "spm", WITHIN),
+        StatTest("anova", "ANOVA", "omnibus", "parametric"),
+        StatTest("kruskal", "Kruskal–Wallis", "omnibus", "nonparametric"),
+        StatTest("friedman", "Friedman", "omnibus", "nonparametric"),
+    )
+}
+
+
+def _design_problem(design: Design) -> str | None:
+    if not design.factors:
+        return "nothing is grouped: add a grouping layer to compare its levels"
+    if design.mixed:
+        names = ", ".join(design.mixed)
+        return f"{names}: some units at one level, some at several; set within or between first"
+    if design.within and design.unit is None:
+        return "a within-subject factor needs an experimental unit (collapse to one, without weight by N)"
+    return None
+
+
+def why_unavailable(test_id: str, design: Design) -> str | None:
+    """``None`` if ``test_id`` can run on ``design``, else the reason."""
+    info = TESTS.get(test_id)
+    if info is None:
+        return f"unknown test {test_id!r}"
+    problem = _design_problem(design)
+    if problem:
+        return problem
+    if info.role == "pairwise":
+        primary = design.factors[0]
+        if len(primary.levels) < 2:
+            return f"{primary.name} has fewer than 2 levels here"
+        if primary.kind != info.kind:
+            return f"{info.label} needs a {info.kind}-subject factor; {primary.name} is {primary.kind}"
+        return None
+    # Omnibus
+    if test_id == "kruskal":
+        if len(design.factors) != 1 or design.factors[0].kind != BETWEEN:
+            return "Kruskal–Wallis takes one between-subject factor"
+        if len(design.factors[0].levels) < 2:
+            return f"{design.factors[0].name} has fewer than 2 levels here"
+    if test_id == "friedman":
+        if len(design.factors) != 1 or design.factors[0].kind != WITHIN:
+            return "Friedman takes one within-subject factor"
+        if len(design.factors[0].levels) < 3:
+            return "Friedman needs 3+ levels; with 2, the Wilcoxon test is the test"
+    if test_id == "anova":
+        thin = [f.name for f in design.factors if len(f.levels) < 2]
+        if thin:
+            return f"{', '.join(thin)} has fewer than 2 levels here"
+    return None
+
+
+@dataclass(frozen=True)
+class Selection:
+    omnibus: str | None
+    omnibus_reason: str | None
+    pairwise: str | None
+    pairwise_reason: str | None
+    #: Why these tests (shown in the GUI and the report).
+    why: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"omnibus": self.omnibus, "omnibus_reason": self.omnibus_reason,
+                "pairwise": self.pairwise, "pairwise_reason": self.pairwise_reason, "why": self.why}
+
+
+def _pairwise_default(analysis: Analysis, design: Design) -> tuple[str | None, str]:
+    primary = design.factors[0] if design.factors else None
+    if primary is None:
+        return None, "no grouping layer"
+    within = primary.kind == WITHIN
+    if analysis.family == "spm":
+        test = "spm_ttest_paired" if within else "spm_ttest2"
+    elif analysis.family == "nonparametric":
+        test = "wilcoxon" if within else "mann_whitney"
+    elif within:
+        test = "ttest_paired"
+    else:
+        test = "ttest_ind_student" if analysis.equal_var else "ttest_ind_welch"
+    because = f"{primary.name} is {primary.kind}-subject" + (" (overridden)" if primary.overridden else "")
+    if test.startswith("ttest_ind"):
+        because += ", equal variances assumed" if analysis.equal_var else ", equal variances not assumed"
+    return test, because
+
+
+def _omnibus_default(analysis: Analysis, design: Design) -> tuple[str | None, str]:
+    if not design.factors:
+        return None, "no grouping layer"
+    if analysis.family == "spm":
+        return None, "SPM on curves runs the pairwise tests; an SPM ANOVA is not yet in csv-stats"
+    if analysis.family == "parametric":
+        return "anova", "parametric family"
+    if len(design.factors) == 1:
+        f = design.factors[0]
+        if f.kind == WITHIN:
+            return "friedman", f"{f.name} is within-subject"
+        return "kruskal", f"{f.name} is between-subject"
+    return None, "no rank-based test for a factorial design; the pairwise rank tests still run"
+
+
+def select_tests(analysis: Analysis, design: Design) -> Selection:
+    """The omnibus and pairwise tests for ``analysis`` on ``design``.
+
+    An override (``omnibus_test`` / ``pairwise_test``) wins, and is still
+    checked by :func:`why_unavailable`.
+    """
+    omnibus = omnibus_reason = pairwise = pairwise_reason = None
+    whys: list[str] = []
+    if analysis.omnibus:
+        if analysis.omnibus_test:
+            omnibus, because = analysis.omnibus_test, "chosen"
+            if TESTS.get(omnibus) and TESTS[omnibus].role != "omnibus":
+                omnibus_reason = f"{TESTS[omnibus].label} is not an omnibus test"
+        else:
+            omnibus, because = _omnibus_default(analysis, design)
+        if omnibus is None:
+            omnibus_reason = because
+        elif omnibus_reason is None:
+            omnibus_reason = why_unavailable(omnibus, design)
+        if omnibus:
+            whys.append(f"omnibus: {TESTS[omnibus].label if omnibus in TESTS else omnibus} ({because})")
+    if analysis.pairwise:
+        if analysis.pairwise_test:
+            pairwise, because = analysis.pairwise_test, "chosen"
+            if TESTS.get(pairwise) and TESTS[pairwise].role != "pairwise":
+                pairwise_reason = f"{TESTS[pairwise].label} is not a pairwise test"
+        else:
+            pairwise, because = _pairwise_default(analysis, design)
+        if pairwise is None:
+            pairwise_reason = because
+        elif pairwise_reason is None:
+            pairwise_reason = why_unavailable(pairwise, design)
+        if pairwise:
+            whys.append(f"pairwise: {TESTS[pairwise].label if pairwise in TESTS else pairwise} ({because})")
+    return Selection(omnibus, omnibus_reason, pairwise, pairwise_reason, "; ".join(whys) or "nothing selected")
+
+
+def available_tests(design: Design) -> list[dict[str, Any]]:
+    """Every test with its availability, for the GUI's "+ Add test" menu."""
+    return [
+        {"id": t.id, "label": t.label, "role": t.role, "family": t.family, "reason": why_unavailable(t.id, design)}
+        for t in TESTS.values()
+    ]

scistackstats-0.1.41/src/scistackstats/codegen.py

@@ -0,0 +1,148 @@
+"""Code export: a ``stat_`` function that reproduces a StatStudio table (Stage 11).
+
+The generated function **calls scistackstats** with the spec embedded as a
+literal. It does not restate the statistics: design derivation, test
+selection, correction families and every csv-stats call stay in their one
+owner, so the exported code cannot drift from what StatStudio showed. It
+receives the long-format table ``for_each`` hands a ``stat_`` function
+(schema keys as columns, ``as_table`` on), rebuilds the ``LongTable`` with
+the level order and schema the preview used, and returns the report dict
+(stored by scidb's ``stat_`` machinery: lineage, ``finalized``, ``scidb
+report``).
+
+First pass: a DV must be a single-column Variable whose factors are its
+schema keys. :func:`why_not_exportable` refuses, with the reason, for chosen
+``ColName`` columns, joined grouping variables, variant rows or variant
+axes, and an x variable. Rebuilding those tables from ``for_each`` inputs
+would be a second copy of the source's join and melt logic.
+"""
+
+from __future__ import annotations
+
+import keyword
+import pprint
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from scistacklog import Log
+
+from scistackplot import LongTable
+
+from .spec import StatsSpec
+
+LAYER = "scistackstats"
+
+
+class NotExportable(ValueError):
+    """This stats spec cannot be exported as code yet; the message says why."""
+
+
+def _ident(text: str) -> str:
+    name = re.sub(r"\W+", "_", str(text)).strip("_").lower() or "stats"
+    if name[0].isdigit() or keyword.iskeyword(name):
+        name = f"v_{name}"
+    return name
+
+
+def default_function_name(stats_spec: StatsSpec) -> str:
+    return f"stat_{_ident(stats_spec.targets[0].variable)}"
+
+
+def input_params(stats_spec: StatsSpec) -> dict[str, str]:
+    """``{parameter: variable}``: ``df`` for one variable, ``df_<name>`` for several."""
+    variables = list(dict.fromkeys(dv.variable for dv in stats_spec.targets))
+    if len(variables) == 1:
+        return {"df": variables[0]}
+    return {f"df_{_ident(v)}": v for v in variables}
+
+
+def why_not_exportable(stats_spec: StatsSpec, tables: dict[str, LongTable]) -> str | None:
+    design = stats_spec.design
+    if any(dv.columns is not None for dv in stats_spec.targets):
+        return "chosen columns of a multi-column variable are not exported as code yet"
+    if design.factor_variables:
+        return "joined grouping variables are not exported as code yet"
+    if design.x_measure:
+        return "an x variable is not exported as code yet"
+    if design.variant_sets:
+        return "variant rows are not exported as code yet"
+    for dv in stats_spec.targets:
+        table = tables.get(dv.variable)
+        if table is None:
+            return f"no table loaded for {dv.variable}"
+        if table.variant_factors:
+            names = [f.name for f in table.variant_factors]
+            return f"{dv.variable} has variant axes ({', '.join(names)}); they are not exported as code yet"
+        if table.field_factors:
+            return f"{dv.variable} is a multi-column variable; it is not exported as code yet"
+    return None
+
+
+@dataclass
+class StatsFunction:
+    name: str
+    source: str
+    #: ``{parameter: variable}`` the function takes.
+    inputs: dict[str, str] = field(default_factory=dict)
+
+
+def _table_literal(variable: str, param: str, table: LongTable) -> list[str]:
+    factors = list(table.factor_names)
+    levels = {f: [str(v) for v in table.factor(f).levels] for f in factors}
+    return [
+        f"    {variable!r}: LongTable.from_frame(",
+        f"        {param},",
+        f"        factors=[c for c in {factors!r} if c in {param}.columns],",
+        f"        measures=[{variable!r}],",
+        f"        name={variable!r},",
+        f"        schema_levels={list(table.schema_levels)!r},",
+        f"        level_order={levels!r},",
+        "    ),",
+    ]
+
+
+def generate_stats_function(stats_spec: StatsSpec, tables: dict[str, LongTable], *,
+                            function_name: str | None = None) -> StatsFunction:
+    """The ``stat_`` function source (module docstring). Raises
+    :class:`NotExportable` with the reason when the spec is not exportable."""
+    reason = why_not_exportable(stats_spec, tables)
+    if reason:
+        Log.info("[stats codegen] not exportable: %s", reason, layer=LAYER)
+        raise NotExportable(reason)
+    name = function_name or default_function_name(stats_spec)
+    if not name.isidentifier() or keyword.iskeyword(name):
+        raise NotExportable(f"{name!r} is not a valid function name")
+    params = input_params(stats_spec)
+    spec_literal = pprint.pformat(stats_spec.to_dict(), indent=1, width=96, sort_dicts=True)
+    dvs = ", ".join(dv.key for dv in stats_spec.targets)
+    analyses = ", ".join(f"{a.id} ({a.family})" for a in stats_spec.comparisons) or "descriptives only"
+    lines = [
+        f"def {name}({', '.join(params)}):",
+        f'    """StatStudio statistics for {dvs}: {analyses}.',
+        "",
+        "    Generated by scistackstats. The statistics are scistackstats' and",
+        "    csv-stats' own (this function calls them with the saved settings), so",
+        "    the result is what StatStudio showed for the same rows.",
+        '    """',
+        "    from scistackplot import LongTable",
+        "    from scistackstats import StatsSpec, describe_table, run",
+        "",
+        "    spec = StatsSpec.from_dict(",
+        *("        " + line for line in spec_literal.splitlines()),
+        "    )",
+        "    tables = {",
+    ]
+    for param, variable in params.items():
+        lines += ["    " + line for line in _table_literal(variable, param, tables[variable])]
+    lines += [
+        "    }",
+        "    report = run(spec, tables).to_dict()",
+        '    report["descriptives"] = describe_table(spec, tables).to_dict()',
+        "    return report",
+        "",
+    ]
+    source = "\n".join(lines)
+    Log.info("[stats codegen] %s: %d input(s), %d analysis(es), %d line(s)", name, len(params),
+             len(stats_spec.comparisons), len(lines), layer=LAYER)
+    return StatsFunction(name=name, source=source, inputs=params)