scistackstatsdb 0.1.41__tar.gz

scistackstatsdb-0.1.41/.gitignore

@@ -0,0 +1,37 @@
+.venv
+.DS_Store
+*/*/__pycache__
+others_projects/sciforge
+scistack-gui/extension/node_modules/
+scistack-gui/frontend/node_modules/__pycache__/ 
+*.pyc 
+__pycache__/ 
+*.pyc
+*.pyo
+*.egg-info/
+# Python package build output (rebuilt fresh by CI on every publish; a
+# committed dist/ gets deleted by `rm -rf dist` mid-build and dirties the
+# tree for hatch-vcs). Excludes scistack-gui/extension/dist/, which is an
+# intentionally committed compiled bundle, not a Python build artifact.
+/dist/
+/*/dist/
+# Generated database artifacts (DuckDB data/lineage + write-ahead logs)
+*.duckdb
+*.duckdb.wal
+*.wal
+scistack-gui/frontend/node_modules/
+# Compiled output of the frontend's React-free unit tests (npm test in
+# scistack-gui/frontend). Regenerated by `tsc -p tsconfig.test.json`; unlike
+# extension/dist/, nothing loads it at runtime.
+scistack-gui/frontend/dist/
+# mkdocs build output (regenerated by `mkdocs build`)
+/site/
+# Runtime output of code_export_service (pipeline-to-code export) — timestamped
+# per-run files, not source.
+/exports/
+*.log
+output.txt
+scimatlab/tests/matlab/test-failures.txt
+
+# csv-stats is its own repo (github.com/mtillman14/csv-stats), cloned here for editable dev
+/csv-stats/

scistackstatsdb-0.1.41/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.5
+Name: scistackstatsdb
+Version: 0.1.41
+Summary: scidb-backed statistics: saved StatStudio specs, linked plots and stat_ endpoints
+Author: SciStack Contributors
+License-Expression: MIT
+Keywords: provenance,scidb,scistack,statistics,visualization
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: scistack-db==0.1.41
+Requires-Dist: scistacklog==0.1.41
+Requires-Dist: scistackplot==0.1.41
+Requires-Dist: scistackplotdb==0.1.41
+Requires-Dist: scistackstats==0.1.41
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# scistackstatsdb
+
+scidb-backed statistics: loads StatStudio tables through `scistackplotdb`,
+and (later stages) saved stats specs, linked plots and `stat_` endpoints.

scistackstatsdb-0.1.41/README.md

@@ -0,0 +1,4 @@
+# scistackstatsdb
+
+scidb-backed statistics: loads StatStudio tables through `scistackplotdb`,
+and (later stages) saved stats specs, linked plots and `stat_` endpoints.

scistackstatsdb-0.1.41/pyproject.toml

@@ -0,0 +1,71 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs", "hatch-dependency-coversion==0.0.1a4"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+source = "vcs"
+raw-options = { search_parent_directories = true, local_scheme = "no-local-version" }
+
+# Sibling SciStack packages are pinned to EXACTLY this build's version at build
+# time (lockstep releases), so [project] dependencies lists them unversioned. The
+# extra "dynamic" entry is what makes hatchling run the hook. Every sibling in
+# dependencies must be named here: CI's build job fails a wheel whose internal
+# dependency is not "==<its own version>".
+[tool.hatch.metadata.hooks.dependency-coversion]
+override-versions-of = ["scistackstats", "scistackplot", "scistackplotdb", "scistack-db", "scistacklog"]
+
+[project]
+name = "scistackstatsdb"
+dynamic = ["version", "dependency-coversioning"]
+description = "scidb-backed statistics: saved StatStudio specs, linked plots and stat_ endpoints"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+    { name = "SciStack Contributors" }
+]
+keywords = [
+    "statistics",
+    "visualization",
+    "scidb",
+    "scistack",
+    "provenance",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Typing :: Typed",
+]
+dependencies = [
+    "scistackstats",
+    "scistackplot",
+    "scistackplotdb",
+    "scistack-db",
+    "scistacklog",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/src",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scistackstatsdb"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

scistackstatsdb-0.1.41/src/scistackstatsdb/__init__.py

@@ -0,0 +1,53 @@
+"""
+scistackstatsdb — scidb-backed statistics.
+
+Loads StatStudio tables from a scidb database through scistackplotdb's
+``ScidbSource`` (the same loading the figures use), saves stats tables and
+stats presets (``saved``), finds the stats tables linked to a saved plot, and
+carries them in project bundles (``bundle_section``). The statistics
+themselves are :mod:`scistackstats`.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from scistackstats import StatsSpec, load_tables
+
+from .endpoint import StatsEndpointCode, generate_stats_endpoint, required_declarations
+from .saved import (
+    SavedStats,
+    SavedStatsError,
+    SavedStatsExists,
+    SavedStatsInfo,
+    StatsPresetInfo,
+    apply_stats_preset,
+    find_saved_stats,
+    hide_saved_stats,
+    hide_stats_preset,
+    linked_stats_for,
+    list_saved_stats,
+    list_stats_presets,
+    load_saved_stats,
+    rename_saved_stats,
+    rename_stats_preset,
+    save_stats,
+    save_stats_preset,
+    saved_stats_history,
+)
+
+
+def load_stats_tables(db: Any, stats_spec: StatsSpec) -> dict:
+    """``{variable: LongTable}`` for every DV of ``stats_spec``, from ``db``."""
+    from scistackplotdb import ScidbSource
+
+    return load_tables(ScidbSource(db), stats_spec)
+
+
+__all__ = [
+    "StatsEndpointCode", "generate_stats_endpoint", "required_declarations",
+    "SavedStats", "SavedStatsError", "SavedStatsExists", "SavedStatsInfo", "StatsPresetInfo",
+    "apply_stats_preset", "find_saved_stats", "hide_saved_stats", "hide_stats_preset", "linked_stats_for",
+    "list_saved_stats", "list_stats_presets", "load_saved_stats", "load_stats_tables", "rename_saved_stats",
+    "rename_stats_preset", "save_stats", "save_stats_preset", "saved_stats_history",
+]

scistackstatsdb-0.1.41/src/scistackstatsdb/bundle_section.py

@@ -0,0 +1,42 @@
+"""The ``stats`` section of a project bundle: saved stats tables and stats
+presets, copied row for row (``VersionedStore.dump_rows`` / ``load_rows``),
+like scistackplotdb's ``plots`` section. A schema-key rename is applied with
+the same exact-string remap the plots section uses."""
+
+from __future__ import annotations
+
+import json
+
+from scistacklog import Log
+from scistackplotdb.bundle_section import _remap_row
+
+from .saved import PRESET_STORE, STORE
+
+LAYER = "scistackstatsdb"
+
+_STORES = {"saved_stats.json": STORE, "stats_presets.json": PRESET_STORE}
+
+
+class StatsSection:
+    name = "stats"
+
+    def export(self, ctx) -> "dict[str, bytes]":
+        files: dict[str, bytes] = {}
+        for filename, store in _STORES.items():
+            rows = store.dump_rows(ctx.db)
+            if rows:
+                files[filename] = json.dumps(rows, indent=2, default=str).encode("utf-8")
+        Log.info("[bundle_section] stats export: %s", sorted(files), layer=LAYER)
+        return files
+
+    def import_(self, ctx, files: "dict[str, bytes]") -> dict:
+        report = {}
+        key_map = getattr(ctx, "key_map", None)
+        for filename, store in _STORES.items():
+            if filename in files:
+                rows = json.loads(files[filename].decode("utf-8"))
+                if key_map is not None and not key_map.is_identity:
+                    rows = [_remap_row(row, store, key_map, ctx.map_report) for row in rows]
+                report[store.table] = store.load_rows(ctx.db, rows)
+        Log.info("[bundle_section] stats import: %s", report, layer=LAYER)
+        return report

scistackstatsdb-0.1.41/src/scistackstatsdb/endpoint.py

@@ -0,0 +1,75 @@
+"""StatStudio → pipeline: the ``stat_`` function plus its ``for_each`` call.
+
+The function is ``scistackstats.codegen.generate_stats_function``. This
+module writes the call around it:
+
+- **no iteration keywords** (P7): one call over the whole fan-out, so a
+  correction family that spans strata or DVs is computed in one place. scidb
+  stores one record per call (a grand aggregation);
+- one input per DV variable, loaded by name;
+- ``finalized`` (record mode) by default, so the result is a lineage record
+  that goes stale when its inputs change and that ``scidb report`` lists.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from scistacklog import Log
+from scistackplot import LongTable
+
+from scistackstats import StatsSpec
+from scistackstats.codegen import generate_stats_function
+
+LAYER = "scistackstatsdb"
+
+
+@dataclass
+class StatsEndpointCode:
+    function_name: str
+    function_source: str
+    foreach_source: str
+    output_variable: str
+    inputs: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def source(self) -> str:
+        return f"{self.function_source}\n\n{self.foreach_source}"
+
+
+def default_output_variable(stats_spec: StatsSpec) -> str:
+    """``Ratings`` -> ``RatingsStats``."""
+    return f"{stats_spec.targets[0].variable}Stats"
+
+
+def generate_stats_endpoint(
+    stats_spec: StatsSpec,
+    tables: dict[str, LongTable],
+    *,
+    function_name: str | None = None,
+    output_variable: str | None = None,
+    finalized: bool = True,
+) -> StatsEndpointCode:
+    """The ``stat_`` function and its ``for_each`` call. Raises
+    ``scistackstats.codegen.NotExportable`` with the reason when the spec
+    cannot be exported yet."""
+    fn = generate_stats_function(stats_spec, tables, function_name=function_name)
+    output = output_variable or default_output_variable(stats_spec)
+    lines = ["for_each(", f"    {fn.name},", "    inputs={"]
+    lines += [f'        "{param}": {variable},' for param, variable in fn.inputs.items()]
+    lines += [
+        "    },",
+        f"    outputs=[{output}],",
+        f"    as_table={list(fn.inputs)!r},",
+        f"    finalized={finalized},",
+        ")",
+    ]
+    Log.info("[stats endpoint] %s: inputs %s -> %s, finalized=%s (no iteration: one record)",
+             fn.name, fn.inputs, output, finalized, layer=LAYER)
+    return StatsEndpointCode(function_name=fn.name, function_source=fn.source,
+                             foreach_source="\n".join(lines), output_variable=output, inputs=fn.inputs)
+
+
+def required_declarations(code: StatsEndpointCode) -> list[str]:
+    """Variable types the call needs that may not exist yet (the output)."""
+    return [code.output_variable]

scistackstatsdb-0.1.41/src/scistackstatsdb/saved.py

@@ -0,0 +1,236 @@
+"""Saved StatStudio tables and stats presets, kept in the project database.
+
+Both use scistackplotdb's :class:`~scistackplotdb._versioned.VersionedStore`,
+the owner of the saved-plot rules (S20). So a saved stats table behaves like
+a saved plot:
+
+- the id is the identity, the name a label, unique across the project;
+- re-saving a visible name appends a version, and the newest version opens;
+- "remove" hides every version, and nothing is ever deleted;
+- save is strict (``StatsSpec.from_dict``), open salvages
+  (``scistackstats.restore``), and opening never rewrites the stored copy.
+
+A saved stats table may name the saved plot it syncs with
+(``StatsSpec.linked_plot``, S11). The link is stored only here.
+:func:`linked_stats_for` finds a plot's stats tables by query, so the plot
+store never holds a back-reference.
+
+A **stats preset** stores only the stats settings
+(``scistackstats.restore.STATS_SETTINGS``): what to compute and how it reads.
+Applying one to a stats table keeps that table's design, DVs and link.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from scistacklog import Log
+from scistackplotdb._versioned import VersionedStore
+
+from scistackstats import StatsSpec, StatsSpecError
+from scistackstats.restore import restore_stats_spec, stats_settings, with_stats_settings
+
+LAYER = "scistackstatsdb"
+ENVELOPE_FORMAT = 1
+
+
+class SavedStatsError(ValueError):
+    """A save, rename or load that cannot be done as asked."""
+
+
+class SavedStatsExists(SavedStatsError):
+    """``save_stats(overwrite=False)`` found another visible item with that name."""
+
+    def __init__(self, existing: "SavedStatsInfo"):
+        super().__init__(f"There is already a saved stats table named {existing.name!r} (version {existing.version}).")
+        self.existing = existing
+
+
+STORE = VersionedStore(
+    table="_stats_spec", id_column="stats_id", scope_column=None,
+    noun="saved stats table", tag="saved-stats", error=SavedStatsError,
+)
+PRESET_STORE = VersionedStore(
+    table="_stats_preset", id_column="preset_id", scope_column=None,
+    noun="stats preset", tag="stats-preset", error=SavedStatsError,
+)
+
+
+@dataclass(frozen=True)
+class SavedStatsInfo:
+    stats_id: str
+    name: str
+    version: int
+    saved_at: str
+    hidden: bool = False
+    linked_plot: str | None = None
+    dvs: tuple[str, ...] = ()
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"stats_id": self.stats_id, "name": self.name, "version": self.version, "saved_at": self.saved_at,
+                "hidden": self.hidden, "linked_plot": self.linked_plot, "dvs": list(self.dvs)}
+
+
+@dataclass
+class SavedStats:
+    info: SavedStatsInfo
+    spec: StatsSpec
+    #: Settings that did not come back as saved (``RestoreNote`` dicts).
+    notes: list[dict[str, Any]] = field(default_factory=list)
+    stored_format: Any = None
+
+
+def _versions() -> dict[str, str]:
+    import scistackstats
+    from scistackstats import get_engine
+
+    out = {"scistackstats": getattr(scistackstats, "__version__", "dev")}
+    try:
+        out.update(get_engine().versions())
+    except Exception:  # pragma: no cover - versions are display only
+        pass
+    return out
+
+
+def _strict(spec: Any) -> StatsSpec:
+    if isinstance(spec, StatsSpec):
+        return spec
+    try:
+        return StatsSpec.from_dict(spec)
+    except (StatsSpecError, KeyError, TypeError, ValueError) as exc:
+        raise SavedStatsError(f"The stats settings could not be read: {exc}") from exc
+
+
+def _info(row, envelope: dict | None = None) -> SavedStatsInfo:
+    stats_id, _scope, name, version, saved_at, hidden = row
+    envelope = envelope or {}
+    linked = envelope.get("linked_plot")
+    dvs = envelope.get("dvs")
+    return SavedStatsInfo(
+        stats_id=str(stats_id), name=str(name), version=int(version), saved_at=str(saved_at), hidden=bool(hidden),
+        linked_plot=linked if isinstance(linked, str) else None,
+        dvs=tuple(str(d) for d in dvs) if isinstance(dvs, list) else (),
+    )
+
+
+def _newest(db, stats_id: str) -> SavedStatsInfo:
+    row, text = STORE.load(db, stats_id)
+    return _info(row, STORE.parse_envelope(text, f"{row[2]!r} version {row[3]}"))
+
+
+def save_stats(db, name: str, spec: Any, *, overwrite: bool = True, current_stats_id: str | None = None) -> SavedStatsInfo:
+    """Save a stats spec (or its dict, read strictly) as *name*."""
+    name = STORE.check_name(name)
+    spec = _strict(spec)
+    envelope = {
+        "format": ENVELOPE_FORMAT,
+        "spec": spec.to_dict(),
+        "linked_plot": spec.linked_plot,
+        "dvs": [d.key for d in spec.targets],
+        "saved_with": _versions(),
+    }
+    row, _new = STORE.save(db, None, name, envelope, overwrite=overwrite, current_id=current_stats_id,
+                           on_clash=lambda clash: SavedStatsExists(_info(clash)))
+    return _info(row, envelope)
+
+
+def load_saved_stats(db, stats_id: str, *, version: int | None = None) -> SavedStats:
+    """One version (the newest by default), salvaged. Never rewrites it."""
+    row, text = STORE.load(db, stats_id, version)
+    envelope = STORE.parse_envelope(text, f"{row[2]!r} version {row[3]}")
+    info = _info(row, envelope)
+    restored = restore_stats_spec(envelope.get("spec"), fallback_measure=(info.dvs[0].split("[")[0] if info.dvs else None))
+    Log.info("[saved-stats] opened %r version %d (envelope format %r): %d note(s)",
+             info.name, info.version, envelope.get("format"), len(restored.notes), layer=LAYER)
+    return SavedStats(info=info, spec=restored.spec, notes=[n.to_dict() for n in restored.notes],
+                      stored_format=envelope.get("format"))
+
+
+def list_saved_stats(db, *, include_hidden: bool = False) -> list[SavedStatsInfo]:
+    return [_info(row[:6], STORE.parse_envelope(row[6], repr(row[2])))
+            for row in STORE.list_rows(db, include_hidden=include_hidden, with_envelope=True)]
+
+
+def saved_stats_history(db, stats_id: str) -> list[SavedStatsInfo]:
+    return [_info(row[:6], STORE.parse_envelope(row[6], repr(row[2])))
+            for row in STORE.history(db, stats_id, with_envelope=True)]
+
+
+def rename_saved_stats(db, stats_id: str, new_name: str) -> SavedStatsInfo:
+    STORE.rename(db, stats_id, new_name)
+    return _newest(db, stats_id)
+
+
+def hide_saved_stats(db, stats_id: str, hidden: bool = True) -> SavedStatsInfo:
+    STORE.set_hidden(db, stats_id, hidden)
+    return _newest(db, stats_id)
+
+
+def linked_stats_for(db, saved_plot_id: str) -> list[SavedStatsInfo]:
+    """The visible saved stats tables linked to one saved plot (S11)."""
+    found = [info for info in list_saved_stats(db) if info.linked_plot == saved_plot_id]
+    Log.debug("[saved-stats] plot %s: %d linked stats table(s)", saved_plot_id, len(found), layer=LAYER)
+    return found
+
+
+def find_saved_stats(db, name: str) -> SavedStatsInfo | None:
+    row = STORE.visible_by_name(db, None, STORE.check_name(name))
+    return _newest(db, row[0]) if row else None
+
+
+# ---------------------------------------------------------------------------
+# Stats presets
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class StatsPresetInfo:
+    preset_id: str
+    name: str
+    version: int
+    saved_at: str
+    hidden: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        return dict(self.__dict__)
+
+
+def _preset_info(row) -> StatsPresetInfo:
+    preset_id, _scope, name, version, saved_at, hidden = row[:6]
+    return StatsPresetInfo(str(preset_id), str(name), int(version), str(saved_at), bool(hidden))
+
+
+def save_stats_preset(db, name: str, spec: Any, *, overwrite: bool = True,
+                      current_preset_id: str | None = None) -> StatsPresetInfo:
+    """Save *spec*'s stats settings (no design, DVs or link) as a preset."""
+    name = PRESET_STORE.check_name(name)
+    spec = _strict(spec)
+    envelope = {"format": ENVELOPE_FORMAT, "settings": stats_settings(spec), "saved_with": _versions()}
+    row, _ = PRESET_STORE.save(db, None, name, envelope, overwrite=overwrite, current_id=current_preset_id,
+                               on_clash=lambda clash: SavedStatsError(PRESET_STORE.clash_message(None, name)))
+    return _preset_info(row)
+
+
+def list_stats_presets(db, *, include_hidden: bool = False) -> list[StatsPresetInfo]:
+    return [_preset_info(row) for row in PRESET_STORE.list_rows(db, include_hidden=include_hidden)]
+
+
+def hide_stats_preset(db, preset_id: str, hidden: bool = True) -> StatsPresetInfo:
+    PRESET_STORE.set_hidden(db, preset_id, hidden)
+    return _preset_info(PRESET_STORE.latest(db, preset_id))
+
+
+def rename_stats_preset(db, preset_id: str, new_name: str) -> StatsPresetInfo:
+    PRESET_STORE.rename(db, preset_id, new_name)
+    return _preset_info(PRESET_STORE.latest(db, preset_id))
+
+
+def apply_stats_preset(db, preset_id: str, target: StatsSpec, *, version: int | None = None) -> tuple[StatsSpec, list[dict]]:
+    """``target`` with the preset's stats settings; its design, DVs and link kept."""
+    row, text = PRESET_STORE.load(db, preset_id, version)
+    envelope = PRESET_STORE.parse_envelope(text, f"{row[2]!r} version {row[3]}")
+    spec, notes = with_stats_settings(target, envelope.get("settings") or {})
+    Log.info("[stats-preset] %r version %d applied to %s: %d note(s)", row[2], row[3],
+             ", ".join(d.key for d in target.targets), len(notes), layer=LAYER)
+    return spec, [n.to_dict() for n in notes]