scistackplotdb 0.1.26__py3-none-any.whl

scistackplotdb/__init__.py

@@ -0,0 +1,80 @@
+"""
+scistackplotdb — plot scidb variables.
+
+Loads variables into the long format ``scistackplot`` consumes, adds the four
+things a flat table never needed (shape classification, schema-depth joins,
+variants as factors, and a transport budget), and generates pipeline endpoints
+from a finished spec.
+
+::
+
+    from scidb import configure_database
+    from scistackplot import PlotSpec, Role, PlotKind, render
+    from scistackplotdb import ScidbSource
+
+    db = configure_database("experiment.duckdb", ["subject", "session", "trial"])
+    source = ScidbSource(db)
+
+    table = source.get_table(["StepLength"])
+    spec = PlotSpec(
+        measures=["StepLength"],
+        roles={"session": Role.X, "subject": Role.FREE, "trial": Role.FREE},
+        kind=PlotKind.BOX,
+    )
+    figure = render(table, spec)
+
+Everything about recording a figure — ``finalized``, artifact stamping,
+``scidb report`` — belongs to scidb's existing ``plot_`` endpoint machinery and
+is unchanged. See ``docs/claude/plotting-library-design.md``.
+"""
+
+from __future__ import annotations
+
+from .endpoint import (
+    EndpointCode,
+    default_output_variable,
+    default_path_template,
+    generate_endpoint,
+)
+from .hierarchy import join_frames, join_kind, joinable, joined_levels
+from .load import (
+    LATEST_COLUMN,
+    MISSING_VERSION_LEVEL,
+    VERSION_FACTOR_PREFIX,
+    VariableFrame,
+    attach_variants,
+    data_columns_for,
+    load_variable,
+    registered_variables,
+    sample_value,
+    schema_keys,
+)
+from .source import ScidbSource
+from .variants import selection_for, variant_graph, variant_set
+
+__all__ = [
+    "ScidbSource",
+    "variant_set",
+    "variant_graph",
+    "selection_for",
+    "VariableFrame",
+    "VERSION_FACTOR_PREFIX",
+    "MISSING_VERSION_LEVEL",
+    "LATEST_COLUMN",
+    "load_variable",
+    "attach_variants",
+    "registered_variables",
+    "schema_keys",
+    "data_columns_for",
+    "sample_value",
+    "join_kind",
+    "joinable",
+    "join_frames",
+    "joined_levels",
+    "generate_endpoint",
+    "EndpointCode",
+    "default_output_variable",
+    "default_path_template",
+]
+
+__version__ = "0.1.0"

scistackplotdb/endpoint.py

@@ -0,0 +1,257 @@
+"""
+Turning a ``PlotSpec`` into a pipeline endpoint.
+
+The GUI's "Add to pipeline" produces two things: a ``plot_`` function (generated
+by ``scistackplot.codegen`` — literal seaborn, no runtime dependency on this
+package) and the ``for_each`` call that runs it. This module owns the second
+half, and with it the one translation that has to be exactly right:
+
+    Role.ITERATE  ->  a for_each iteration keyword
+
+Interactively, ITERATE fans out through a pandas ``groupby`` inside
+``resolve()``. In the pipeline it fans out through ``for_each`` + ``PathOutput``.
+If those two ever disagree, the exported pipeline is not what the user
+previewed — the worst failure mode this layer has, and what
+``tests/test_fanout_parity.py`` exists to prevent.
+
+Everything else the endpoint needs already exists in scidb: ``finalized``,
+artifact stamping, ``skip_computed`` and ``scidb report`` are untouched.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from scistacklog import Log
+from scistackplot import LongTable, PlotSpec, default_function_name
+from scistackplot import generate_plot_function
+from scistackplot.codegen import group_param, variant_params
+from scistackplot.roles import fanout_keys
+from scistackplot.variants import defined_sets
+
+from .load import LATEST_COLUMN
+
+LAYER = "scistackplotdb"
+
+
+@dataclass
+class EndpointCode:
+    """Generated source for one plotting endpoint."""
+
+    function_name: str
+    function_source: str
+    foreach_source: str
+    iterate_keys: list[str] = field(default_factory=list)
+    path_template: str = ""
+    output_variable: str = ""
+
+    @property
+    def source(self) -> str:
+        """Function and call together, ready to append to a pipeline module."""
+        return f"{self.function_source}\n\n{self.foreach_source}"
+
+
+def generate_endpoint(
+    spec: PlotSpec,
+    table: LongTable,
+    *,
+    input_variable: str,
+    function_name: str | None = None,
+    output_variable: str | None = None,
+    path_template: str | None = None,
+    finalized: bool = True,
+    x_variable: str | None = None,
+) -> EndpointCode:
+    """
+    Generate the ``plot_`` function and its ``for_each`` call.
+
+    ``input_variable`` is the scidb variable type supplying the data;
+    ``x_variable`` is the optional second measure for an x–y scatter.
+    """
+    name = function_name or default_function_name(spec)
+    # NOT spec.iterate_factors: the fan-out includes schema keys promoted
+    # because a nested key iterates, and runs in schema order. The preview and
+    # the generated for_each have to agree on both — that is what
+    # tests/test_fanout_parity.py checks.
+    iterate_keys = fanout_keys(spec, table)
+    output = output_variable or default_output_variable(input_variable)
+    template = path_template or default_path_template(name, iterate_keys)
+
+    function_source = generate_plot_function(spec, table, function_name=name)
+    foreach_source = _foreach_call(
+        spec=spec,
+        function_name=name,
+        input_variable=input_variable,
+        x_variable=x_variable,
+        output_variable=output,
+        path_template=template,
+        iterate_keys=iterate_keys,
+        finalized=finalized,
+    )
+
+    Log.info(
+        "generated endpoint %s: input=%s output=%s iterate=%s finalized=%s",
+        name,
+        input_variable,
+        output,
+        iterate_keys or "none",
+        finalized,
+        layer=LAYER,
+    )
+    return EndpointCode(
+        function_name=name,
+        function_source=function_source,
+        foreach_source=foreach_source,
+        iterate_keys=iterate_keys,
+        path_template=template,
+        output_variable=output,
+    )
+
+
+def default_output_variable(input_variable: str) -> str:
+    """``StepLength`` -> ``StepLengthFigure``."""
+    return f"{input_variable}Figure"
+
+
+def default_path_template(function_name: str, iterate_keys: list[str]) -> str:
+    """
+    Build a PathOutput template that cannot collide.
+
+    Every ITERATE key goes into the filename. Omitting one would make two
+    figures write the same file; for schema keys scidb treats that as
+    pre-existing overwrite behavior (no error), and for variants its collision
+    guard raises before anything renders. Including them all avoids both.
+    """
+    slug = re.sub(r"^plot_", "", function_name)
+    parts = "".join(f"_{{{key}}}" for key in iterate_keys)
+    return f"plots/{slug}{parts}.png"
+
+
+def variant_expression(input_variable: str, variant_set) -> str:
+    """A ``Variant(...)`` call selecting one named variant's records.
+
+    The inverse of :func:`~scistackplotdb.variants.selection_for`, and the thing
+    that makes a variant figure reproducible by hand: what the popup's
+    checkboxes and version dropdowns produced comes back out as the same
+    wrapper a scientist would have typed.
+
+    Selections spanning two producing functions **nest** rather than resorting
+    to dotted-string kwargs::
+
+        Variant(Variant(EMG, fn="loadEMG", code_version="v1"), fn="bandpass", low_hz=20)
+
+    Nesting is documented, merges the two filters (``scidb.variant.Variant``
+    handles it explicitly), and keeps every keyword readable — where
+    ``**{"__code__.loadEMG": "v1"}`` would leak a reserved namespace into code a
+    user is meant to edit.
+    """
+    from scistackplot import CODE_FACTOR_PREFIX, LATEST
+
+    by_function: dict[str, dict[str, object]] = {}
+    for column, value in (variant_set.selection or {}).items():
+        if column.startswith(CODE_FACTOR_PREFIX):
+            fn_name = column[len(CODE_FACTOR_PREFIX) :]
+            by_function.setdefault(fn_name, {})["code_version"] = value
+        elif column == LATEST_COLUMN:
+            # The source's "these are the current records" recommendation. scidb
+            # spells the same thing `code_version="latest"`, resolved per schema
+            # location by the same rule — not "the highest ordinal".
+            by_function.setdefault(None, {})["code_version"] = LATEST
+        else:
+            fn_name, _, param = column.rpartition(".")
+            by_function.setdefault(fn_name or None, {})[param] = value
+
+    expression = input_variable
+    for fn_name in sorted(by_function, key=lambda n: (n is None, n or "")):
+        arguments = []
+        if fn_name:
+            arguments.append(f"fn={fn_name!r}")
+        arguments.extend(f"{key}={value!r}" for key, value in by_function[fn_name].items())
+        expression = f"Variant({expression}, {', '.join(arguments)})"
+    return expression
+
+
+def _single_variant_expression(input_variable: str, spec) -> str:
+    """The lone variant's pin, or the bare variable when nothing is selected.
+
+    The row may name its own variable, in which case that is what the endpoint
+    loads — ``input_variable`` is only the default.
+    """
+    sets = defined_sets(spec.variant_sets)
+    if len(sets) != 1:
+        return input_variable
+    return variant_expression(sets[0].variable or input_variable, sets[0])
+
+
+def _foreach_call(
+    *,
+    spec: PlotSpec,
+    function_name: str,
+    input_variable: str,
+    x_variable: str | None,
+    output_variable: str,
+    path_template: str,
+    iterate_keys: list[str],
+    finalized: bool,
+) -> str:
+    variant_inputs = variant_params(spec)
+    if variant_inputs:
+        # One input per named variant, each loaded through its own pin. See
+        # `codegen.variant_params` for why this cannot be a single `df`.
+        # Zipped against `defined_sets`, not `spec.variant_sets`: unfilled rows
+        # produce no input, so indexing the raw list would pair a parameter with
+        # the wrong variant's selection.
+        inputs = [
+            f'        "{generated.param}": '
+            f"{variant_expression(generated.variable, variant)},"
+            for generated, variant in zip(
+                variant_inputs, defined_sets(spec.variant_sets), strict=True
+            )
+        ]
+        table_inputs = [generated.param for generated in variant_inputs]
+    else:
+        pinned = _single_variant_expression(input_variable, spec)
+        inputs = [f'        "df": {pinned},']
+        table_inputs = ["df"]
+    if x_variable:
+        inputs.append(f'        "df_x": {x_variable},')
+        table_inputs.append("df_x")
+    for group in spec.factor_variables:
+        # A grouping variable arrives as its own input and is merged onto the
+        # data inside the function: `as_table` hands a function schema keys and
+        # data columns only, so a subject-level Condition cannot ride along on
+        # the measure's frame.
+        inputs.append(f'        "{group_param(group)}": {group},')
+        table_inputs.append(group_param(group))
+    inputs.append(f'        "filename": PathOutput("{path_template}"),')
+
+    lines = [
+        "for_each(",
+        f"    {function_name},",
+        "    inputs={",
+        *inputs,
+        "    },",
+        f"    outputs=[{output_variable}],",
+        # A plot_ function receives the long-format table (schema keys as
+        # columns) — as_table defaults ON only for stat_, so say it explicitly.
+        f"    as_table={table_inputs!r},",
+        f"    finalized={finalized},",
+    ]
+    for key in iterate_keys:
+        # [] means "every value present" — the same all-values resolution
+        # scifor applies to an empty iteration list.
+        lines.append(f"    {key}=[],")
+    lines.append(")")
+    return "\n".join(lines)
+
+
+def required_declarations(code: EndpointCode) -> list[str]:
+    """
+    Variable types the generated call needs that may not exist yet.
+
+    The GUI declares these through the normal entity-declaration path before
+    writing the code, so a generated endpoint never references an undeclared
+    type.
+    """
+    return [code.output_variable]

scistackplotdb/hierarchy.py

@@ -0,0 +1,138 @@
+"""
+Schema-depth joins.
+
+Two variables can share a plot only if their schema levels can be joined.
+Because the dataset schema is an ordered, contiguous hierarchy (see
+``docs/claude/schema-hierarchy-contiguity.md``), a variable's levels are always
+a *prefix* of the schema key list — so the join rule is simple and total:
+
+* identical levels  → a straight merge on all of them;
+* one a prefix of the other → **broadcast**: the shallower variable's value is
+  repeated across every deeper row beneath it (subject-level Mass against
+  trial-level Speed);
+* otherwise → refuse, and say why.
+
+Answering this question is also what lets the GUI populate its measure list
+honestly instead of offering combinations that cannot be built.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+import pandas as pd
+from scistacklog import Log
+
+from .load import LATEST_COLUMN, VariableFrame
+
+LAYER = "scistackplotdb"
+
+JoinKind = Literal["identical", "broadcast", "incompatible"]
+
+#: Private carriers for each side's latest-flag across a merge. Never leave
+#: :func:`join_frames` — they are combined into one :data:`LATEST_COLUMN` and
+#: dropped, so nothing downstream has to know a join happened.
+_LEFT_LATEST = "__latest_left"
+_RIGHT_LATEST = "__latest_right"
+
+
+def join_kind(levels_a: list[str], levels_b: list[str]) -> JoinKind:
+    if levels_a == levels_b:
+        return "identical"
+    shallow, deep = (levels_a, levels_b) if len(levels_a) < len(levels_b) else (levels_b, levels_a)
+    if deep[: len(shallow)] == shallow:
+        return "broadcast"
+    return "incompatible"
+
+
+def joinable(levels_a: list[str], levels_b: list[str]) -> bool:
+    return join_kind(levels_a, levels_b) != "incompatible"
+
+
+def join_frames(
+    left: VariableFrame,
+    right: VariableFrame,
+    *,
+    left_value: str,
+    right_value: str,
+) -> pd.DataFrame:
+    """
+    Join two variables into one long frame carrying both measures.
+
+    The merge is on the *shallower* variable's levels, which is exactly what
+    broadcasting means: one subject-level row is reused for each of that
+    subject's trials.
+    """
+    kind = join_kind(left.levels, right.levels)
+    if kind == "incompatible":
+        raise ValueError(
+            f"{left.name} (levels {left.levels}) and {right.name} "
+            f"(levels {right.levels}) cannot share a plot: neither variable's "
+            f"schema levels are a prefix of the other's, so there is no "
+            f"unambiguous way to line their records up."
+        )
+
+    shallow_levels = left.levels if len(left.levels) <= len(right.levels) else right.levels
+    on = list(shallow_levels)
+
+    left_frame = left.frame[[*left.levels, left_value, *left.variant_columns]].copy()
+    right_frame = right.frame[[*right.levels, right_value, *right.variant_columns]].copy()
+
+    # Carry each side's "my chain is current" flag through the join under a
+    # private name, so the two cannot collide and neither is lost. Before this
+    # the flag was simply dropped and a two-measure plot silently fell back to
+    # showing every code version — the one case where the pin-latest default
+    # quietly stopped applying.
+    if left.latest_column:
+        left_frame[_LEFT_LATEST] = left.frame[left.latest_column].values
+    if right.latest_column:
+        right_frame[_RIGHT_LATEST] = right.frame[right.latest_column].values
+
+    # Variant columns can collide by name when both variables carry the same
+    # branch param; suffix the right one so neither is silently dropped.
+    overlap = set(left.variant_columns) & set(right.variant_columns)
+    if overlap:
+        right_frame = right_frame.rename(
+            columns={name: f"{name}::{right.name}" for name in overlap}
+        )
+
+    merged = left_frame.merge(right_frame, on=on, how="inner", suffixes=("", "_right"))
+
+    # A joined row is current only if BOTH of its measures are. Either side
+    # being stale makes the pair a comparison across code versions, which is
+    # exactly what the flag exists to keep out of the default figure.
+    if _LEFT_LATEST in merged.columns or _RIGHT_LATEST in merged.columns:
+        combined = None
+        for column in (_LEFT_LATEST, _RIGHT_LATEST):
+            if column in merged.columns:
+                flag = merged[column].astype(bool)
+                combined = flag if combined is None else (combined & flag)
+        merged[LATEST_COLUMN] = combined
+        merged = merged.drop(
+            columns=[c for c in (_LEFT_LATEST, _RIGHT_LATEST) if c in merged.columns]
+        )
+
+    Log.info(
+        "%s join: %s(%d) x %s(%d) on %s -> %d row(s)",
+        kind,
+        left.name,
+        len(left_frame),
+        right.name,
+        len(right_frame),
+        on,
+        len(merged),
+        layer=LAYER,
+    )
+    if kind == "broadcast":
+        Log.debug(
+            "broadcast: %s is shallower (%s); its values repeat across deeper rows",
+            left.name if len(left.levels) <= len(right.levels) else right.name,
+            shallow_levels,
+            layer=LAYER,
+        )
+    return merged
+
+
+def joined_levels(left: VariableFrame, right: VariableFrame) -> list[str]:
+    """The level set of the joined frame — always the deeper of the two."""
+    return left.levels if len(left.levels) >= len(right.levels) else right.levels