scistackplotdb 0.1.35__tar.gz → 0.1.37__tar.gz

{scistackplotdb-0.1.35 → scistackplotdb-0.1.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scistackplotdb
-Version: 0.1.35
+Version: 0.1.37
 Summary: scidb-backed plotting: load variables into long tables and plot them with scistackplot
 Author: SciStack Contributors
 License-Expression: MIT
@@ -17,10 +17,10 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Visualization
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: sciduckdb==0.1.35
-Requires-Dist: scistack-db==0.1.35
-Requires-Dist: scistacklog==0.1.35
-Requires-Dist: scistackplot==0.1.35
+Requires-Dist: sciduckdb==0.1.37
+Requires-Dist: scistack-db==0.1.37
+Requires-Dist: scistacklog==0.1.37
+Requires-Dist: scistackplot==0.1.37
 Provides-Extra: dev
 Requires-Dist: matplotlib>=3.6; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0; extra == 'dev'

{scistackplotdb-0.1.35 → scistackplotdb-0.1.37}/src/scistackplotdb/__init__.py

@@ -57,6 +57,7 @@ from .saved import (
     SavedPlotExists,
     SavedPlotInfo,
     check_name,
+    follow_latest_saved_plot,
     hide_saved_plot,
     list_saved_plots,
     load_saved_plot,
@@ -97,6 +98,7 @@ __all__ = [
     "rename_saved_plot",
     "hide_saved_plot",
     "saved_plot_history",
+    "follow_latest_saved_plot",
     "check_name",
     "SavedPlot",
     "SavedPlotInfo",

{scistackplotdb-0.1.35 → scistackplotdb-0.1.37}/src/scistackplotdb/endpoint.py

@@ -81,6 +81,7 @@ def generate_endpoint(
     function_source = generate_plot_function(spec, table, function_name=name)
     foreach_source = _foreach_call(
         spec=spec,
+        table=table,
         function_name=name,
         input_variable=input_variable,
         x_variable=x_variable,
@@ -140,6 +141,29 @@ def _pin_literal(value) -> str:
     return repr(list(value) if isinstance(value, tuple) else value)
 
 
+def row_expression(input_variable: str, variant_set, table=None) -> str:
+    """One variant ROW as a ``Variant(...)`` load — through the row's
+    EFFECTIVE selection (``scistackplot.variants.export_selection``), so the
+    exported code loads what the preview drew: current code for a row that
+    names none, the frozen version for a reopened saved plot.
+
+    Grouping variables keep calling :func:`variant_expression` with their raw
+    selection: an empty grouping pin means "nothing filtered", not "current".
+    """
+    from types import SimpleNamespace
+
+    from scistackplot.variants import export_selection
+
+    # The TABLE's default, exactly what the preview applied: a stacked
+    # multi-variable table has none, so its rows export as written.
+    selection = export_selection(
+        variant_set,
+        latest_column=(table.latest_column if table is not None else None) or LATEST_COLUMN,
+        default_pin=table.default_pin if table is not None else None,
+    )
+    return variant_expression(input_variable, SimpleNamespace(selection=selection))
+
+
 def variant_expression(input_variable: str, variant_set) -> str:
     """A ``Variant(...)`` call selecting one named variant's records.
 
@@ -191,7 +215,7 @@ def variant_expression(input_variable: str, variant_set) -> str:
     return expression
 
 
-def _single_variant_expression(input_variable: str, spec) -> str:
+def _single_variant_expression(input_variable: str, spec, table=None) -> str:
     """The lone variant's pin, or the bare variable when nothing is selected.
 
     The row may name its own variable, in which case that is what the endpoint
@@ -200,12 +224,13 @@ def _single_variant_expression(input_variable: str, spec) -> str:
     sets = defined_sets(spec.variant_sets)
     if len(sets) != 1:
         return input_variable
-    return variant_expression(sets[0].variable or input_variable, sets[0])
+    return row_expression(sets[0].variable or input_variable, sets[0], table)
 
 
 def _foreach_call(
     *,
     spec: PlotSpec,
+    table: LongTable | None = None,
     function_name: str,
     input_variable: str,
     x_variable: str | None,
@@ -223,14 +248,14 @@ def _foreach_call(
         # the wrong variant's selection.
         inputs = [
             f'        "{generated.param}": '
-            f"{variant_expression(generated.variable, variant)},"
+            f"{row_expression(generated.variable, variant, table)},"
             for generated, variant in zip(
                 variant_inputs, defined_sets(spec.variant_sets), strict=True
             )
         ]
         table_inputs = [generated.param for generated in variant_inputs]
     else:
-        pinned = _single_variant_expression(input_variable, spec)
+        pinned = _single_variant_expression(input_variable, spec, table)
         inputs = [f'        "df": {pinned},']
         table_inputs = ["df"]
     if x_variable:

{scistackplotdb-0.1.35 → scistackplotdb-0.1.37}/src/scistackplotdb/load.py

@@ -24,6 +24,8 @@ import pandas as pd
 from scistacklog import Log
 from scistackplot import CODE_FACTOR_PREFIX, RUN_FACTOR_PREFIX
 from scistackplot.framesize import format_extent, frame_extent
+from scistackplot.shape import Shape, classify_value
+from scistackplot.table import natural_sort_key
 
 LAYER = "scistackplotdb"
 
@@ -425,7 +427,8 @@ def load_variable(
     ) as timer:
         with timer.phase("column_metadata"):
             keys = schema_keys(db)
-            columns = data_columns_for(db, variable)
+            column_types = data_column_types_for(db, variable)
+            columns = list(column_types)
         if not columns:
             Log.warn("variable %r has no data columns", variable, layer=LAYER)
             return VariableFrame(name=variable, frame=pd.DataFrame())
@@ -513,6 +516,9 @@ def load_variable(
             variant_columns or "none",
             layer=LAYER,
         )
+        if columns and len(frame):
+            with timer.phase("field_coverage"):
+                _log_field_coverage(variable, frame, column_types, levels)
         if len(frame) > 1:
             with timer.phase("identical_content"):
                 _log_identical_content(db, variable, frame, keys)
@@ -527,6 +533,157 @@ def load_variable(
         )
 
 
+#: A scalar field that holds a number in fewer than this share of its
+#: records is WARNed, with where its numbers are.
+SPARSE_FIELD_SHARE = 0.5
+
+#: How many distinct levels of a key, and how many text examples, a coverage
+#: line names before it says "+N more".
+_COVERAGE_LEVELS_SHOWN = 6
+
+
+@dataclass(frozen=True)
+class FieldCoverage:
+    """What one scalar data column holds, record by record: how many cells
+    are a number, NULL/NaN, text (with a few examples), or something else."""
+
+    column: str
+    declared_type: str
+    records: int
+    numeric: int
+    null: int
+    text: int
+    other: int
+    text_examples: tuple[str, ...] = ()
+
+    @property
+    def missing(self) -> int:
+        return self.records - self.numeric
+
+    def describe(self) -> str:
+        parts = [f"{self.null} NULL"] if self.null else []
+        if self.text:
+            examples = ", ".join(repr(t) for t in self.text_examples)
+            parts.append(f"{self.text} text (e.g. {examples})")
+        if self.other:
+            parts.append(f"{self.other} other")
+        return (
+            f"{self.column} ({self.declared_type}) {self.numeric}/{self.records} numeric"
+            + (f" ({', '.join(parts)})" if parts else "")
+        )
+
+
+def field_coverage(
+    frame: pd.DataFrame, column_types: dict[str, str]
+) -> tuple[list[FieldCoverage], pd.DataFrame]:
+    """Per scalar column of ``frame``: what its cells hold. Array (1-D/2-D)
+    columns are skipped — a signal is not "missing a number". Also returns a
+    boolean frame, one column per covered field, True where the cell is a
+    number (for :func:`_log_field_coverage` to say WHERE the numbers are)."""
+
+    out: list[FieldCoverage] = []
+    is_number = pd.DataFrame(index=frame.index)
+    for column, declared in column_types.items():
+        if column not in frame.columns:
+            continue
+        values = frame[column]
+        if pd.api.types.is_numeric_dtype(values) and not pd.api.types.is_bool_dtype(values):
+            mask = values.notna()
+            out.append(
+                FieldCoverage(column, declared, len(values), int(mask.sum()), int((~mask).sum()), 0, 0)
+            )
+            is_number[column] = mask.to_numpy()
+            continue
+        numeric = null = text = other = 0
+        arrays = False
+        examples: list[str] = []
+        mask = []
+        for value in values.to_numpy():
+            if value is None or (isinstance(value, float) and value != value):
+                null += 1
+                mask.append(False)
+            elif classify_value(value) is Shape.SCALAR:
+                numeric += 1
+                mask.append(True)
+            elif isinstance(value, str):
+                text += 1
+                mask.append(False)
+                if value not in examples and len(examples) < 3:
+                    examples.append(value)
+            elif isinstance(value, (list, tuple, np.ndarray)):
+                arrays = True
+                break
+            else:
+                other += 1
+                mask.append(False)
+        if arrays:
+            continue
+        out.append(
+            FieldCoverage(column, declared, len(values), numeric, null, text, other, tuple(examples))
+        )
+        is_number[column] = mask
+    return out, is_number
+
+
+def _levels_text(values: pd.Series) -> str:
+    levels = sorted({str(v) for v in values if v is not None}, key=natural_sort_key)
+    shown = levels[:_COVERAGE_LEVELS_SHOWN]
+    more = len(levels) - len(shown)
+    return "{" + ", ".join(shown) + (f", +{more} more" if more else "") + "}"
+
+
+def _log_field_coverage(
+    variable: str, frame: pd.DataFrame, column_types: dict[str, str], levels: list[str]
+) -> None:
+    """INFO: every scalar field that is not a number in every record. WARN,
+    per field, when fewer than :data:`SPARSE_FIELD_SHARE` of its records hold
+    one — naming the levels those records sit at, because a plot of such a
+    field shows only them and reads as a broken grouping ("Dist_m showed one
+    group/one subject": 6 of 714 records held a number, scidb.log
+    2026-10-15). The plot is right; the data is what to look at.
+
+    Never raises: a diagnostic must not break a load.
+    """
+    try:
+        coverage, is_number = field_coverage(frame, column_types)
+        gaps = [c for c in coverage if c.missing]
+        if not gaps:
+            return
+        Log.info(
+            "field coverage of %s: %d of %d scalar field(s) are not a number in every "
+            "record — %s",
+            variable,
+            len(gaps),
+            len(coverage),
+            "; ".join(c.describe() for c in gaps),
+            layer=LAYER,
+        )
+        for c in gaps:
+            if c.numeric >= SPARSE_FIELD_SHARE * c.records:
+                continue
+            where = frame.loc[is_number[c.column].to_numpy(dtype=bool)]
+            located = ", ".join(
+                f"{key} ∈ {_levels_text(where[key])}"
+                for key in levels
+                if key in where.columns
+            )
+            Log.warn(
+                "%s.%s holds a number in only %d of %d record(s) — %s. A plot of it can "
+                "show only those records%s. If the source has values for the others, "
+                "they were not saved as numbers: check what the loader wrote for "
+                "this column.",
+                variable,
+                c.column,
+                c.numeric,
+                c.records,
+                c.describe(),
+                f", at {located}" if c.numeric and located else "",
+                layer=LAYER,
+            )
+    except Exception as exc:  # noqa: BLE001 — diagnostic only
+        Log.debug("field coverage of %s failed: %s", variable, exc, layer=LAYER)
+
+
 def _log_identical_content(db, variable: str, frame: pd.DataFrame, keys) -> None:
     """Warn when records of ``variable`` hold byte-identical payloads, naming
     the schema keys that vary across each duplicate group.

scistackplotdb-0.1.37/src/scistackplotdb/saved.py

@@ -0,0 +1,686 @@
+"""
+Saved plots: named, per-variable Plot Studio figures kept in the project
+database.
+
+A saved plot is **display intent**. It changes nothing a run computes, so it is
+not an aspect of the GUI's intent store (docs/claude/intent-and-fact.md §1).
+It lives here, in the layer that knows both the database and the spec, so a
+script can reopen a figure the GUI saved::
+
+    from scistackplotdb import list_saved_plots, load_saved_plot
+    saved = load_saved_plot(db, list_saved_plots(db, "StepLength")[0].plot_id)
+    render(ScidbSource(db).get_table(saved.spec.variant_variables()), saved.spec)
+
+**One table, append-only.** ``_saved_plot`` holds one row per *version*.
+Saving under a name that is already showing appends a version to that plot;
+the newest is what the list shows and what opens. Nothing is ever deleted
+(feedback_never_delete_mark_hidden): "remove" sets ``hidden`` on every
+version of the plot, and the rows stay.
+
+``plot_id`` is the plot's identity, stable across renames. A name is a label.
+Keying by name would make rename an update of an identity column, and would
+make "a new plot reusing a removed plot's name" silently inherit its history.
+
+**The stored spec is always the current format.** :func:`save_plot` reads the
+incoming spec strictly, so what lands in the table is exactly what this
+version writes. Old formats are only ever *read*, by
+:func:`scistackplot.restore_spec`. The stored copy is never rewritten on
+open; only an explicit save writes. See ``.claude/plan-saved-plots.md``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from scistacklog import Log
+
+from ._versioned import MAX_NAME_LENGTH, VersionedStore  # noqa: F401  (re-exported)
+
+LAYER = "scistackplotdb"
+
+TABLE = "_saved_plot"
+
+#: The envelope's layout version. Written on every save and logged on every
+#: load; nothing branches on it (salvage, never migrate).
+ENVELOPE_FORMAT = 1
+
+
+class SavedPlotError(ValueError):
+    """A save, rename or load that cannot be done as asked."""
+
+
+class SavedPlotExists(SavedPlotError):
+    """``save_plot(overwrite=False)`` found ANOTHER visible plot with that name.
+
+    ``existing`` is that plot, so the caller can ask the user and retry with
+    ``overwrite=True``. The name comparison stays here, where names are
+    normalised, instead of in each caller.
+    """
+
+    def __init__(self, existing: "SavedPlotInfo"):
+        super().__init__(
+            f"{existing.variable} already has a saved plot named "
+            f"{existing.name!r} (version {existing.version})."
+        )
+        self.existing = existing
+
+
+@dataclass(frozen=True)
+class SavedPlotInfo:
+    """One version of a saved plot, without its contents: a list row."""
+
+    plot_id: str
+    variable: str
+    name: str
+    version: int
+    saved_at: str
+    hidden: bool = False
+
+    def to_dict(self) -> dict:
+        return {
+            "plot_id": self.plot_id,
+            "variable": self.variable,
+            "name": self.name,
+            "version": self.version,
+            "saved_at": self.saved_at,
+            "hidden": self.hidden,
+        }
+
+
+@dataclass(frozen=True)
+class SavedPlot:
+    """A saved plot, opened: the restored spec plus what did not survive."""
+
+    info: SavedPlotInfo
+    spec: Any  # scistackplot.PlotSpec
+    #: The panel's own view settings (preview mode, aspect choice, figure
+    #: index), stored verbatim. The frontend owns their meaning and reads them
+    #: as leniently as the spec is read here.
+    view: dict = field(default_factory=dict)
+    #: ``scistackplot.RestoreNote`` s from restoring the spec and, when a table
+    #: was given, from reconciling it with today's data.
+    notes: list = field(default_factory=list)
+    #: The envelope format this version was written in.
+    stored_format: Any = None
+    #: Code versions in today's data NEWER than the ones this plot was saved
+    #: with: ``[{row, column, function, frozen, newer, newest, locations}]``
+    #: (:func:`thaw_code`). Non-empty is the GUI's cue to ask "switch to
+    #: latest?". Empty for a plot that follows latest or recorded no version.
+    newer: list = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            **self.info.to_dict(),
+            "spec": self.spec.to_dict(),
+            "view": dict(self.view),
+            "notes": [note.to_dict() for note in self.notes],
+            "stored_format": self.stored_format,
+            "newer": [dict(entry) for entry in self.newer],
+        }
+
+
+# ---------------------------------------------------------------------------
+# Table
+# ---------------------------------------------------------------------------
+
+#: The rules (names, versions, hiding, never deleting) live in `_versioned`,
+#: shared with presets so the two stores cannot drift apart.
+STORE = VersionedStore(
+    table=TABLE,
+    id_column="plot_id",
+    scope_column="variable",
+    noun="saved plot",
+    tag="saved_plot",
+    error=SavedPlotError,
+)
+
+
+def ensure_table(db) -> None:
+    """Create ``_saved_plot`` if absent. Called by every write."""
+    STORE.ensure_table(db)
+
+
+def _table_exists(db) -> bool:
+    """Reads never create the table: listing an untouched database must not
+    write to it."""
+    return STORE.table_exists(db)
+
+
+# ---------------------------------------------------------------------------
+# Writes
+# ---------------------------------------------------------------------------
+
+
+def save_plot(
+    db,
+    variable: str,
+    name: str,
+    spec,
+    view: dict | None = None,
+    *,
+    overwrite: bool = True,
+    current_plot_id: str | None = None,
+    table=None,
+    table_for: Callable[[Any], Any] | None = None,
+) -> SavedPlotInfo:
+    """Save *spec* under *name* for *variable*, returning the new version.
+
+    **Freezes the code it drew.** Given the table the plot draws from
+    (*table*, or *table_for* to load it from the spec), every variant row not
+    following latest records the function hashes of its rows
+    (:func:`freeze_code`), so the plot reopens on that version after the code
+    changes. Without a table nothing is frozen and the plot reopens on current
+    code; that is logged.
+
+    A visible plot of that name gets a new version, so the previous one is
+    kept as history. Otherwise a new plot is started, even when a removed
+    (hidden) plot had that name, because a new plot does not inherit an old
+    plot's history.
+
+    With ``overwrite=False``, adding a version to an existing plot is allowed
+    only when that plot is *current_plot_id* (the one the user has open).
+    Saving onto any OTHER plot's name raises :class:`SavedPlotExists` so the
+    caller can ask first.
+
+    *spec* is a ``PlotSpec`` or its dict. A dict is read **strictly**: the
+    panel always sends the current format, and a spec that does not parse is
+    a bug to surface, not something to store.
+    """
+    from scistackplot import PlotSpec, __version__ as plot_version
+
+    variable = _check_text(variable, "variable")
+    name = check_name(name)
+    if isinstance(spec, dict):
+        try:
+            spec = PlotSpec.from_dict(spec)
+        except Exception as exc:
+            raise SavedPlotError(f"The plot's settings could not be read: {exc}") from exc
+    if view is None:
+        view = {}
+    if not isinstance(view, dict):
+        raise SavedPlotError(f"view must be a table of settings, got {type(view).__name__}")
+
+    if table is None and table_for is not None:
+        try:
+            table = table_for(spec)
+        except Exception as exc:
+            Log.warn(
+                "[saved_plot] %s / %r: the data could not be loaded to freeze its "
+                "code version (%s); saving unfrozen",
+                variable,
+                name,
+                exc,
+                layer=LAYER,
+            )
+    if table is not None:
+        frozen_code = freeze_code(db, spec, table)
+    else:
+        frozen_code = {}
+        Log.info(
+            "[saved_plot] %s / %r: no table given, so no code version is frozen "
+            "(the plot reopens on current code)",
+            variable,
+            name,
+            layer=LAYER,
+        )
+
+    envelope = {
+        "format": ENVELOPE_FORMAT,
+        "spec": _strip_frozen(spec).to_dict(),
+        "view": view,
+        "saved_with": {"scistackplot": plot_version},
+        FROZEN_KEY: frozen_code,
+    }
+    row, _new = STORE.save(
+        db,
+        variable,
+        name,
+        envelope,
+        overwrite=overwrite,
+        current_id=current_plot_id,
+        on_clash=lambda clash: SavedPlotExists(_info(clash)),
+    )
+    return _info(row)
+
+
+def rename_saved_plot(db, plot_id: str, new_name: str) -> SavedPlotInfo:
+    """Rename a plot (every version). Refused when another visible plot of
+    the same variable already has that name."""
+    return _info(STORE.rename(db, plot_id, new_name))
+
+
+def hide_saved_plot(db, plot_id: str, hidden: bool = True) -> SavedPlotInfo:
+    """Remove a plot from the list, keeping every row. ``hidden=False``
+    brings it back."""
+    return _info(STORE.set_hidden(db, plot_id, hidden))
+
+
+# ---------------------------------------------------------------------------
+# Reads
+# ---------------------------------------------------------------------------
+
+
+def list_saved_plots(db, variable: str, *, include_hidden: bool = False) -> list[SavedPlotInfo]:
+    """The newest version of each saved plot of *variable*, newest first."""
+    return [_info(row) for row in STORE.list_rows(db, variable, include_hidden=include_hidden)]
+
+
+def saved_plot_history(db, plot_id: str) -> list[SavedPlotInfo]:
+    """Every version of one plot, newest first."""
+    return [_info(row) for row in STORE.history(db, plot_id)]
+
+
+def load_saved_plot(
+    db,
+    plot_id: str,
+    *,
+    version: int | None = None,
+    table=None,
+    table_for: Callable[[Any], Any] | None = None,
+) -> SavedPlot:
+    """Open a saved plot (its newest version unless *version* is given).
+
+    The spec is read with :func:`scistackplot.restore_spec`, so a plot saved
+    by an older version of the plotting layer still opens. Settings that no
+    longer exist or no longer parse come back as notes.
+
+    To also reconcile the spec with today's data, pass either *table* (the
+    ``LongTable`` the plot will draw from) or *table_for*, a callable that
+    loads that table from the RESTORED spec. The table depends on the spec
+    (its variant rows and factor variables decide what gets loaded), so a
+    caller that cannot know the spec in advance hands over the loader. The
+    GUI does this with its cached source. A loader that raises does not fail
+    the open: the plot opens unreconciled, with a note saying why.
+
+    Never rewrites the stored copy: a later build may salvage more of it.
+    """
+    from scistackplot import NoteKind, RestoreNote, reconcile, restore_spec
+
+    row, text = STORE.load(db, plot_id, version)
+    info = _info(row)
+    envelope = STORE.parse_envelope(
+        text, f"{info.variable} / {info.name!r} version {info.version}"
+    )
+
+    restored = restore_spec(envelope.get("spec"), fallback_measure=info.variable)
+    spec, notes = restored.spec, list(restored.notes)
+    if table is None and table_for is not None:
+        try:
+            table = table_for(spec)
+        except Exception as exc:
+            Log.warn(
+                "[saved_plot] %s / %r: today's data could not be loaded for "
+                "reconciling (%s); opening unreconciled",
+                info.variable,
+                info.name,
+                exc,
+                layer=LAYER,
+            )
+            notes.append(
+                RestoreNote(
+                    "",
+                    NoteKind.NOT_IN_DATA,
+                    f"today's data could not be loaded, so the settings were "
+                    f"not checked against it: {exc}",
+                )
+            )
+    newer: list = []
+    frozen_code = envelope.get(FROZEN_KEY)
+    if table is not None and isinstance(frozen_code, dict) and frozen_code:
+        # Before reconcile, so it checks the roles against the table the
+        # FROZEN rows derive (docs/claude/variant-row-selection.md).
+        spec, newer, thaw_notes = thaw_code(
+            db, spec, table, frozen_code, label=f"{info.variable} / {info.name!r}"
+        )
+        notes.extend(thaw_notes)
+    elif not frozen_code:
+        # Saved before freezing existed, or with no table: opens on current
+        # code with no prompt. Nothing is back-filled (no migrations).
+        Log.info(
+            "[saved_plot] %s / %r: no frozen code version stored; opening on "
+            "current code",
+            info.variable,
+            info.name,
+            layer=LAYER,
+        )
+    if table is not None:
+        reconciled = reconcile(spec, table)
+        spec, notes = reconciled.spec, notes + list(reconciled.notes)
+
+    view = envelope.get("view")
+    if not isinstance(view, dict):
+        Log.warn(
+            "[saved_plot] %s / %r: stored view settings are not a table (%r); "
+            "opening with the panel's defaults",
+            info.variable,
+            info.name,
+            view,
+            layer=LAYER,
+        )
+        view = {}
+
+    stored_format = envelope.get("format")
+    Log.info(
+        "[saved_plot] opened %s / %r version %d (envelope format %r, saved with %s): "
+        "%d note(s)",
+        info.variable,
+        info.name,
+        info.version,
+        stored_format,
+        envelope.get("saved_with"),
+        len(notes),
+        layer=LAYER,
+    )
+    return SavedPlot(
+        info=info,
+        spec=spec,
+        view=view,
+        notes=notes,
+        stored_format=stored_format,
+        newer=newer,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def check_name(name: Any) -> str:
+    """A saved plot's name, trimmed, or :class:`SavedPlotError` saying why not.
+
+    The one statement of what a name may be (``VersionedStore.check_name``);
+    the GUI asks the backend rather than repeating the rule.
+    """
+    return STORE.check_name(name)
+
+
+def _check_text(value: Any, what: str) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise SavedPlotError(f"A saved plot needs a {what}.")
+    return value.strip()
+
+
+def _info(row) -> SavedPlotInfo:
+    plot_id, variable, name, version, saved_at, hidden = row
+    return SavedPlotInfo(
+        plot_id=str(plot_id),
+        variable=str(variable),
+        name=str(name),
+        version=int(version),
+        saved_at=str(saved_at),
+        hidden=bool(hidden),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Frozen code versions (docs/claude/variant-row-selection.md)
+# ---------------------------------------------------------------------------
+#
+# A saved plot remembers WHICH CODE drew it. At save time every variant row
+# that does not follow latest records the function hashes of the rows it drew
+# (``frozen_code`` in the envelope). On open, those hashes become today's
+# ordinals in ``VariantSet.frozen`` — so the figure reopens on the version it
+# was saved with — and anything newer is reported as ``SavedPlot.newer`` for
+# the GUI's "switch to latest?" prompt. Hashes, not ordinals, are stored
+# because ordinals renumber after a real Variants-popup delete.
+
+#: Envelope key: ``{str(row_index): {function_name: [function_hash, ...]}}``.
+FROZEN_KEY = "frozen_code"
+
+
+def _row_masks(spec, table) -> list[tuple[int, Any]]:
+    """``[(row_index, mask)]`` — the rows each variant row DRAWS, by the same
+    owner and first-match-wins order the figure uses
+    (``scistackplot.variants.row_mask`` / ``apply_variant_sets``).
+
+    With no row concrete, the figure draws an empty row's effective selection;
+    that is frozen as row 0 (synthesised on open if the spec has none).
+    """
+    import pandas as pd
+    from scistackplot.spec import VariantSet
+    from scistackplot.variants import defined_sets, is_stated, row_mask
+
+    frame = table.frame
+    if not defined_sets(spec.variant_sets):
+        first = spec.variant_sets[0] if spec.variant_sets else VariantSet()
+        return [(0, row_mask(table, spec, first))]
+    claimed = pd.Series(False, index=frame.index)
+    out = []
+    for index, variant in enumerate(spec.variant_sets):
+        if not is_stated(variant, index):
+            continue
+        fresh = row_mask(table, spec, variant) & ~claimed
+        claimed |= fresh
+        out.append((index, fresh))
+    return out
+
+
+def freeze_code(db, spec, table) -> dict[str, dict[str, list[str]]]:
+    """The envelope's ``frozen_code``: per variant row not following latest,
+    every function in the drawn records' chains and the hash(es) it ran.
+
+    Works with ONE version too (the usual state at save time): there is no
+    ``Code:`` column then, but the chain still names the hash, and that is
+    what lets the plot reopen on it after a second version appears.
+    """
+    from scidb.provenance_query import code_versions_batch
+
+    frame = table.frame
+    if "record_id" not in frame.columns:
+        Log.warn(
+            "[saved_plot] %r: the table has no record_id column, so no code "
+            "version can be frozen — the plot will reopen on current code",
+            getattr(table, "name", None),
+            layer=LAYER,
+        )
+        return {}
+    rows = _row_masks(spec, table)
+    follow = {
+        index for index, variant in enumerate(spec.variant_sets) if variant.follow_latest
+    }
+    wanted = {index: mask for index, mask in rows if index not in follow}
+    record_ids = sorted(
+        {
+            str(rid)
+            for mask in wanted.values()
+            for rid in frame.loc[mask, "record_id"].dropna().unique()
+        }
+    )
+    chains = code_versions_batch(db._duck, record_ids) if record_ids else {}
+    frozen: dict[str, dict[str, list[str]]] = {}
+    for index, mask in wanted.items():
+        by_fn: dict[str, set] = {}
+        for rid in frame.loc[mask, "record_id"].dropna().unique():
+            for fn_name, fn_hash in (chains.get(str(rid)) or {}).items():
+                by_fn.setdefault(fn_name, set()).add(fn_hash)
+        if by_fn:
+            frozen[str(index)] = {fn: sorted(hashes) for fn, hashes in sorted(by_fn.items())}
+    Log.info(
+        "[saved_plot] froze code for %d row(s) over %d record(s): %s%s",
+        len(frozen),
+        len(record_ids),
+        {
+            index: {fn: [h[:12] for h in hashes] for fn, hashes in fns.items()}
+            for index, fns in frozen.items()
+        },
+        f" (rows following latest: {sorted(follow)})" if follow else "",
+        layer=LAYER,
+    )
+    return frozen
+
+
+def _strip_frozen(spec):
+    """The spec as stored: ``frozen`` ordinals are derived on open from the
+    envelope's hashes, never stored (an ordinal can renumber)."""
+    from dataclasses import replace
+
+    return replace(
+        spec,
+        variant_sets=[replace(v, frozen={}) for v in spec.variant_sets],
+    )
+
+
+def _code_columns(table) -> dict[str, str]:
+    """``{function_name: column}`` for every code axis the table carries."""
+    out = {}
+    for factor in table.factors:
+        origin = factor.origin or {}
+        if origin.get("kind") == "code" and origin.get("function"):
+            out[origin["function"]] = factor.name
+    return out
+
+
+def thaw_code(db, spec, table, frozen_code: dict, *, label: str = ""):
+    """``(spec, newer, notes)`` — apply a stored freeze to today's data.
+
+    For each frozen row and function that is CURRENTLY an axis (two or more
+    versions, so a ``Code:`` column), the stored hashes become today's
+    ordinals in ``VariantSet.frozen``. A single-version function needs
+    nothing. ``newer`` lists, per row and function, the versions present in
+    the data that are newer than every frozen one — what the GUI asks about.
+
+    A frozen hash that no longer exists (its version was deleted) is NOT
+    silently replaced: it gets a note, and the row falls back to the
+    remaining frozen versions or, with none left, to current code — which the
+    note says, and ``newer`` offers to make permanent.
+    """
+    from dataclasses import replace
+
+    from scidb.provenance_query import function_versions
+    from scistackplot import NoteKind, RestoreNote
+    from scistackplot.spec import VariantSet
+    from scistackplot.variants import _ordinal_rank
+
+    if not frozen_code:
+        return spec, [], []
+    columns = _code_columns(table)
+    fn_names = sorted({fn for fns in frozen_code.values() for fn in fns})
+    versions = function_versions(db._duck, fn_names)
+    sets = list(spec.variant_sets)
+    newer: list[dict] = []
+    notes: list = []
+    frame = table.frame
+    levels = [key for key in table.schema_levels if key in frame.columns]
+
+    for raw_index, fns in sorted(frozen_code.items(), key=lambda kv: int(kv[0])):
+        index = int(raw_index)
+        if index >= len(sets):
+            if index == 0 and not sets:
+                sets.append(VariantSet())
+            else:
+                continue
+        variant = sets[index]
+        if variant.follow_latest:
+            continue
+        frozen: dict[str, list[str]] = {}
+        for fn_name, hashes in fns.items():
+            by_hash = {v["function_hash"]: v["version"] for v in versions.get(fn_name, [])}
+            ordinals = [by_hash[h] for h in hashes if h in by_hash]
+            missing = [h for h in hashes if h not in by_hash]
+            if missing:
+                notes.append(
+                    RestoreNote(
+                        f"variant_sets[{index}]",
+                        NoteKind.NOT_IN_DATA,
+                        f"the {fn_name} version(s) this plot was saved with "
+                        f"({', '.join(h[:12] for h in missing)}) no longer exist"
+                        + ("" if ordinals else "; showing the current version instead"),
+                    )
+                )
+            column = columns.get(fn_name)
+            if column is None:
+                continue  # one version today: nothing to choose between
+            if ordinals:
+                frozen[column] = sorted(set(ordinals), key=_ordinal_rank)
+            present = frame[column].dropna().astype(str)
+            top = max((_ordinal_rank(o) for o in ordinals), default=0)
+            later = sorted(
+                {level for level in present.unique() if _ordinal_rank(level) > top},
+                key=_ordinal_rank,
+            )
+            if later:
+                at = frame[frame[column].astype(str).isin(later)]
+                newer.append(
+                    {
+                        "row": index,
+                        "column": column,
+                        "function": fn_name,
+                        "frozen": list(frozen.get(column, [])),
+                        "newer": later,
+                        "newest": later[-1],
+                        "locations": int(len(at[levels].drop_duplicates())) if levels else int(len(at)),
+                    }
+                )
+        if frozen:
+            sets[index] = replace(variant, frozen=frozen)
+
+    Log.info(
+        "[saved_plot] %s: frozen code -> %s; newer: %s",
+        label,
+        {i: v.frozen for i, v in enumerate(sets) if v.frozen} or "nothing to pin",
+        [f"row {n['row']} {n['function']} {n['frozen']} < {n['newer']}" for n in newer]
+        or "none",
+        layer=LAYER,
+    )
+    return replace(spec, variant_sets=sets), newer, notes
+
+
+def follow_latest_saved_plot(db, plot_id: str, *, table=None) -> SavedPlotInfo:
+    """Answer "switch to latest" for a saved plot: append a version whose rows
+    follow latest code from now on (never frozen again, never asked again).
+
+    Each row drops its frozen pins AND its own code choices (``Code:``/``Run:``
+    keys and the latest flag) — a row still saying ``Code:f = v1`` would keep
+    drawing v1. What remains (branch params, the variable) is kept, and the
+    row then draws current code by the effective-selection rule. Appends, as
+    any re-save does; older versions stay.
+    """
+    from dataclasses import replace
+
+    from scistackplot import restore_spec
+    from scistackplot.spec import VariantSet
+    from scistackplot.variants import is_function_axis
+
+    from .load import LATEST_COLUMN
+
+    row, text = STORE.load(db, plot_id, None)
+    info = _info(row)
+    envelope = STORE.parse_envelope(text, f"{info.variable} / {info.name!r}")
+    spec = restore_spec(envelope.get("spec"), fallback_measure=info.variable).spec
+    sets = list(spec.variant_sets) or [VariantSet()]
+    sets = [
+        replace(
+            variant,
+            frozen={},
+            follow_latest=True,
+            selection={
+                key: value
+                for key, value in variant.selection.items()
+                if not is_function_axis(key) and key != LATEST_COLUMN
+            },
+        )
+        for variant in sets
+    ]
+    spec = replace(spec, variant_sets=sets)
+    view = envelope.get("view") if isinstance(envelope.get("view"), dict) else {}
+    new_info = save_plot(
+        db,
+        info.variable,
+        info.name,
+        spec,
+        view,
+        overwrite=True,
+        current_plot_id=plot_id,
+        table=table,
+    )
+    Log.info(
+        "[saved_plot] %s / %r now follows latest code (version %d -> %d)",
+        info.variable,
+        info.name,
+        info.version,
+        new_info.version,
+        layer=LAYER,
+    )
+    return new_info

scistackplotdb-0.1.35/src/scistackplotdb/saved.py

@@ -1,350 +0,0 @@
-"""
-Saved plots: named, per-variable Plot Studio figures kept in the project
-database.
-
-A saved plot is **display intent**. It changes nothing a run computes, so it is
-not an aspect of the GUI's intent store (docs/claude/intent-and-fact.md §1).
-It lives here, in the layer that knows both the database and the spec, so a
-script can reopen a figure the GUI saved::
-
-    from scistackplotdb import list_saved_plots, load_saved_plot
-    saved = load_saved_plot(db, list_saved_plots(db, "StepLength")[0].plot_id)
-    render(ScidbSource(db).get_table(saved.spec.variant_variables()), saved.spec)
-
-**One table, append-only.** ``_saved_plot`` holds one row per *version*.
-Saving under a name that is already showing appends a version to that plot;
-the newest is what the list shows and what opens. Nothing is ever deleted
-(feedback_never_delete_mark_hidden): "remove" sets ``hidden`` on every
-version of the plot, and the rows stay.
-
-``plot_id`` is the plot's identity, stable across renames. A name is a label.
-Keying by name would make rename an update of an identity column, and would
-make "a new plot reusing a removed plot's name" silently inherit its history.
-
-**The stored spec is always the current format.** :func:`save_plot` reads the
-incoming spec strictly, so what lands in the table is exactly what this
-version writes. Old formats are only ever *read*, by
-:func:`scistackplot.restore_spec`. The stored copy is never rewritten on
-open; only an explicit save writes. See ``.claude/plan-saved-plots.md``.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Any, Callable
-
-from scistacklog import Log
-
-from ._versioned import MAX_NAME_LENGTH, VersionedStore  # noqa: F401  (re-exported)
-
-LAYER = "scistackplotdb"
-
-TABLE = "_saved_plot"
-
-#: The envelope's layout version. Written on every save and logged on every
-#: load; nothing branches on it (salvage, never migrate).
-ENVELOPE_FORMAT = 1
-
-
-class SavedPlotError(ValueError):
-    """A save, rename or load that cannot be done as asked."""
-
-
-class SavedPlotExists(SavedPlotError):
-    """``save_plot(overwrite=False)`` found ANOTHER visible plot with that name.
-
-    ``existing`` is that plot, so the caller can ask the user and retry with
-    ``overwrite=True``. The name comparison stays here, where names are
-    normalised, instead of in each caller.
-    """
-
-    def __init__(self, existing: "SavedPlotInfo"):
-        super().__init__(
-            f"{existing.variable} already has a saved plot named "
-            f"{existing.name!r} (version {existing.version})."
-        )
-        self.existing = existing
-
-
-@dataclass(frozen=True)
-class SavedPlotInfo:
-    """One version of a saved plot, without its contents: a list row."""
-
-    plot_id: str
-    variable: str
-    name: str
-    version: int
-    saved_at: str
-    hidden: bool = False
-
-    def to_dict(self) -> dict:
-        return {
-            "plot_id": self.plot_id,
-            "variable": self.variable,
-            "name": self.name,
-            "version": self.version,
-            "saved_at": self.saved_at,
-            "hidden": self.hidden,
-        }
-
-
-@dataclass(frozen=True)
-class SavedPlot:
-    """A saved plot, opened: the restored spec plus what did not survive."""
-
-    info: SavedPlotInfo
-    spec: Any  # scistackplot.PlotSpec
-    #: The panel's own view settings (preview mode, aspect choice, figure
-    #: index), stored verbatim. The frontend owns their meaning and reads them
-    #: as leniently as the spec is read here.
-    view: dict = field(default_factory=dict)
-    #: ``scistackplot.RestoreNote`` s from restoring the spec and, when a table
-    #: was given, from reconciling it with today's data.
-    notes: list = field(default_factory=list)
-    #: The envelope format this version was written in.
-    stored_format: Any = None
-
-    def to_dict(self) -> dict:
-        return {
-            **self.info.to_dict(),
-            "spec": self.spec.to_dict(),
-            "view": dict(self.view),
-            "notes": [note.to_dict() for note in self.notes],
-            "stored_format": self.stored_format,
-        }
-
-
-# ---------------------------------------------------------------------------
-# Table
-# ---------------------------------------------------------------------------
-
-#: The rules (names, versions, hiding, never deleting) live in `_versioned`,
-#: shared with presets so the two stores cannot drift apart.
-STORE = VersionedStore(
-    table=TABLE,
-    id_column="plot_id",
-    scope_column="variable",
-    noun="saved plot",
-    tag="saved_plot",
-    error=SavedPlotError,
-)
-
-
-def ensure_table(db) -> None:
-    """Create ``_saved_plot`` if absent. Called by every write."""
-    STORE.ensure_table(db)
-
-
-def _table_exists(db) -> bool:
-    """Reads never create the table: listing an untouched database must not
-    write to it."""
-    return STORE.table_exists(db)
-
-
-# ---------------------------------------------------------------------------
-# Writes
-# ---------------------------------------------------------------------------
-
-
-def save_plot(
-    db,
-    variable: str,
-    name: str,
-    spec,
-    view: dict | None = None,
-    *,
-    overwrite: bool = True,
-    current_plot_id: str | None = None,
-) -> SavedPlotInfo:
-    """Save *spec* under *name* for *variable*, returning the new version.
-
-    A visible plot of that name gets a new version, so the previous one is
-    kept as history. Otherwise a new plot is started, even when a removed
-    (hidden) plot had that name, because a new plot does not inherit an old
-    plot's history.
-
-    With ``overwrite=False``, adding a version to an existing plot is allowed
-    only when that plot is *current_plot_id* (the one the user has open).
-    Saving onto any OTHER plot's name raises :class:`SavedPlotExists` so the
-    caller can ask first.
-
-    *spec* is a ``PlotSpec`` or its dict. A dict is read **strictly**: the
-    panel always sends the current format, and a spec that does not parse is
-    a bug to surface, not something to store.
-    """
-    from scistackplot import PlotSpec, __version__ as plot_version
-
-    variable = _check_text(variable, "variable")
-    name = check_name(name)
-    if isinstance(spec, dict):
-        try:
-            spec = PlotSpec.from_dict(spec)
-        except Exception as exc:
-            raise SavedPlotError(f"The plot's settings could not be read: {exc}") from exc
-    if view is None:
-        view = {}
-    if not isinstance(view, dict):
-        raise SavedPlotError(f"view must be a table of settings, got {type(view).__name__}")
-
-    envelope = {
-        "format": ENVELOPE_FORMAT,
-        "spec": spec.to_dict(),
-        "view": view,
-        "saved_with": {"scistackplot": plot_version},
-    }
-    row, _new = STORE.save(
-        db,
-        variable,
-        name,
-        envelope,
-        overwrite=overwrite,
-        current_id=current_plot_id,
-        on_clash=lambda clash: SavedPlotExists(_info(clash)),
-    )
-    return _info(row)
-
-
-def rename_saved_plot(db, plot_id: str, new_name: str) -> SavedPlotInfo:
-    """Rename a plot (every version). Refused when another visible plot of
-    the same variable already has that name."""
-    return _info(STORE.rename(db, plot_id, new_name))
-
-
-def hide_saved_plot(db, plot_id: str, hidden: bool = True) -> SavedPlotInfo:
-    """Remove a plot from the list, keeping every row. ``hidden=False``
-    brings it back."""
-    return _info(STORE.set_hidden(db, plot_id, hidden))
-
-
-# ---------------------------------------------------------------------------
-# Reads
-# ---------------------------------------------------------------------------
-
-
-def list_saved_plots(db, variable: str, *, include_hidden: bool = False) -> list[SavedPlotInfo]:
-    """The newest version of each saved plot of *variable*, newest first."""
-    return [_info(row) for row in STORE.list_rows(db, variable, include_hidden=include_hidden)]
-
-
-def saved_plot_history(db, plot_id: str) -> list[SavedPlotInfo]:
-    """Every version of one plot, newest first."""
-    return [_info(row) for row in STORE.history(db, plot_id)]
-
-
-def load_saved_plot(
-    db,
-    plot_id: str,
-    *,
-    version: int | None = None,
-    table=None,
-    table_for: Callable[[Any], Any] | None = None,
-) -> SavedPlot:
-    """Open a saved plot (its newest version unless *version* is given).
-
-    The spec is read with :func:`scistackplot.restore_spec`, so a plot saved
-    by an older version of the plotting layer still opens. Settings that no
-    longer exist or no longer parse come back as notes.
-
-    To also reconcile the spec with today's data, pass either *table* (the
-    ``LongTable`` the plot will draw from) or *table_for*, a callable that
-    loads that table from the RESTORED spec. The table depends on the spec
-    (its variant rows and factor variables decide what gets loaded), so a
-    caller that cannot know the spec in advance hands over the loader. The
-    GUI does this with its cached source. A loader that raises does not fail
-    the open: the plot opens unreconciled, with a note saying why.
-
-    Never rewrites the stored copy: a later build may salvage more of it.
-    """
-    from scistackplot import NoteKind, RestoreNote, reconcile, restore_spec
-
-    row, text = STORE.load(db, plot_id, version)
-    info = _info(row)
-    envelope = STORE.parse_envelope(
-        text, f"{info.variable} / {info.name!r} version {info.version}"
-    )
-
-    restored = restore_spec(envelope.get("spec"), fallback_measure=info.variable)
-    spec, notes = restored.spec, list(restored.notes)
-    if table is None and table_for is not None:
-        try:
-            table = table_for(spec)
-        except Exception as exc:
-            Log.warn(
-                "[saved_plot] %s / %r: today's data could not be loaded for "
-                "reconciling (%s); opening unreconciled",
-                info.variable,
-                info.name,
-                exc,
-                layer=LAYER,
-            )
-            notes.append(
-                RestoreNote(
-                    "",
-                    NoteKind.NOT_IN_DATA,
-                    f"today's data could not be loaded, so the settings were "
-                    f"not checked against it: {exc}",
-                )
-            )
-    if table is not None:
-        reconciled = reconcile(spec, table)
-        spec, notes = reconciled.spec, notes + list(reconciled.notes)
-
-    view = envelope.get("view")
-    if not isinstance(view, dict):
-        Log.warn(
-            "[saved_plot] %s / %r: stored view settings are not a table (%r); "
-            "opening with the panel's defaults",
-            info.variable,
-            info.name,
-            view,
-            layer=LAYER,
-        )
-        view = {}
-
-    stored_format = envelope.get("format")
-    Log.info(
-        "[saved_plot] opened %s / %r version %d (envelope format %r, saved with %s): "
-        "%d note(s)",
-        info.variable,
-        info.name,
-        info.version,
-        stored_format,
-        envelope.get("saved_with"),
-        len(notes),
-        layer=LAYER,
-    )
-    return SavedPlot(
-        info=info, spec=spec, view=view, notes=notes, stored_format=stored_format
-    )
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def check_name(name: Any) -> str:
-    """A saved plot's name, trimmed, or :class:`SavedPlotError` saying why not.
-
-    The one statement of what a name may be (``VersionedStore.check_name``);
-    the GUI asks the backend rather than repeating the rule.
-    """
-    return STORE.check_name(name)
-
-
-def _check_text(value: Any, what: str) -> str:
-    if not isinstance(value, str) or not value.strip():
-        raise SavedPlotError(f"A saved plot needs a {what}.")
-    return value.strip()
-
-
-def _info(row) -> SavedPlotInfo:
-    plot_id, variable, name, version, saved_at, hidden = row
-    return SavedPlotInfo(
-        plot_id=str(plot_id),
-        variable=str(variable),
-        name=str(name),
-        version=int(version),
-        saved_at=str(saved_at),
-        hidden=bool(hidden),
-    )