scistackplotdb 0.1.26__tar.gz → 0.1.29__tar.gz

{scistackplotdb-0.1.26 → scistackplotdb-0.1.29}/.gitignore

@@ -20,9 +20,14 @@ __pycache__/
 *.duckdb.wal
 *.wal
 scistack-gui/frontend/node_modules/
+# Compiled output of the frontend's React-free unit tests (npm test in
+# scistack-gui/frontend). Regenerated by `tsc -p tsconfig.test.json`; unlike
+# extension/dist/, nothing loads it at runtime.
+scistack-gui/frontend/dist/
 # mkdocs build output (regenerated by `mkdocs build`)
 /site/
 # Runtime output of code_export_service (pipeline-to-code export) — timestamped
 # per-run files, not source.
 /exports/
 *.log
+output.txt

{scistackplotdb-0.1.26 → scistackplotdb-0.1.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scistackplotdb
-Version: 0.1.26
+Version: 0.1.29
 Summary: scidb-backed plotting: load variables into long tables and plot them with scistackplot
 Author: SciStack Contributors
 License-Expression: MIT
@@ -10,8 +10,10 @@ Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Visualization
 Classifier: Typing :: Typed
 Requires-Python: >=3.10

{scistackplotdb-0.1.26 → scistackplotdb-0.1.29}/pyproject.toml

@@ -29,8 +29,10 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Scientific/Engineering :: Visualization",
     "Typing :: Typed",
 ]

{scistackplotdb-0.1.26 → scistackplotdb-0.1.29}/src/scistackplotdb/__init__.py

@@ -50,13 +50,19 @@ from .load import (
     schema_keys,
 )
 from .source import ScidbSource
-from .variants import selection_for, variant_graph, variant_set
+from .variants import (
+    branch_params_for,
+    selection_for,
+    variant_graph,
+    variant_set,
+)
 
 __all__ = [
     "ScidbSource",
     "variant_set",
     "variant_graph",
     "selection_for",
+    "branch_params_for",
     "VariableFrame",
     "VERSION_FACTOR_PREFIX",
     "MISSING_VERSION_LEVEL",

{scistackplotdb-0.1.26 → scistackplotdb-0.1.29}/src/scistackplotdb/load.py

@@ -7,8 +7,10 @@ columns once a variable is joined to ``_schema``, which is the same shape
 part a flat CSV never needed: attaching branch params as columns, and knowing
 which schema keys a given variable actually occupies.
 
-Queries go through ``_fetchall``/``_fetchone`` (never ``_execute(...).fetchall()``
-— see docs/claude on DuckDB fetch locking) and batch the branch-params walk
+Queries go through ``_fetchall``/``_fetchone``/``_fetchdf`` (never
+``_execute(...).fetchall()`` — see docs/claude on DuckDB fetch locking); the
+payload itself only ever through ``_fetchdf``, which is what keeps a sample from
+becoming a Python float (see ``load_variable``) and batch the branch-params walk
 rather than asking per record (the N+1 trap).
 """
 
@@ -17,9 +19,11 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from typing import Any
 
+import numpy as np
 import pandas as pd
 from scistacklog import Log
 from scistackplot import CODE_FACTOR_PREFIX
+from scistackplot.framesize import format_extent, frame_extent
 
 LAYER = "scistackplotdb"
 
@@ -179,17 +183,125 @@ def variable_levels(db, variable: str) -> list[str]:
     return [key for key, count in zip(keys, row, strict=True) if count]
 
 
-def load_variable(db, variable: str, *, with_variants: bool = True) -> VariableFrame:
-    """Load every non-excluded record of ``variable`` as a long frame."""
-    with Log.timer("load_variable", layer=LAYER, extra=variable):
-        keys = schema_keys(db)
-        columns = data_columns_for(db, variable)
+def _normalize_cell(value: Any) -> Any:
+    """One data cell as the shape downstream expects, converted PER CELL only.
+
+    A DataFrame fetch already delivers a ``DOUBLE[]`` cell as an ndarray; this
+    leaves it alone. What it handles:
+
+    * a ``DOUBLE[][]`` cell, which DuckDB hands over as an object ndarray of
+      row ndarrays — stacked into one 2-D float array when the rows are
+      rectangular, so ``np.asarray(cell).shape`` is ``(rows, cols)`` as the
+      heatmap path expects; ragged rows become a list of row arrays;
+    * a Python ``list`` cell, from a DuckDB build that still boxes LIST columns
+      on the pandas path — converted once, so the rest of the stack sees one
+      contract. The boxing has already been paid by then; the "loaded" log line
+      says so (``boxed``), which is the signal to look at the DuckDB version.
+
+    A NULL cell, which the DataFrame fetch spells ``pd.NA``, becomes None — the
+    spelling every downstream check (``value is None``) already knows. Scalars,
+    None and NaN pass through.
+    """
+    if value is pd.NA:
+        return None
+    if isinstance(value, np.ndarray):
+        if (
+            value.dtype == object
+            and value.size
+            and isinstance(value.flat[0], (np.ndarray, list))
+        ):
+            try:
+                return np.stack([_float_row(row) for row in value])
+            except ValueError:
+                # Ragged rows: a Python list OF row arrays (one object per row,
+                # not per sample), which `shape.classify_value` still reads
+                # as MATRIX_2D — an object ndarray of ndim 1 would read as a
+                # 1-D series and the heatmap would silently become a line.
+                return list(value)
+        if isinstance(value, np.ma.MaskedArray):
+            # A NULL ELEMENT inside a list — which is what scidb's single-record
+            # INSERT binding stores a NaN as — comes back masked, and the first
+            # `np.asarray` downstream silently drops the mask and exposes the
+            # fill value: a NaN sample became a number, and every "NaN rows are
+            # dropped" comparison in the parity suite failed (2026-09-13).
+            # Filled here, once, with the NaN every consumer already handles.
+            return _float_row(value)
+        return value
+    if isinstance(value, list):
+        try:
+            return np.asarray(value, dtype=float)
+        except (TypeError, ValueError):
+            return np.asarray(value, dtype=object)
+    return value
+
+
+def _float_row(row: Any) -> np.ndarray:
+    """One 1-D float64 array from a row/cell, a masked element becoming NaN."""
+    masked = np.ma.asarray(row, dtype="float64")
+    return np.ma.filled(masked, np.nan)
+
+
+def _object_column(cells: list, index) -> pd.Series:
+    """A Series of exactly these cells, dtype object, no inference.
+
+    Not ``Series.map`` and not ``pd.Series(cells)``: both run dtype inference
+    over the values, and a column of same-length float arrays is precisely the
+    input that inference reinterprets (a 2-D block, a scalar per cell, a string
+    dtype for the keys). Filling a preallocated object array one cell at a time
+    is the one construction every pandas version leaves alone.
+    """
+    out = np.empty(len(cells), dtype=object)
+    for i, cell in enumerate(cells):
+        out[i] = cell
+    return pd.Series(out, index=index, dtype=object)
+
+
+def _key_text(value: Any) -> "str | None":
+    """A schema key as text, or None for a NULL — whether pandas spelled that
+    NULL as None or as NaN."""
+    if value is None:
+        return None
+    if isinstance(value, float) and value != value:
+        return None
+    return str(value)
+
+
+def load_variable(
+    db, variable: str, *, with_variants: bool = True, include_data: bool = True
+) -> VariableFrame:
+    """Load every non-excluded record of ``variable`` as a long frame.
+
+    ``include_data=False`` selects the record ids, schema keys and variant
+    columns but **no data columns** — everything needed to answer a question
+    about a variable's identity and variants, and none of the payload. It exists
+    because the payload is where all the cost is: on 2026-09-13 the data columns
+    of one variable were 174 million samples / ~5.2 GB, and the surfaces that
+    loaded them to read variant metadata simply never returned
+    (.claude/plot-at-scale-plan.md §7). The returned frame reports
+    ``data_columns=[]``, so it is NOT a plottable table and ``_build_table`` will
+    (correctly) refuse it.
+    """
+    with Log.timer(
+        "load_variable",
+        layer=LAYER,
+        extra=variable if include_data else f"{variable} (metadata only)",
+    ) as timer:
+        with timer.phase("column_metadata"):
+            keys = schema_keys(db)
+            columns = data_columns_for(db, variable)
         if not columns:
             Log.warn("variable %r has no data columns", variable, layer=LAYER)
             return VariableFrame(name=variable, frame=pd.DataFrame())
+        if not include_data:
+            columns = []
 
         table = table_name_for(db, variable)
-        schema_select = "".join(f', s."{key}"' for key in keys)
+        # Schema keys as VARCHAR in the query, not stringified after: a
+        # DataFrame fetch types each column, and an integer key with one NULL
+        # among its rows would arrive as float64 and stringify as "1.0".
+        schema_select = "".join(
+            f', CAST(s."{key}" AS VARCHAR) AS "{key}"' for key in keys
+        )
         data_select = "".join(f', t."{column}"' for column in columns)
         query = (
             f"SELECT t.record_id{data_select}{schema_select} "
@@ -198,27 +310,57 @@ def load_variable(db, variable: str, *, with_variants: bool = True) -> VariableF
             f"LEFT JOIN _schema s ON r.schema_id = s.schema_id "
             f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE"
         )
-        rows = db._duck._fetchall(query, [variable])
-
-        frame = pd.DataFrame(
-            rows, columns=["record_id", *columns, *keys]
-        )
-        for key in keys:
-            frame[key] = frame[key].map(lambda v: None if v is None else str(v))
+        # NOTE: no schema filter, no LIMIT, no column projection — this reads
+        # EVERY non-excluded record of the variable with EVERY data column,
+        # whatever the caller intends to plot (pushdown is a later stage of
+        # .claude/plan-plot-minimal-load-examples.md).
+        #
+        # `_fetchdf`, never `_fetchall`, for the data columns. Measured on the
+        # real database 2026-09-13 (plan §7): one DOUBLE[] column of 17.4 M
+        # samples took 4.7 s through `fetchall` — a Python float per sample —
+        # and 0.26 s through `.df()`, which hands each cell over as one numpy
+        # buffer. That 18x was 86 of the 91 s `plot_describe` spent before it
+        # timed out. The "loaded" line below says `ndarray` or `boxed` so a
+        # regression to per-sample boxing is visible in the log, not inferred.
+        with timer.phase("fetch"):
+            frame = db._duck._fetchdf(query, [variable])
+
+        with timer.phase("dataframe"):
+            frame = frame[["record_id", *columns, *keys]].copy()
+            for column in columns:
+                if pd.api.types.is_numeric_dtype(frame[column]):
+                    continue  # a scalar column: already one float64 buffer
+                frame[column] = _object_column(
+                    [_normalize_cell(v) for v in frame[column].to_numpy()],
+                    frame.index,
+                )
+        with timer.phase("stringify_keys"):
+            for key in keys:
+                frame[key] = _object_column(
+                    [_key_text(v) for v in frame[key].to_numpy()], frame.index
+                )
 
         levels = [key for key in keys if frame[key].notna().any()]
         variant_columns: list[str] = []
         variant_axes: list[dict] = []
         latest_column: str | None = None
         if with_variants and len(frame):
-            frame, variant_columns, latest_column, variant_axes = attach_variants(
-                db, frame
-            )
-
+            with timer.phase("attach_variants"):
+                frame, variant_columns, latest_column, variant_axes = attach_variants(
+                    db, frame
+                )
+
+        # Cells and SAMPLES alongside the record count: 419 records is the same
+        # number whether each holds 200 samples or 250,000, and only the second
+        # explains a plot that never returns. Measured over the data columns
+        # only, one O(cells) pass (scistackplot.framesize).
+        with timer.phase("measure_extent"):
+            extent = frame_extent(frame, columns)
         Log.info(
-            "loaded %s: %d record(s), levels=%s, variants=%s",
+            "loaded %s: %d record(s), %s, levels=%s, variants=%s",
             variable,
             len(frame),
+            format_extent(extent),
             levels,
             variant_columns or "none",
             layer=LAYER,

{scistackplotdb-0.1.26 → scistackplotdb-0.1.29}/src/scistackplotdb/source.py

@@ -22,6 +22,8 @@ from scistackplot import (
     is_plottable,
     natural_sort_key,
 )
+from scistackplot.dedup import SingleFlight
+from scistackplot.framesize import format_extent, frame_extent
 from scistackplot.sources import BaseSource
 from scistackplot.variants import VARIABLE_COLUMN
 
@@ -56,7 +58,9 @@ class ScidbSource(BaseSource):
     run-ownership work resolved).
     """
 
-    def __init__(self, db, *, name: str | None = None) -> None:
+    def __init__(
+        self, db, *, name: str | None = None, fast: bool = True
+    ) -> None:
         self._db = db
         # `dataset_db_path`, not `db_path` — DatabaseManager has never had the
         # latter, so this silently fell through to "scidb" for every project.
@@ -64,6 +68,30 @@ class ScidbSource(BaseSource):
         self._frames: dict[str, Any] = {}
         self._shapes: dict[str, Shape] = {}
         self._levels: dict[str, list[str]] = {}
+        # Whether tables from this source carry the numpy reducer
+        # (``scistackplot.NumpyReducer``, over the ndarray cells `load_variable`
+        # delivers) or the pandas reference. On by default — the reference is
+        # the path that spent 500 s on 174 M samples — and switchable so the
+        # two can be run side by side on one database, which is how the parity
+        # suite works and how a suspected disagreement gets bisected.
+        #
+        # A DuckDB-SQL reducer sat here for one day (2026-09-13) and lost every
+        # measurement to numpy once the fetch stopped boxing samples
+        # (.claude/plan-plot-minimal-load-examples.md §8). DuckDB selects rows;
+        # numpy reduces them — so `resolve` no longer touches the database and
+        # the GUI's connection hold ends when the frames are loaded.
+        self._fast = fast
+        self._reducer_instance = None
+
+    def _reducer(self):
+        """The reducer every table from this source carries (one per source)."""
+        if not self._fast:
+            return None  # -> reducer_for() supplies the pandas reference
+        if self._reducer_instance is None:
+            from scistackplot.reducer import NumpyReducer
+
+            self._reducer_instance = NumpyReducer()
+        return self._reducer_instance
 
     # ---- description -----------------------------------------------------
 
@@ -164,12 +192,124 @@ class ScidbSource(BaseSource):
             return sorted(unique, key=numeric_key)
         return sorted(unique, key=natural_sort_key)
 
+    # ---- metadata --------------------------------------------------------
+
+    def variant_table(self, variable: str) -> LongTable:
+        """A table carrying this variable's VARIANT structure and no payload.
+
+        Same variant columns, levels, ``default_pin`` and latest-flag a full
+        ``get_table`` would produce, built from a query that selects no data
+        columns at all. ``measures`` is empty, so this is not plottable and must
+        not be handed to ``resolve`` — it answers metadata questions only.
+
+        Why it exists: :func:`scistackplot.default_selection` reads
+        ``default_pin``, ``latest_column`` and the variant factors' levels, and
+        nothing else — it never touches a measure column. Answering it through
+        ``get_table`` meant loading 174 million samples / ~5.2 GB to read a
+        handful of variant levels, which is why the schema-location picker timed
+        out on a 419-location variable (.claude/plot-at-scale-plan.md §7).
+
+        Cached and deduplicated like any other table, under its own key, so the
+        picker opening twice costs one query.
+        """
+        key = ("__variants__", variable)
+        memo = self._table_cache()
+        label = f"variant_table({variable})"
+        if key in memo:
+            Log.info("%s: table cache HIT", label, layer=LAYER)
+            return memo[key]
+
+        def _build():
+            if key in memo:
+                Log.info("%s: table cache HIT (filled while waiting)", label, layer=LAYER)
+                return memo[key]
+            Log.info("%s: table cache MISS — building (no data columns)", label, layer=LAYER)
+            variable_frame = load_variable(self._db, variable, include_data=False)
+            frame = variable_frame.frame
+            latest = variable_frame.latest_column
+            # Variant columns first, then schema keys — the same order and the
+            # same `_ordered` level sorting `_build_table` uses, so a selection
+            # derived here and one derived from the full table cannot disagree
+            # about which level is "first".
+            variant_columns = [
+                c for c in variable_frame.variant_columns if c in frame.columns
+            ]
+            factors = list(variant_columns)
+            factors.extend(key for key in variable_frame.levels if key in frame.columns)
+            level_order = {
+                name: self._ordered(
+                    name, [str(v) for v in frame[name].dropna().unique()]
+                )
+                for name in factors
+            }
+            table = LongTable.from_frame(
+                frame,
+                factors=factors,
+                measures=[],
+                level_order=level_order,
+                variant_factors=variant_columns,
+                name=variable,
+                default_pin={latest: True} if latest else None,
+                latest_column=latest,
+                factor_origins={
+                    axis["column"]: axis for axis in variable_frame.variant_axes
+                },
+                schema_levels=variable_frame.levels,
+            )
+            memo[key] = table
+            return table
+
+        return self._table_single_flight().run(
+            key,
+            _build,
+            on_wait=lambda: Log.info(
+                "%s: build already in flight — waiting for it", label, layer=LAYER
+            ),
+        )
+
     # ---- data ------------------------------------------------------------
 
     def _variable_frame(self, variable: str):
-        if variable not in self._frames:
-            self._frames[variable] = load_variable(self._db, variable)
-        return self._frames[variable]
+        # Hit/miss at INFO: this cache is the difference between a request that
+        # reads the whole variable and one that reads nothing, and a plot request
+        # that returned in milliseconds is otherwise unattributable
+        # (.claude/plot-at-scale-plan.md §1). The layer below (load_variable)
+        # reports what a miss actually cost.
+        if variable in self._frames:
+            Log.info("variable frame cache HIT: %s", variable, layer=LAYER)
+            return self._frames[variable]
+
+        def _load():
+            if variable in self._frames:
+                Log.info(
+                    "variable frame cache HIT: %s (filled while waiting)",
+                    variable,
+                    layer=LAYER,
+                )
+                return self._frames[variable]
+            Log.info("variable frame cache MISS: %s — loading", variable, layer=LAYER)
+            frame = load_variable(self._db, variable)
+            self._frames[variable] = frame
+            return frame
+
+        # Deduped for the same reason get_table is, one layer down: two DIFFERENT
+        # table keys over the same variable (a plot's measure and a factor join,
+        # say) both land here, and each would otherwise read the whole variable.
+        return self._frame_single_flight().run(
+            variable,
+            _load,
+            on_wait=lambda: Log.info(
+                "variable frame load already in flight: %s — waiting for it",
+                variable,
+                layer=LAYER,
+            ),
+        )
+
+    def _frame_single_flight(self):
+        """The in-flight map for :meth:`_variable_frame`, created on first use."""
+        from scistackplot.sources.base import _lazy_attr
+
+        return _lazy_attr(self, "_frame_inflight", SingleFlight)
 
     def _build_table(
         self,
@@ -330,6 +470,7 @@ class ScidbSource(BaseSource):
             factors,
             layer=LAYER,
         )
+        table.reducer = self._reducer()
         return table
 
     def _attach_factor_variables(
@@ -546,6 +687,7 @@ class ScidbSource(BaseSource):
             levels,
             layer=LAYER,
         )
+        table.reducer = self._reducer()
         return table
 
     def variant_graph(self, variable: str, functions: list[str] | None = None) -> dict:
@@ -607,17 +749,27 @@ class ScidbSource(BaseSource):
         while field_factor in id_vars:  # never shadow a schema key
             field_factor += "_"
 
-        melted = frame.melt(
-            id_vars=id_vars,
-            value_vars=usable,
-            var_name=field_factor,
-            value_name=measure,
-        )
+        # Timed because this is where the row count multiplies by the field
+        # count — a 10-field record becomes 10 rows, each still holding a whole
+        # signal — and every one of those cells is carried through the rest of
+        # the pipeline whether or not the plot asks for that field.
+        with Log.timer(
+            f"melt_fields({variable_frame.name})",
+            layer=LAYER,
+            extra=f"{len(usable)} field(s)",
+        ):
+            melted = frame.melt(
+                id_vars=id_vars,
+                value_vars=usable,
+                var_name=field_factor,
+                value_name=measure,
+            )
         Log.info(
-            "melted %r: %d field(s) -> %d row(s), field factor %r",
+            "melted %r: %d field(s) -> %d row(s), %s, field factor %r",
             variable_frame.name,
             len(usable),
             len(melted),
+            format_extent(frame_extent(melted, [measure])),
             field_factor,
             layer=LAYER,
         )

{scistackplotdb-0.1.26 → scistackplotdb-0.1.29}/src/scistackplotdb/variants.py

@@ -95,6 +95,55 @@ def selection_for(
     return selection
 
 
+def branch_params_for(selection: dict[str, Any]) -> dict[str, Any]:
+    """The inverse of :func:`selection_for`: a column-keyed selection →
+    ``Variant.branch_params``.
+
+    Needed because the schema location picker asks **scidb** a question about a
+    variant the plotting layer is holding: "which locations have this one?"
+    ``scidb.locations.location_states`` takes a ``branch_params_filter``, the
+    same dict ``Variant`` builds and ``find_record_id`` matches on, so the
+    picker's dots and a ``Variant(...).load()`` cannot disagree about which
+    records a selection names.
+
+    Three mappings, and the third is the one worth reading twice:
+
+    * ``Code:<fn>`` → ``__code__.<fn>``;
+    * anything else is already ``fn.param``, scidb's own namespacing, and
+      passes through — including a **list** value, which scidb already reads as
+      membership (``database.py``'s list-valued branch params), so the picker's
+      multi-checkbox needs no new scidb work;
+    * the :data:`~scistackplotdb.load.LATEST_COLUMN` flag → ``__code__ =
+      "latest"``. Both spell the same **per-location** rule (each schema
+      location contributes its own newest record, rather than the global highest
+      ordinal), which is exactly the meaning this picker needs: a location never
+      re-run under the newest code must show up as the record it actually has,
+      not as red.
+
+    ``CodeIsLatest: False`` has no scidb spelling — "not the latest" is not a
+    pin — so it is dropped with a warning rather than silently inverted.
+    """
+    from scidb.variant import CODE_PIN_PREFIX, LATEST_VERSION
+
+    out: dict[str, Any] = {}
+    for key, value in (selection or {}).items():
+        if key == LATEST_COLUMN:
+            if value is False:
+                Log.warn(
+                    "selection pins %s=False, which scidb cannot express "
+                    "(there is no 'not the latest' pin) — ignoring it",
+                    LATEST_COLUMN,
+                    layer=LAYER,
+                )
+                continue
+            out[CODE_PIN_PREFIX] = LATEST_VERSION
+        elif key.startswith(CODE_FACTOR_PREFIX):
+            out[f"{CODE_PIN_PREFIX}.{key[len(CODE_FACTOR_PREFIX):]}"] = value
+        else:
+            out[key] = value
+    return out
+
+
 def _axes_of(table: LongTable | None) -> list[dict]:
     if table is None:
         return []