scistackplotdb 0.1.66__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scistackplotdb
3
- Version: 0.1.66
3
+ Version: 0.2.0
4
4
  Summary: scidb-backed plotting: load variables into long tables and plot them with scistackplot
5
5
  Author: SciStack Contributors
6
6
  License-Expression: MIT
@@ -17,10 +17,11 @@ Classifier: Programming Language :: Python :: 3.13
17
17
  Classifier: Topic :: Scientific/Engineering :: Visualization
18
18
  Classifier: Typing :: Typed
19
19
  Requires-Python: >=3.10
20
- Requires-Dist: sciduckdb==0.1.66
21
- Requires-Dist: scistack-db==0.1.66
22
- Requires-Dist: scistacklog==0.1.66
23
- Requires-Dist: scistackplot==0.1.66
20
+ Requires-Dist: sciduckdb==0.2.0
21
+ Requires-Dist: scifor==0.2.0
22
+ Requires-Dist: scistack-db==0.2.0
23
+ Requires-Dist: scistacklog==0.2.0
24
+ Requires-Dist: scistackplot==0.2.0
24
25
  Provides-Extra: dev
25
26
  Requires-Dist: matplotlib>=3.6; extra == 'dev'
26
27
  Requires-Dist: pytest-cov>=4.0; extra == 'dev'
@@ -184,7 +185,7 @@ else goes through a natural sort so zero-padded IDs land as
184
185
  `01, 02, … 10` instead of `01, 10, 02`.
185
186
 
186
187
  A project that declares its levels (`[schema_keys]` in scistack.toml, read by
187
- `scidb.schema_order`) wins over both: declared levels first, in that order,
188
+ `scifor.project.schema_order`) wins over both: declared levels first, in that order,
188
189
  the rest after them by the rule above. The declaration is read live, and a
189
190
  change drops the source's built tables, so an edit shows on the next plot
190
191
  request. Exported seaborn code states every order it relies on (`order=`,
@@ -154,7 +154,7 @@ else goes through a natural sort so zero-padded IDs land as
154
154
  `01, 02, … 10` instead of `01, 10, 02`.
155
155
 
156
156
  A project that declares its levels (`[schema_keys]` in scistack.toml, read by
157
- `scidb.schema_order`) wins over both: declared levels first, in that order,
157
+ `scifor.project.schema_order`) wins over both: declared levels first, in that order,
158
158
  the rest after them by the rule above. The declaration is read live, and a
159
159
  change drops the source's built tables, so an edit shows on the next plot
160
160
  request. Exported seaborn code states every order it relies on (`order=`,
@@ -12,7 +12,7 @@ raw-options = { search_parent_directories = true, local_scheme = "no-local-versi
12
12
  # dependencies must be named here: CI's build job fails a wheel whose internal
13
13
  # dependency is not "==<its own version>".
14
14
  [tool.hatch.metadata.hooks.dependency-coversion]
15
- override-versions-of = ["scistackplot", "scistack-db", "sciduckdb", "scistacklog"]
15
+ override-versions-of = ["scistackplot", "scistack-db", "sciduckdb", "scifor", "scistacklog"]
16
16
 
17
17
  [project]
18
18
  name = "scistackplotdb"
@@ -48,6 +48,7 @@ dependencies = [
48
48
  "scistackplot",
49
49
  "scistack-db",
50
50
  "sciduckdb",
51
+ "scifor",
51
52
  "scistacklog",
52
53
  ]
53
54
 
@@ -65,6 +65,7 @@ from .saved import (
65
65
  save_plot,
66
66
  saved_plot_history,
67
67
  )
68
+ from .designs import DesignError, StoredDesign, apply_design, is_linked, link_design, load_design, save_design
68
69
  from .presets import (
69
70
  AppliedPreset,
70
71
  Preset,
@@ -91,6 +92,13 @@ from .variants import (
91
92
  )
92
93
 
93
94
  __all__ = [
95
+ "DesignError",
96
+ "StoredDesign",
97
+ "apply_design",
98
+ "is_linked",
99
+ "link_design",
100
+ "load_design",
101
+ "save_design",
94
102
  "ScidbSource",
95
103
  # a wide variable's text fields as their own measure
96
104
  "TEXT_SUFFIX",
@@ -1,5 +1,5 @@
1
1
  """
2
- The ``plots`` section of a project bundle (``scidb.bundle``): saved plots and
2
+ The ``plots`` section of a project bundle (``scidb.project.bundle``): saved plots and
3
3
  plot presets, copied row for row.
4
4
 
5
5
  Both stores key nothing by canvas ids or database record ids -- a saved plot
@@ -15,11 +15,16 @@ import json
15
15
 
16
16
  from scistacklog import Log
17
17
 
18
- from . import presets, saved
18
+ from . import designs, presets, saved
19
19
 
20
20
  LAYER = "scistackplotdb"
21
21
 
22
- _STORES = {"saved_plots.json": saved.STORE, "presets.json": presets.STORE}
22
+ _STORES = {
23
+ "saved_plots.json": saved.STORE,
24
+ "presets.json": presets.STORE,
25
+ # The shared design of a linked figure + table (designs.py).
26
+ "designs.json": designs.STORE,
27
+ }
23
28
 
24
29
 
25
30
  class PlotsSection:
@@ -41,19 +46,7 @@ class PlotsSection:
41
46
  if filename in files:
42
47
  rows = json.loads(files[filename].decode("utf-8"))
43
48
  if key_map is not None and not key_map.is_identity:
44
- rows = [_remap_row(row, store, key_map, ctx.map_report) for row in rows]
49
+ rows = [store.remap_row(row, key_map, ctx.map_report) for row in rows]
45
50
  report[store.table] = store.load_rows(ctx.db, rows)
46
51
  Log.info("[bundle_section] plots import: %s", report, layer=LAYER)
47
52
  return report
48
-
49
-
50
- def _remap_row(row: dict, store, key_map, map_report) -> dict:
51
- """A saved plot or preset carried into another schema (portability Stage
52
- 6). A spec names factors in many fields (roles, groups, facets, colours,
53
- filters, difference bars, ...), so every string EQUAL to an exporter key
54
- is renamed (``KeyMap.exact_strings``) and the item is flagged for review
55
- -- a factor that merely equals a key name cannot be told apart."""
56
- envelope = json.loads(row["envelope_json"])
57
- where = f"{store.noun} {row.get('name')!r}"
58
- mapped = key_map.exact_strings(envelope, where, map_report)
59
- return {**row, "envelope_json": json.dumps(mapped)}
@@ -0,0 +1,148 @@
1
+ """
2
+ The shared design of a linked figure and table: one owner
3
+ (docs/claude/studio-unification.md, user decision 2026-11-01).
4
+
5
+ A saved plot and the StatStudio tables linked to it used to each keep their
6
+ own copy of the design, with "Sync to…" banners to copy one onto the other.
7
+ Now the design of a linked pair lives in ONE row set here, keyed by the
8
+ saved plot's ``plot_id``. Either studio writes it; both read it on open, and
9
+ an edit in one reaches the other through the GUI's broadcast.
10
+
11
+ What the design is, field by field, is still
12
+ :data:`scistackplot.model.design.DESIGN_FIELDS`. This store keeps a whole
13
+ ``PlotSpec`` dict (so it reads back through ``restore_spec``) but only its
14
+ design fields are ever applied (:func:`apply_design`, through
15
+ ``copy_design``). Display settings stay with each artifact.
16
+
17
+ **A design exists only once something is linked.** :func:`link_design` is
18
+ called when a StatStudio table is set up from a saved plot. An unlinked plot
19
+ has no row here and its saved envelope stays the owner of its design.
20
+
21
+ Rows are the shared ``VersionedStore`` (append-only, never deleted): the
22
+ item's NAME is the plot id, so "the design of plot X" is a name lookup, and
23
+ each change is a new version. A write that would not change the design
24
+ (same ``design_fingerprint``) appends nothing.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ from dataclasses import dataclass
30
+ from typing import Any
31
+
32
+ from scistacklog import Log
33
+
34
+ from scidb.store.versioned_store import VersionedStore
35
+
36
+ LAYER = "scistackplotdb"
37
+
38
+ TABLE = "_plot_design"
39
+
40
+ ENVELOPE_FORMAT = 1
41
+
42
+
43
+ class DesignError(ValueError):
44
+ """A design that cannot be stored or read as asked."""
45
+
46
+
47
+ STORE = VersionedStore(
48
+ table=TABLE,
49
+ id_column="design_id",
50
+ scope_column=None,
51
+ noun="shared design",
52
+ tag="design",
53
+ error=DesignError,
54
+ owner="scistackplotdb",
55
+ )
56
+
57
+
58
+ @dataclass(frozen=True)
59
+ class StoredDesign:
60
+ plot_id: str
61
+ design_id: str
62
+ version: int
63
+ #: The stored spec (a full PlotSpec; only its design fields apply).
64
+ spec: Any
65
+ fingerprint: str
66
+
67
+
68
+ def _row_for(db, plot_id: str):
69
+ if not plot_id:
70
+ return None
71
+ return STORE.visible_by_name(db, None, plot_id)
72
+
73
+
74
+ def load_design(db, plot_id: str) -> StoredDesign | None:
75
+ """The newest shared design of saved plot *plot_id*, or None when it is
76
+ not linked to anything (or the row cannot be read)."""
77
+ from scistackplot import restore_spec
78
+ from scistackplot.model.design import design_fingerprint
79
+
80
+ row = _row_for(db, plot_id)
81
+ if row is None:
82
+ return None
83
+ _, text = STORE.load(db, row[0])
84
+ envelope = STORE.parse_envelope(text, f"design of {plot_id[:8]}")
85
+ raw = envelope.get("spec")
86
+ if not isinstance(raw, dict):
87
+ Log.warn("[design] %s: stored design has no spec; ignoring it", plot_id[:8], layer=LAYER)
88
+ return None
89
+ restored = restore_spec(raw, fallback_measure=(raw.get("measures") or [None])[0])
90
+ for note in restored.notes:
91
+ Log.info("[design] %s: restore note: %s", plot_id[:8], getattr(note, "message", note), layer=LAYER)
92
+ spec = restored.spec
93
+ return StoredDesign(plot_id, row[0], row[3], spec, design_fingerprint(spec))
94
+
95
+
96
+ def is_linked(db, plot_id: str | None) -> bool:
97
+ return bool(plot_id) and _row_for(db, plot_id) is not None
98
+
99
+
100
+ def save_design(db, plot_id: str, spec) -> tuple[StoredDesign, bool]:
101
+ """Write *spec*'s design as the shared design of *plot_id*.
102
+
103
+ Returns ``(stored, changed)``. When the design fields are the same as
104
+ the newest stored version, nothing is appended and ``changed`` is
105
+ False, so a broadcast echo between two open studios stops here.
106
+ """
107
+ from scistackplot.model.design import design_fingerprint
108
+
109
+ if not plot_id:
110
+ raise DesignError("A shared design needs the saved plot it belongs to.")
111
+ fingerprint = design_fingerprint(spec)
112
+ current = load_design(db, plot_id)
113
+ if current is not None and current.fingerprint == fingerprint:
114
+ Log.debug("[design] %s: unchanged (fingerprint %s); nothing appended", plot_id[:8], fingerprint,
115
+ layer=LAYER)
116
+ return current, False
117
+ envelope = {"format": ENVELOPE_FORMAT, "plot_id": plot_id, "spec": spec.to_dict()}
118
+ row, is_new = STORE.save(db, None, plot_id, envelope, overwrite=True, current_id=None,
119
+ on_clash=lambda r: DesignError(f"design clash for {plot_id}"))
120
+ Log.info("[design] %s: %s version %d (fingerprint %s)", plot_id[:8],
121
+ "linked, first" if is_new else "changed, new", row[3], fingerprint, layer=LAYER)
122
+ return StoredDesign(plot_id, row[0], row[3], spec, fingerprint), True
123
+
124
+
125
+ def link_design(db, plot_id: str, spec) -> StoredDesign:
126
+ """Make *plot_id*'s design shared, starting from *spec* when nothing is
127
+ stored yet. An already-linked plot keeps its stored design."""
128
+ current = load_design(db, plot_id)
129
+ if current is not None:
130
+ Log.info("[design] %s: already linked (version %d)", plot_id[:8], current.version, layer=LAYER)
131
+ return current
132
+ stored, _ = save_design(db, plot_id, spec)
133
+ return stored
134
+
135
+
136
+ def apply_design(db, plot_id: str | None, spec) -> tuple[Any, list, int | None]:
137
+ """*spec* with the shared design of *plot_id* copied onto it
138
+ (``copy_design``: design fields only, display kept). Unchanged, with
139
+ version None, when *plot_id* is not linked."""
140
+ from scistackplot.model.design import copy_design
141
+
142
+ stored = load_design(db, plot_id) if plot_id else None
143
+ if stored is None:
144
+ return spec, [], None
145
+ merged, notes = copy_design(stored.spec, spec)
146
+ Log.info("[design] %s: applied version %d onto a %s spec (%d note(s))", plot_id[:8], stored.version,
147
+ getattr(spec.kind, "value", spec.kind), len(notes), layer=LAYER)
148
+ return merged, notes, stored.version
@@ -2,7 +2,7 @@
2
2
  Turning a ``PlotSpec`` into a pipeline endpoint.
3
3
 
4
4
  The GUI's "Add to pipeline" produces two things: a ``plot_`` function (generated
5
- by ``scistackplot.codegen`` — literal seaborn, no runtime dependency on this
5
+ by ``scistackplot.export.codegen`` — literal seaborn, no runtime dependency on this
6
6
  package) and the ``for_each`` call that runs it. This module owns the second
7
7
  half, and with it the one translation that has to be exactly right:
8
8
 
@@ -26,9 +26,9 @@ from dataclasses import dataclass, field
26
26
  from scistacklog import Log
27
27
  from scistackplot import LongTable, PlotSpec, default_function_name
28
28
  from scistackplot import generate_plot_function
29
- from scistackplot.codegen import group_param, variant_params
30
- from scistackplot.roles import fanout_keys
31
- from scistackplot.variants import defined_sets
29
+ from scistackplot.export.codegen import group_param, variant_params
30
+ from scistackplot.model.roles import fanout_keys
31
+ from scistackplot.reduce.variants import defined_sets
32
32
 
33
33
  from .load import LATEST_COLUMN
34
34
  from .text_fields import base_variable
@@ -147,7 +147,7 @@ def _pin_literal(value) -> str:
147
147
 
148
148
  def row_expression(input_variable: str, variant_set, table=None) -> str:
149
149
  """One variant ROW as a ``Variant(...)`` load — through the row's
150
- EFFECTIVE selection (``scistackplot.variants.export_selection``), so the
150
+ EFFECTIVE selection (``scistackplot.reduce.variants.export_selection``), so the
151
151
  exported code loads what the preview drew: current code for a row that
152
152
  names none, the frozen version for a reopened saved plot.
153
153
 
@@ -156,7 +156,7 @@ def row_expression(input_variable: str, variant_set, table=None) -> str:
156
156
  """
157
157
  from types import SimpleNamespace
158
158
 
159
- from scistackplot.variants import export_selection
159
+ from scistackplot.reduce.variants import export_selection
160
160
 
161
161
  # The TABLE's default, exactly what the preview applied: a table with
162
162
  # none (nothing superseded) exports its rows as written.
@@ -181,7 +181,7 @@ def variant_expression(input_variable: str, variant_set) -> str:
181
181
 
182
182
  Variant(Variant(EMG, fn="loadEMG", code_version="v1"), fn="bandpass", low_hz=20)
183
183
 
184
- Nesting is documented, merges the two filters (``scidb.variant.Variant``
184
+ Nesting is documented, merges the two filters (``scidb.model.variant.Variant``
185
185
  handles it explicitly), and keeps every keyword readable — where
186
186
  ``**{"__code__.loadEMG": "v1"}`` would leak a reserved namespace into code a
187
187
  user is meant to edit.
@@ -7,11 +7,12 @@ columns once a variable is joined to ``_schema``, which is the same shape
7
7
  part a flat CSV never needed: attaching branch params as columns, and knowing
8
8
  which schema keys a given variable actually occupies.
9
9
 
10
- Queries go through ``_fetchall``/``_fetchone``/``_fetchdf`` (never
11
- ``_execute(...).fetchall()`` — see docs/claude on DuckDB fetch locking); the
12
- payload itself only ever through ``_fetchdf``, which is what keeps a sample from
13
- becoming a Python float (see ``load_variable``) and batch the branch-params walk
14
- rather than asking per record (the N+1 trap).
10
+ The storage SQL is scidb's: every read of a variable's records goes through
11
+ ``db.record_store`` (``scidb.store.record_store.RecordStore``: the catalog, one
12
+ sampled cell, the long frame), which fetches the payload only through
13
+ ``fetchdf`` so a sample never becomes a Python float (see ``load_variable``).
14
+ The branch-params walk is batched rather than asked per record (the N+1
15
+ trap).
15
16
  """
16
17
 
17
18
  from __future__ import annotations
@@ -24,9 +25,9 @@ import pandas as pd
24
25
  from sciduckdb import merge_tz_columns, stored_tz_companions, tz_column_name
25
26
  from scistacklog import Log
26
27
  from scistackplot import CODE_FACTOR_PREFIX, RUN_FACTOR_PREFIX, element_values
27
- from scistackplot.framesize import format_extent, frame_extent
28
- from scistackplot.shape import Shape, classify_value
29
- from scistackplot.table import natural_sort_key
28
+ from scistackplot.text.framesize import format_extent, frame_extent
29
+ from scistackplot.model.shape import Shape, classify_value
30
+ from scistackplot.model.table import natural_sort_key
30
31
 
31
32
  LAYER = "scistackplotdb"
32
33
 
@@ -100,7 +101,7 @@ class VariableFrame:
100
101
  #: Data columns stored as ``row_list`` ("combine rows": one element per
101
102
  #: source row, sciduckdb.row_lists). Their cells arrive as arrays; the
102
103
  #: source splits them into rows with a ``Row`` factor
103
- #: (scistackplot.rows), never plots them as 1-D series.
104
+ #: (scistackplot.reduce.rows), never plots them as 1-D series.
104
105
  row_list_columns: list[str] = field(default_factory=list)
105
106
 
106
107
  @property
@@ -109,12 +110,11 @@ class VariableFrame:
109
110
 
110
111
 
111
112
  def schema_keys(db) -> list[str]:
112
- return list(db._duck.dataset_schema)
113
+ return db.record_store.schema_keys()
113
114
 
114
115
 
115
116
  def registered_variables(db) -> list[str]:
116
- rows = db._duck._fetchall("SELECT variable_name FROM _variables ORDER BY variable_name")
117
- return [row[0] for row in rows]
117
+ return db.record_store.variable_names()
118
118
 
119
119
 
120
120
  def table_name_for(db, variable: str) -> str:
@@ -126,10 +126,7 @@ def table_name_for(db, variable: str) -> str:
126
126
  filters on ``_record.type`` — reading a shared table without that filter
127
127
  would silently mix two variables' records into one plot.
128
128
  """
129
- row = db._duck._fetchone(
130
- "SELECT table_name FROM _registered_types WHERE type_name = ?", [variable]
131
- )
132
- return row[0] if row and row[0] else f"{variable}_data"
129
+ return db.record_store.table_name(variable)
133
130
 
134
131
 
135
132
  def data_columns_for(db, variable: str) -> list[str]:
@@ -139,14 +136,7 @@ def data_columns_for(db, variable: str) -> list[str]:
139
136
  def _stored_column_types(db, variable: str) -> dict[str, str]:
140
137
  """``{column: DuckDB type}`` of the variable's table as stored (zone
141
138
  companions included), in ordinal order."""
142
- table = table_name_for(db, variable)
143
- rows = db._duck._fetchall(
144
- "SELECT column_name, data_type FROM information_schema.columns "
145
- "WHERE table_name = ? AND column_name != 'record_id' "
146
- "ORDER BY ordinal_position",
147
- [table],
148
- )
149
- return {row[0]: str(row[1] or "") for row in rows}
139
+ return db.record_store.stored_column_types(variable)
150
140
 
151
141
 
152
142
  def datetime_zone_meta(db, variable: str, columns) -> dict:
@@ -283,15 +273,7 @@ def sample_value(db, variable: str) -> Any:
283
273
  columns = data_columns_for(db, variable)
284
274
  if not columns:
285
275
  return None
286
- table = table_name_for(db, variable)
287
- row = db._duck._fetchone(
288
- f'SELECT t."{columns[0]}" FROM "{table}" t '
289
- f"JOIN _record r ON t.record_id = r.record_id "
290
- f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE "
291
- f"AND t.\"{columns[0]}\" IS NOT NULL LIMIT 1",
292
- [variable],
293
- )
294
- value = row[0] if row else None
276
+ value = db.record_store.sample_cell(variable, columns[0])
295
277
  # A row list's cell is its source rows, not a series: classify by one
296
278
  # ELEMENT, the shape the source turns it into (rows + a Row factor).
297
279
  if value is not None and _row_list_columns(db, variable, columns[:1]):
@@ -308,15 +290,7 @@ def sample_column_value(db, variable: str, column: str) -> Any:
308
290
  in every column, so "what is this variable" cannot answer "may this column
309
291
  group a figure".
310
292
  """
311
- table = table_name_for(db, variable)
312
- row = db._duck._fetchone(
313
- f'SELECT t."{column}" FROM "{table}" t '
314
- f"JOIN _record r ON t.record_id = r.record_id "
315
- f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE "
316
- f'AND t."{column}" IS NOT NULL LIMIT 1',
317
- [variable],
318
- )
319
- return row[0] if row else None
293
+ return db.record_store.sample_cell(variable, column)
320
294
 
321
295
 
322
296
  def field_shapes(db, variable: str) -> dict[str, Shape]:
@@ -361,16 +335,7 @@ def column_levels(db, variable: str, column: str, *, limit: int) -> list[str]:
361
335
  NULL arrives as float64 in pandas and would spell its integer levels
362
336
  ``1.0``, which nothing downstream matches.
363
337
  """
364
- table = table_name_for(db, variable)
365
- rows = db._duck._fetchall(
366
- f'SELECT DISTINCT CAST(t."{column}" AS VARCHAR) AS level FROM "{table}" t '
367
- f"JOIN _record r ON t.record_id = r.record_id "
368
- f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE "
369
- f'AND t."{column}" IS NOT NULL '
370
- f"LIMIT {int(limit)}",
371
- [variable],
372
- )
373
- return [row[0] for row in rows]
338
+ return db.record_store.column_levels(variable, column, limit=limit)
374
339
 
375
340
 
376
341
  def variable_levels(db, variable: str) -> list[str]:
@@ -382,19 +347,7 @@ def variable_levels(db, variable: str) -> list[str]:
382
347
  database just to open the panel. COUNT ignores NULLs, so one row of counts
383
348
  says exactly which keys are populated.
384
349
  """
385
- keys = schema_keys(db)
386
- if not keys:
387
- return []
388
- counts = ", ".join(f'COUNT(s."{key}")' for key in keys)
389
- row = db._duck._fetchone(
390
- f"SELECT {counts} FROM _record r "
391
- f"LEFT JOIN _schema s ON r.schema_id = s.schema_id "
392
- f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE",
393
- [variable],
394
- )
395
- if row is None:
396
- return []
397
- return [key for key, count in zip(keys, row, strict=True) if count]
350
+ return db.record_store.occupied_keys(variable)
398
351
 
399
352
 
400
353
  def _normalize_cell(value: Any) -> Any:
@@ -493,7 +446,7 @@ def _row_list_columns(db, variable: str, columns) -> list[str]:
493
446
  ``on_duplicate="combine"``). This reader selects the stored cells as they
494
447
  are, so EVERY cell of such a column is an array (``[7]``, ``[5, 12]``).
495
448
  The source splits them into rows with a ``Row`` factor
496
- (scistackplot.rows) — they are observations, not a 1-D series. This
449
+ (scistackplot.reduce.rows) — they are observations, not a 1-D series. This
497
450
  replaced the "read as 1-D" decision of
498
451
  .claude/plan-combine-duplicate-locations.md §6 on 2026-10-30
499
452
  (.claude/plan-row-list-plots.md)."""
@@ -541,32 +494,19 @@ def load_variable(
541
494
  if not include_data:
542
495
  columns = []
543
496
 
544
- table = table_name_for(db, variable)
545
- # Schema keys as VARCHAR in the query, not stringified after: a
546
- # DataFrame fetch types each column, and an integer key with one NULL
547
- # among its rows would arrive as float64 and stringify as "1.0".
548
- schema_select = "".join(
549
- f', CAST(s."{key}" AS VARCHAR) AS "{key}"' for key in keys
550
- )
551
497
  # Datetime columns bring their __tz zone column, folded back right
552
498
  # after the fetch (sciduckdb.merge_tz_columns): without it every
553
499
  # aware value reached a plot as a naive UTC instant.
554
500
  zone_meta = datetime_zone_meta(db, variable, columns) if columns else {"columns": {}}
555
501
  zone_select = [tz_column_name(c) for c in zone_meta["columns"]]
556
- data_select = "".join(f', t."{column}"' for column in [*columns, *zone_select])
557
- query = (
558
- f"SELECT t.record_id{data_select}{schema_select} "
559
- f'FROM "{table}" t '
560
- f"JOIN _record r ON t.record_id = r.record_id "
561
- f"LEFT JOIN _schema s ON r.schema_id = s.schema_id "
562
- f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE"
563
- )
564
502
  # NOTE: no schema filter, no LIMIT, no column projection — this reads
565
503
  # EVERY non-excluded record of the variable with EVERY data column,
566
504
  # whatever the caller intends to plot (pushdown is a later stage of
567
505
  # .claude/plan-plot-minimal-load-examples.md).
568
506
  #
569
- # `_fetchdf`, never `_fetchall`, for the data columns. Measured on the
507
+ # The read is scidb's (`RecordStore.long_frame`, schema keys cast to
508
+ # text in the query), through `fetchdf`, never `fetchall`, for the
509
+ # data columns. Measured on the
570
510
  # real database 2026-09-13 (plan §7): one DOUBLE[] column of 17.4 M
571
511
  # samples took 4.7 s through `fetchall` — a Python float per sample —
572
512
  # and 0.26 s through `.df()`, which hands each cell over as one numpy
@@ -574,7 +514,7 @@ def load_variable(
574
514
  # timed out. The "loaded" line below says `ndarray` or `boxed` so a
575
515
  # regression to per-sample boxing is visible in the log, not inferred.
576
516
  with timer.phase("fetch"):
577
- frame = db._duck._fetchdf(query, [variable])
517
+ frame = db.record_store.long_frame(variable, [*columns, *zone_select])
578
518
  if zone_meta["columns"]:
579
519
  with timer.phase("datetime_zones"):
580
520
  merge_tz_columns(frame, zone_meta, where=f"load_variable({variable})")
@@ -618,7 +558,7 @@ def load_variable(
618
558
  # Cells and SAMPLES alongside the record count: 419 records is the same
619
559
  # number whether each holds 200 samples or 250,000, and only the second
620
560
  # explains a plot that never returns. Measured over the data columns
621
- # only, one O(cells) pass (scistackplot.framesize).
561
+ # only, one O(cells) pass (scistackplot.text.framesize).
622
562
  with timer.phase("measure_extent"):
623
563
  extent = frame_extent(frame, columns)
624
564
  Log.info(
@@ -870,7 +810,7 @@ def attach_variants(
870
810
  See ``docs/claude/variant-selection.md`` and
871
811
  ``docs/claude/function-version-variants.md``.
872
812
  """
873
- from scidb.provenance_query import variant_identity_batch
813
+ from scidb.provenance.query import variant_identity_batch
874
814
 
875
815
  record_ids = frame["record_id"].tolist()
876
816
  ident = variant_identity_batch(db._duck, record_ids)
@@ -1131,9 +1071,9 @@ def _truncate(value: Any, limit: int = 60) -> Any:
1131
1071
 
1132
1072
  def _drop_excluded_rows(db, frame: pd.DataFrame, keys: list[str], variable: str) -> pd.DataFrame:
1133
1073
  """``frame`` minus the rows at a schema location the project excluded
1134
- (:func:`scidb.exclusions.filter_excluded_combos`, the rule ``for_each``
1074
+ (:func:`scidb.store.exclusions.filter_excluded_combos`, the rule ``for_each``
1135
1075
  applies). One pass over the DISTINCT locations, not the rows."""
1136
- from scidb.exclusions import filter_excluded_combos
1076
+ from scidb.store.exclusions import filter_excluded_combos
1137
1077
 
1138
1078
  present = [key for key in keys if key in frame.columns]
1139
1079
  if frame.empty or not present:
@@ -4,7 +4,7 @@ database and applied to any variable.
4
4
 
5
5
  A saved plot (:mod:`scistackplotdb.saved`) belongs to one variable. A preset
6
6
  belongs to the project: it is the saved plot's settings with everything that
7
- belongs to the variable taken out (``scistackplot.presets`` decides what that
7
+ belongs to the variable taken out (``scistackplot.model.presets`` decides what that
8
8
  is, once). Apply it to another variable and the figure is drawn the same way::
9
9
 
10
10
  from scistackplotdb import ScidbSource, apply_preset_to, list_presets
@@ -16,7 +16,7 @@ is, once). Apply it to another variable and the figure is drawn the same way::
16
16
  )
17
17
 
18
18
  Storage follows the saved-plot rules exactly, through the same
19
- :class:`~scistackplotdb._versioned.VersionedStore`: one append-only table,
19
+ :class:`~scidb.store.versioned_store.VersionedStore`: one append-only table,
20
20
  ``_plot_preset``; the id is the identity and the name a label (unique across
21
21
  the project); re-saving a visible name appends a version; "remove" hides,
22
22
  nothing is ever deleted; save is strict, open salvages, and the stored copy
@@ -30,7 +30,7 @@ from typing import Any, Callable
30
30
 
31
31
  from scistacklog import Log
32
32
 
33
- from ._versioned import VersionedStore
33
+ from scidb.store.versioned_store import VersionedStore
34
34
 
35
35
  LAYER = "scistackplotdb"
36
36
 
@@ -117,6 +117,7 @@ STORE = VersionedStore(
117
117
  noun="preset",
118
118
  tag="preset",
119
119
  error=PresetError,
120
+ owner="scistackplotdb",
120
121
  )
121
122
 
122
123
 
@@ -35,7 +35,7 @@ from typing import Any, Callable
35
35
 
36
36
  from scistacklog import Log
37
37
 
38
- from ._versioned import MAX_NAME_LENGTH, VersionedStore # noqa: F401 (re-exported)
38
+ from scidb.store.versioned_store import VersionedStore
39
39
 
40
40
  LAYER = "scistackplotdb"
41
41
 
@@ -124,7 +124,7 @@ class SavedPlot:
124
124
  # Table
125
125
  # ---------------------------------------------------------------------------
126
126
 
127
- #: The rules (names, versions, hiding, never deleting) live in `_versioned`,
127
+ #: The rules (names, versions, hiding, never deleting) live in `scidb.store.versioned_store`,
128
128
  #: shared with presets so the two stores cannot drift apart.
129
129
  STORE = VersionedStore(
130
130
  table=TABLE,
@@ -133,6 +133,7 @@ STORE = VersionedStore(
133
133
  noun="saved plot",
134
134
  tag="saved_plot",
135
135
  error=SavedPlotError,
136
+ owner="scistackplotdb",
136
137
  )
137
138
 
138
139
 
@@ -433,14 +434,14 @@ FROZEN_KEY = "frozen_code"
433
434
  def _row_masks(spec, table) -> list[tuple[int, Any]]:
434
435
  """``[(row_index, mask)]`` — the rows each variant row DRAWS, by the same
435
436
  owner and first-match-wins order the figure uses
436
- (``scistackplot.variants.row_mask`` / ``apply_variant_sets``).
437
+ (``scistackplot.reduce.variants.row_mask`` / ``apply_variant_sets``).
437
438
 
438
439
  With no row concrete, the figure draws an empty row's effective selection;
439
440
  that is frozen as row 0 (synthesised on open if the spec has none).
440
441
  """
441
442
  import pandas as pd
442
- from scistackplot.spec import VariantSet
443
- from scistackplot.variants import defined_sets, is_stated, row_mask
443
+ from scistackplot.model.spec import VariantSet
444
+ from scistackplot.reduce.variants import defined_sets, is_stated, row_mask
444
445
 
445
446
  frame = table.frame
446
447
  if not defined_sets(spec.variant_sets):
@@ -465,7 +466,7 @@ def freeze_code(db, spec, table) -> dict[str, dict[str, list[str]]]:
465
466
  ``Code:`` column then, but the chain still names the hash, and that is
466
467
  what lets the plot reopen on it after a second version appears.
467
468
  """
468
- from scidb.provenance_query import code_versions_batch
469
+ from scidb.provenance.query import code_versions_batch
469
470
 
470
471
  frame = table.frame
471
472
  if "record_id" not in frame.columns:
@@ -548,10 +549,10 @@ def thaw_code(db, spec, table, frozen_code: dict, *, label: str = ""):
548
549
  """
549
550
  from dataclasses import replace
550
551
 
551
- from scidb.provenance_query import function_versions
552
+ from scidb.provenance.query import function_versions
552
553
  from scistackplot import NoteKind, RestoreNote
553
- from scistackplot.spec import VariantSet
554
- from scistackplot.variants import _ordinal_rank
554
+ from scistackplot.model.spec import VariantSet
555
+ from scistackplot.reduce.variants import _ordinal_rank
555
556
 
556
557
  if not frozen_code:
557
558
  return spec, [], []
@@ -640,8 +641,8 @@ def follow_latest_saved_plot(db, plot_id: str, *, table=None) -> SavedPlotInfo:
640
641
  from dataclasses import replace
641
642
 
642
643
  from scistackplot import restore_spec
643
- from scistackplot.spec import VariantSet
644
- from scistackplot.variants import is_function_axis
644
+ from scistackplot.model.spec import VariantSet
645
+ from scistackplot.reduce.variants import is_function_axis
645
646
 
646
647
  from .load import LATEST_COLUMN
647
648
 
@@ -27,12 +27,12 @@ from scistackplot import (
27
27
  natural_sort_key,
28
28
  row_origin,
29
29
  )
30
- from scistackplot.categorical import MAX_CATEGORY_LEVELS
31
- from scistackplot.dedup import SingleFlight
32
- from scistackplot.framesize import format_extent, frame_extent
30
+ from scistackplot.reduce.categorical import MAX_CATEGORY_LEVELS
31
+ from scistackplot.sources.dedup import SingleFlight
32
+ from scistackplot.text.framesize import format_extent, frame_extent
33
33
  from scistackplot.sources import BaseSource, UnknownMeasureError
34
- from scistackplot.variants import VARIABLE_COLUMN
35
- from scistackplot.temporal import is_temporal_column
34
+ from scistackplot.reduce.variants import VARIABLE_COLUMN
35
+ from scistackplot.reduce.temporal import is_temporal_column
36
36
 
37
37
  from .hierarchy import join_frames, joinable, joined_levels
38
38
  from .load import (
@@ -75,7 +75,7 @@ def _as_levels(column):
75
75
  keys are cast to VARCHAR on load for the same reason
76
76
  (``load.load_variable``).
77
77
 
78
- Deliberately these two calls and nothing cleverer: ``scistackplot.codegen``
78
+ Deliberately these two calls and nothing cleverer: ``scistackplot.export.codegen``
79
79
  emits ``fillna(...).astype(str)`` into the exported endpoint, so anything
80
80
  extra here (normalising ``70.0`` to ``70``, say) would spell a level one way
81
81
  in the preview and another way in the export — the disagreement that
@@ -85,7 +85,7 @@ def _as_levels(column):
85
85
 
86
86
  A DATETIME column is the exception: its values stay datetimes (missing
87
87
  ones named), and ``LongTable.from_frame`` gives each its identity label
88
- in time order (``scistackplot.temporal.level_labels``) — text that says
88
+ in time order (``scistackplot.reduce.temporal.level_labels``) — text that says
89
89
  when, not ``2026-07-01 14:00:00+02:00`` sorted as text. The generated code
90
90
  emits the same labels (``temporal.factor_label_code``).
91
91
  """
@@ -150,7 +150,7 @@ class ScidbSource(BaseSource):
150
150
  if not self._fast:
151
151
  return None # -> reducer_for() supplies the pandas reference
152
152
  if self._reducer_instance is None:
153
- from scistackplot.reducer import NumpyReducer
153
+ from scistackplot.reduce.reducer import NumpyReducer
154
154
 
155
155
  self._reducer_instance = NumpyReducer()
156
156
  return self._reducer_instance
@@ -279,10 +279,7 @@ class ScidbSource(BaseSource):
279
279
  return self._levels[variable]
280
280
 
281
281
  def _schema_levels(self, key: str) -> list[str]:
282
- rows = self._db._duck._fetchall(
283
- f'SELECT DISTINCT s."{key}" FROM _schema s WHERE s."{key}" IS NOT NULL'
284
- )
285
- values = [str(row[0]) for row in rows]
282
+ values = [str(v) for v in self._db.distinct_schema_values(key)]
286
283
  return self._ordered(key, values)
287
284
 
288
285
  def _aliases_source(self):
@@ -383,7 +380,7 @@ class ScidbSource(BaseSource):
383
380
  # of vanishing. scidb owns the declaration; this only applies it.
384
381
  order = getattr(self._db, "dataset_schema_key_order", None) or {}
385
382
  if order.get(key):
386
- from scidb.schema_order import order_levels
383
+ from scifor.project.schema_order import order_levels
387
384
 
388
385
  ordered = order_levels(key, unique, declared=order, fallback=default)
389
386
  else:
@@ -478,7 +475,7 @@ class ScidbSource(BaseSource):
478
475
  than one drawn from a cache that is almost certainly still good.
479
476
  """
480
477
  try:
481
- from scidb.provenance_query import variable_content_fingerprint
478
+ from scidb.provenance.query import variable_content_fingerprint
482
479
 
483
480
  return variable_content_fingerprint(self._db._duck, variable)
484
481
  except Exception as exc:
@@ -603,7 +600,7 @@ class ScidbSource(BaseSource):
603
600
  memoizes the result — this method always does the full work.
604
601
 
605
602
  ``measures`` are the y variables. Several of them **stack**: one value
606
- column plus a :data:`~scistackplot.variants.VARIABLE_COLUMN` saying
603
+ column plus a :data:`~scistackplot.reduce.variants.VARIABLE_COLUMN` saying
607
604
  which variable each row came from, which is what lets a variant row
608
605
  claim its own variable's rows and no others.
609
606
 
@@ -809,8 +806,30 @@ class ScidbSource(BaseSource):
809
806
  layer=LAYER,
810
807
  )
811
808
  table.reducer = self._reducer()
809
+ if len(measures) == 1:
810
+ table.absent = self._absent_locations(frame, levels)
812
811
  return table
813
812
 
813
+ def _absent_locations(self, frame, levels: list[str]):
814
+ """The variable's "No record" locations (`scidb.provenance.inventory.
815
+ absent_locations`, the owner) as a frame of its schema-key columns:
816
+ places the dataset has where this variable has no record. None when
817
+ it cannot be told (no schema keys, or the query failed — said in the
818
+ log, never fatal: the figure draws either way)."""
819
+ import pandas as pd
820
+ from scidb.provenance.inventory import absent_locations
821
+
822
+ keys = [key for key in levels if key in frame.columns]
823
+ if not keys:
824
+ return None
825
+ try:
826
+ present = frame[keys].dropna().astype(str).drop_duplicates().itertuples(index=False, name=None)
827
+ rows = absent_locations(self._db, keys, present)
828
+ except Exception as exc: # noqa: BLE001 - a readout, never a reason to fail the plot
829
+ Log.warn("absent locations for %s not known: %s", keys, exc, layer=LAYER)
830
+ return None
831
+ return pd.DataFrame(rows, columns=keys)
832
+
814
833
  def _own_column_groups(
815
834
  self, primary, factor_variables: list[FactorVariable]
816
835
  ) -> list[FactorVariable]:
@@ -1071,7 +1090,7 @@ class ScidbSource(BaseSource):
1071
1090
  ``drop_duplicates`` silently kept whichever came first — a figure
1072
1091
  stratified by a version of the spreadsheet nobody chose.
1073
1092
 
1074
- The pin is applied with :func:`scistackplot.variants.variant_set_mask`,
1093
+ The pin is applied with :func:`scistackplot.reduce.variants.variant_set_mask`,
1075
1094
  the same one definition the measure's own variant rows use, so "what a
1076
1095
  selection keeps" cannot mean two things one call apart.
1077
1096
 
@@ -1080,7 +1099,7 @@ class ScidbSource(BaseSource):
1080
1099
  (``plot-variant-rows.md`` § "An unfilled row is inert"). What used to be
1081
1100
  silent about that state is now :meth:`_one_label_per_location`.
1082
1101
  """
1083
- from scistackplot.variants import variant_set_mask
1102
+ from scistackplot.reduce.variants import variant_set_mask
1084
1103
 
1085
1104
  selection = group.selection
1086
1105
  frame = variable_frame.frame
@@ -1409,12 +1428,12 @@ class ScidbSource(BaseSource):
1409
1428
  def default_variant_for(self, variable: str) -> dict:
1410
1429
  """The variant a GROUPING by ``variable`` should open on: latest.
1411
1430
 
1412
- Exactly :func:`scistackplot.variants.default_selection` over that
1431
+ Exactly :func:`scistackplot.reduce.variants.default_selection` over that
1413
1432
  variable's own variant table — the same rule, and the same *code*, that
1414
1433
  decides what a plot opens on. Two rules for "latest" would be two
1415
1434
  answers, and this one has to agree with the Variants section beside it.
1416
1435
 
1417
- Read :func:`~scistackplot.variants.default_selection` before touching
1436
+ Read :func:`~scistackplot.reduce.variants.default_selection` before touching
1418
1437
  this: "latest" here is the per-schema-location **boolean flag**, never
1419
1438
  the string ``"latest"``. The string stops resolving through the flag the
1420
1439
  moment anything else is pinned in the same selection, and silently drops
@@ -1425,7 +1444,7 @@ class ScidbSource(BaseSource):
1425
1444
  metadata queries rather than reading any of them.
1426
1445
  """
1427
1446
  variable = base_variable(variable) # a text measure loads its variable
1428
- from scistackplot.variants import default_selection
1447
+ from scistackplot.reduce.variants import default_selection
1429
1448
 
1430
1449
  return default_selection(self.variant_table(variable))
1431
1450
 
@@ -1561,7 +1580,7 @@ class ScidbSource(BaseSource):
1561
1580
  text_measure_name(variable_frame.name), len(usable), layer=LAYER,
1562
1581
  )
1563
1582
  # Datetime fields melt into ONE value column with the rest, which is a
1564
- # time axis only if every field is a datetime (scistackplot.temporal).
1583
+ # time axis only if every field is a datetime (scistackplot.reduce.temporal).
1565
1584
  # Beside numeric fields they would read as numbers (NaN), so they are
1566
1585
  # left out and named — group by one (Variable.column) instead.
1567
1586
  datetimes = [c for c in usable if is_temporal_column(frame[c])]
@@ -1637,7 +1656,7 @@ class ScidbSource(BaseSource):
1637
1656
  field_factor: str | None,
1638
1657
  ):
1639
1658
  """``(frame, row column or None, its origin or None)`` — every
1640
- row-list cell split into one row per element (scistackplot.rows,
1659
+ row-list cell split into one row per element (scistackplot.reduce.rows,
1641
1660
  the one owner of the reshape; codegen mirrors it from the origin).
1642
1661
 
1643
1662
  The Row factor is kept only when some cell held two or more rows: a
@@ -5,7 +5,7 @@ text fields (``Gait``: ``StepLength``, ``Cadence``, ``Outcome``, ``Side``)
5
5
  cannot melt them all under one ``ColName`` factor. The catalog lists it twice
6
6
  (user decision, 2026-11-01): ``Gait`` holds the numeric fields, as it always
7
7
  has, and ``Gait (text)`` holds the text fields, drawn as counts, %s or
8
- categories (``scistackplot.categorical``). Every field is reachable; none is
8
+ categories (``scistackplot.reduce.categorical``). Every field is reachable; none is
9
9
  dropped. See docs/claude/categorical-measures.md.
10
10
 
11
11
  This module is the one owner of the NAME: how a text measure is spelled, and
@@ -47,7 +47,7 @@ def _check_column_vocabulary() -> None:
47
47
  here would not raise anywhere: it would make every pin a no-op and quietly
48
48
  plot every variant at once.
49
49
  """
50
- from scidb.variant import (
50
+ from scidb.model.variant import (
51
51
  CODE_COLUMN_PREFIX,
52
52
  LATEST_COLUMN_NAME,
53
53
  RUN_COLUMN_PREFIX,
@@ -79,7 +79,7 @@ def variant_set(
79
79
  table: LongTable | None = None,
80
80
  ) -> VariantSet:
81
81
  """
82
- Build a named :class:`~scistackplot.spec.VariantSet` from a ``scidb.Variant``.
82
+ Build a named :class:`~scistackplot.model.spec.VariantSet` from a ``scidb.Variant``.
83
83
 
84
84
  ::
85
85
 
@@ -123,7 +123,7 @@ def selection_for(
123
123
  become ``Run:fn``; branch params keep their namespaced name, which is
124
124
  already the column name.
125
125
  """
126
- from scidb.variant import VariantAxes
126
+ from scidb.model.variant import VariantAxes
127
127
 
128
128
  # The three dimensions, parsed by their owner (docs/claude/variant-space.md
129
129
  # §2) rather than by re-reading the prefixes here — this was the fourth
@@ -159,7 +159,7 @@ def branch_params_for(selection: dict[str, Any]) -> dict[str, Any]:
159
159
 
160
160
  Needed because the schema location picker asks **scidb** a question about a
161
161
  variant the plotting layer is holding: "which locations have this one?"
162
- ``scidb.locations.location_states`` takes a ``branch_params_filter``, the
162
+ ``scidb.provenance.locations.location_states`` takes a ``branch_params_filter``, the
163
163
  same dict ``Variant`` builds and ``find_record_id`` matches on, so the
164
164
  picker's dots and a ``Variant(...).load()`` cannot disagree about which
165
165
  records a selection names.
@@ -182,7 +182,7 @@ def branch_params_for(selection: dict[str, Any]) -> dict[str, Any]:
182
182
  ``CodeIsLatest: False`` has no scidb spelling — "not the latest" is not a
183
183
  pin — so it is dropped with a warning rather than silently inverted.
184
184
 
185
- The mapping itself is **scidb's** (:func:`scidb.variant.normalize_selection`)
185
+ The mapping itself is **scidb's** (:func:`scidb.model.variant.normalize_selection`)
186
186
  rather than this module's, because three consumers now hand scidb a
187
187
  column-keyed selection — this picker, the GUI's provenance RPC and
188
188
  ``scidb trace --variant`` — and a second copy of the rule is how the two
@@ -190,7 +190,7 @@ def branch_params_for(selection: dict[str, Any]) -> dict[str, Any]:
190
190
  *check* that the two layers still spell the columns the same way: this
191
191
  module is the only place that can see both sides.
192
192
  """
193
- from scidb.variant import normalize_selection
193
+ from scidb.model.variant import normalize_selection
194
194
 
195
195
  return normalize_selection(selection)
196
196
 
@@ -204,7 +204,7 @@ def _axes_of(table: LongTable | None) -> list[dict]:
204
204
  def _bare_code_column(code_axes: list[dict]) -> str:
205
205
  """The column a bare ``code_version=`` means, or a message naming the choice.
206
206
 
207
- Same rule as ``scidb.database._filter_records_by_code_version``: exactly one
207
+ Same rule as ``scidb.store.database._filter_records_by_code_version``: exactly one
208
208
  versioned function is unambiguous, several are not.
209
209
  """
210
210
  if len(code_axes) == 1:
@@ -283,7 +283,7 @@ def variant_graph(db, variable_frame, functions: list[str] | None = None) -> dic
283
283
  function node on the canvas, so nodes outside this variable's chain can
284
284
  still say what they have run.
285
285
  """
286
- from scidb.provenance_query import code_versions_batch, function_versions
286
+ from scidb.provenance.query import code_versions_batch, function_versions
287
287
 
288
288
  frame = variable_frame.frame
289
289
  record_ids = frame["record_id"].tolist() if "record_id" in frame.columns else []
@@ -1,392 +0,0 @@
1
- """
2
- Named, versioned, hideable rows: the store behind saved plots and presets.
3
-
4
- Both are "a JSON envelope the user names, re-saves and removes", with the same
5
- rules (docs/claude/saved-plots.md, docs/claude/plot-presets.md):
6
-
7
- * **The id is the identity; the name is a label.** Rename is an UPDATE of
8
- ``name``. A new item reusing a removed item's name gets a new id.
9
- * **Re-saving a visible name appends a version.** The newest version is what
10
- lists and opens. The version number is computed inside the INSERT, so two
11
- racing saves cannot claim the same one.
12
- * **Nothing is ever deleted** (feedback_never_delete_mark_hidden): "remove"
13
- sets ``hidden`` on every version.
14
- * **Reads never create the table.** Writes call :meth:`ensure_table`.
15
- * All reads go through ``_fetchall``/``_fetchone`` (the fetch-locking guard).
16
-
17
- This module owns those rules once, so the two stores cannot drift apart
18
- (CLAUDE.md NOTE 4). A store with a *scope column* (saved plots: ``variable``)
19
- keeps names unique per scope; a store without one (presets) keeps them unique
20
- across the project. Callers turn the raw rows this returns into their own
21
- info objects.
22
- """
23
-
24
- from __future__ import annotations
25
-
26
- import json
27
- import uuid
28
- from dataclasses import dataclass
29
- from datetime import datetime, timezone
30
- from typing import Any, Callable
31
-
32
- from scistacklog import Log
33
-
34
- LAYER = "scistackplotdb"
35
-
36
- #: Longest name accepted. A name is a list label, not a description.
37
- MAX_NAME_LENGTH = 120
38
-
39
- #: ``(item_id, scope, name, version, saved_at, hidden)``; scope is None for an
40
- #: unscoped store.
41
- Row = tuple
42
-
43
-
44
- @dataclass(frozen=True)
45
- class VersionedStore:
46
- """One append-only table of named, versioned envelopes."""
47
-
48
- table: str
49
- #: Column holding the item's identity (``plot_id``, ``preset_id``).
50
- id_column: str
51
- #: Column names are unique within, or None for project-wide names.
52
- scope_column: str | None
53
- #: What an item is called in messages ("saved plot", "preset").
54
- noun: str
55
- #: Log tag, without brackets.
56
- tag: str
57
- #: The error every refusal raises.
58
- error: type[Exception]
59
-
60
- # ---- table ----------------------------------------------------------
61
-
62
- def ensure_table(self, db) -> None:
63
- scope = f"{self.scope_column} VARCHAR NOT NULL," if self.scope_column else ""
64
- db._duck._execute(f"""
65
- CREATE TABLE IF NOT EXISTS {self.table} (
66
- {self.id_column} VARCHAR NOT NULL,
67
- {scope}
68
- name VARCHAR NOT NULL,
69
- version INTEGER NOT NULL,
70
- saved_at VARCHAR NOT NULL,
71
- hidden BOOLEAN NOT NULL DEFAULT FALSE,
72
- envelope_json VARCHAR NOT NULL,
73
- PRIMARY KEY ({self.id_column}, version)
74
- )
75
- """)
76
-
77
- def table_exists(self, db) -> bool:
78
- row = db._duck._fetchone(
79
- "SELECT count(*) FROM information_schema.tables WHERE table_name = ?",
80
- [self.table],
81
- )
82
- return bool(row and row[0])
83
-
84
- # ---- names ----------------------------------------------------------
85
-
86
- def check_name(self, name: Any) -> str:
87
- """*name* trimmed, or the store's error saying why it is not one."""
88
- if not isinstance(name, str):
89
- raise self.error(f"A {self.noun} needs a name.")
90
- name = " ".join(name.split()) # trim, and no tabs/newlines in a list label
91
- if not name:
92
- raise self.error(f"A {self.noun} needs a name.")
93
- if len(name) > MAX_NAME_LENGTH:
94
- raise self.error(
95
- f"A {self.noun}'s name is at most {MAX_NAME_LENGTH} characters "
96
- f"(this one is {len(name)})."
97
- )
98
- return name
99
-
100
- def clash_message(self, scope: str | None, name: str) -> str:
101
- if self.scope_column:
102
- return f"{scope} already has a {self.noun} named {name!r}."
103
- return f"There is already a {self.noun} named {name!r}."
104
-
105
- def describe(self, scope: str | None, name: str) -> str:
106
- """``StepLength / 'Fig 3'`` or ``'Session box'``, for logs."""
107
- return f"{scope} / {name!r}" if self.scope_column else repr(name)
108
-
109
- # ---- writes ---------------------------------------------------------
110
-
111
- def save(
112
- self,
113
- db,
114
- scope: str | None,
115
- name: str,
116
- envelope: dict,
117
- *,
118
- overwrite: bool,
119
- current_id: str | None,
120
- on_clash: Callable[[Row], Exception],
121
- ) -> tuple[Row, bool]:
122
- """Append *envelope* as a new version, returning ``(row, is_new)``.
123
-
124
- A visible item called *name* (in *scope*) gets a new version.
125
- Otherwise a new item is started. With ``overwrite=False``, appending
126
- to an item other than *current_id* raises ``on_clash(that row)`` so
127
- the caller can ask first.
128
- """
129
- try:
130
- text = json.dumps(envelope, sort_keys=True)
131
- except (TypeError, ValueError) as exc:
132
- raise self.error(f"The settings are not storable as JSON: {exc}") from exc
133
-
134
- self.ensure_table(db)
135
- current = self.visible_by_name(db, scope, name)
136
- if current is not None and not overwrite and current[0] != current_id:
137
- Log.info(
138
- "[%s] save of %s refused: another %s (%s) has that name and "
139
- "overwrite was not confirmed",
140
- self.tag,
141
- self.describe(scope, name),
142
- self.noun,
143
- current[0][:8],
144
- layer=LAYER,
145
- )
146
- raise on_clash(current)
147
- item_id = current[0] if current else uuid.uuid4().hex
148
- saved_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
149
- scope_col = f"{self.scope_column}, " if self.scope_column else ""
150
- scope_val = [scope] if self.scope_column else []
151
- # One statement: the next version number is decided inside the insert,
152
- # so two saves racing on one item cannot both claim the same version.
153
- db._duck._execute(
154
- f"""
155
- INSERT INTO {self.table}
156
- ({self.id_column}, {scope_col}name, version, saved_at, hidden, envelope_json)
157
- SELECT ?, {"?, " if self.scope_column else ""}?,
158
- COALESCE(MAX(version), 0) + 1, ?, FALSE, ?
159
- FROM {self.table} WHERE {self.id_column} = ?
160
- """,
161
- [item_id, *scope_val, name, saved_at, text, item_id],
162
- )
163
- row = self.latest(db, item_id)
164
- Log.info(
165
- "[%s] saved %s as version %d (%s, %d bytes, envelope format %r)",
166
- self.tag,
167
- self.describe(scope, name),
168
- row[3],
169
- f"new {self.noun}" if current is None else f"{self.noun} {item_id[:8]}",
170
- len(text),
171
- envelope.get("format"),
172
- layer=LAYER,
173
- )
174
- return row, current is None
175
-
176
- def rename(self, db, item_id: str, new_name: str) -> Row:
177
- new_name = self.check_name(new_name)
178
- row = self.require(db, item_id)
179
- clash = self.visible_by_name(db, row[1], new_name)
180
- if clash is not None and clash[0] != item_id:
181
- raise self.error(self.clash_message(row[1], new_name))
182
- db._duck._execute(
183
- f"UPDATE {self.table} SET name = ? WHERE {self.id_column} = ?",
184
- [new_name, item_id],
185
- )
186
- Log.info(
187
- "[%s] renamed %s -> %r (%s)",
188
- self.tag,
189
- self.describe(row[1], row[2]),
190
- new_name,
191
- item_id[:8],
192
- layer=LAYER,
193
- )
194
- return self.latest(db, item_id)
195
-
196
- def set_hidden(self, db, item_id: str, hidden: bool) -> Row:
197
- row = self.require(db, item_id)
198
- if not hidden:
199
- clash = self.visible_by_name(db, row[1], row[2])
200
- if clash is not None and clash[0] != item_id:
201
- raise self.error(
202
- f"{self.clash_message(row[1], row[2])[:-1]}; rename one of them first."
203
- )
204
- db._duck._execute(
205
- f"UPDATE {self.table} SET hidden = ? WHERE {self.id_column} = ?",
206
- [bool(hidden), item_id],
207
- )
208
- Log.info(
209
- "[%s] %s %s (%s)",
210
- self.tag,
211
- "hid" if hidden else "unhid",
212
- self.describe(row[1], row[2]),
213
- item_id[:8],
214
- layer=LAYER,
215
- )
216
- return self.latest(db, item_id)
217
-
218
- # ---- reads ----------------------------------------------------------
219
-
220
- def _columns(self) -> str:
221
- scope = self.scope_column or "NULL"
222
- return f"{self.id_column}, {scope}, name, version, saved_at, hidden"
223
-
224
- def list_rows(
225
- self,
226
- db,
227
- scope: str | None = None,
228
- *,
229
- include_hidden: bool = False,
230
- with_envelope: bool = False,
231
- ) -> list[Row]:
232
- """The newest version of each item (in *scope*), newest first.
233
-
234
- With *with_envelope*, each row carries its envelope JSON as a 7th
235
- element, for a list that shows something stored inside it.
236
- """
237
- envelope = ", envelope_json" if with_envelope else ""
238
- if not self.table_exists(db):
239
- return []
240
- where = f"{self.scope_column} = ? AND " if self.scope_column else ""
241
- rows = db._duck._fetchall(
242
- f"""
243
- SELECT {self._columns()}{envelope}
244
- FROM {self.table} AS t
245
- WHERE {where}version = (
246
- SELECT MAX(version) FROM {self.table}
247
- WHERE {self.id_column} = t.{self.id_column}
248
- )
249
- {"" if include_hidden else "AND NOT hidden"}
250
- ORDER BY saved_at DESC, name
251
- """,
252
- [scope] if self.scope_column else [],
253
- )
254
- Log.debug(
255
- "[%s] %s: %d %s(s) listed%s",
256
- self.tag,
257
- scope if self.scope_column else "project",
258
- len(rows),
259
- self.noun,
260
- " (hidden included)" if include_hidden else "",
261
- layer=LAYER,
262
- )
263
- return list(rows)
264
-
265
- def history(self, db, item_id: str, *, with_envelope: bool = False) -> list[Row]:
266
- if not self.table_exists(db):
267
- return []
268
- envelope = ", envelope_json" if with_envelope else ""
269
- return list(db._duck._fetchall(
270
- f"""
271
- SELECT {self._columns()}{envelope}
272
- FROM {self.table} WHERE {self.id_column} = ? ORDER BY version DESC
273
- """,
274
- [item_id],
275
- ))
276
-
277
- def load(self, db, item_id: str, version: int | None = None) -> tuple[Row, str]:
278
- """``(row, envelope_json)`` of one version (the newest by default)."""
279
- if not self.table_exists(db):
280
- raise self.error(f"No {self.noun} {item_id!r}: nothing has been saved yet.")
281
- if version is None:
282
- row = db._duck._fetchone(
283
- f"""
284
- SELECT {self._columns()}, envelope_json
285
- FROM {self.table} WHERE {self.id_column} = ?
286
- ORDER BY version DESC LIMIT 1
287
- """,
288
- [item_id],
289
- )
290
- else:
291
- row = db._duck._fetchone(
292
- f"""
293
- SELECT {self._columns()}, envelope_json
294
- FROM {self.table} WHERE {self.id_column} = ? AND version = ?
295
- """,
296
- [item_id, int(version)],
297
- )
298
- if row is None:
299
- which = f"version {version} of " if version is not None else ""
300
- raise self.error(f"No {self.noun} {which}{item_id!r}.")
301
- return tuple(row[:6]), row[6]
302
-
303
- def visible_by_name(self, db, scope: str | None, name: str) -> Row | None:
304
- if not self.table_exists(db):
305
- return None
306
- where = f"{self.scope_column} = ? AND " if self.scope_column else ""
307
- row = db._duck._fetchone(
308
- f"""
309
- SELECT {self._columns()}
310
- FROM {self.table}
311
- WHERE {where}name = ? AND NOT hidden
312
- ORDER BY version DESC LIMIT 1
313
- """,
314
- [*([scope] if self.scope_column else []), name],
315
- )
316
- return tuple(row) if row else None
317
-
318
- def latest(self, db, item_id: str) -> Row:
319
- row = db._duck._fetchone(
320
- f"""
321
- SELECT {self._columns()}
322
- FROM {self.table} WHERE {self.id_column} = ?
323
- ORDER BY version DESC LIMIT 1
324
- """,
325
- [item_id],
326
- )
327
- if row is None:
328
- raise self.error(f"No {self.noun} {item_id!r}.")
329
- return tuple(row)
330
-
331
- def require(self, db, item_id: str) -> Row:
332
- if not self.table_exists(db):
333
- raise self.error(f"No {self.noun} {item_id!r}: nothing has been saved yet.")
334
- return self.latest(db, item_id)
335
-
336
- def parse_envelope(self, text: str, label: str) -> dict:
337
- """The stored envelope as a dict. A row that is not one opens on
338
- defaults rather than failing the open."""
339
- try:
340
- envelope = json.loads(text)
341
- except (TypeError, ValueError) as exc:
342
- Log.warn("[%s] %s: stored envelope is not JSON (%s)", self.tag, label, exc,
343
- layer=LAYER)
344
- return {}
345
- if not isinstance(envelope, dict):
346
- Log.warn(
347
- "[%s] %s: stored envelope is %s, not a table",
348
- self.tag,
349
- label,
350
- type(envelope).__name__,
351
- layer=LAYER,
352
- )
353
- return {}
354
- return envelope
355
-
356
- # ---- copying a whole store (project bundles) ------------------------
357
-
358
- def _all_columns(self) -> list[str]:
359
- scope = [self.scope_column] if self.scope_column else []
360
- return [self.id_column, *scope, "name", "version", "saved_at", "hidden", "envelope_json"]
361
-
362
- def dump_rows(self, db) -> list[dict]:
363
- """Every row, every version, hidden ones too, as plain dicts: what a
364
- project bundle carries (``scistackplotdb.bundle_section``). Nothing
365
- is filtered -- a copy keeps the history the original kept."""
366
- if not self.table_exists(db):
367
- return []
368
- cols = self._all_columns()
369
- rows = db._duck._fetchall(
370
- f"SELECT {', '.join(cols)} FROM {self.table} ORDER BY {self.id_column}, version"
371
- )
372
- Log.info("[%s] dump_rows: %d row(s)", self.tag, len(rows), layer=LAYER)
373
- return [dict(zip(cols, row)) for row in rows]
374
-
375
- def load_rows(self, db, rows: list[dict]) -> int:
376
- """Insert *rows* (from :meth:`dump_rows`) verbatim, ids and versions
377
- kept; a row already present is left as it is. Returns rows inserted."""
378
- if not rows:
379
- return 0
380
- self.ensure_table(db)
381
- cols = self._all_columns()
382
- before = db._duck._fetchone(f"SELECT count(*) FROM {self.table}")[0]
383
- for row in rows:
384
- db._duck._execute(
385
- f"INSERT INTO {self.table} ({', '.join(cols)}) "
386
- f"VALUES ({', '.join('?' for _ in cols)}) ON CONFLICT DO NOTHING",
387
- [row.get(c) for c in cols],
388
- )
389
- after = db._duck._fetchone(f"SELECT count(*) FROM {self.table}")[0]
390
- Log.info("[%s] load_rows: %d of %d row(s) inserted", self.tag, after - before,
391
- len(rows), layer=LAYER)
392
- return after - before