scistackplotdb 0.1.66__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/PKG-INFO +7 -6
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/README.md +1 -1
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/pyproject.toml +2 -1
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/__init__.py +8 -0
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/bundle_section.py +9 -16
- scistackplotdb-0.2.0/src/scistackplotdb/designs.py +148 -0
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/endpoint.py +7 -7
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/load.py +27 -87
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/presets.py +4 -3
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/saved.py +12 -11
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/source.py +41 -22
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/text_fields.py +1 -1
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/variants.py +8 -8
- scistackplotdb-0.1.66/src/scistackplotdb/_versioned.py +0 -392
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/.gitignore +0 -0
- {scistackplotdb-0.1.66 → scistackplotdb-0.2.0}/src/scistackplotdb/hierarchy.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scistackplotdb
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: scidb-backed plotting: load variables into long tables and plot them with scistackplot
|
|
5
5
|
Author: SciStack Contributors
|
|
6
6
|
License-Expression: MIT
|
|
@@ -17,10 +17,11 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
17
17
|
Classifier: Topic :: Scientific/Engineering :: Visualization
|
|
18
18
|
Classifier: Typing :: Typed
|
|
19
19
|
Requires-Python: >=3.10
|
|
20
|
-
Requires-Dist: sciduckdb==0.
|
|
21
|
-
Requires-Dist:
|
|
22
|
-
Requires-Dist:
|
|
23
|
-
Requires-Dist:
|
|
20
|
+
Requires-Dist: sciduckdb==0.2.0
|
|
21
|
+
Requires-Dist: scifor==0.2.0
|
|
22
|
+
Requires-Dist: scistack-db==0.2.0
|
|
23
|
+
Requires-Dist: scistacklog==0.2.0
|
|
24
|
+
Requires-Dist: scistackplot==0.2.0
|
|
24
25
|
Provides-Extra: dev
|
|
25
26
|
Requires-Dist: matplotlib>=3.6; extra == 'dev'
|
|
26
27
|
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
@@ -184,7 +185,7 @@ else goes through a natural sort so zero-padded IDs land as
|
|
|
184
185
|
`01, 02, … 10` instead of `01, 10, 02`.
|
|
185
186
|
|
|
186
187
|
A project that declares its levels (`[schema_keys]` in scistack.toml, read by
|
|
187
|
-
`
|
|
188
|
+
`scifor.project.schema_order`) wins over both: declared levels first, in that order,
|
|
188
189
|
the rest after them by the rule above. The declaration is read live, and a
|
|
189
190
|
change drops the source's built tables, so an edit shows on the next plot
|
|
190
191
|
request. Exported seaborn code states every order it relies on (`order=`,
|
|
@@ -154,7 +154,7 @@ else goes through a natural sort so zero-padded IDs land as
|
|
|
154
154
|
`01, 02, … 10` instead of `01, 10, 02`.
|
|
155
155
|
|
|
156
156
|
A project that declares its levels (`[schema_keys]` in scistack.toml, read by
|
|
157
|
-
`
|
|
157
|
+
`scifor.project.schema_order`) wins over both: declared levels first, in that order,
|
|
158
158
|
the rest after them by the rule above. The declaration is read live, and a
|
|
159
159
|
change drops the source's built tables, so an edit shows on the next plot
|
|
160
160
|
request. Exported seaborn code states every order it relies on (`order=`,
|
|
@@ -12,7 +12,7 @@ raw-options = { search_parent_directories = true, local_scheme = "no-local-versi
|
|
|
12
12
|
# dependencies must be named here: CI's build job fails a wheel whose internal
|
|
13
13
|
# dependency is not "==<its own version>".
|
|
14
14
|
[tool.hatch.metadata.hooks.dependency-coversion]
|
|
15
|
-
override-versions-of = ["scistackplot", "scistack-db", "sciduckdb", "scistacklog"]
|
|
15
|
+
override-versions-of = ["scistackplot", "scistack-db", "sciduckdb", "scifor", "scistacklog"]
|
|
16
16
|
|
|
17
17
|
[project]
|
|
18
18
|
name = "scistackplotdb"
|
|
@@ -48,6 +48,7 @@ dependencies = [
|
|
|
48
48
|
"scistackplot",
|
|
49
49
|
"scistack-db",
|
|
50
50
|
"sciduckdb",
|
|
51
|
+
"scifor",
|
|
51
52
|
"scistacklog",
|
|
52
53
|
]
|
|
53
54
|
|
|
@@ -65,6 +65,7 @@ from .saved import (
|
|
|
65
65
|
save_plot,
|
|
66
66
|
saved_plot_history,
|
|
67
67
|
)
|
|
68
|
+
from .designs import DesignError, StoredDesign, apply_design, is_linked, link_design, load_design, save_design
|
|
68
69
|
from .presets import (
|
|
69
70
|
AppliedPreset,
|
|
70
71
|
Preset,
|
|
@@ -91,6 +92,13 @@ from .variants import (
|
|
|
91
92
|
)
|
|
92
93
|
|
|
93
94
|
__all__ = [
|
|
95
|
+
"DesignError",
|
|
96
|
+
"StoredDesign",
|
|
97
|
+
"apply_design",
|
|
98
|
+
"is_linked",
|
|
99
|
+
"link_design",
|
|
100
|
+
"load_design",
|
|
101
|
+
"save_design",
|
|
94
102
|
"ScidbSource",
|
|
95
103
|
# a wide variable's text fields as their own measure
|
|
96
104
|
"TEXT_SUFFIX",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"""
|
|
2
|
-
The ``plots`` section of a project bundle (``scidb.bundle``): saved plots and
|
|
2
|
+
The ``plots`` section of a project bundle (``scidb.project.bundle``): saved plots and
|
|
3
3
|
plot presets, copied row for row.
|
|
4
4
|
|
|
5
5
|
Both stores key nothing by canvas ids or database record ids -- a saved plot
|
|
@@ -15,11 +15,16 @@ import json
|
|
|
15
15
|
|
|
16
16
|
from scistacklog import Log
|
|
17
17
|
|
|
18
|
-
from . import presets, saved
|
|
18
|
+
from . import designs, presets, saved
|
|
19
19
|
|
|
20
20
|
LAYER = "scistackplotdb"
|
|
21
21
|
|
|
22
|
-
_STORES = {
|
|
22
|
+
_STORES = {
|
|
23
|
+
"saved_plots.json": saved.STORE,
|
|
24
|
+
"presets.json": presets.STORE,
|
|
25
|
+
# The shared design of a linked figure + table (designs.py).
|
|
26
|
+
"designs.json": designs.STORE,
|
|
27
|
+
}
|
|
23
28
|
|
|
24
29
|
|
|
25
30
|
class PlotsSection:
|
|
@@ -41,19 +46,7 @@ class PlotsSection:
|
|
|
41
46
|
if filename in files:
|
|
42
47
|
rows = json.loads(files[filename].decode("utf-8"))
|
|
43
48
|
if key_map is not None and not key_map.is_identity:
|
|
44
|
-
rows = [
|
|
49
|
+
rows = [store.remap_row(row, key_map, ctx.map_report) for row in rows]
|
|
45
50
|
report[store.table] = store.load_rows(ctx.db, rows)
|
|
46
51
|
Log.info("[bundle_section] plots import: %s", report, layer=LAYER)
|
|
47
52
|
return report
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
def _remap_row(row: dict, store, key_map, map_report) -> dict:
|
|
51
|
-
"""A saved plot or preset carried into another schema (portability Stage
|
|
52
|
-
6). A spec names factors in many fields (roles, groups, facets, colours,
|
|
53
|
-
filters, difference bars, ...), so every string EQUAL to an exporter key
|
|
54
|
-
is renamed (``KeyMap.exact_strings``) and the item is flagged for review
|
|
55
|
-
-- a factor that merely equals a key name cannot be told apart."""
|
|
56
|
-
envelope = json.loads(row["envelope_json"])
|
|
57
|
-
where = f"{store.noun} {row.get('name')!r}"
|
|
58
|
-
mapped = key_map.exact_strings(envelope, where, map_report)
|
|
59
|
-
return {**row, "envelope_json": json.dumps(mapped)}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The shared design of a linked figure and table: one owner
|
|
3
|
+
(docs/claude/studio-unification.md, user decision 2026-11-01).
|
|
4
|
+
|
|
5
|
+
A saved plot and the StatStudio tables linked to it used to each keep their
|
|
6
|
+
own copy of the design, with "Sync to…" banners to copy one onto the other.
|
|
7
|
+
Now the design of a linked pair lives in ONE row set here, keyed by the
|
|
8
|
+
saved plot's ``plot_id``. Either studio writes it; both read it on open, and
|
|
9
|
+
an edit in one reaches the other through the GUI's broadcast.
|
|
10
|
+
|
|
11
|
+
What the design is, field by field, is still
|
|
12
|
+
:data:`scistackplot.model.design.DESIGN_FIELDS`. This store keeps a whole
|
|
13
|
+
``PlotSpec`` dict (so it reads back through ``restore_spec``) but only its
|
|
14
|
+
design fields are ever applied (:func:`apply_design`, through
|
|
15
|
+
``copy_design``). Display settings stay with each artifact.
|
|
16
|
+
|
|
17
|
+
**A design exists only once something is linked.** :func:`link_design` is
|
|
18
|
+
called when a StatStudio table is set up from a saved plot. An unlinked plot
|
|
19
|
+
has no row here and its saved envelope stays the owner of its design.
|
|
20
|
+
|
|
21
|
+
Rows are the shared ``VersionedStore`` (append-only, never deleted): the
|
|
22
|
+
item's NAME is the plot id, so "the design of plot X" is a name lookup, and
|
|
23
|
+
each change is a new version. A write that would not change the design
|
|
24
|
+
(same ``design_fingerprint``) appends nothing.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
from scistacklog import Log
|
|
33
|
+
|
|
34
|
+
from scidb.store.versioned_store import VersionedStore
|
|
35
|
+
|
|
36
|
+
LAYER = "scistackplotdb"
|
|
37
|
+
|
|
38
|
+
TABLE = "_plot_design"
|
|
39
|
+
|
|
40
|
+
ENVELOPE_FORMAT = 1
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class DesignError(ValueError):
|
|
44
|
+
"""A design that cannot be stored or read as asked."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
STORE = VersionedStore(
|
|
48
|
+
table=TABLE,
|
|
49
|
+
id_column="design_id",
|
|
50
|
+
scope_column=None,
|
|
51
|
+
noun="shared design",
|
|
52
|
+
tag="design",
|
|
53
|
+
error=DesignError,
|
|
54
|
+
owner="scistackplotdb",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class StoredDesign:
|
|
60
|
+
plot_id: str
|
|
61
|
+
design_id: str
|
|
62
|
+
version: int
|
|
63
|
+
#: The stored spec (a full PlotSpec; only its design fields apply).
|
|
64
|
+
spec: Any
|
|
65
|
+
fingerprint: str
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _row_for(db, plot_id: str):
|
|
69
|
+
if not plot_id:
|
|
70
|
+
return None
|
|
71
|
+
return STORE.visible_by_name(db, None, plot_id)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def load_design(db, plot_id: str) -> StoredDesign | None:
|
|
75
|
+
"""The newest shared design of saved plot *plot_id*, or None when it is
|
|
76
|
+
not linked to anything (or the row cannot be read)."""
|
|
77
|
+
from scistackplot import restore_spec
|
|
78
|
+
from scistackplot.model.design import design_fingerprint
|
|
79
|
+
|
|
80
|
+
row = _row_for(db, plot_id)
|
|
81
|
+
if row is None:
|
|
82
|
+
return None
|
|
83
|
+
_, text = STORE.load(db, row[0])
|
|
84
|
+
envelope = STORE.parse_envelope(text, f"design of {plot_id[:8]}")
|
|
85
|
+
raw = envelope.get("spec")
|
|
86
|
+
if not isinstance(raw, dict):
|
|
87
|
+
Log.warn("[design] %s: stored design has no spec; ignoring it", plot_id[:8], layer=LAYER)
|
|
88
|
+
return None
|
|
89
|
+
restored = restore_spec(raw, fallback_measure=(raw.get("measures") or [None])[0])
|
|
90
|
+
for note in restored.notes:
|
|
91
|
+
Log.info("[design] %s: restore note: %s", plot_id[:8], getattr(note, "message", note), layer=LAYER)
|
|
92
|
+
spec = restored.spec
|
|
93
|
+
return StoredDesign(plot_id, row[0], row[3], spec, design_fingerprint(spec))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def is_linked(db, plot_id: str | None) -> bool:
|
|
97
|
+
return bool(plot_id) and _row_for(db, plot_id) is not None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def save_design(db, plot_id: str, spec) -> tuple[StoredDesign, bool]:
|
|
101
|
+
"""Write *spec*'s design as the shared design of *plot_id*.
|
|
102
|
+
|
|
103
|
+
Returns ``(stored, changed)``. When the design fields are the same as
|
|
104
|
+
the newest stored version, nothing is appended and ``changed`` is
|
|
105
|
+
False, so a broadcast echo between two open studios stops here.
|
|
106
|
+
"""
|
|
107
|
+
from scistackplot.model.design import design_fingerprint
|
|
108
|
+
|
|
109
|
+
if not plot_id:
|
|
110
|
+
raise DesignError("A shared design needs the saved plot it belongs to.")
|
|
111
|
+
fingerprint = design_fingerprint(spec)
|
|
112
|
+
current = load_design(db, plot_id)
|
|
113
|
+
if current is not None and current.fingerprint == fingerprint:
|
|
114
|
+
Log.debug("[design] %s: unchanged (fingerprint %s); nothing appended", plot_id[:8], fingerprint,
|
|
115
|
+
layer=LAYER)
|
|
116
|
+
return current, False
|
|
117
|
+
envelope = {"format": ENVELOPE_FORMAT, "plot_id": plot_id, "spec": spec.to_dict()}
|
|
118
|
+
row, is_new = STORE.save(db, None, plot_id, envelope, overwrite=True, current_id=None,
|
|
119
|
+
on_clash=lambda r: DesignError(f"design clash for {plot_id}"))
|
|
120
|
+
Log.info("[design] %s: %s version %d (fingerprint %s)", plot_id[:8],
|
|
121
|
+
"linked, first" if is_new else "changed, new", row[3], fingerprint, layer=LAYER)
|
|
122
|
+
return StoredDesign(plot_id, row[0], row[3], spec, fingerprint), True
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def link_design(db, plot_id: str, spec) -> StoredDesign:
|
|
126
|
+
"""Make *plot_id*'s design shared, starting from *spec* when nothing is
|
|
127
|
+
stored yet. An already-linked plot keeps its stored design."""
|
|
128
|
+
current = load_design(db, plot_id)
|
|
129
|
+
if current is not None:
|
|
130
|
+
Log.info("[design] %s: already linked (version %d)", plot_id[:8], current.version, layer=LAYER)
|
|
131
|
+
return current
|
|
132
|
+
stored, _ = save_design(db, plot_id, spec)
|
|
133
|
+
return stored
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def apply_design(db, plot_id: str | None, spec) -> tuple[Any, list, int | None]:
|
|
137
|
+
"""*spec* with the shared design of *plot_id* copied onto it
|
|
138
|
+
(``copy_design``: design fields only, display kept). Unchanged, with
|
|
139
|
+
version None, when *plot_id* is not linked."""
|
|
140
|
+
from scistackplot.model.design import copy_design
|
|
141
|
+
|
|
142
|
+
stored = load_design(db, plot_id) if plot_id else None
|
|
143
|
+
if stored is None:
|
|
144
|
+
return spec, [], None
|
|
145
|
+
merged, notes = copy_design(stored.spec, spec)
|
|
146
|
+
Log.info("[design] %s: applied version %d onto a %s spec (%d note(s))", plot_id[:8], stored.version,
|
|
147
|
+
getattr(spec.kind, "value", spec.kind), len(notes), layer=LAYER)
|
|
148
|
+
return merged, notes, stored.version
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Turning a ``PlotSpec`` into a pipeline endpoint.
|
|
3
3
|
|
|
4
4
|
The GUI's "Add to pipeline" produces two things: a ``plot_`` function (generated
|
|
5
|
-
by ``scistackplot.codegen`` — literal seaborn, no runtime dependency on this
|
|
5
|
+
by ``scistackplot.export.codegen`` — literal seaborn, no runtime dependency on this
|
|
6
6
|
package) and the ``for_each`` call that runs it. This module owns the second
|
|
7
7
|
half, and with it the one translation that has to be exactly right:
|
|
8
8
|
|
|
@@ -26,9 +26,9 @@ from dataclasses import dataclass, field
|
|
|
26
26
|
from scistacklog import Log
|
|
27
27
|
from scistackplot import LongTable, PlotSpec, default_function_name
|
|
28
28
|
from scistackplot import generate_plot_function
|
|
29
|
-
from scistackplot.codegen import group_param, variant_params
|
|
30
|
-
from scistackplot.roles import fanout_keys
|
|
31
|
-
from scistackplot.variants import defined_sets
|
|
29
|
+
from scistackplot.export.codegen import group_param, variant_params
|
|
30
|
+
from scistackplot.model.roles import fanout_keys
|
|
31
|
+
from scistackplot.reduce.variants import defined_sets
|
|
32
32
|
|
|
33
33
|
from .load import LATEST_COLUMN
|
|
34
34
|
from .text_fields import base_variable
|
|
@@ -147,7 +147,7 @@ def _pin_literal(value) -> str:
|
|
|
147
147
|
|
|
148
148
|
def row_expression(input_variable: str, variant_set, table=None) -> str:
|
|
149
149
|
"""One variant ROW as a ``Variant(...)`` load — through the row's
|
|
150
|
-
EFFECTIVE selection (``scistackplot.variants.export_selection``), so the
|
|
150
|
+
EFFECTIVE selection (``scistackplot.reduce.variants.export_selection``), so the
|
|
151
151
|
exported code loads what the preview drew: current code for a row that
|
|
152
152
|
names none, the frozen version for a reopened saved plot.
|
|
153
153
|
|
|
@@ -156,7 +156,7 @@ def row_expression(input_variable: str, variant_set, table=None) -> str:
|
|
|
156
156
|
"""
|
|
157
157
|
from types import SimpleNamespace
|
|
158
158
|
|
|
159
|
-
from scistackplot.variants import export_selection
|
|
159
|
+
from scistackplot.reduce.variants import export_selection
|
|
160
160
|
|
|
161
161
|
# The TABLE's default, exactly what the preview applied: a table with
|
|
162
162
|
# none (nothing superseded) exports its rows as written.
|
|
@@ -181,7 +181,7 @@ def variant_expression(input_variable: str, variant_set) -> str:
|
|
|
181
181
|
|
|
182
182
|
Variant(Variant(EMG, fn="loadEMG", code_version="v1"), fn="bandpass", low_hz=20)
|
|
183
183
|
|
|
184
|
-
Nesting is documented, merges the two filters (``scidb.variant.Variant``
|
|
184
|
+
Nesting is documented, merges the two filters (``scidb.model.variant.Variant``
|
|
185
185
|
handles it explicitly), and keeps every keyword readable — where
|
|
186
186
|
``**{"__code__.loadEMG": "v1"}`` would leak a reserved namespace into code a
|
|
187
187
|
user is meant to edit.
|
|
@@ -7,11 +7,12 @@ columns once a variable is joined to ``_schema``, which is the same shape
|
|
|
7
7
|
part a flat CSV never needed: attaching branch params as columns, and knowing
|
|
8
8
|
which schema keys a given variable actually occupies.
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
``
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
rather than
|
|
10
|
+
The storage SQL is scidb's: every read of a variable's records goes through
|
|
11
|
+
``db.record_store`` (``scidb.store.record_store.RecordStore``: the catalog, one
|
|
12
|
+
sampled cell, the long frame), which fetches the payload only through
|
|
13
|
+
``fetchdf`` so a sample never becomes a Python float (see ``load_variable``).
|
|
14
|
+
The branch-params walk is batched rather than asked per record (the N+1
|
|
15
|
+
trap).
|
|
15
16
|
"""
|
|
16
17
|
|
|
17
18
|
from __future__ import annotations
|
|
@@ -24,9 +25,9 @@ import pandas as pd
|
|
|
24
25
|
from sciduckdb import merge_tz_columns, stored_tz_companions, tz_column_name
|
|
25
26
|
from scistacklog import Log
|
|
26
27
|
from scistackplot import CODE_FACTOR_PREFIX, RUN_FACTOR_PREFIX, element_values
|
|
27
|
-
from scistackplot.framesize import format_extent, frame_extent
|
|
28
|
-
from scistackplot.shape import Shape, classify_value
|
|
29
|
-
from scistackplot.table import natural_sort_key
|
|
28
|
+
from scistackplot.text.framesize import format_extent, frame_extent
|
|
29
|
+
from scistackplot.model.shape import Shape, classify_value
|
|
30
|
+
from scistackplot.model.table import natural_sort_key
|
|
30
31
|
|
|
31
32
|
LAYER = "scistackplotdb"
|
|
32
33
|
|
|
@@ -100,7 +101,7 @@ class VariableFrame:
|
|
|
100
101
|
#: Data columns stored as ``row_list`` ("combine rows": one element per
|
|
101
102
|
#: source row, sciduckdb.row_lists). Their cells arrive as arrays; the
|
|
102
103
|
#: source splits them into rows with a ``Row`` factor
|
|
103
|
-
#: (scistackplot.rows), never plots them as 1-D series.
|
|
104
|
+
#: (scistackplot.reduce.rows), never plots them as 1-D series.
|
|
104
105
|
row_list_columns: list[str] = field(default_factory=list)
|
|
105
106
|
|
|
106
107
|
@property
|
|
@@ -109,12 +110,11 @@ class VariableFrame:
|
|
|
109
110
|
|
|
110
111
|
|
|
111
112
|
def schema_keys(db) -> list[str]:
|
|
112
|
-
return
|
|
113
|
+
return db.record_store.schema_keys()
|
|
113
114
|
|
|
114
115
|
|
|
115
116
|
def registered_variables(db) -> list[str]:
|
|
116
|
-
|
|
117
|
-
return [row[0] for row in rows]
|
|
117
|
+
return db.record_store.variable_names()
|
|
118
118
|
|
|
119
119
|
|
|
120
120
|
def table_name_for(db, variable: str) -> str:
|
|
@@ -126,10 +126,7 @@ def table_name_for(db, variable: str) -> str:
|
|
|
126
126
|
filters on ``_record.type`` — reading a shared table without that filter
|
|
127
127
|
would silently mix two variables' records into one plot.
|
|
128
128
|
"""
|
|
129
|
-
|
|
130
|
-
"SELECT table_name FROM _registered_types WHERE type_name = ?", [variable]
|
|
131
|
-
)
|
|
132
|
-
return row[0] if row and row[0] else f"{variable}_data"
|
|
129
|
+
return db.record_store.table_name(variable)
|
|
133
130
|
|
|
134
131
|
|
|
135
132
|
def data_columns_for(db, variable: str) -> list[str]:
|
|
@@ -139,14 +136,7 @@ def data_columns_for(db, variable: str) -> list[str]:
|
|
|
139
136
|
def _stored_column_types(db, variable: str) -> dict[str, str]:
|
|
140
137
|
"""``{column: DuckDB type}`` of the variable's table as stored (zone
|
|
141
138
|
companions included), in ordinal order."""
|
|
142
|
-
|
|
143
|
-
rows = db._duck._fetchall(
|
|
144
|
-
"SELECT column_name, data_type FROM information_schema.columns "
|
|
145
|
-
"WHERE table_name = ? AND column_name != 'record_id' "
|
|
146
|
-
"ORDER BY ordinal_position",
|
|
147
|
-
[table],
|
|
148
|
-
)
|
|
149
|
-
return {row[0]: str(row[1] or "") for row in rows}
|
|
139
|
+
return db.record_store.stored_column_types(variable)
|
|
150
140
|
|
|
151
141
|
|
|
152
142
|
def datetime_zone_meta(db, variable: str, columns) -> dict:
|
|
@@ -283,15 +273,7 @@ def sample_value(db, variable: str) -> Any:
|
|
|
283
273
|
columns = data_columns_for(db, variable)
|
|
284
274
|
if not columns:
|
|
285
275
|
return None
|
|
286
|
-
|
|
287
|
-
row = db._duck._fetchone(
|
|
288
|
-
f'SELECT t."{columns[0]}" FROM "{table}" t '
|
|
289
|
-
f"JOIN _record r ON t.record_id = r.record_id "
|
|
290
|
-
f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE "
|
|
291
|
-
f"AND t.\"{columns[0]}\" IS NOT NULL LIMIT 1",
|
|
292
|
-
[variable],
|
|
293
|
-
)
|
|
294
|
-
value = row[0] if row else None
|
|
276
|
+
value = db.record_store.sample_cell(variable, columns[0])
|
|
295
277
|
# A row list's cell is its source rows, not a series: classify by one
|
|
296
278
|
# ELEMENT, the shape the source turns it into (rows + a Row factor).
|
|
297
279
|
if value is not None and _row_list_columns(db, variable, columns[:1]):
|
|
@@ -308,15 +290,7 @@ def sample_column_value(db, variable: str, column: str) -> Any:
|
|
|
308
290
|
in every column, so "what is this variable" cannot answer "may this column
|
|
309
291
|
group a figure".
|
|
310
292
|
"""
|
|
311
|
-
|
|
312
|
-
row = db._duck._fetchone(
|
|
313
|
-
f'SELECT t."{column}" FROM "{table}" t '
|
|
314
|
-
f"JOIN _record r ON t.record_id = r.record_id "
|
|
315
|
-
f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE "
|
|
316
|
-
f'AND t."{column}" IS NOT NULL LIMIT 1',
|
|
317
|
-
[variable],
|
|
318
|
-
)
|
|
319
|
-
return row[0] if row else None
|
|
293
|
+
return db.record_store.sample_cell(variable, column)
|
|
320
294
|
|
|
321
295
|
|
|
322
296
|
def field_shapes(db, variable: str) -> dict[str, Shape]:
|
|
@@ -361,16 +335,7 @@ def column_levels(db, variable: str, column: str, *, limit: int) -> list[str]:
|
|
|
361
335
|
NULL arrives as float64 in pandas and would spell its integer levels
|
|
362
336
|
``1.0``, which nothing downstream matches.
|
|
363
337
|
"""
|
|
364
|
-
|
|
365
|
-
rows = db._duck._fetchall(
|
|
366
|
-
f'SELECT DISTINCT CAST(t."{column}" AS VARCHAR) AS level FROM "{table}" t '
|
|
367
|
-
f"JOIN _record r ON t.record_id = r.record_id "
|
|
368
|
-
f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE "
|
|
369
|
-
f'AND t."{column}" IS NOT NULL '
|
|
370
|
-
f"LIMIT {int(limit)}",
|
|
371
|
-
[variable],
|
|
372
|
-
)
|
|
373
|
-
return [row[0] for row in rows]
|
|
338
|
+
return db.record_store.column_levels(variable, column, limit=limit)
|
|
374
339
|
|
|
375
340
|
|
|
376
341
|
def variable_levels(db, variable: str) -> list[str]:
|
|
@@ -382,19 +347,7 @@ def variable_levels(db, variable: str) -> list[str]:
|
|
|
382
347
|
database just to open the panel. COUNT ignores NULLs, so one row of counts
|
|
383
348
|
says exactly which keys are populated.
|
|
384
349
|
"""
|
|
385
|
-
|
|
386
|
-
if not keys:
|
|
387
|
-
return []
|
|
388
|
-
counts = ", ".join(f'COUNT(s."{key}")' for key in keys)
|
|
389
|
-
row = db._duck._fetchone(
|
|
390
|
-
f"SELECT {counts} FROM _record r "
|
|
391
|
-
f"LEFT JOIN _schema s ON r.schema_id = s.schema_id "
|
|
392
|
-
f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE",
|
|
393
|
-
[variable],
|
|
394
|
-
)
|
|
395
|
-
if row is None:
|
|
396
|
-
return []
|
|
397
|
-
return [key for key, count in zip(keys, row, strict=True) if count]
|
|
350
|
+
return db.record_store.occupied_keys(variable)
|
|
398
351
|
|
|
399
352
|
|
|
400
353
|
def _normalize_cell(value: Any) -> Any:
|
|
@@ -493,7 +446,7 @@ def _row_list_columns(db, variable: str, columns) -> list[str]:
|
|
|
493
446
|
``on_duplicate="combine"``). This reader selects the stored cells as they
|
|
494
447
|
are, so EVERY cell of such a column is an array (``[7]``, ``[5, 12]``).
|
|
495
448
|
The source splits them into rows with a ``Row`` factor
|
|
496
|
-
(scistackplot.rows) — they are observations, not a 1-D series. This
|
|
449
|
+
(scistackplot.reduce.rows) — they are observations, not a 1-D series. This
|
|
497
450
|
replaced the "read as 1-D" decision of
|
|
498
451
|
.claude/plan-combine-duplicate-locations.md §6 on 2026-10-30
|
|
499
452
|
(.claude/plan-row-list-plots.md)."""
|
|
@@ -541,32 +494,19 @@ def load_variable(
|
|
|
541
494
|
if not include_data:
|
|
542
495
|
columns = []
|
|
543
496
|
|
|
544
|
-
table = table_name_for(db, variable)
|
|
545
|
-
# Schema keys as VARCHAR in the query, not stringified after: a
|
|
546
|
-
# DataFrame fetch types each column, and an integer key with one NULL
|
|
547
|
-
# among its rows would arrive as float64 and stringify as "1.0".
|
|
548
|
-
schema_select = "".join(
|
|
549
|
-
f', CAST(s."{key}" AS VARCHAR) AS "{key}"' for key in keys
|
|
550
|
-
)
|
|
551
497
|
# Datetime columns bring their __tz zone column, folded back right
|
|
552
498
|
# after the fetch (sciduckdb.merge_tz_columns): without it every
|
|
553
499
|
# aware value reached a plot as a naive UTC instant.
|
|
554
500
|
zone_meta = datetime_zone_meta(db, variable, columns) if columns else {"columns": {}}
|
|
555
501
|
zone_select = [tz_column_name(c) for c in zone_meta["columns"]]
|
|
556
|
-
data_select = "".join(f', t."{column}"' for column in [*columns, *zone_select])
|
|
557
|
-
query = (
|
|
558
|
-
f"SELECT t.record_id{data_select}{schema_select} "
|
|
559
|
-
f'FROM "{table}" t '
|
|
560
|
-
f"JOIN _record r ON t.record_id = r.record_id "
|
|
561
|
-
f"LEFT JOIN _schema s ON r.schema_id = s.schema_id "
|
|
562
|
-
f"WHERE r.type = ? AND r.excluded IS DISTINCT FROM TRUE"
|
|
563
|
-
)
|
|
564
502
|
# NOTE: no schema filter, no LIMIT, no column projection — this reads
|
|
565
503
|
# EVERY non-excluded record of the variable with EVERY data column,
|
|
566
504
|
# whatever the caller intends to plot (pushdown is a later stage of
|
|
567
505
|
# .claude/plan-plot-minimal-load-examples.md).
|
|
568
506
|
#
|
|
569
|
-
#
|
|
507
|
+
# The read is scidb's (`RecordStore.long_frame`, schema keys cast to
|
|
508
|
+
# text in the query), through `fetchdf`, never `fetchall`, for the
|
|
509
|
+
# data columns. Measured on the
|
|
570
510
|
# real database 2026-09-13 (plan §7): one DOUBLE[] column of 17.4 M
|
|
571
511
|
# samples took 4.7 s through `fetchall` — a Python float per sample —
|
|
572
512
|
# and 0.26 s through `.df()`, which hands each cell over as one numpy
|
|
@@ -574,7 +514,7 @@ def load_variable(
|
|
|
574
514
|
# timed out. The "loaded" line below says `ndarray` or `boxed` so a
|
|
575
515
|
# regression to per-sample boxing is visible in the log, not inferred.
|
|
576
516
|
with timer.phase("fetch"):
|
|
577
|
-
frame = db.
|
|
517
|
+
frame = db.record_store.long_frame(variable, [*columns, *zone_select])
|
|
578
518
|
if zone_meta["columns"]:
|
|
579
519
|
with timer.phase("datetime_zones"):
|
|
580
520
|
merge_tz_columns(frame, zone_meta, where=f"load_variable({variable})")
|
|
@@ -618,7 +558,7 @@ def load_variable(
|
|
|
618
558
|
# Cells and SAMPLES alongside the record count: 419 records is the same
|
|
619
559
|
# number whether each holds 200 samples or 250,000, and only the second
|
|
620
560
|
# explains a plot that never returns. Measured over the data columns
|
|
621
|
-
# only, one O(cells) pass (scistackplot.framesize).
|
|
561
|
+
# only, one O(cells) pass (scistackplot.text.framesize).
|
|
622
562
|
with timer.phase("measure_extent"):
|
|
623
563
|
extent = frame_extent(frame, columns)
|
|
624
564
|
Log.info(
|
|
@@ -870,7 +810,7 @@ def attach_variants(
|
|
|
870
810
|
See ``docs/claude/variant-selection.md`` and
|
|
871
811
|
``docs/claude/function-version-variants.md``.
|
|
872
812
|
"""
|
|
873
|
-
from scidb.
|
|
813
|
+
from scidb.provenance.query import variant_identity_batch
|
|
874
814
|
|
|
875
815
|
record_ids = frame["record_id"].tolist()
|
|
876
816
|
ident = variant_identity_batch(db._duck, record_ids)
|
|
@@ -1131,9 +1071,9 @@ def _truncate(value: Any, limit: int = 60) -> Any:
|
|
|
1131
1071
|
|
|
1132
1072
|
def _drop_excluded_rows(db, frame: pd.DataFrame, keys: list[str], variable: str) -> pd.DataFrame:
|
|
1133
1073
|
"""``frame`` minus the rows at a schema location the project excluded
|
|
1134
|
-
(:func:`scidb.exclusions.filter_excluded_combos`, the rule ``for_each``
|
|
1074
|
+
(:func:`scidb.store.exclusions.filter_excluded_combos`, the rule ``for_each``
|
|
1135
1075
|
applies). One pass over the DISTINCT locations, not the rows."""
|
|
1136
|
-
from scidb.exclusions import filter_excluded_combos
|
|
1076
|
+
from scidb.store.exclusions import filter_excluded_combos
|
|
1137
1077
|
|
|
1138
1078
|
present = [key for key in keys if key in frame.columns]
|
|
1139
1079
|
if frame.empty or not present:
|
|
@@ -4,7 +4,7 @@ database and applied to any variable.
|
|
|
4
4
|
|
|
5
5
|
A saved plot (:mod:`scistackplotdb.saved`) belongs to one variable. A preset
|
|
6
6
|
belongs to the project: it is the saved plot's settings with everything that
|
|
7
|
-
belongs to the variable taken out (``scistackplot.presets`` decides what that
|
|
7
|
+
belongs to the variable taken out (``scistackplot.model.presets`` decides what that
|
|
8
8
|
is, once). Apply it to another variable and the figure is drawn the same way::
|
|
9
9
|
|
|
10
10
|
from scistackplotdb import ScidbSource, apply_preset_to, list_presets
|
|
@@ -16,7 +16,7 @@ is, once). Apply it to another variable and the figure is drawn the same way::
|
|
|
16
16
|
)
|
|
17
17
|
|
|
18
18
|
Storage follows the saved-plot rules exactly, through the same
|
|
19
|
-
:class:`~
|
|
19
|
+
:class:`~scidb.store.versioned_store.VersionedStore`: one append-only table,
|
|
20
20
|
``_plot_preset``; the id is the identity and the name a label (unique across
|
|
21
21
|
the project); re-saving a visible name appends a version; "remove" hides,
|
|
22
22
|
nothing is ever deleted; save is strict, open salvages, and the stored copy
|
|
@@ -30,7 +30,7 @@ from typing import Any, Callable
|
|
|
30
30
|
|
|
31
31
|
from scistacklog import Log
|
|
32
32
|
|
|
33
|
-
from .
|
|
33
|
+
from scidb.store.versioned_store import VersionedStore
|
|
34
34
|
|
|
35
35
|
LAYER = "scistackplotdb"
|
|
36
36
|
|
|
@@ -117,6 +117,7 @@ STORE = VersionedStore(
|
|
|
117
117
|
noun="preset",
|
|
118
118
|
tag="preset",
|
|
119
119
|
error=PresetError,
|
|
120
|
+
owner="scistackplotdb",
|
|
120
121
|
)
|
|
121
122
|
|
|
122
123
|
|
|
@@ -35,7 +35,7 @@ from typing import Any, Callable
|
|
|
35
35
|
|
|
36
36
|
from scistacklog import Log
|
|
37
37
|
|
|
38
|
-
from .
|
|
38
|
+
from scidb.store.versioned_store import VersionedStore
|
|
39
39
|
|
|
40
40
|
LAYER = "scistackplotdb"
|
|
41
41
|
|
|
@@ -124,7 +124,7 @@ class SavedPlot:
|
|
|
124
124
|
# Table
|
|
125
125
|
# ---------------------------------------------------------------------------
|
|
126
126
|
|
|
127
|
-
#: The rules (names, versions, hiding, never deleting) live in `
|
|
127
|
+
#: The rules (names, versions, hiding, never deleting) live in `scidb.store.versioned_store`,
|
|
128
128
|
#: shared with presets so the two stores cannot drift apart.
|
|
129
129
|
STORE = VersionedStore(
|
|
130
130
|
table=TABLE,
|
|
@@ -133,6 +133,7 @@ STORE = VersionedStore(
|
|
|
133
133
|
noun="saved plot",
|
|
134
134
|
tag="saved_plot",
|
|
135
135
|
error=SavedPlotError,
|
|
136
|
+
owner="scistackplotdb",
|
|
136
137
|
)
|
|
137
138
|
|
|
138
139
|
|
|
@@ -433,14 +434,14 @@ FROZEN_KEY = "frozen_code"
|
|
|
433
434
|
def _row_masks(spec, table) -> list[tuple[int, Any]]:
|
|
434
435
|
"""``[(row_index, mask)]`` — the rows each variant row DRAWS, by the same
|
|
435
436
|
owner and first-match-wins order the figure uses
|
|
436
|
-
(``scistackplot.variants.row_mask`` / ``apply_variant_sets``).
|
|
437
|
+
(``scistackplot.reduce.variants.row_mask`` / ``apply_variant_sets``).
|
|
437
438
|
|
|
438
439
|
With no row concrete, the figure draws an empty row's effective selection;
|
|
439
440
|
that is frozen as row 0 (synthesised on open if the spec has none).
|
|
440
441
|
"""
|
|
441
442
|
import pandas as pd
|
|
442
|
-
from scistackplot.spec import VariantSet
|
|
443
|
-
from scistackplot.variants import defined_sets, is_stated, row_mask
|
|
443
|
+
from scistackplot.model.spec import VariantSet
|
|
444
|
+
from scistackplot.reduce.variants import defined_sets, is_stated, row_mask
|
|
444
445
|
|
|
445
446
|
frame = table.frame
|
|
446
447
|
if not defined_sets(spec.variant_sets):
|
|
@@ -465,7 +466,7 @@ def freeze_code(db, spec, table) -> dict[str, dict[str, list[str]]]:
|
|
|
465
466
|
``Code:`` column then, but the chain still names the hash, and that is
|
|
466
467
|
what lets the plot reopen on it after a second version appears.
|
|
467
468
|
"""
|
|
468
|
-
from scidb.
|
|
469
|
+
from scidb.provenance.query import code_versions_batch
|
|
469
470
|
|
|
470
471
|
frame = table.frame
|
|
471
472
|
if "record_id" not in frame.columns:
|
|
@@ -548,10 +549,10 @@ def thaw_code(db, spec, table, frozen_code: dict, *, label: str = ""):
|
|
|
548
549
|
"""
|
|
549
550
|
from dataclasses import replace
|
|
550
551
|
|
|
551
|
-
from scidb.
|
|
552
|
+
from scidb.provenance.query import function_versions
|
|
552
553
|
from scistackplot import NoteKind, RestoreNote
|
|
553
|
-
from scistackplot.spec import VariantSet
|
|
554
|
-
from scistackplot.variants import _ordinal_rank
|
|
554
|
+
from scistackplot.model.spec import VariantSet
|
|
555
|
+
from scistackplot.reduce.variants import _ordinal_rank
|
|
555
556
|
|
|
556
557
|
if not frozen_code:
|
|
557
558
|
return spec, [], []
|
|
@@ -640,8 +641,8 @@ def follow_latest_saved_plot(db, plot_id: str, *, table=None) -> SavedPlotInfo:
|
|
|
640
641
|
from dataclasses import replace
|
|
641
642
|
|
|
642
643
|
from scistackplot import restore_spec
|
|
643
|
-
from scistackplot.spec import VariantSet
|
|
644
|
-
from scistackplot.variants import is_function_axis
|
|
644
|
+
from scistackplot.model.spec import VariantSet
|
|
645
|
+
from scistackplot.reduce.variants import is_function_axis
|
|
645
646
|
|
|
646
647
|
from .load import LATEST_COLUMN
|
|
647
648
|
|
|
@@ -27,12 +27,12 @@ from scistackplot import (
|
|
|
27
27
|
natural_sort_key,
|
|
28
28
|
row_origin,
|
|
29
29
|
)
|
|
30
|
-
from scistackplot.categorical import MAX_CATEGORY_LEVELS
|
|
31
|
-
from scistackplot.dedup import SingleFlight
|
|
32
|
-
from scistackplot.framesize import format_extent, frame_extent
|
|
30
|
+
from scistackplot.reduce.categorical import MAX_CATEGORY_LEVELS
|
|
31
|
+
from scistackplot.sources.dedup import SingleFlight
|
|
32
|
+
from scistackplot.text.framesize import format_extent, frame_extent
|
|
33
33
|
from scistackplot.sources import BaseSource, UnknownMeasureError
|
|
34
|
-
from scistackplot.variants import VARIABLE_COLUMN
|
|
35
|
-
from scistackplot.temporal import is_temporal_column
|
|
34
|
+
from scistackplot.reduce.variants import VARIABLE_COLUMN
|
|
35
|
+
from scistackplot.reduce.temporal import is_temporal_column
|
|
36
36
|
|
|
37
37
|
from .hierarchy import join_frames, joinable, joined_levels
|
|
38
38
|
from .load import (
|
|
@@ -75,7 +75,7 @@ def _as_levels(column):
|
|
|
75
75
|
keys are cast to VARCHAR on load for the same reason
|
|
76
76
|
(``load.load_variable``).
|
|
77
77
|
|
|
78
|
-
Deliberately these two calls and nothing cleverer: ``scistackplot.codegen``
|
|
78
|
+
Deliberately these two calls and nothing cleverer: ``scistackplot.export.codegen``
|
|
79
79
|
emits ``fillna(...).astype(str)`` into the exported endpoint, so anything
|
|
80
80
|
extra here (normalising ``70.0`` to ``70``, say) would spell a level one way
|
|
81
81
|
in the preview and another way in the export — the disagreement that
|
|
@@ -85,7 +85,7 @@ def _as_levels(column):
|
|
|
85
85
|
|
|
86
86
|
A DATETIME column is the exception: its values stay datetimes (missing
|
|
87
87
|
ones named), and ``LongTable.from_frame`` gives each its identity label
|
|
88
|
-
in time order (``scistackplot.temporal.level_labels``) — text that says
|
|
88
|
+
in time order (``scistackplot.reduce.temporal.level_labels``) — text that says
|
|
89
89
|
when, not ``2026-07-01 14:00:00+02:00`` sorted as text. The generated code
|
|
90
90
|
emits the same labels (``temporal.factor_label_code``).
|
|
91
91
|
"""
|
|
@@ -150,7 +150,7 @@ class ScidbSource(BaseSource):
|
|
|
150
150
|
if not self._fast:
|
|
151
151
|
return None # -> reducer_for() supplies the pandas reference
|
|
152
152
|
if self._reducer_instance is None:
|
|
153
|
-
from scistackplot.reducer import NumpyReducer
|
|
153
|
+
from scistackplot.reduce.reducer import NumpyReducer
|
|
154
154
|
|
|
155
155
|
self._reducer_instance = NumpyReducer()
|
|
156
156
|
return self._reducer_instance
|
|
@@ -279,10 +279,7 @@ class ScidbSource(BaseSource):
|
|
|
279
279
|
return self._levels[variable]
|
|
280
280
|
|
|
281
281
|
def _schema_levels(self, key: str) -> list[str]:
|
|
282
|
-
|
|
283
|
-
f'SELECT DISTINCT s."{key}" FROM _schema s WHERE s."{key}" IS NOT NULL'
|
|
284
|
-
)
|
|
285
|
-
values = [str(row[0]) for row in rows]
|
|
282
|
+
values = [str(v) for v in self._db.distinct_schema_values(key)]
|
|
286
283
|
return self._ordered(key, values)
|
|
287
284
|
|
|
288
285
|
def _aliases_source(self):
|
|
@@ -383,7 +380,7 @@ class ScidbSource(BaseSource):
|
|
|
383
380
|
# of vanishing. scidb owns the declaration; this only applies it.
|
|
384
381
|
order = getattr(self._db, "dataset_schema_key_order", None) or {}
|
|
385
382
|
if order.get(key):
|
|
386
|
-
from
|
|
383
|
+
from scifor.project.schema_order import order_levels
|
|
387
384
|
|
|
388
385
|
ordered = order_levels(key, unique, declared=order, fallback=default)
|
|
389
386
|
else:
|
|
@@ -478,7 +475,7 @@ class ScidbSource(BaseSource):
|
|
|
478
475
|
than one drawn from a cache that is almost certainly still good.
|
|
479
476
|
"""
|
|
480
477
|
try:
|
|
481
|
-
from scidb.
|
|
478
|
+
from scidb.provenance.query import variable_content_fingerprint
|
|
482
479
|
|
|
483
480
|
return variable_content_fingerprint(self._db._duck, variable)
|
|
484
481
|
except Exception as exc:
|
|
@@ -603,7 +600,7 @@ class ScidbSource(BaseSource):
|
|
|
603
600
|
memoizes the result — this method always does the full work.
|
|
604
601
|
|
|
605
602
|
``measures`` are the y variables. Several of them **stack**: one value
|
|
606
|
-
column plus a :data:`~scistackplot.variants.VARIABLE_COLUMN` saying
|
|
603
|
+
column plus a :data:`~scistackplot.reduce.variants.VARIABLE_COLUMN` saying
|
|
607
604
|
which variable each row came from, which is what lets a variant row
|
|
608
605
|
claim its own variable's rows and no others.
|
|
609
606
|
|
|
@@ -809,8 +806,30 @@ class ScidbSource(BaseSource):
|
|
|
809
806
|
layer=LAYER,
|
|
810
807
|
)
|
|
811
808
|
table.reducer = self._reducer()
|
|
809
|
+
if len(measures) == 1:
|
|
810
|
+
table.absent = self._absent_locations(frame, levels)
|
|
812
811
|
return table
|
|
813
812
|
|
|
813
|
+
def _absent_locations(self, frame, levels: list[str]):
|
|
814
|
+
"""The variable's "No record" locations (`scidb.provenance.inventory.
|
|
815
|
+
absent_locations`, the owner) as a frame of its schema-key columns:
|
|
816
|
+
places the dataset has where this variable has no record. None when
|
|
817
|
+
it cannot be told (no schema keys, or the query failed — said in the
|
|
818
|
+
log, never fatal: the figure draws either way)."""
|
|
819
|
+
import pandas as pd
|
|
820
|
+
from scidb.provenance.inventory import absent_locations
|
|
821
|
+
|
|
822
|
+
keys = [key for key in levels if key in frame.columns]
|
|
823
|
+
if not keys:
|
|
824
|
+
return None
|
|
825
|
+
try:
|
|
826
|
+
present = frame[keys].dropna().astype(str).drop_duplicates().itertuples(index=False, name=None)
|
|
827
|
+
rows = absent_locations(self._db, keys, present)
|
|
828
|
+
except Exception as exc: # noqa: BLE001 - a readout, never a reason to fail the plot
|
|
829
|
+
Log.warn("absent locations for %s not known: %s", keys, exc, layer=LAYER)
|
|
830
|
+
return None
|
|
831
|
+
return pd.DataFrame(rows, columns=keys)
|
|
832
|
+
|
|
814
833
|
def _own_column_groups(
|
|
815
834
|
self, primary, factor_variables: list[FactorVariable]
|
|
816
835
|
) -> list[FactorVariable]:
|
|
@@ -1071,7 +1090,7 @@ class ScidbSource(BaseSource):
|
|
|
1071
1090
|
``drop_duplicates`` silently kept whichever came first — a figure
|
|
1072
1091
|
stratified by a version of the spreadsheet nobody chose.
|
|
1073
1092
|
|
|
1074
|
-
The pin is applied with :func:`scistackplot.variants.variant_set_mask`,
|
|
1093
|
+
The pin is applied with :func:`scistackplot.reduce.variants.variant_set_mask`,
|
|
1075
1094
|
the same one definition the measure's own variant rows use, so "what a
|
|
1076
1095
|
selection keeps" cannot mean two things one call apart.
|
|
1077
1096
|
|
|
@@ -1080,7 +1099,7 @@ class ScidbSource(BaseSource):
|
|
|
1080
1099
|
(``plot-variant-rows.md`` § "An unfilled row is inert"). What used to be
|
|
1081
1100
|
silent about that state is now :meth:`_one_label_per_location`.
|
|
1082
1101
|
"""
|
|
1083
|
-
from scistackplot.variants import variant_set_mask
|
|
1102
|
+
from scistackplot.reduce.variants import variant_set_mask
|
|
1084
1103
|
|
|
1085
1104
|
selection = group.selection
|
|
1086
1105
|
frame = variable_frame.frame
|
|
@@ -1409,12 +1428,12 @@ class ScidbSource(BaseSource):
|
|
|
1409
1428
|
def default_variant_for(self, variable: str) -> dict:
|
|
1410
1429
|
"""The variant a GROUPING by ``variable`` should open on: latest.
|
|
1411
1430
|
|
|
1412
|
-
Exactly :func:`scistackplot.variants.default_selection` over that
|
|
1431
|
+
Exactly :func:`scistackplot.reduce.variants.default_selection` over that
|
|
1413
1432
|
variable's own variant table — the same rule, and the same *code*, that
|
|
1414
1433
|
decides what a plot opens on. Two rules for "latest" would be two
|
|
1415
1434
|
answers, and this one has to agree with the Variants section beside it.
|
|
1416
1435
|
|
|
1417
|
-
Read :func:`~scistackplot.variants.default_selection` before touching
|
|
1436
|
+
Read :func:`~scistackplot.reduce.variants.default_selection` before touching
|
|
1418
1437
|
this: "latest" here is the per-schema-location **boolean flag**, never
|
|
1419
1438
|
the string ``"latest"``. The string stops resolving through the flag the
|
|
1420
1439
|
moment anything else is pinned in the same selection, and silently drops
|
|
@@ -1425,7 +1444,7 @@ class ScidbSource(BaseSource):
|
|
|
1425
1444
|
metadata queries rather than reading any of them.
|
|
1426
1445
|
"""
|
|
1427
1446
|
variable = base_variable(variable) # a text measure loads its variable
|
|
1428
|
-
from scistackplot.variants import default_selection
|
|
1447
|
+
from scistackplot.reduce.variants import default_selection
|
|
1429
1448
|
|
|
1430
1449
|
return default_selection(self.variant_table(variable))
|
|
1431
1450
|
|
|
@@ -1561,7 +1580,7 @@ class ScidbSource(BaseSource):
|
|
|
1561
1580
|
text_measure_name(variable_frame.name), len(usable), layer=LAYER,
|
|
1562
1581
|
)
|
|
1563
1582
|
# Datetime fields melt into ONE value column with the rest, which is a
|
|
1564
|
-
# time axis only if every field is a datetime (scistackplot.temporal).
|
|
1583
|
+
# time axis only if every field is a datetime (scistackplot.reduce.temporal).
|
|
1565
1584
|
# Beside numeric fields they would read as numbers (NaN), so they are
|
|
1566
1585
|
# left out and named — group by one (Variable.column) instead.
|
|
1567
1586
|
datetimes = [c for c in usable if is_temporal_column(frame[c])]
|
|
@@ -1637,7 +1656,7 @@ class ScidbSource(BaseSource):
|
|
|
1637
1656
|
field_factor: str | None,
|
|
1638
1657
|
):
|
|
1639
1658
|
"""``(frame, row column or None, its origin or None)`` — every
|
|
1640
|
-
row-list cell split into one row per element (scistackplot.rows,
|
|
1659
|
+
row-list cell split into one row per element (scistackplot.reduce.rows,
|
|
1641
1660
|
the one owner of the reshape; codegen mirrors it from the origin).
|
|
1642
1661
|
|
|
1643
1662
|
The Row factor is kept only when some cell held two or more rows: a
|
|
@@ -5,7 +5,7 @@ text fields (``Gait``: ``StepLength``, ``Cadence``, ``Outcome``, ``Side``)
|
|
|
5
5
|
cannot melt them all under one ``ColName`` factor. The catalog lists it twice
|
|
6
6
|
(user decision, 2026-11-01): ``Gait`` holds the numeric fields, as it always
|
|
7
7
|
has, and ``Gait (text)`` holds the text fields, drawn as counts, %s or
|
|
8
|
-
categories (``scistackplot.categorical``). Every field is reachable; none is
|
|
8
|
+
categories (``scistackplot.reduce.categorical``). Every field is reachable; none is
|
|
9
9
|
dropped. See docs/claude/categorical-measures.md.
|
|
10
10
|
|
|
11
11
|
This module is the one owner of the NAME: how a text measure is spelled, and
|
|
@@ -47,7 +47,7 @@ def _check_column_vocabulary() -> None:
|
|
|
47
47
|
here would not raise anywhere: it would make every pin a no-op and quietly
|
|
48
48
|
plot every variant at once.
|
|
49
49
|
"""
|
|
50
|
-
from scidb.variant import (
|
|
50
|
+
from scidb.model.variant import (
|
|
51
51
|
CODE_COLUMN_PREFIX,
|
|
52
52
|
LATEST_COLUMN_NAME,
|
|
53
53
|
RUN_COLUMN_PREFIX,
|
|
@@ -79,7 +79,7 @@ def variant_set(
|
|
|
79
79
|
table: LongTable | None = None,
|
|
80
80
|
) -> VariantSet:
|
|
81
81
|
"""
|
|
82
|
-
Build a named :class:`~scistackplot.spec.VariantSet` from a ``scidb.Variant``.
|
|
82
|
+
Build a named :class:`~scistackplot.model.spec.VariantSet` from a ``scidb.Variant``.
|
|
83
83
|
|
|
84
84
|
::
|
|
85
85
|
|
|
@@ -123,7 +123,7 @@ def selection_for(
|
|
|
123
123
|
become ``Run:fn``; branch params keep their namespaced name, which is
|
|
124
124
|
already the column name.
|
|
125
125
|
"""
|
|
126
|
-
from scidb.variant import VariantAxes
|
|
126
|
+
from scidb.model.variant import VariantAxes
|
|
127
127
|
|
|
128
128
|
# The three dimensions, parsed by their owner (docs/claude/variant-space.md
|
|
129
129
|
# §2) rather than by re-reading the prefixes here — this was the fourth
|
|
@@ -159,7 +159,7 @@ def branch_params_for(selection: dict[str, Any]) -> dict[str, Any]:
|
|
|
159
159
|
|
|
160
160
|
Needed because the schema location picker asks **scidb** a question about a
|
|
161
161
|
variant the plotting layer is holding: "which locations have this one?"
|
|
162
|
-
``scidb.locations.location_states`` takes a ``branch_params_filter``, the
|
|
162
|
+
``scidb.provenance.locations.location_states`` takes a ``branch_params_filter``, the
|
|
163
163
|
same dict ``Variant`` builds and ``find_record_id`` matches on, so the
|
|
164
164
|
picker's dots and a ``Variant(...).load()`` cannot disagree about which
|
|
165
165
|
records a selection names.
|
|
@@ -182,7 +182,7 @@ def branch_params_for(selection: dict[str, Any]) -> dict[str, Any]:
|
|
|
182
182
|
``CodeIsLatest: False`` has no scidb spelling — "not the latest" is not a
|
|
183
183
|
pin — so it is dropped with a warning rather than silently inverted.
|
|
184
184
|
|
|
185
|
-
The mapping itself is **scidb's** (:func:`scidb.variant.normalize_selection`)
|
|
185
|
+
The mapping itself is **scidb's** (:func:`scidb.model.variant.normalize_selection`)
|
|
186
186
|
rather than this module's, because three consumers now hand scidb a
|
|
187
187
|
column-keyed selection — this picker, the GUI's provenance RPC and
|
|
188
188
|
``scidb trace --variant`` — and a second copy of the rule is how the two
|
|
@@ -190,7 +190,7 @@ def branch_params_for(selection: dict[str, Any]) -> dict[str, Any]:
|
|
|
190
190
|
*check* that the two layers still spell the columns the same way: this
|
|
191
191
|
module is the only place that can see both sides.
|
|
192
192
|
"""
|
|
193
|
-
from scidb.variant import normalize_selection
|
|
193
|
+
from scidb.model.variant import normalize_selection
|
|
194
194
|
|
|
195
195
|
return normalize_selection(selection)
|
|
196
196
|
|
|
@@ -204,7 +204,7 @@ def _axes_of(table: LongTable | None) -> list[dict]:
|
|
|
204
204
|
def _bare_code_column(code_axes: list[dict]) -> str:
|
|
205
205
|
"""The column a bare ``code_version=`` means, or a message naming the choice.
|
|
206
206
|
|
|
207
|
-
Same rule as ``scidb.database._filter_records_by_code_version``: exactly one
|
|
207
|
+
Same rule as ``scidb.store.database._filter_records_by_code_version``: exactly one
|
|
208
208
|
versioned function is unambiguous, several are not.
|
|
209
209
|
"""
|
|
210
210
|
if len(code_axes) == 1:
|
|
@@ -283,7 +283,7 @@ def variant_graph(db, variable_frame, functions: list[str] | None = None) -> dic
|
|
|
283
283
|
function node on the canvas, so nodes outside this variable's chain can
|
|
284
284
|
still say what they have run.
|
|
285
285
|
"""
|
|
286
|
-
from scidb.
|
|
286
|
+
from scidb.provenance.query import code_versions_batch, function_versions
|
|
287
287
|
|
|
288
288
|
frame = variable_frame.frame
|
|
289
289
|
record_ids = frame["record_id"].tolist() if "record_id" in frame.columns else []
|
|
@@ -1,392 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Named, versioned, hideable rows: the store behind saved plots and presets.
|
|
3
|
-
|
|
4
|
-
Both are "a JSON envelope the user names, re-saves and removes", with the same
|
|
5
|
-
rules (docs/claude/saved-plots.md, docs/claude/plot-presets.md):
|
|
6
|
-
|
|
7
|
-
* **The id is the identity; the name is a label.** Rename is an UPDATE of
|
|
8
|
-
``name``. A new item reusing a removed item's name gets a new id.
|
|
9
|
-
* **Re-saving a visible name appends a version.** The newest version is what
|
|
10
|
-
lists and opens. The version number is computed inside the INSERT, so two
|
|
11
|
-
racing saves cannot claim the same one.
|
|
12
|
-
* **Nothing is ever deleted** (feedback_never_delete_mark_hidden): "remove"
|
|
13
|
-
sets ``hidden`` on every version.
|
|
14
|
-
* **Reads never create the table.** Writes call :meth:`ensure_table`.
|
|
15
|
-
* All reads go through ``_fetchall``/``_fetchone`` (the fetch-locking guard).
|
|
16
|
-
|
|
17
|
-
This module owns those rules once, so the two stores cannot drift apart
|
|
18
|
-
(CLAUDE.md NOTE 4). A store with a *scope column* (saved plots: ``variable``)
|
|
19
|
-
keeps names unique per scope; a store without one (presets) keeps them unique
|
|
20
|
-
across the project. Callers turn the raw rows this returns into their own
|
|
21
|
-
info objects.
|
|
22
|
-
"""
|
|
23
|
-
|
|
24
|
-
from __future__ import annotations
|
|
25
|
-
|
|
26
|
-
import json
|
|
27
|
-
import uuid
|
|
28
|
-
from dataclasses import dataclass
|
|
29
|
-
from datetime import datetime, timezone
|
|
30
|
-
from typing import Any, Callable
|
|
31
|
-
|
|
32
|
-
from scistacklog import Log
|
|
33
|
-
|
|
34
|
-
LAYER = "scistackplotdb"
|
|
35
|
-
|
|
36
|
-
#: Longest name accepted. A name is a list label, not a description.
|
|
37
|
-
MAX_NAME_LENGTH = 120
|
|
38
|
-
|
|
39
|
-
#: ``(item_id, scope, name, version, saved_at, hidden)``; scope is None for an
|
|
40
|
-
#: unscoped store.
|
|
41
|
-
Row = tuple
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
@dataclass(frozen=True)
|
|
45
|
-
class VersionedStore:
|
|
46
|
-
"""One append-only table of named, versioned envelopes."""
|
|
47
|
-
|
|
48
|
-
table: str
|
|
49
|
-
#: Column holding the item's identity (``plot_id``, ``preset_id``).
|
|
50
|
-
id_column: str
|
|
51
|
-
#: Column names are unique within, or None for project-wide names.
|
|
52
|
-
scope_column: str | None
|
|
53
|
-
#: What an item is called in messages ("saved plot", "preset").
|
|
54
|
-
noun: str
|
|
55
|
-
#: Log tag, without brackets.
|
|
56
|
-
tag: str
|
|
57
|
-
#: The error every refusal raises.
|
|
58
|
-
error: type[Exception]
|
|
59
|
-
|
|
60
|
-
# ---- table ----------------------------------------------------------
|
|
61
|
-
|
|
62
|
-
def ensure_table(self, db) -> None:
|
|
63
|
-
scope = f"{self.scope_column} VARCHAR NOT NULL," if self.scope_column else ""
|
|
64
|
-
db._duck._execute(f"""
|
|
65
|
-
CREATE TABLE IF NOT EXISTS {self.table} (
|
|
66
|
-
{self.id_column} VARCHAR NOT NULL,
|
|
67
|
-
{scope}
|
|
68
|
-
name VARCHAR NOT NULL,
|
|
69
|
-
version INTEGER NOT NULL,
|
|
70
|
-
saved_at VARCHAR NOT NULL,
|
|
71
|
-
hidden BOOLEAN NOT NULL DEFAULT FALSE,
|
|
72
|
-
envelope_json VARCHAR NOT NULL,
|
|
73
|
-
PRIMARY KEY ({self.id_column}, version)
|
|
74
|
-
)
|
|
75
|
-
""")
|
|
76
|
-
|
|
77
|
-
def table_exists(self, db) -> bool:
|
|
78
|
-
row = db._duck._fetchone(
|
|
79
|
-
"SELECT count(*) FROM information_schema.tables WHERE table_name = ?",
|
|
80
|
-
[self.table],
|
|
81
|
-
)
|
|
82
|
-
return bool(row and row[0])
|
|
83
|
-
|
|
84
|
-
# ---- names ----------------------------------------------------------
|
|
85
|
-
|
|
86
|
-
def check_name(self, name: Any) -> str:
|
|
87
|
-
"""*name* trimmed, or the store's error saying why it is not one."""
|
|
88
|
-
if not isinstance(name, str):
|
|
89
|
-
raise self.error(f"A {self.noun} needs a name.")
|
|
90
|
-
name = " ".join(name.split()) # trim, and no tabs/newlines in a list label
|
|
91
|
-
if not name:
|
|
92
|
-
raise self.error(f"A {self.noun} needs a name.")
|
|
93
|
-
if len(name) > MAX_NAME_LENGTH:
|
|
94
|
-
raise self.error(
|
|
95
|
-
f"A {self.noun}'s name is at most {MAX_NAME_LENGTH} characters "
|
|
96
|
-
f"(this one is {len(name)})."
|
|
97
|
-
)
|
|
98
|
-
return name
|
|
99
|
-
|
|
100
|
-
def clash_message(self, scope: str | None, name: str) -> str:
|
|
101
|
-
if self.scope_column:
|
|
102
|
-
return f"{scope} already has a {self.noun} named {name!r}."
|
|
103
|
-
return f"There is already a {self.noun} named {name!r}."
|
|
104
|
-
|
|
105
|
-
def describe(self, scope: str | None, name: str) -> str:
|
|
106
|
-
"""``StepLength / 'Fig 3'`` or ``'Session box'``, for logs."""
|
|
107
|
-
return f"{scope} / {name!r}" if self.scope_column else repr(name)
|
|
108
|
-
|
|
109
|
-
# ---- writes ---------------------------------------------------------
|
|
110
|
-
|
|
111
|
-
def save(
|
|
112
|
-
self,
|
|
113
|
-
db,
|
|
114
|
-
scope: str | None,
|
|
115
|
-
name: str,
|
|
116
|
-
envelope: dict,
|
|
117
|
-
*,
|
|
118
|
-
overwrite: bool,
|
|
119
|
-
current_id: str | None,
|
|
120
|
-
on_clash: Callable[[Row], Exception],
|
|
121
|
-
) -> tuple[Row, bool]:
|
|
122
|
-
"""Append *envelope* as a new version, returning ``(row, is_new)``.
|
|
123
|
-
|
|
124
|
-
A visible item called *name* (in *scope*) gets a new version.
|
|
125
|
-
Otherwise a new item is started. With ``overwrite=False``, appending
|
|
126
|
-
to an item other than *current_id* raises ``on_clash(that row)`` so
|
|
127
|
-
the caller can ask first.
|
|
128
|
-
"""
|
|
129
|
-
try:
|
|
130
|
-
text = json.dumps(envelope, sort_keys=True)
|
|
131
|
-
except (TypeError, ValueError) as exc:
|
|
132
|
-
raise self.error(f"The settings are not storable as JSON: {exc}") from exc
|
|
133
|
-
|
|
134
|
-
self.ensure_table(db)
|
|
135
|
-
current = self.visible_by_name(db, scope, name)
|
|
136
|
-
if current is not None and not overwrite and current[0] != current_id:
|
|
137
|
-
Log.info(
|
|
138
|
-
"[%s] save of %s refused: another %s (%s) has that name and "
|
|
139
|
-
"overwrite was not confirmed",
|
|
140
|
-
self.tag,
|
|
141
|
-
self.describe(scope, name),
|
|
142
|
-
self.noun,
|
|
143
|
-
current[0][:8],
|
|
144
|
-
layer=LAYER,
|
|
145
|
-
)
|
|
146
|
-
raise on_clash(current)
|
|
147
|
-
item_id = current[0] if current else uuid.uuid4().hex
|
|
148
|
-
saved_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
149
|
-
scope_col = f"{self.scope_column}, " if self.scope_column else ""
|
|
150
|
-
scope_val = [scope] if self.scope_column else []
|
|
151
|
-
# One statement: the next version number is decided inside the insert,
|
|
152
|
-
# so two saves racing on one item cannot both claim the same version.
|
|
153
|
-
db._duck._execute(
|
|
154
|
-
f"""
|
|
155
|
-
INSERT INTO {self.table}
|
|
156
|
-
({self.id_column}, {scope_col}name, version, saved_at, hidden, envelope_json)
|
|
157
|
-
SELECT ?, {"?, " if self.scope_column else ""}?,
|
|
158
|
-
COALESCE(MAX(version), 0) + 1, ?, FALSE, ?
|
|
159
|
-
FROM {self.table} WHERE {self.id_column} = ?
|
|
160
|
-
""",
|
|
161
|
-
[item_id, *scope_val, name, saved_at, text, item_id],
|
|
162
|
-
)
|
|
163
|
-
row = self.latest(db, item_id)
|
|
164
|
-
Log.info(
|
|
165
|
-
"[%s] saved %s as version %d (%s, %d bytes, envelope format %r)",
|
|
166
|
-
self.tag,
|
|
167
|
-
self.describe(scope, name),
|
|
168
|
-
row[3],
|
|
169
|
-
f"new {self.noun}" if current is None else f"{self.noun} {item_id[:8]}",
|
|
170
|
-
len(text),
|
|
171
|
-
envelope.get("format"),
|
|
172
|
-
layer=LAYER,
|
|
173
|
-
)
|
|
174
|
-
return row, current is None
|
|
175
|
-
|
|
176
|
-
def rename(self, db, item_id: str, new_name: str) -> Row:
|
|
177
|
-
new_name = self.check_name(new_name)
|
|
178
|
-
row = self.require(db, item_id)
|
|
179
|
-
clash = self.visible_by_name(db, row[1], new_name)
|
|
180
|
-
if clash is not None and clash[0] != item_id:
|
|
181
|
-
raise self.error(self.clash_message(row[1], new_name))
|
|
182
|
-
db._duck._execute(
|
|
183
|
-
f"UPDATE {self.table} SET name = ? WHERE {self.id_column} = ?",
|
|
184
|
-
[new_name, item_id],
|
|
185
|
-
)
|
|
186
|
-
Log.info(
|
|
187
|
-
"[%s] renamed %s -> %r (%s)",
|
|
188
|
-
self.tag,
|
|
189
|
-
self.describe(row[1], row[2]),
|
|
190
|
-
new_name,
|
|
191
|
-
item_id[:8],
|
|
192
|
-
layer=LAYER,
|
|
193
|
-
)
|
|
194
|
-
return self.latest(db, item_id)
|
|
195
|
-
|
|
196
|
-
def set_hidden(self, db, item_id: str, hidden: bool) -> Row:
|
|
197
|
-
row = self.require(db, item_id)
|
|
198
|
-
if not hidden:
|
|
199
|
-
clash = self.visible_by_name(db, row[1], row[2])
|
|
200
|
-
if clash is not None and clash[0] != item_id:
|
|
201
|
-
raise self.error(
|
|
202
|
-
f"{self.clash_message(row[1], row[2])[:-1]}; rename one of them first."
|
|
203
|
-
)
|
|
204
|
-
db._duck._execute(
|
|
205
|
-
f"UPDATE {self.table} SET hidden = ? WHERE {self.id_column} = ?",
|
|
206
|
-
[bool(hidden), item_id],
|
|
207
|
-
)
|
|
208
|
-
Log.info(
|
|
209
|
-
"[%s] %s %s (%s)",
|
|
210
|
-
self.tag,
|
|
211
|
-
"hid" if hidden else "unhid",
|
|
212
|
-
self.describe(row[1], row[2]),
|
|
213
|
-
item_id[:8],
|
|
214
|
-
layer=LAYER,
|
|
215
|
-
)
|
|
216
|
-
return self.latest(db, item_id)
|
|
217
|
-
|
|
218
|
-
# ---- reads ----------------------------------------------------------
|
|
219
|
-
|
|
220
|
-
def _columns(self) -> str:
|
|
221
|
-
scope = self.scope_column or "NULL"
|
|
222
|
-
return f"{self.id_column}, {scope}, name, version, saved_at, hidden"
|
|
223
|
-
|
|
224
|
-
def list_rows(
|
|
225
|
-
self,
|
|
226
|
-
db,
|
|
227
|
-
scope: str | None = None,
|
|
228
|
-
*,
|
|
229
|
-
include_hidden: bool = False,
|
|
230
|
-
with_envelope: bool = False,
|
|
231
|
-
) -> list[Row]:
|
|
232
|
-
"""The newest version of each item (in *scope*), newest first.
|
|
233
|
-
|
|
234
|
-
With *with_envelope*, each row carries its envelope JSON as a 7th
|
|
235
|
-
element, for a list that shows something stored inside it.
|
|
236
|
-
"""
|
|
237
|
-
envelope = ", envelope_json" if with_envelope else ""
|
|
238
|
-
if not self.table_exists(db):
|
|
239
|
-
return []
|
|
240
|
-
where = f"{self.scope_column} = ? AND " if self.scope_column else ""
|
|
241
|
-
rows = db._duck._fetchall(
|
|
242
|
-
f"""
|
|
243
|
-
SELECT {self._columns()}{envelope}
|
|
244
|
-
FROM {self.table} AS t
|
|
245
|
-
WHERE {where}version = (
|
|
246
|
-
SELECT MAX(version) FROM {self.table}
|
|
247
|
-
WHERE {self.id_column} = t.{self.id_column}
|
|
248
|
-
)
|
|
249
|
-
{"" if include_hidden else "AND NOT hidden"}
|
|
250
|
-
ORDER BY saved_at DESC, name
|
|
251
|
-
""",
|
|
252
|
-
[scope] if self.scope_column else [],
|
|
253
|
-
)
|
|
254
|
-
Log.debug(
|
|
255
|
-
"[%s] %s: %d %s(s) listed%s",
|
|
256
|
-
self.tag,
|
|
257
|
-
scope if self.scope_column else "project",
|
|
258
|
-
len(rows),
|
|
259
|
-
self.noun,
|
|
260
|
-
" (hidden included)" if include_hidden else "",
|
|
261
|
-
layer=LAYER,
|
|
262
|
-
)
|
|
263
|
-
return list(rows)
|
|
264
|
-
|
|
265
|
-
def history(self, db, item_id: str, *, with_envelope: bool = False) -> list[Row]:
|
|
266
|
-
if not self.table_exists(db):
|
|
267
|
-
return []
|
|
268
|
-
envelope = ", envelope_json" if with_envelope else ""
|
|
269
|
-
return list(db._duck._fetchall(
|
|
270
|
-
f"""
|
|
271
|
-
SELECT {self._columns()}{envelope}
|
|
272
|
-
FROM {self.table} WHERE {self.id_column} = ? ORDER BY version DESC
|
|
273
|
-
""",
|
|
274
|
-
[item_id],
|
|
275
|
-
))
|
|
276
|
-
|
|
277
|
-
def load(self, db, item_id: str, version: int | None = None) -> tuple[Row, str]:
|
|
278
|
-
"""``(row, envelope_json)`` of one version (the newest by default)."""
|
|
279
|
-
if not self.table_exists(db):
|
|
280
|
-
raise self.error(f"No {self.noun} {item_id!r}: nothing has been saved yet.")
|
|
281
|
-
if version is None:
|
|
282
|
-
row = db._duck._fetchone(
|
|
283
|
-
f"""
|
|
284
|
-
SELECT {self._columns()}, envelope_json
|
|
285
|
-
FROM {self.table} WHERE {self.id_column} = ?
|
|
286
|
-
ORDER BY version DESC LIMIT 1
|
|
287
|
-
""",
|
|
288
|
-
[item_id],
|
|
289
|
-
)
|
|
290
|
-
else:
|
|
291
|
-
row = db._duck._fetchone(
|
|
292
|
-
f"""
|
|
293
|
-
SELECT {self._columns()}, envelope_json
|
|
294
|
-
FROM {self.table} WHERE {self.id_column} = ? AND version = ?
|
|
295
|
-
""",
|
|
296
|
-
[item_id, int(version)],
|
|
297
|
-
)
|
|
298
|
-
if row is None:
|
|
299
|
-
which = f"version {version} of " if version is not None else ""
|
|
300
|
-
raise self.error(f"No {self.noun} {which}{item_id!r}.")
|
|
301
|
-
return tuple(row[:6]), row[6]
|
|
302
|
-
|
|
303
|
-
def visible_by_name(self, db, scope: str | None, name: str) -> Row | None:
|
|
304
|
-
if not self.table_exists(db):
|
|
305
|
-
return None
|
|
306
|
-
where = f"{self.scope_column} = ? AND " if self.scope_column else ""
|
|
307
|
-
row = db._duck._fetchone(
|
|
308
|
-
f"""
|
|
309
|
-
SELECT {self._columns()}
|
|
310
|
-
FROM {self.table}
|
|
311
|
-
WHERE {where}name = ? AND NOT hidden
|
|
312
|
-
ORDER BY version DESC LIMIT 1
|
|
313
|
-
""",
|
|
314
|
-
[*([scope] if self.scope_column else []), name],
|
|
315
|
-
)
|
|
316
|
-
return tuple(row) if row else None
|
|
317
|
-
|
|
318
|
-
def latest(self, db, item_id: str) -> Row:
|
|
319
|
-
row = db._duck._fetchone(
|
|
320
|
-
f"""
|
|
321
|
-
SELECT {self._columns()}
|
|
322
|
-
FROM {self.table} WHERE {self.id_column} = ?
|
|
323
|
-
ORDER BY version DESC LIMIT 1
|
|
324
|
-
""",
|
|
325
|
-
[item_id],
|
|
326
|
-
)
|
|
327
|
-
if row is None:
|
|
328
|
-
raise self.error(f"No {self.noun} {item_id!r}.")
|
|
329
|
-
return tuple(row)
|
|
330
|
-
|
|
331
|
-
def require(self, db, item_id: str) -> Row:
|
|
332
|
-
if not self.table_exists(db):
|
|
333
|
-
raise self.error(f"No {self.noun} {item_id!r}: nothing has been saved yet.")
|
|
334
|
-
return self.latest(db, item_id)
|
|
335
|
-
|
|
336
|
-
def parse_envelope(self, text: str, label: str) -> dict:
|
|
337
|
-
"""The stored envelope as a dict. A row that is not one opens on
|
|
338
|
-
defaults rather than failing the open."""
|
|
339
|
-
try:
|
|
340
|
-
envelope = json.loads(text)
|
|
341
|
-
except (TypeError, ValueError) as exc:
|
|
342
|
-
Log.warn("[%s] %s: stored envelope is not JSON (%s)", self.tag, label, exc,
|
|
343
|
-
layer=LAYER)
|
|
344
|
-
return {}
|
|
345
|
-
if not isinstance(envelope, dict):
|
|
346
|
-
Log.warn(
|
|
347
|
-
"[%s] %s: stored envelope is %s, not a table",
|
|
348
|
-
self.tag,
|
|
349
|
-
label,
|
|
350
|
-
type(envelope).__name__,
|
|
351
|
-
layer=LAYER,
|
|
352
|
-
)
|
|
353
|
-
return {}
|
|
354
|
-
return envelope
|
|
355
|
-
|
|
356
|
-
# ---- copying a whole store (project bundles) ------------------------
|
|
357
|
-
|
|
358
|
-
def _all_columns(self) -> list[str]:
|
|
359
|
-
scope = [self.scope_column] if self.scope_column else []
|
|
360
|
-
return [self.id_column, *scope, "name", "version", "saved_at", "hidden", "envelope_json"]
|
|
361
|
-
|
|
362
|
-
def dump_rows(self, db) -> list[dict]:
|
|
363
|
-
"""Every row, every version, hidden ones too, as plain dicts: what a
|
|
364
|
-
project bundle carries (``scistackplotdb.bundle_section``). Nothing
|
|
365
|
-
is filtered -- a copy keeps the history the original kept."""
|
|
366
|
-
if not self.table_exists(db):
|
|
367
|
-
return []
|
|
368
|
-
cols = self._all_columns()
|
|
369
|
-
rows = db._duck._fetchall(
|
|
370
|
-
f"SELECT {', '.join(cols)} FROM {self.table} ORDER BY {self.id_column}, version"
|
|
371
|
-
)
|
|
372
|
-
Log.info("[%s] dump_rows: %d row(s)", self.tag, len(rows), layer=LAYER)
|
|
373
|
-
return [dict(zip(cols, row)) for row in rows]
|
|
374
|
-
|
|
375
|
-
def load_rows(self, db, rows: list[dict]) -> int:
|
|
376
|
-
"""Insert *rows* (from :meth:`dump_rows`) verbatim, ids and versions
|
|
377
|
-
kept; a row already present is left as it is. Returns rows inserted."""
|
|
378
|
-
if not rows:
|
|
379
|
-
return 0
|
|
380
|
-
self.ensure_table(db)
|
|
381
|
-
cols = self._all_columns()
|
|
382
|
-
before = db._duck._fetchone(f"SELECT count(*) FROM {self.table}")[0]
|
|
383
|
-
for row in rows:
|
|
384
|
-
db._duck._execute(
|
|
385
|
-
f"INSERT INTO {self.table} ({', '.join(cols)}) "
|
|
386
|
-
f"VALUES ({', '.join('?' for _ in cols)}) ON CONFLICT DO NOTHING",
|
|
387
|
-
[row.get(c) for c in cols],
|
|
388
|
-
)
|
|
389
|
-
after = db._duck._fetchone(f"SELECT count(*) FROM {self.table}")[0]
|
|
390
|
-
Log.info("[%s] load_rows: %d of %d row(s) inserted", self.tag, after - before,
|
|
391
|
-
len(rows), layer=LAYER)
|
|
392
|
-
return after - before
|
|
File without changes
|
|
File without changes
|