scistackplotdb 0.1.65__tar.gz → 0.1.66__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scistackplotdb
3
- Version: 0.1.65
3
+ Version: 0.1.66
4
4
  Summary: scidb-backed plotting: load variables into long tables and plot them with scistackplot
5
5
  Author: SciStack Contributors
6
6
  License-Expression: MIT
@@ -17,10 +17,10 @@ Classifier: Programming Language :: Python :: 3.13
17
17
  Classifier: Topic :: Scientific/Engineering :: Visualization
18
18
  Classifier: Typing :: Typed
19
19
  Requires-Python: >=3.10
20
- Requires-Dist: sciduckdb==0.1.65
21
- Requires-Dist: scistack-db==0.1.65
22
- Requires-Dist: scistacklog==0.1.65
23
- Requires-Dist: scistackplot==0.1.65
20
+ Requires-Dist: sciduckdb==0.1.66
21
+ Requires-Dist: scistack-db==0.1.66
22
+ Requires-Dist: scistacklog==0.1.66
23
+ Requires-Dist: scistackplot==0.1.66
24
24
  Provides-Extra: dev
25
25
  Requires-Dist: matplotlib>=3.6; extra == 'dev'
26
26
  Requires-Dist: pytest-cov>=4.0; extra == 'dev'
@@ -94,6 +94,18 @@ validate(spec.with_roles(**{"scale.factor": Role.COLLAPSE}), table)
94
94
  # PlotSpec.variant_sets.
95
95
  ```
96
96
 
97
+ **Text fields are their own measure.** One melted value column holds one
98
+ shape, so a wide variable with numeric AND text fields is listed twice:
99
+ `Gait` (its numeric fields, one `ColName` level each) and `Gait (text)` (its
100
+ text fields), drawn as counts, %s or categories (`scistackplot`'s "Text
101
+ measures"). A variable whose fields are all text is itself the text measure.
102
+ `scistackplotdb.text_fields` owns the name (`base_variable("Gait (text)")`
103
+ is `"Gait"`, which is what a generated endpoint's `for_each` reads):
104
+
105
+ ```python
106
+ source.get_table(["Gait (text)"]) # Side, Outcome, ... melted under ColName
107
+ ```
108
+
97
109
  **A transport budget.** 1-D data across hundreds of trials is megabytes.
98
110
  `resolve(..., max_points=N)` downsamples for the interactive panel; export
99
111
  never does.
@@ -64,6 +64,18 @@ validate(spec.with_roles(**{"scale.factor": Role.COLLAPSE}), table)
64
64
  # PlotSpec.variant_sets.
65
65
  ```
66
66
 
67
+ **Text fields are their own measure.** One melted value column holds one
68
+ shape, so a wide variable with numeric AND text fields is listed twice:
69
+ `Gait` (its numeric fields, one `ColName` level each) and `Gait (text)` (its
70
+ text fields), drawn as counts, %s or categories (`scistackplot`'s "Text
71
+ measures"). A variable whose fields are all text is itself the text measure.
72
+ `scistackplotdb.text_fields` owns the name (`base_variable("Gait (text)")`
73
+ is `"Gait"`, which is what a generated endpoint's `for_each` reads):
74
+
75
+ ```python
76
+ source.get_table(["Gait (text)"]) # Side, Outcome, ... melted under ColName
77
+ ```
78
+
67
79
  **A transport budget.** 1-D data across hundreds of trials is megabytes.
68
80
  `resolve(..., max_points=N)` downsamples for the interactive panel; export
69
81
  never does.
@@ -82,6 +82,7 @@ from .presets import (
82
82
  save_preset,
83
83
  )
84
84
  from .source import ScidbSource
85
+ from .text_fields import TEXT_SUFFIX, base_variable, is_text_measure, text_measure_name
85
86
  from .variants import (
86
87
  branch_params_for,
87
88
  selection_for,
@@ -91,6 +92,11 @@ from .variants import (
91
92
 
92
93
  __all__ = [
93
94
  "ScidbSource",
95
+ # a wide variable's text fields as their own measure
96
+ "TEXT_SUFFIX",
97
+ "base_variable",
98
+ "is_text_measure",
99
+ "text_measure_name",
94
100
  # saved plots
95
101
  "save_plot",
96
102
  "list_saved_plots",
@@ -31,6 +31,7 @@ from scistackplot.roles import fanout_keys
31
31
  from scistackplot.variants import defined_sets
32
32
 
33
33
  from .load import LATEST_COLUMN
34
+ from .text_fields import base_variable
34
35
 
35
36
  LAYER = "scistackplotdb"
36
37
 
@@ -69,6 +70,9 @@ def generate_endpoint(
69
70
  ``input_variable`` is the scidb variable type supplying the data;
70
71
  ``x_variable`` is the optional second measure for an x–y scatter.
71
72
  """
73
+ # A text measure (`Gait (text)`) is its variable's text fields: the
74
+ # for_each input is the variable, and the plot function melts the fields.
75
+ input_variable = base_variable(input_variable)
72
76
  name = function_name or default_function_name(spec)
73
77
  # NOT spec.iterate_factors: the fan-out includes schema keys promoted
74
78
  # because a nested key iterates, and runs in schema order. The preview and
@@ -23,7 +23,7 @@ import numpy as np
23
23
  import pandas as pd
24
24
  from sciduckdb import merge_tz_columns, stored_tz_companions, tz_column_name
25
25
  from scistacklog import Log
26
- from scistackplot import CODE_FACTOR_PREFIX, RUN_FACTOR_PREFIX
26
+ from scistackplot import CODE_FACTOR_PREFIX, RUN_FACTOR_PREFIX, element_values
27
27
  from scistackplot.framesize import format_extent, frame_extent
28
28
  from scistackplot.shape import Shape, classify_value
29
29
  from scistackplot.table import natural_sort_key
@@ -97,6 +97,11 @@ class VariableFrame:
97
97
  #: Name of the per-row "this is my location's newest code version" flag, or
98
98
  #: None when the variable holds only one version. See :data:`LATEST_COLUMN`.
99
99
  latest_column: str | None = None
100
+ #: Data columns stored as ``row_list`` ("combine rows": one element per
101
+ #: source row, sciduckdb.row_lists). Their cells arrive as arrays; the
102
+ #: source splits them into rows with a ``Row`` factor
103
+ #: (scistackplot.rows), never plots them as 1-D series.
104
+ row_list_columns: list[str] = field(default_factory=list)
100
105
 
101
106
  @property
102
107
  def value_column(self) -> str:
@@ -286,7 +291,13 @@ def sample_value(db, variable: str) -> Any:
286
291
  f"AND t.\"{columns[0]}\" IS NOT NULL LIMIT 1",
287
292
  [variable],
288
293
  )
289
- return row[0] if row else None
294
+ value = row[0] if row else None
295
+ # A row list's cell is its source rows, not a series: classify by one
296
+ # ELEMENT, the shape the source turns it into (rows + a Row factor).
297
+ if value is not None and _row_list_columns(db, variable, columns[:1]):
298
+ elements = element_values(pd.Series([value], dtype=object)).dropna()
299
+ value = elements.iloc[0] if len(elements) else None
300
+ return value
290
301
 
291
302
 
292
303
  def sample_column_value(db, variable: str, column: str) -> Any:
@@ -308,6 +319,36 @@ def sample_column_value(db, variable: str, column: str) -> Any:
308
319
  return row[0] if row else None
309
320
 
310
321
 
322
+ def field_shapes(db, variable: str) -> dict[str, Shape]:
323
+ """``{data column: Shape}`` for every field of a variable, from one sampled
324
+ value each — what the catalog needs to list a wide variable's TEXT fields
325
+ as their own measure (``source.text_measure_name``) without loading it.
326
+
327
+ By value, as :func:`sample_value` classifies (a row list by one ELEMENT).
328
+ A container-typed column is SERIES_1D from its type alone and never
329
+ sampled (:func:`is_container_type`: the EMG-trace cost), unless it is a
330
+ row list, whose type is a list but whose elements are the values.
331
+ The table load re-classifies every field by its values
332
+ (``ScidbSource._melt_fields``), which is the owner; this is the cheap
333
+ answer the catalog lists by, and it can only be wrong in the direction of
334
+ a "(text)" entry that then says it has no text fields.
335
+ """
336
+ types = data_column_types_for(db, variable)
337
+ row_lists = set(_row_list_columns(db, variable, list(types)))
338
+ shapes: dict[str, Shape] = {}
339
+ with Log.timer(f"field_shapes({variable})", layer=LAYER, extra=f"{len(types)} column(s)"):
340
+ for column, data_type in types.items():
341
+ if is_container_type(data_type) and column not in row_lists:
342
+ shapes[column] = Shape.SERIES_1D
343
+ continue
344
+ value = sample_column_value(db, variable, column)
345
+ if value is not None and column in row_lists:
346
+ elements = element_values(pd.Series([value], dtype=object)).dropna()
347
+ value = elements.iloc[0] if len(elements) else None
348
+ shapes[column] = classify_value(value)
349
+ return shapes
350
+
351
+
311
352
  def column_levels(db, variable: str, column: str, *, limit: int) -> list[str]:
312
353
  """Distinct values of one column, as text, at most ``limit`` of them.
313
354
 
@@ -447,23 +488,26 @@ def _key_text(value: Any) -> "str | None":
447
488
  return str(value)
448
489
 
449
490
 
450
- def _log_row_list_columns(db, variable: str, columns) -> None:
451
- """Say which columns are row lists (sciduckdb.row_lists: rows combined by
491
+ def _row_list_columns(db, variable: str, columns) -> list[str]:
492
+ """Which columns are row lists (sciduckdb.row_lists: rows combined by
452
493
  ``on_duplicate="combine"``). This reader selects the stored cells as they
453
- are, so EVERY cell of such a column is a 1-D array (``[7]``, ``[5, 12]``)
454
- and the column is plotted as a 1-D measure (the collapse kinds take its
455
- mean/median per location) - a decision recorded in
456
- .claude/plan-combine-duplicate-locations.md §6, not an accident."""
494
+ are, so EVERY cell of such a column is an array (``[7]``, ``[5, 12]``).
495
+ The source splits them into rows with a ``Row`` factor
496
+ (scistackplot.rows) — they are observations, not a 1-D series. This
497
+ replaced the "read as 1-D" decision of
498
+ .claude/plan-combine-duplicate-locations.md §6 on 2026-10-30
499
+ (.claude/plan-row-list-plots.md)."""
457
500
  from sciduckdb import is_row_list_meta
458
501
 
459
502
  stored = (db.get_dtype_meta(variable) or {}).get("columns", {})
460
503
  row_lists = [c for c in columns if is_row_list_meta(stored.get(c))]
461
504
  if row_lists:
462
505
  Log.debug(
463
- "variable %r: row-list column(s) %s (combined rows) are read as "
464
- "1-D arrays, one element per source row",
506
+ "variable %r: row-list column(s) %s (combined rows) — split into "
507
+ "one row per element by the source",
465
508
  variable, row_lists, layer=LAYER,
466
509
  )
510
+ return row_lists
467
511
 
468
512
 
469
513
  def load_variable(
@@ -490,7 +534,7 @@ def load_variable(
490
534
  keys = schema_keys(db)
491
535
  column_types = data_column_types_for(db, variable)
492
536
  columns = list(column_types)
493
- _log_row_list_columns(db, variable, columns)
537
+ row_list_columns = _row_list_columns(db, variable, columns)
494
538
  if not columns:
495
539
  Log.warn("variable %r has no data columns", variable, layer=LAYER)
496
540
  return VariableFrame(name=variable, frame=pd.DataFrame())
@@ -597,6 +641,7 @@ def load_variable(
597
641
  variant_columns=variant_columns,
598
642
  variant_axes=variant_axes,
599
643
  latest_column=latest_column,
644
+ row_list_columns=[c for c in row_list_columns if c in columns],
600
645
  )
601
646
 
602
647
 
@@ -21,9 +21,13 @@ from scistackplot import (
21
21
  Shape,
22
22
  classify_column,
23
23
  classify_value,
24
+ element_values,
25
+ explode_row_lists,
24
26
  is_plottable,
25
27
  natural_sort_key,
28
+ row_origin,
26
29
  )
30
+ from scistackplot.categorical import MAX_CATEGORY_LEVELS
27
31
  from scistackplot.dedup import SingleFlight
28
32
  from scistackplot.framesize import format_extent, frame_extent
29
33
  from scistackplot.sources import BaseSource, UnknownMeasureError
@@ -36,6 +40,7 @@ from .load import (
36
40
  column_levels,
37
41
  data_column_types_for,
38
42
  data_columns_for,
43
+ field_shapes,
39
44
  is_container_type,
40
45
  load_variable,
41
46
  registered_variables,
@@ -44,6 +49,7 @@ from .load import (
44
49
  schema_keys,
45
50
  variable_levels,
46
51
  )
52
+ from .text_fields import base_variable, is_text_measure, text_measure_name
47
53
 
48
54
  LAYER = "scistackplotdb"
49
55
 
@@ -59,7 +65,7 @@ FIELD_FACTOR = "ColName"
59
65
  #: unreadable and a facet per level is hundreds of empty panels. Not a hard
60
66
  #: limit — a spec naming such a column still works — only what
61
67
  #: :meth:`ScidbSource.groupable_report` will put in front of a user.
62
- MAX_GROUP_LEVELS = 50
68
+ MAX_GROUP_LEVELS = MAX_CATEGORY_LEVELS # one threshold: "are these categories?"
63
69
 
64
70
 
65
71
  def _as_levels(column):
@@ -112,6 +118,10 @@ class ScidbSource(BaseSource):
112
118
  # in `_frames` would hand a later plot a frame with no measure in it.
113
119
  self._variant_frames: dict[str, Any] = {}
114
120
  self._shapes: dict[str, Shape] = {}
121
+ # variable -> {field: Shape} for a multi-column variable, sampled once
122
+ # (`load.field_shapes`): which fields are text, for the catalog's
123
+ # "Var (text)" entry, without loading the variable.
124
+ self._field_shapes: dict[str, dict[str, Shape]] = {}
115
125
  self._levels: dict[str, list[str]] = {}
116
126
  # variable -> the content fingerprint its cached frames were built at.
117
127
  # Checked on every cache hit, because nothing tells this process when
@@ -160,19 +170,34 @@ class ScidbSource(BaseSource):
160
170
  measures = []
161
171
  for variable in registered_variables(self._db):
162
172
  shape = self._shape_of(variable)
163
- columns = data_columns_for(self._db, variable)
173
+ numeric, text = self._field_split(variable)
174
+ columns = numeric if text and numeric else data_columns_for(self._db, variable)
164
175
  measures.append(
165
176
  {
166
177
  "name": variable,
167
178
  "display": variable,
168
179
  "shape": str(shape),
169
180
  "exploded": False,
170
- "plottable": shape
171
- in (Shape.SCALAR, Shape.SERIES_1D, Shape.MATRIX_2D),
181
+ "plottable": is_plottable(shape),
172
182
  "columns": columns,
173
183
  "levels": self._levels_of(variable),
174
184
  }
175
185
  )
186
+ if text and numeric:
187
+ # A MIXED wide variable: its text fields are their own
188
+ # measure (`text_fields`), so none is hidden.
189
+ measures.append(
190
+ {
191
+ "name": text_measure_name(variable),
192
+ "display": text_measure_name(variable),
193
+ "shape": str(Shape.CATEGORICAL),
194
+ "exploded": False,
195
+ "plottable": True,
196
+ "columns": text,
197
+ "levels": self._levels_of(variable),
198
+ "text_of": variable,
199
+ }
200
+ )
176
201
 
177
202
  factors = []
178
203
  for key in keys:
@@ -202,10 +227,47 @@ class ScidbSource(BaseSource):
202
227
  }
203
228
 
204
229
  def _shape_of(self, variable: str) -> Shape:
230
+ """What one cell of ``variable`` (a measure name) holds. A text
231
+ measure (`text_fields`) is categorical; a multi-column variable is
232
+ what its NUMERIC fields hold (the measure the melt makes of them),
233
+ and categorical only when every field is text."""
234
+ if is_text_measure(variable):
235
+ return Shape.CATEGORICAL
205
236
  if variable not in self._shapes:
206
- self._shapes[variable] = classify_value(sample_value(self._db, variable))
237
+ numeric, text = self._field_split(variable)
238
+ if text and not numeric:
239
+ shape = Shape.CATEGORICAL
240
+ elif text:
241
+ shape = self._field_shapes[variable][numeric[0]]
242
+ else:
243
+ shape = classify_value(sample_value(self._db, variable))
244
+ self._shapes[variable] = shape
207
245
  return self._shapes[variable]
208
246
 
247
+ def _field_split(self, variable: str) -> tuple[list[str], list[str]]:
248
+ """``(other fields, text fields)`` of a MULTI-column variable, from
249
+ one sampled value per field (`load.field_shapes`), cached. A
250
+ single-column variable answers ``(its column, [])`` with no query —
251
+ its shape is `_shape_of`'s. Text fields are what `text_fields` lists
252
+ as ``Var (text)``."""
253
+ columns = data_columns_for(self._db, variable)
254
+ if len(columns) < 2:
255
+ return list(columns), []
256
+ if variable not in self._field_shapes:
257
+ self._field_shapes[variable] = field_shapes(self._db, variable)
258
+ shapes = self._field_shapes[variable]
259
+ text = [c for c, s in shapes.items() if s is Shape.CATEGORICAL]
260
+ if text:
261
+ Log.info(
262
+ "variable %r: %d text field(s) %s of %d — listed as %r",
263
+ variable, len(text), text[:8], len(shapes),
264
+ text_measure_name(variable) if len(text) < len(shapes) else variable,
265
+ layer=LAYER,
266
+ )
267
+ shapes = self._field_shapes[variable]
268
+ text = [c for c, s in shapes.items() if s is Shape.CATEGORICAL]
269
+ return [c for c in shapes if c not in set(text)], text
270
+
209
271
  def _levels_of(self, variable: str) -> list[str]:
210
272
  """Schema depth WITHOUT loading the variable — describe() asks for every
211
273
  registered variable, and loading each frame would read the whole
@@ -348,6 +410,7 @@ class ScidbSource(BaseSource):
348
410
  Cached and deduplicated like any other table, under its own key, so the
349
411
  picker opening twice costs one query.
350
412
  """
413
+ variable = base_variable(variable) # a text measure loads its variable
351
414
  key = ("__variants__", variable)
352
415
  memo = self._current_table_cache()
353
416
  label = f"variant_table({variable})"
@@ -476,6 +539,7 @@ class ScidbSource(BaseSource):
476
539
  return False
477
540
 
478
541
  def _variable_frame(self, variable: str):
542
+ variable = base_variable(variable) # a text measure loads its variable
479
543
  # Hit/miss at INFO: this cache is the difference between a request that
480
544
  # reads the whole variable and one that reads nothing, and a plot request
481
545
  # that returned in milliseconds is otherwise unattributable
@@ -569,8 +633,9 @@ class ScidbSource(BaseSource):
569
633
 
570
634
  factor_variables = list(factor_variables or [])
571
635
  known = registered_variables(self._db)
636
+ # A text measure (`Gait (text)`) loads its variable (`text_fields`).
572
637
  requested = [
573
- *measures,
638
+ *(base_variable(name) for name in measures),
574
639
  *([x_measure] if x_measure else []),
575
640
  *(group.variable for group in factor_variables),
576
641
  ]
@@ -580,21 +645,46 @@ class ScidbSource(BaseSource):
580
645
  # callers (and the GUI) handle one kind of "no such measure".
581
646
  raise UnknownMeasureError(f"Unknown variable(s) {unknown}. Available: {known}")
582
647
 
648
+ text_measures = [name for name in [*measures, x_measure] if name and is_text_measure(name)]
649
+ if text_measures and (len(measures) > 1 or x_measure is not None):
650
+ raise ValueError(
651
+ f"{text_measures} hold a variable's TEXT fields, which plot on "
652
+ f"their own: they cannot stack with another variable or pair with "
653
+ f"an x measure."
654
+ )
583
655
  if len(measures) > 1:
584
656
  return self._stacked_table(measures, factor_variables)
585
657
 
586
- primary = self._variable_frame(measures[0])
658
+ text_only = is_text_measure(measures[0])
659
+ primary = self._variable_frame(base_variable(measures[0]))
587
660
  own = self._own_column_groups(primary, factor_variables)
588
661
  field_columns: list[str] = []
662
+ row_column: str | None = None
663
+ row_info: dict | None = None
589
664
 
590
665
  if x_measure is None:
591
666
  if len(primary.data_columns) > 1:
592
667
  frame, field_factor = self._melt_fields(
593
- primary, measures[0], carry=[group.column for group in own]
668
+ primary,
669
+ measures[0],
670
+ carry=[group.column for group in own],
671
+ text=text_only,
594
672
  )
595
673
  field_columns = [field_factor]
674
+ melted = set(frame[field_factor].unique())
675
+ row_fields = [c for c in primary.row_list_columns if c in melted]
596
676
  else:
597
677
  frame = self._named_frame(primary, measures[0])
678
+ field_factor = None
679
+ row_fields = (
680
+ [measures[0]]
681
+ if primary.value_column in primary.row_list_columns
682
+ else []
683
+ )
684
+ if row_fields:
685
+ frame, row_column, row_info = self._split_row_lists(
686
+ primary.name, frame, measures[0], row_fields, field_factor
687
+ )
598
688
  levels = primary.levels
599
689
  variant_columns = list(primary.variant_columns)
600
690
  variant_axes = list(primary.variant_axes)
@@ -605,6 +695,22 @@ class ScidbSource(BaseSource):
605
695
  else:
606
696
  secondary = self._variable_frame(x_measure)
607
697
  pair = (measures[0], x_measure)
698
+ combined = [
699
+ name
700
+ for name, vf in zip(pair, (primary, secondary), strict=True)
701
+ if vf.row_list_columns
702
+ ]
703
+ if combined:
704
+ # Not split here: pairing one x per y row needs the two
705
+ # variables' rows to line up, and nothing says row i of one
706
+ # is row i of the other. Said, not silent.
707
+ Log.warn(
708
+ "%s hold combined rows (row lists); an x-y plot reads "
709
+ "them as 1-D arrays — plot each on its own to see one "
710
+ "point per source row",
711
+ combined,
712
+ layer=LAYER,
713
+ )
608
714
  multi = [
609
715
  name
610
716
  for name, frame in zip(pair, (primary, secondary), strict=True)
@@ -661,6 +767,15 @@ class ScidbSource(BaseSource):
661
767
  # Keys: the attached factor names, in the order they were joined.
662
768
  factors.extend(group_depths)
663
769
  factors.extend(c for c in field_columns if c in frame.columns)
770
+ factor_origins = {axis["column"]: axis for axis in variant_axes}
771
+ factor_depths = dict(group_depths)
772
+ if row_column is not None and row_info is not None:
773
+ factors.append(row_column)
774
+ factor_origins[row_column] = row_info
775
+ # Pinned by every schema key the record sits under plus itself:
776
+ # one level below the variable's own, so the scalar default
777
+ # groups it (one mark per source row) and iterates the rest.
778
+ factor_depths[row_column] = len(levels) + 1
664
779
 
665
780
  level_order = {
666
781
  name: self._level_order(name, frame, is_field=name in field_columns)
@@ -677,10 +792,10 @@ class ScidbSource(BaseSource):
677
792
  name=measures[0],
678
793
  default_pin=default_pin,
679
794
  latest_column=latest_column,
680
- factor_origins={axis["column"]: axis for axis in variant_axes},
795
+ factor_origins=factor_origins,
681
796
  # Where each joined grouping sits in the hierarchy, so a nested x
682
797
  # axis puts a subject-level grouping OUTSIDE the sessions inside it.
683
- factor_depths=group_depths,
798
+ factor_depths=factor_depths,
684
799
  # The variable's own schema depth, outermost first — the nesting
685
800
  # that decides which keys a fan-out has to iterate together and in
686
801
  # which order (roles.fanout_keys).
@@ -1093,6 +1208,17 @@ class ScidbSource(BaseSource):
1093
1208
  ``spec.measures`` stop naming a real variable.
1094
1209
  """
1095
1210
  frames = [self._variable_frame(name) for name in measures]
1211
+ combined = [f.name for f in frames if f.row_list_columns]
1212
+ if combined:
1213
+ # Splitting is per variable, and stacked variables share one
1214
+ # value column; not done here yet. Said, not silent.
1215
+ Log.warn(
1216
+ "%s hold combined rows (row lists); stacked with other "
1217
+ "variables they are read as 1-D arrays — plot each on its own "
1218
+ "to see one point per source row",
1219
+ combined,
1220
+ layer=LAYER,
1221
+ )
1096
1222
 
1097
1223
  # Dict/struct variables stack too, provided they carry the SAME fields:
1098
1224
  # RawEMG and FilteredEMG, both keyed by muscle, are the archetypal
@@ -1298,6 +1424,7 @@ class ScidbSource(BaseSource):
1298
1424
  offering a default for every groupable variable costs a handful of
1299
1425
  metadata queries rather than reading any of them.
1300
1426
  """
1427
+ variable = base_variable(variable) # a text measure loads its variable
1301
1428
  from scistackplot.variants import default_selection
1302
1429
 
1303
1430
  return default_selection(self.variant_table(variable))
@@ -1318,6 +1445,7 @@ class ScidbSource(BaseSource):
1318
1445
  Its own cache, never ``_frames``: a data-less frame stored there would
1319
1446
  be handed to a later plot as though it held the measure.
1320
1447
  """
1448
+ variable = base_variable(variable) # a text measure loads its variable
1321
1449
  # Same self-validation as the data frame: this cache is what served a
1322
1450
  # stale `variant_table` for a whole session after a re-run (2026-09-15).
1323
1451
  if (
@@ -1361,6 +1489,7 @@ class ScidbSource(BaseSource):
1361
1489
  *,
1362
1490
  fields: list[str] | None = None,
1363
1491
  carry: list[str] | None = None,
1492
+ text: bool = False,
1364
1493
  ):
1365
1494
  """
1366
1495
  Turn a dict/struct variable's columns into ONE measure plus a field
@@ -1379,17 +1508,58 @@ class ScidbSource(BaseSource):
1379
1508
  of a ``Gait`` record — ``FactorVariable.is_own_column``): they stay on
1380
1509
  every melted row as id columns instead of becoming a field level or
1381
1510
  being dropped as unplottable.
1511
+
1512
+ ``text`` melts the TEXT fields instead (the measure `Gait (text)`,
1513
+ `text_fields`). One melted column holds one shape, so a mixed variable
1514
+ is two measures: by default its other fields, and its text fields
1515
+ under the text name — said in the log, never dropped. A variable whose
1516
+ fields are ALL text melts them by default (it has nothing else).
1382
1517
  """
1383
1518
  frame = variable_frame.frame
1384
1519
  carried = [c for c in (carry or []) if c in frame.columns]
1385
1520
  usable, skipped = [], []
1521
+ texts: list[str] = []
1522
+ row_lists = set(variable_frame.row_list_columns)
1386
1523
  for column in variable_frame.data_columns:
1387
1524
  if column in carried:
1388
1525
  continue
1389
- if is_plottable(classify_column(frame[column])):
1526
+ # A row list is classified by its ELEMENTS (it is split into rows
1527
+ # after the melt): a VARCHAR row list is text and joins the text
1528
+ # fields, instead of melting in as a "1-D" column.
1529
+ values = (
1530
+ element_values(frame[column]) if column in row_lists else frame[column]
1531
+ )
1532
+ shape = classify_column(values)
1533
+ if shape is Shape.CATEGORICAL:
1534
+ texts.append(column)
1535
+ elif is_plottable(shape):
1390
1536
  usable.append(column)
1391
1537
  else:
1392
1538
  skipped.append(column)
1539
+ if text or not usable:
1540
+ # The text measure, or a variable that has only text fields.
1541
+ if texts and usable:
1542
+ Log.info(
1543
+ "variable %r: plotting its %d text field(s) %s; its other "
1544
+ "field(s) %s are the measure %r",
1545
+ variable_frame.name, len(texts), texts, usable,
1546
+ variable_frame.name, layer=LAYER,
1547
+ )
1548
+ usable = texts
1549
+ if text and not texts:
1550
+ raise ValueError(
1551
+ f"Variable {variable_frame.name!r} has no text fields "
1552
+ f"(columns: {variable_frame.data_columns}); plot "
1553
+ f"{variable_frame.name!r} itself."
1554
+ )
1555
+ elif texts:
1556
+ # INFO, not a drop: they are reachable, under their own name.
1557
+ Log.info(
1558
+ "variable %r: %d text field(s) %s are plotted as %r, not with "
1559
+ "the %d other field(s)",
1560
+ variable_frame.name, len(texts), texts,
1561
+ text_measure_name(variable_frame.name), len(usable), layer=LAYER,
1562
+ )
1393
1563
  # Datetime fields melt into ONE value column with the rest, which is a
1394
1564
  # time axis only if every field is a datetime (scistackplot.temporal).
1395
1565
  # Beside numeric fields they would read as numbers (NaN), so they are
@@ -1458,6 +1628,51 @@ class ScidbSource(BaseSource):
1458
1628
  )
1459
1629
  return melted, field_factor
1460
1630
 
1631
+ def _split_row_lists(
1632
+ self,
1633
+ variable: str,
1634
+ frame,
1635
+ measure: str,
1636
+ row_fields: list[str],
1637
+ field_factor: str | None,
1638
+ ):
1639
+ """``(frame, row column or None, its origin or None)`` — every
1640
+ row-list cell split into one row per element (scistackplot.rows,
1641
+ the one owner of the reshape; codegen mirrors it from the origin).
1642
+
1643
+ The Row factor is kept only when some cell held two or more rows: a
1644
+ one-level factor is a constant (synthetic-factors.md invariant 6).
1645
+ The cells are unwrapped either way, so ``[v]`` reads as ``v``."""
1646
+ with Log.timer(
1647
+ f"split_row_lists({variable})",
1648
+ layer=LAYER,
1649
+ extra=f"{len(row_fields)} field(s)",
1650
+ ):
1651
+ split = explode_row_lists(
1652
+ frame, measure, row_fields, field_factor=field_factor
1653
+ )
1654
+ Log.info(
1655
+ "split row-list field(s) %s of %r: %d cell(s), %d -> %d row(s), "
1656
+ "at most %d per cell; %s",
1657
+ row_fields,
1658
+ variable,
1659
+ split.cells,
1660
+ split.rows_in,
1661
+ split.rows_out,
1662
+ split.max_rows,
1663
+ f"factor {split.row_factor!r}"
1664
+ if split.row_factor
1665
+ else "no cell held two rows, so no Row factor",
1666
+ layer=LAYER,
1667
+ )
1668
+ if split.row_factor is None:
1669
+ return split.frame, None, None
1670
+ return (
1671
+ split.frame,
1672
+ split.row_factor,
1673
+ row_origin(row_fields, field_factor),
1674
+ )
1675
+
1461
1676
  def _named_frame(self, variable_frame, measure: str):
1462
1677
  """The variable's frame with its data column renamed to the measure."""
1463
1678
  return variable_frame.frame.rename(
@@ -1511,6 +1726,19 @@ class ScidbSource(BaseSource):
1511
1726
  cannot offer has to say **why** in place rather than be absent. "EMG is
1512
1727
  not clickable" with no reason is indistinguishable from a broken dialog.
1513
1728
  """
1729
+ if is_text_measure(measure) or self._shape_of(measure) is Shape.CATEGORICAL:
1730
+ # A text measure plots on its own (`_build_table` refuses a stack):
1731
+ # counts of one variable's categories beside another's are not one
1732
+ # axis. Every candidate says so.
1733
+ reason = "a text measure is plotted on its own, not stacked"
1734
+ return {
1735
+ "offered": [],
1736
+ "rejected": {
1737
+ name: reason
1738
+ for name in registered_variables(self._db)
1739
+ if name != base_variable(measure)
1740
+ },
1741
+ }
1514
1742
  own_shape = self._shape_of(measure)
1515
1743
  own_levels = self._levels_of(measure)
1516
1744
  own_columns = data_columns_for(self._db, measure)
@@ -1554,6 +1782,7 @@ class ScidbSource(BaseSource):
1554
1782
 
1555
1783
  def groupable_with(self, measure: str) -> list[FactorVariable]:
1556
1784
  """Groupings offered for ``measure``. See :meth:`groupable_report`."""
1785
+ measure = base_variable(measure) # a text measure groups as its variable
1557
1786
  return [
1558
1787
  FactorVariable(offer["variable"], offer["column"])
1559
1788
  for offer in self.groupable_report(measure)["offered"]
@@ -1591,6 +1820,7 @@ class ScidbSource(BaseSource):
1591
1820
  same question in one call — but it is the expensive form, and the GUI
1592
1821
  no longer opens a panel with it.
1593
1822
  """
1823
+ measure = base_variable(measure) # a text measure groups as its variable
1594
1824
  report = self.groupable_variables(measure)
1595
1825
  rejected: dict[str, str] = dict(report["rejected"])
1596
1826
  buckets: dict[str, list[dict]] = {
@@ -1650,6 +1880,7 @@ class ScidbSource(BaseSource):
1650
1880
  categorical groups lead — so the two cannot drift about what comes
1651
1881
  first.
1652
1882
  """
1883
+ measure = base_variable(measure) # a text measure groups as its variable
1653
1884
  own = self._levels_of(measure)
1654
1885
  offered: list[dict] = []
1655
1886
  rejected: dict[str, str] = {}
@@ -1799,6 +2030,7 @@ class ScidbSource(BaseSource):
1799
2030
  when ``variable`` IS the measure (its own ``Side`` column grouping it),
1800
2031
  a numeric column is a plotted field, not a numeric sheet column.
1801
2032
  """
2033
+ measure = base_variable(measure) # a text measure groups as its variable
1802
2034
  rejected: dict[str, str] = {}
1803
2035
  types = data_column_types_for(self._db, variable)
1804
2036
  # The plotted variable's own numeric columns are its FIELDS — what the
@@ -1826,6 +2058,7 @@ class ScidbSource(BaseSource):
1826
2058
  onto the analysis rows, where the measure's own fields are not
1827
2059
  carried.
1828
2060
  """
2061
+ measure = base_variable(measure) # a text measure groups as its variable
1829
2062
  found = self.groupable_variables(measure)
1830
2063
  offered, rejected = [], dict(found["rejected"])
1831
2064
  for entry in found["offered"]:
@@ -1843,6 +2076,7 @@ class ScidbSource(BaseSource):
1843
2076
  """Which COLUMNS of ``variable`` may be a covariate: the numeric ones
1844
2077
  (the mirror of :meth:`groupable_columns`, which offers the text
1845
2078
  ones). Same ``{"offered", "rejected"}`` shape, same type pre-filter."""
2079
+ measure = base_variable(measure) # a text measure groups as its variable
1846
2080
  rejected: dict[str, str] = {}
1847
2081
  offered: list[dict] = []
1848
2082
  types = data_column_types_for(self._db, variable)
@@ -1943,6 +2177,8 @@ class ScidbSource(BaseSource):
1943
2177
 
1944
2178
  def joinable_with(self, measure: str) -> list[str]:
1945
2179
  """Variables that can supply an x axis for ``measure``."""
2180
+ if is_text_measure(measure) or self._shape_of(measure) is Shape.CATEGORICAL:
2181
+ return [] # a text measure has no y values for an x axis to pair with
1946
2182
  own = self._levels_of(measure)
1947
2183
  result = []
1948
2184
  for candidate in registered_variables(self._db):
@@ -1968,6 +2204,7 @@ class ScidbSource(BaseSource):
1968
2204
  self._frames.clear()
1969
2205
  self._variant_frames.clear()
1970
2206
  self._shapes.clear()
2207
+ self._field_shapes.clear()
1971
2208
  self._levels.clear()
1972
2209
  self._fingerprints.clear()
1973
2210
  else:
@@ -1977,6 +2214,7 @@ class ScidbSource(BaseSource):
1977
2214
  # holds the answer to.
1978
2215
  self._variant_frames.pop(variable, None)
1979
2216
  self._shapes.pop(variable, None)
2217
+ self._field_shapes.pop(variable, None)
1980
2218
  self._levels.pop(variable, None)
1981
2219
  self._fingerprints.pop(variable, None)
1982
2220
  # Built tables are derived from those frames, so they are stale too —
@@ -0,0 +1,36 @@
1
+ """A wide variable's TEXT fields, plotted as their own measure.
2
+
3
+ One melted value column holds one shape, so a wide variable with numeric AND
4
+ text fields (``Gait``: ``StepLength``, ``Cadence``, ``Outcome``, ``Side``)
5
+ cannot melt them all under one ``ColName`` factor. The catalog lists it twice
6
+ (user decision, 2026-11-01): ``Gait`` holds the numeric fields, as it always
7
+ has, and ``Gait (text)`` holds the text fields, drawn as counts, %s or
8
+ categories (``scistackplot.categorical``). Every field is reachable; none is
9
+ dropped. See docs/claude/categorical-measures.md.
10
+
11
+ This module is the one owner of the NAME: how a text measure is spelled, and
12
+ which variable a measure name loads. Everything that turns a measure name back
13
+ into a scidb variable — the source's loaders, the endpoint generator's
14
+ ``for_each`` input, the GUI — asks :func:`base_variable`.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ #: Appended to a variable's name for the measure holding its text fields.
20
+ TEXT_SUFFIX = " (text)"
21
+
22
+
23
+ def text_measure_name(variable: str) -> str:
24
+ """The measure name of ``variable``'s text fields: ``"Gait (text)"``."""
25
+ return f"{variable}{TEXT_SUFFIX}"
26
+
27
+
28
+ def is_text_measure(name: str) -> bool:
29
+ """Whether ``name`` is a text-fields measure rather than a variable."""
30
+ return name.endswith(TEXT_SUFFIX) and len(name) > len(TEXT_SUFFIX)
31
+
32
+
33
+ def base_variable(name: str) -> str:
34
+ """The scidb variable a measure name loads: ``"Gait (text)"`` → ``"Gait"``,
35
+ any other name unchanged."""
36
+ return name[: -len(TEXT_SUFFIX)] if is_text_measure(name) else name