scistackplot 0.3.3__tar.gz → 0.3.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scistackplot-0.3.3 → scistackplot-0.3.6}/.gitignore +5 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/PKG-INFO +3 -3
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/export/reduction_code.py +19 -6
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/table.py +7 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/cell.py +29 -20
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/omissions.py +38 -1
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/reduction.py +48 -7
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/resolve.py +19 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/series_stats.py +13 -19
- {scistackplot-0.3.3 → scistackplot-0.3.6}/README.md +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/pyproject.toml +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/__init__.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/export/__init__.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/export/codegen.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/export/custom.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/export/figure_file.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/export/plot_data.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/__init__.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/capability.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/design.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/presets.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/restore.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/role_labels.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/roles.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/shape.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/model/spec.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/__init__.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/bins.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/categorical.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/chain.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/compare.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/diffbars.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/fits.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/groups.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/kde.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/numeric.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/panels.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/reducer.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/resolved.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/spaghetti.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/spread.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/stacking.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/temporal.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/variants.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/weights.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/reduce/ylimits.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/render/__init__.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/render/base.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/render/mpl.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/render/plotly_.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/sources/__init__.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/sources/base.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/sources/csv.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/sources/dedup.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/sources/frame.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/__init__.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/aliases.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/autosize.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/colors.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/figsize.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/framesize.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/paper.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/textsize.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/ticklabels.py +0 -0
- {scistackplot-0.3.3 → scistackplot-0.3.6}/src/scistackplot/text/xaxis.py +0 -0
|
@@ -35,3 +35,8 @@ scimatlab/tests/matlab/test-failures.txt
|
|
|
35
35
|
|
|
36
36
|
# csv-stats is its own repo (github.com/mtillman14/csv-stats), cloned here for editable dev
|
|
37
37
|
/csv-stats/
|
|
38
|
+
# Scratch files at the repo root (CLAUDE.md NOTE 1: tmp.py is where pasted
|
|
39
|
+
# Python goes; test-failures.txt is a local test summary). Untracked
|
|
40
|
+
# 2026-11-10 (grug stage repo_hygiene); the files stay on disk.
|
|
41
|
+
/tmp.py
|
|
42
|
+
/test-failures.txt
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scistackplot
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.6
|
|
4
4
|
Summary: Spec-driven plotting for long-format scientific data — standalone, GUI-friendly
|
|
5
5
|
Author: SciStack Contributors
|
|
6
6
|
License-Expression: MIT
|
|
@@ -20,8 +20,8 @@ Requires-Python: >=3.10
|
|
|
20
20
|
Requires-Dist: csv-stats>=0.5.0
|
|
21
21
|
Requires-Dist: numpy>=1.21
|
|
22
22
|
Requires-Dist: pandas>=1.5
|
|
23
|
-
Requires-Dist: scicanonicalhash==0.3.
|
|
24
|
-
Requires-Dist: scistacklog==0.3.
|
|
23
|
+
Requires-Dist: scicanonicalhash==0.3.6
|
|
24
|
+
Requires-Dist: scistacklog==0.3.6
|
|
25
25
|
Provides-Extra: dev
|
|
26
26
|
Requires-Dist: matplotlib>=3.6; extra == 'dev'
|
|
27
27
|
Requires-Dist: plotly>=5.0; extra == 'dev'
|
|
@@ -168,9 +168,13 @@ def emit_collapse_cells(step: CollapseCells) -> list[str]:
|
|
|
168
168
|
Written as a named helper rather than ``df[y].map(np.nanmean)``: the
|
|
169
169
|
export is meant to READ as what it does, and ``np.nanmean`` warns on an
|
|
170
170
|
all-NaN cell (the preview silences it). A cell is read as
|
|
171
|
-
``series_stats.cell_array`` reads it
|
|
172
|
-
|
|
173
|
-
|
|
171
|
+
``series_stats.cell_array`` reads it, by ``scicanonicalhash._cells``' rule
|
|
172
|
+
printed out (an export imports nothing of scistack): an array or
|
|
173
|
+
(nested) list of numbers, bools and None (as NaN) is its samples;
|
|
174
|
+
anything else (a lone number, None, text, complex, datetimes, ragged
|
|
175
|
+
nesting) has none and becomes a NaN gap. NaN samples are skipped, as in
|
|
176
|
+
pandas' ``Series.mean()``. ``test_reduction_collapse_cells`` runs this
|
|
177
|
+
over ``_cells.EDGE_CASES`` against the preview.
|
|
174
178
|
"""
|
|
175
179
|
y = step.measure
|
|
176
180
|
centre = "median" if step.statistic is Statistic.MEDIAN else "mean"
|
|
@@ -192,14 +196,23 @@ def emit_collapse_cells(step: CollapseCells) -> list[str]:
|
|
|
192
196
|
return [
|
|
193
197
|
f"# {y}: one value per record, the {centre} of each vector",
|
|
194
198
|
f"def {name}(_cell):",
|
|
195
|
-
"
|
|
196
|
-
' return float("nan")',
|
|
199
|
+
" # numbers only: text, complex, datetimes and ragged lists are gaps",
|
|
197
200
|
" if not isinstance(_cell, (np.ndarray, list, tuple)):",
|
|
198
201
|
' return float("nan")',
|
|
199
202
|
" try:",
|
|
200
|
-
|
|
203
|
+
" _samples = np.asarray(_cell)",
|
|
201
204
|
" except (TypeError, ValueError):",
|
|
202
205
|
' return float("nan")',
|
|
206
|
+
" if _samples.ndim == 0 or _samples.size == 0:",
|
|
207
|
+
' return float("nan")',
|
|
208
|
+
' if _samples.dtype.kind == "O":',
|
|
209
|
+
" _ok = (bool, int, float, np.bool_, np.integer, np.floating, type(None))",
|
|
210
|
+
" if not all(isinstance(_v, _ok) for _v in _samples.ravel()):",
|
|
211
|
+
' return float("nan")',
|
|
212
|
+
' _samples = np.array([np.nan if _v is None else _v for _v in _samples.ravel()], dtype="float64")',
|
|
213
|
+
' elif _samples.dtype.kind not in "biuf":',
|
|
214
|
+
' return float("nan")',
|
|
215
|
+
' _samples = _samples.astype("float64").ravel()',
|
|
203
216
|
" _samples = _samples[~np.isnan(_samples)]",
|
|
204
217
|
f' return float({reduction}) if _samples.size else float("nan")',
|
|
205
218
|
"",
|
|
@@ -251,6 +251,13 @@ class LongTable:
|
|
|
251
251
|
#: attached by a database source). The "No record" reason of a figure's
|
|
252
252
|
#: "Not plotted" list (`omissions`); None when the source cannot tell.
|
|
253
253
|
absent: "pd.DataFrame | None" = None
|
|
254
|
+
#: Records whose 1-D cell was averaged over FEWER elements than it holds
|
|
255
|
+
#: (some, not all, NaN), set by the cell collapse (`cell.apply_cell_collapse`):
|
|
256
|
+
#: the collapsed rows plus ``reduction.CELL_NAN_COLUMN`` / ``CELL_SIZE_COLUMN``.
|
|
257
|
+
#: A side frame, like ``absent``, so it never becomes a grouping column of
|
|
258
|
+
#: the collapse chain. The "NaN elements" reason of a scalar figure's
|
|
259
|
+
#: "Not plotted" list (`omissions.cell_nans`); None when no cell was partial.
|
|
260
|
+
cell_nans: "pd.DataFrame | None" = None
|
|
254
261
|
#: How the per-sample reductions a plot needs should be performed — y
|
|
255
262
|
#: extents, exploding a 1-D measure, striding for transport, averaging
|
|
256
263
|
#: matrices. None means the pandas reference (``reducer.PandasReducer``).
|
|
@@ -50,7 +50,8 @@ from scistacklog import Log
|
|
|
50
50
|
|
|
51
51
|
from .reduction import CollapseCells
|
|
52
52
|
from .series_stats import cell_array
|
|
53
|
-
from
|
|
53
|
+
from scicanonicalhash._cells import numeric_elements
|
|
54
|
+
from ..model.shape import Shape
|
|
54
55
|
from ..model.spec import SCALAR_KINDS, PlotSpec
|
|
55
56
|
from ..model.table import LongTable
|
|
56
57
|
|
|
@@ -120,7 +121,8 @@ def apply_cell_collapse(spec: PlotSpec, table: LongTable) -> LongTable:
|
|
|
120
121
|
if not step.exploded:
|
|
121
122
|
_warn_non_numeric_cells(frame[measure], measure)
|
|
122
123
|
# The rule is the step's (reduction.CollapseCells); codegen prints it.
|
|
123
|
-
collapsed, rows, samples = step.apply(frame)
|
|
124
|
+
collapsed, rows, samples, cell_nans = step.apply(frame)
|
|
125
|
+
partial = 0 if cell_nans is None else len(cell_nans)
|
|
124
126
|
|
|
125
127
|
empty = int(collapsed[measure].isna().sum()) if len(collapsed) else 0
|
|
126
128
|
|
|
@@ -130,12 +132,13 @@ def apply_cell_collapse(spec: PlotSpec, table: LongTable) -> LongTable:
|
|
|
130
132
|
# line per resolve; the phase timer beside it reports the same elapsed.
|
|
131
133
|
Log.info(
|
|
132
134
|
"collapsed 1-D measure %r to scalar by %s: %d row(s) from %d sample(s) "
|
|
133
|
-
"in %.3fs",
|
|
135
|
+
"in %.3fs; %d record(s) averaged over fewer elements (NaN inside)",
|
|
134
136
|
measure,
|
|
135
137
|
statistic,
|
|
136
138
|
rows,
|
|
137
139
|
samples,
|
|
138
140
|
time.perf_counter() - started,
|
|
141
|
+
partial,
|
|
139
142
|
layer=LAYER,
|
|
140
143
|
)
|
|
141
144
|
if empty:
|
|
@@ -165,6 +168,9 @@ def apply_cell_collapse(spec: PlotSpec, table: LongTable) -> LongTable:
|
|
|
165
168
|
table,
|
|
166
169
|
frame=collapsed,
|
|
167
170
|
measures=measures,
|
|
171
|
+
# "Not plotted" lists these (omissions.cell_nans): averaged over fewer
|
|
172
|
+
# elements than the cell holds. Filtered there like `absent`.
|
|
173
|
+
cell_nans=cell_nans,
|
|
168
174
|
# The within-observation axis belongs to the samples that are now gone.
|
|
169
175
|
# Left in place it would be an ordinary column of a scalar table, i.e. a
|
|
170
176
|
# factor-shaped thing with one level per sample position.
|
|
@@ -172,26 +178,29 @@ def apply_cell_collapse(spec: PlotSpec, table: LongTable) -> LongTable:
|
|
|
172
178
|
)
|
|
173
179
|
|
|
174
180
|
|
|
181
|
+
def _unreadable(value) -> bool:
|
|
182
|
+
"""A non-empty array or list holding no readable numbers."""
|
|
183
|
+
if isinstance(value, np.ndarray):
|
|
184
|
+
if value.ndim == 0 or not value.size or value.dtype.kind in "biuf":
|
|
185
|
+
return False
|
|
186
|
+
elif not isinstance(value, (list, tuple)) or not value:
|
|
187
|
+
return False
|
|
188
|
+
return numeric_elements(value) is None
|
|
189
|
+
|
|
190
|
+
|
|
175
191
|
def _warn_non_numeric_cells(values: pd.Series, measure: str) -> None:
|
|
176
|
-
"""WARN when cells of the measure hold
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
costs one dtype check per cell.
|
|
192
|
+
"""WARN when cells of the measure hold arrays or lists that are not
|
|
193
|
+
numbers (text, complex, datetimes, ragged nesting): they collapse to NaN
|
|
194
|
+
gaps (``cell_array``), never to a crash, and the log says how many and
|
|
195
|
+
shows a few.
|
|
196
|
+
|
|
197
|
+
"Not numbers" is ``scicanonicalhash._cells.numeric_elements``'s rule, the
|
|
198
|
+
one ``cell_array`` reads with, so this count and the gaps cannot
|
|
199
|
+
disagree. A numeric ndarray costs one dtype check per cell.
|
|
184
200
|
"""
|
|
185
|
-
bad = [
|
|
186
|
-
v
|
|
187
|
-
for v in values
|
|
188
|
-
if isinstance(v, np.ndarray)
|
|
189
|
-
and v.dtype.kind in "OUS"
|
|
190
|
-
and v.size
|
|
191
|
-
and classify_value(v) is not Shape.SERIES_1D
|
|
192
|
-
]
|
|
201
|
+
bad = [v for v in values if _unreadable(v)]
|
|
193
202
|
if bad:
|
|
194
|
-
examples = [str(v.ravel()[0])[:40] for v in bad[:3]]
|
|
203
|
+
examples = [str(np.asarray(v, dtype=object).ravel()[0])[:40] for v in bad[:3]]
|
|
195
204
|
Log.warn(
|
|
196
205
|
"%d cell(s) of %r hold non-numeric values and are drawn as gaps "
|
|
197
206
|
"(e.g. %s)",
|
|
@@ -11,7 +11,7 @@ footnote and the log line only display it.
|
|
|
11
11
|
| ``no_value`` | the record exists but its value is empty (NaN / NULL) | yes |
|
|
12
12
|
| ``no_record`` | the dataset has the location; this variable has no record there (``scidb.provenance.inventory.absent_locations``) | yes |
|
|
13
13
|
| ``outside_range`` | a histogram value outside the typed From / To range | yes |
|
|
14
|
-
| ``nan_elements`` | some entries inside a vector are NaN | no: thinned |
|
|
14
|
+
| ``nan_elements`` | some entries inside a vector are NaN: a 1-D figure skips those positions; a collapsed (scalar) figure averages the rest (``cell_nans``) | no: thinned |
|
|
15
15
|
| ``partly_averaged`` | a collapsed average used fewer members than exist | no: thinned |
|
|
16
16
|
|
|
17
17
|
Removed on purpose (location filter, Filters, exclusions) is never here.
|
|
@@ -207,6 +207,43 @@ def nan_elements(records: pd.DataFrame, measure: str, keys: list[str]) -> list[O
|
|
|
207
207
|
return items
|
|
208
208
|
|
|
209
209
|
|
|
210
|
+
def cell_nans(
|
|
211
|
+
frame: pd.DataFrame | None,
|
|
212
|
+
figure_key: dict[str, Any],
|
|
213
|
+
measure: str,
|
|
214
|
+
keys: list[str],
|
|
215
|
+
statistic: str = "mean",
|
|
216
|
+
) -> list[Omission]:
|
|
217
|
+
"""Records of a COLLAPSED 1-D measure whose cell was averaged over fewer
|
|
218
|
+
elements than it holds (``LongTable.cell_nans``, already through the
|
|
219
|
+
spec's filters), inside this figure (its ITERATE values).
|
|
220
|
+
|
|
221
|
+
The value is drawn, so this is thinned, not lost (``NAN_ELEMENTS``). The
|
|
222
|
+
counts are the cell collapse's, from the one cell reader
|
|
223
|
+
(``scicanonicalhash._cells``) ``scidb nans`` also uses."""
|
|
224
|
+
from .reduction import CELL_NAN_COLUMN, CELL_SIZE_COLUMN
|
|
225
|
+
|
|
226
|
+
if frame is None or frame.empty:
|
|
227
|
+
return []
|
|
228
|
+
rows = frame
|
|
229
|
+
for name, value in figure_key.items():
|
|
230
|
+
if name in rows.columns:
|
|
231
|
+
rows = rows[rows[name].astype(str) == str(value)]
|
|
232
|
+
present = [k for k in keys if k in rows.columns]
|
|
233
|
+
items = []
|
|
234
|
+
for row in rows.to_dict(orient="records"):
|
|
235
|
+
nan, size = int(row[CELL_NAN_COLUMN]), int(row[CELL_SIZE_COLUMN])
|
|
236
|
+
items.append(
|
|
237
|
+
Omission(
|
|
238
|
+
_keys_of(row, present),
|
|
239
|
+
measure,
|
|
240
|
+
NAN_ELEMENTS,
|
|
241
|
+
f"{nan} of {size} elements NaN; the {statistic} of the other {size - nan}",
|
|
242
|
+
)
|
|
243
|
+
)
|
|
244
|
+
return items
|
|
245
|
+
|
|
246
|
+
|
|
210
247
|
def outside_range(
|
|
211
248
|
sample: pd.DataFrame, measure: str, keys: list[str], low: float, high: float, regular
|
|
212
249
|
) -> list[Omission]:
|
|
@@ -38,6 +38,8 @@ import pandas as pd
|
|
|
38
38
|
from ..model.spec import CategoricalStat, Filter, LevelGroup, LocationFilter, PlotSpec, Statistic
|
|
39
39
|
from ..model.table import category_text
|
|
40
40
|
from .compare import ComparisonPlan
|
|
41
|
+
from scicanonicalhash._cells import nan_count
|
|
42
|
+
|
|
41
43
|
from .series_stats import cell_arrays, collapse_cells
|
|
42
44
|
|
|
43
45
|
|
|
@@ -150,6 +152,12 @@ class CombineLevels:
|
|
|
150
152
|
return order
|
|
151
153
|
|
|
152
154
|
|
|
155
|
+
#: The columns ``CollapseCells.apply`` adds to its partial-NaN rows: NaN
|
|
156
|
+
#: elements, and all elements, of the record's cell (``LongTable.cell_nans``).
|
|
157
|
+
CELL_NAN_COLUMN = "__nan"
|
|
158
|
+
CELL_SIZE_COLUMN = "__size"
|
|
159
|
+
|
|
160
|
+
|
|
153
161
|
@dataclass(frozen=True)
|
|
154
162
|
class CollapseCells:
|
|
155
163
|
"""A 1-D measure reduced to one value per record by its ``statistic``,
|
|
@@ -168,24 +176,57 @@ class CollapseCells:
|
|
|
168
176
|
exploded: bool = False
|
|
169
177
|
index_column: str | None = None
|
|
170
178
|
|
|
171
|
-
def apply(
|
|
172
|
-
|
|
179
|
+
def apply(
|
|
180
|
+
self, frame: pd.DataFrame
|
|
181
|
+
) -> tuple[pd.DataFrame, int, int, "pd.DataFrame | None"]:
|
|
182
|
+
"""The collapsed frame, its row count, the samples it read, and the
|
|
183
|
+
collapsed rows whose cell had SOME NaN elements (``CELL_NAN_COLUMN`` of
|
|
184
|
+
``CELL_SIZE_COLUMN``; None when no cell did). An all-NaN or empty cell
|
|
185
|
+
is not among them: it collapses to NaN, which is a missing value of
|
|
186
|
+
its own. The counts are the same arrays' (one reader,
|
|
187
|
+
``scicanonicalhash._cells``), so they say exactly what the average
|
|
188
|
+
skipped."""
|
|
173
189
|
if self.exploded:
|
|
174
190
|
return self._apply_exploded(frame)
|
|
175
191
|
arrays = cell_arrays(frame[self.measure])
|
|
176
192
|
out = frame.copy()
|
|
177
193
|
out[self.measure] = collapse_cells(arrays, self.statistic)
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
194
|
+
counts = [nan_count(a) for a in arrays]
|
|
195
|
+
partial = [i for i, (nan, size) in enumerate(counts) if 0 < nan < size]
|
|
196
|
+
nans = None
|
|
197
|
+
if partial:
|
|
198
|
+
nans = out.iloc[partial].copy()
|
|
199
|
+
nans[CELL_NAN_COLUMN] = [counts[i][0] for i in partial]
|
|
200
|
+
nans[CELL_SIZE_COLUMN] = [counts[i][1] for i in partial]
|
|
201
|
+
return out, len(out), int(sum(size for _, size in counts)), nans
|
|
202
|
+
|
|
203
|
+
def _apply_exploded(
|
|
204
|
+
self, frame: pd.DataFrame
|
|
205
|
+
) -> tuple[pd.DataFrame, int, int, "pd.DataFrame | None"]:
|
|
181
206
|
keys = [c for c in frame.columns if c not in (self.measure, self.index_column)]
|
|
182
207
|
median = self.statistic is Statistic.MEDIAN
|
|
208
|
+
missing = frame[self.measure].isna()
|
|
183
209
|
if not keys:
|
|
184
210
|
value = frame[self.measure].median() if median else frame[self.measure].mean()
|
|
185
|
-
|
|
211
|
+
out = pd.DataFrame({self.measure: [value]})
|
|
212
|
+
nan, size = int(missing.sum()), len(frame)
|
|
213
|
+
nans = None
|
|
214
|
+
if 0 < nan < size:
|
|
215
|
+
nans = out.assign(**{CELL_NAN_COLUMN: [nan], CELL_SIZE_COLUMN: [size]})
|
|
216
|
+
return out, 1, len(frame), nans
|
|
186
217
|
grouped = frame.groupby(keys, dropna=False, sort=False)[self.measure]
|
|
187
218
|
out = (grouped.median() if median else grouped.mean()).reset_index()
|
|
188
|
-
|
|
219
|
+
# The same groups in the same order (sort=False, dropna=False).
|
|
220
|
+
stats = missing.groupby([frame[k] for k in keys], dropna=False, sort=False).agg(["sum", "size"])
|
|
221
|
+
nan = stats["sum"].to_numpy(dtype=int)
|
|
222
|
+
size = stats["size"].to_numpy(dtype=int)
|
|
223
|
+
partial = np.flatnonzero((nan > 0) & (nan < size))
|
|
224
|
+
nans = None
|
|
225
|
+
if partial.size:
|
|
226
|
+
nans = out.iloc[partial].copy()
|
|
227
|
+
nans[CELL_NAN_COLUMN] = nan[partial]
|
|
228
|
+
nans[CELL_SIZE_COLUMN] = size[partial]
|
|
229
|
+
return out, len(out), len(frame), nans
|
|
189
230
|
|
|
190
231
|
|
|
191
232
|
#: The working column holding each record's category text during the one-hot.
|
|
@@ -2142,6 +2142,25 @@ def _omissions(
|
|
|
2142
2142
|
if raw is not None and not custom:
|
|
2143
2143
|
for measure in measures:
|
|
2144
2144
|
items.extend(om.partly_averaged(raw, sample, measure, keys))
|
|
2145
|
+
cell_nans = table.cell_nans
|
|
2146
|
+
if cell_nans is not None and not cell_nans.empty and not custom:
|
|
2147
|
+
# A collapsed 1-D measure: records averaged over fewer elements
|
|
2148
|
+
# than their cell holds. Their own location (every key the record
|
|
2149
|
+
# has), not the sample's: the record is what holds the NaNs.
|
|
2150
|
+
record_keys = [
|
|
2151
|
+
name
|
|
2152
|
+
for name in dict.fromkeys([*table.schema_levels, *table.factor_names])
|
|
2153
|
+
if name in cell_nans.columns
|
|
2154
|
+
]
|
|
2155
|
+
items.extend(
|
|
2156
|
+
om.cell_nans(
|
|
2157
|
+
apply_filters(cell_nans, spec),
|
|
2158
|
+
figure_key,
|
|
2159
|
+
spec.y_measure,
|
|
2160
|
+
record_keys,
|
|
2161
|
+
str(spec.cell_statistic),
|
|
2162
|
+
)
|
|
2163
|
+
)
|
|
2145
2164
|
drawn = max(len(sample) - lost_rows - len(outside), 0)
|
|
2146
2165
|
elif records is not None:
|
|
2147
2166
|
if not custom:
|
|
@@ -36,31 +36,25 @@ import warnings
|
|
|
36
36
|
from typing import Any
|
|
37
37
|
|
|
38
38
|
import numpy as np
|
|
39
|
+
from scicanonicalhash._cells import numeric_elements
|
|
39
40
|
|
|
40
41
|
from ..model.spec import ErrorBand, Statistic
|
|
41
42
|
from .spread import SD_DDOF, ci95_spread, sem_spread
|
|
42
43
|
|
|
43
44
|
|
|
44
45
|
def cell_array(value: Any) -> np.ndarray:
|
|
45
|
-
"""One cell as a 1-D float64 array; empty
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
return np.empty(0)
|
|
58
|
-
if isinstance(value, (list, tuple)):
|
|
59
|
-
try:
|
|
60
|
-
return np.asarray(value, dtype="float64").ravel()
|
|
61
|
-
except (TypeError, ValueError):
|
|
62
|
-
return np.empty(0)
|
|
63
|
-
return np.empty(0)
|
|
46
|
+
"""One cell as a 1-D float64 array; empty when the cell holds no numbers.
|
|
47
|
+
|
|
48
|
+
The reading rule is ``scicanonicalhash._cells.numeric_elements``'s, the
|
|
49
|
+
one reader shared with scidb (``scidb nans``, the data preview), so a
|
|
50
|
+
plot and the data inspection never disagree about what a cell holds:
|
|
51
|
+
numbers, bools and None (as NaN) in an array or (nested) list; text,
|
|
52
|
+
complex, datetimes, ragged nesting and scalars are not numbers. Those
|
|
53
|
+
become empty here: a gap, counted by the caller, never a crash (a raw
|
|
54
|
+
numpy ValueError once failed a whole resolve, run 0c164454).
|
|
55
|
+
"""
|
|
56
|
+
elements = numeric_elements(value)
|
|
57
|
+
return np.empty(0) if elements is None else elements
|
|
64
58
|
|
|
65
59
|
|
|
66
60
|
def cell_arrays(values) -> list[np.ndarray]:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|