apmtools 0.2.56__tar.gz → 0.2.57__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {apmtools-0.2.56 → apmtools-0.2.57}/PKG-INFO +1 -1
- {apmtools-0.2.56 → apmtools-0.2.57}/pyproject.toml +1 -1
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/classes.py +19 -32
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/plot_configs.py +6 -25
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/data.py +7 -7
- apmtools-0.2.57/src/apmtools/shiny/derived_views.py +35 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/inherited_source.py +29 -9
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/matplotlib_renderer.py +13 -8
- apmtools-0.2.57/src/apmtools/shiny/multi_stat_plot.py +53 -0
- apmtools-0.2.57/src/apmtools/shiny/profile_report.py +51 -0
- apmtools-0.2.57/src/apmtools/shiny/profiling.py +103 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/state.py +14 -0
- apmtools-0.2.57/src/apmtools/shiny/tab_initialization.py +74 -0
- apmtools-0.2.57/src/apmtools/shiny/text_editors.py +23 -0
- apmtools-0.2.57/src/apmtools/shiny/transformations.py +208 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/ui.py +485 -230
- apmtools-0.2.57/src/apmtools/shiny/view_aggregation.py +124 -0
- apmtools-0.2.57/src/apmtools/shiny/view_controller.py +91 -0
- apmtools-0.2.57/src/apmtools/shiny/view_history.py +106 -0
- apmtools-0.2.57/src/apmtools/shiny/workspace_controller.py +58 -0
- apmtools-0.2.57/src/apmtools/view_schema.py +106 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/.gitignore +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/LICENSE +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/README.md +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/__init__.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/data_processing.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/functions.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/legacy.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/plot_app.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/__init__.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/appearance.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/color_editor.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/columns.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/config.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/elements.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/export.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/filter_config.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/histogram_multi.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/legend_identity.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/multi_axis.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/multi_y.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/shiny/summary.py +0 -0
- {apmtools-0.2.56 → apmtools-0.2.57}/src/apmtools/units.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: apmtools
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.57
|
|
4
4
|
Summary: A collection of tools for processing air pollution monitoring data
|
|
5
5
|
Project-URL: Homepage, https://github.com/federlorenz/apmtools
|
|
6
6
|
Author-email: Federico Lorenzetti <lorenzetti.federico@gmail.com>
|
|
@@ -1224,7 +1224,12 @@ class Summary(pd.DataFrame):
|
|
|
1224
1224
|
print(f"{v} not in Summary columns")
|
|
1225
1225
|
return None
|
|
1226
1226
|
|
|
1227
|
-
|
|
1227
|
+
# Non-temporal grouping also works on previously aggregated frames.
|
|
1228
|
+
# Only operations involving a frequency need the source time column.
|
|
1229
|
+
if frequency is not None and time_col not in self.columns:
|
|
1230
|
+
raise KeyError(f"Temporal aggregation requires column {time_col!r}")
|
|
1231
|
+
df = (self.dropna(subset=[time_col]).copy()
|
|
1232
|
+
if frequency is not None else self.copy())
|
|
1228
1233
|
|
|
1229
1234
|
if frequency != None:
|
|
1230
1235
|
|
|
@@ -1555,38 +1560,20 @@ class Summary(pd.DataFrame):
|
|
|
1555
1560
|
df[freq] = df["datetime"].map(
|
|
1556
1561
|
lambda x: (x.weekday//mult))*mult
|
|
1557
1562
|
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
if
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
else:
|
|
1572
|
-
output = df
|
|
1563
|
+
# A frequency may also be an explicitly selected grouping variable.
|
|
1564
|
+
# pandas allows repeated group keys but reset_index() then attempts to
|
|
1565
|
+
# insert the same column twice (e.g. "cannot insert hour, already exists").
|
|
1566
|
+
# Preserve the requested order while grouping each key only once.
|
|
1567
|
+
grouping_keys = list(dict.fromkeys(
|
|
1568
|
+
list(grouping_by or []) +
|
|
1569
|
+
([frequency] if isinstance(frequency, str) else
|
|
1570
|
+
list(frequency) if isinstance(frequency, list) else [])
|
|
1571
|
+
))
|
|
1572
|
+
if frequency is not None or grouping_by is not None:
|
|
1573
|
+
output = df.groupby(grouping_keys, dropna=False) if aggregate is not None else df[grouping_keys]
|
|
1574
|
+
else:
|
|
1575
|
+
output = df
|
|
1573
1576
|
|
|
1574
|
-
if isinstance(frequency, str) and aggregate is None:
|
|
1575
|
-
if grouping_by != None:
|
|
1576
|
-
output = df[grouping_by+[frequency]]
|
|
1577
|
-
else:
|
|
1578
|
-
output = df[[frequency]]
|
|
1579
|
-
if isinstance(frequency, list) and aggregate is None:
|
|
1580
|
-
if grouping_by != None:
|
|
1581
|
-
output = df[grouping_by+frequency]
|
|
1582
|
-
else:
|
|
1583
|
-
output = df[frequency]
|
|
1584
|
-
if frequency == None and aggregate is None:
|
|
1585
|
-
if grouping_by != None:
|
|
1586
|
-
output = df[grouping_by]
|
|
1587
|
-
else:
|
|
1588
|
-
output = df
|
|
1589
|
-
|
|
1590
1577
|
if aggregate is not None:
|
|
1591
1578
|
output = output[value_cols].agg(aggregate)
|
|
1592
1579
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Plot configuration lifecycle: reactive drafts and explicitly retained presets."""
|
|
2
2
|
from __future__ import annotations
|
|
3
|
+
from .shiny.profiling import traced
|
|
3
4
|
import copy
|
|
4
5
|
import json
|
|
5
6
|
import uuid
|
|
@@ -12,27 +13,10 @@ def _json_ready(value):
|
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
def validate_saved_plots(plots):
|
|
15
|
-
"""
|
|
16
|
+
"""Backward-compatible name for strict saved-view structural validation."""
|
|
17
|
+
from .view_schema import validate_saved_views
|
|
18
|
+
return validate_saved_views(plots)
|
|
16
19
|
|
|
17
|
-
Old snapshots remain supported: unknown settings are deliberately retained.
|
|
18
|
-
"""
|
|
19
|
-
if plots is None:
|
|
20
|
-
return []
|
|
21
|
-
if not isinstance(plots, list):
|
|
22
|
-
raise TypeError("Summary.views must be a list or None")
|
|
23
|
-
seen = set()
|
|
24
|
-
for item in plots:
|
|
25
|
-
if not isinstance(item, dict):
|
|
26
|
-
raise ValueError("Saved plot entries must be dictionaries")
|
|
27
|
-
if item.get("schema_version") != 1 or item.get("type") not in ("plot", "table"):
|
|
28
|
-
raise ValueError("Unsupported saved plot configuration")
|
|
29
|
-
if not isinstance(item.get("settings"), dict):
|
|
30
|
-
raise ValueError("Saved plot settings must be a dictionary")
|
|
31
|
-
identifier = item.get("id")
|
|
32
|
-
if not isinstance(identifier, str) or not identifier or identifier in seen:
|
|
33
|
-
raise ValueError("Saved plot IDs must be unique nonempty strings")
|
|
34
|
-
seen.add(identifier)
|
|
35
|
-
return copy.deepcopy(plots)
|
|
36
20
|
|
|
37
21
|
class PlotConfigManager:
|
|
38
22
|
"""Keep live tab drafts separate from explicitly retained Summary.plots.
|
|
@@ -195,11 +179,8 @@ class PlotConfigManager:
|
|
|
195
179
|
self.revision += 1
|
|
196
180
|
|
|
197
181
|
def saved(self):
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
if item.get('type') == 'table':
|
|
201
|
-
item.pop('frame', None) # migrate legacy snapshot records
|
|
202
|
-
return items
|
|
182
|
+
from .view_schema import migrate_saved_views
|
|
183
|
+
return migrate_saved_views(self.summary.views)
|
|
203
184
|
|
|
204
185
|
def restore(self, preset_id, tab_id):
|
|
205
186
|
item = next((p for p in self.saved() if p.get('id') == preset_id), None)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Filtering, aggregation and temporal segmentation."""
|
|
2
2
|
from __future__ import annotations
|
|
3
|
+
from .profiling import traced
|
|
3
4
|
import threading
|
|
4
5
|
from datetime import date, datetime, time, timedelta
|
|
5
6
|
from collections.abc import Sequence
|
|
@@ -82,6 +83,7 @@ def _parse_datetime_filter(value):
|
|
|
82
83
|
|
|
83
84
|
|
|
84
85
|
|
|
86
|
+
@traced('_apply_filters', kind="data" if 'shiny/data.py' not in ("plot_configs.py", "shiny/matplotlib_renderer.py") else "service")
|
|
85
87
|
def _apply_filters(data, subset_filters, datetime_start=None, datetime_end=None):
|
|
86
88
|
"""Apply Summary filters, accepting dates, times and full datetimes.
|
|
87
89
|
|
|
@@ -123,6 +125,7 @@ def _apply_filters(data, subset_filters, datetime_start=None, datetime_end=None)
|
|
|
123
125
|
|
|
124
126
|
|
|
125
127
|
|
|
128
|
+
@traced('_aggregate', kind="data" if 'shiny/data.py' not in ("plot_configs.py", "shiny/matplotlib_renderer.py") else "service")
|
|
126
129
|
def _aggregate(data, value_columns, frequency, grouping_by, aggregate, *,
|
|
127
130
|
only_complete=True, flooring_timestamp=True):
|
|
128
131
|
"""Aggregate through ``Summary.group_stat``.
|
|
@@ -138,13 +141,10 @@ def _aggregate(data, value_columns, frequency, grouping_by, aggregate, *,
|
|
|
138
141
|
# Return empty rather than breaking Shiny's reactive graph.
|
|
139
142
|
return pd.DataFrame()
|
|
140
143
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
frequency
|
|
144
|
-
|
|
145
|
-
aggregate=[aggregate],
|
|
146
|
-
**({"only_complete": only_complete, "flooring_timestamp": flooring_timestamp}
|
|
147
|
-
if frequency is not None else {}),
|
|
144
|
+
from .view_aggregation import aggregate_view
|
|
145
|
+
result = aggregate_view(
|
|
146
|
+
data, value_columns, frequency, grouping_by, aggregate,
|
|
147
|
+
only_complete=only_complete, flooring_timestamp=flooring_timestamp,
|
|
148
148
|
)
|
|
149
149
|
if result is None:
|
|
150
150
|
# group_stat() may return None when no rows survive completeness
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Headless, configuration-only table source preparation for Shiny tab actions.
|
|
2
|
+
|
|
3
|
+
The application creates tables from the owning Summary and ordered view
|
|
4
|
+
settings, never from snapshots of already rendered plot/table data.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
from .inherited_source import source_frame, source_steps
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def prepare_table_source(summary, *, parent_kind=None, parent_settings=None,
|
|
14
|
+
parent_steps=None, restored_settings=None):
|
|
15
|
+
"""Return (display-ready frame, lineage steps) for new/loaded tables.
|
|
16
|
+
|
|
17
|
+
``parent_kind`` is ``'plot'``, ``'table'`` or ``None`` (Info tab).
|
|
18
|
+
For restoration, pass ``restored_settings`` instead; its saved lineage is
|
|
19
|
+
replayed without modification. No parent DataFrame is copied or cached.
|
|
20
|
+
"""
|
|
21
|
+
if restored_settings is not None:
|
|
22
|
+
steps = restored_settings.get('source_steps', [])
|
|
23
|
+
elif parent_kind in ('plot', 'table'):
|
|
24
|
+
if parent_settings is None:
|
|
25
|
+
raise ValueError('Parent view settings must be supplied')
|
|
26
|
+
steps = source_steps(parent_steps if parent_kind == 'table' else [],
|
|
27
|
+
parent_settings, parent_kind)
|
|
28
|
+
elif parent_kind is None:
|
|
29
|
+
steps = []
|
|
30
|
+
else:
|
|
31
|
+
raise ValueError(f'Unknown parent view kind: {parent_kind!r}')
|
|
32
|
+
frame = source_frame(summary, steps)
|
|
33
|
+
frame = frame.reset_index(drop=True)
|
|
34
|
+
frame.columns = [str(column) for column in frame.columns]
|
|
35
|
+
return frame, steps
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
"""Replay configuration-only source transformations for derived tables."""
|
|
2
2
|
from __future__ import annotations
|
|
3
|
+
from .profiling import traced
|
|
3
4
|
import copy
|
|
5
|
+
import os
|
|
4
6
|
import pandas as pd
|
|
5
7
|
import numpy as np
|
|
6
8
|
from .data import _subset_choices, _apply_filters, _aggregate
|
|
7
9
|
from .config import AGGREGATES, FREQUENCIES
|
|
8
10
|
from .columns import _numeric_columns
|
|
11
|
+
from .view_aggregation import normalize_statistics
|
|
9
12
|
|
|
10
13
|
|
|
11
14
|
def _plot_columns(current, step):
|
|
@@ -19,15 +22,16 @@ def _plot_columns(current, step):
|
|
|
19
22
|
values = [str(v) for v in values if str(v) in current.columns]
|
|
20
23
|
groups = step.get('group_by') or []
|
|
21
24
|
groups = [groups] if isinstance(groups, str) else list(groups)
|
|
22
|
-
groups = [str(g) for g in groups if g not in ('variables_x', 'variables_y') and str(g) in current.columns]
|
|
25
|
+
groups = [str(g) for g in groups if g not in ('variables_x', 'variables_y', 'statistics') and str(g) in current.columns]
|
|
23
26
|
x_columns = [str(v) for v in x if str(v) in current.columns]
|
|
24
|
-
if kind in ('bar', 'box') and
|
|
27
|
+
if kind in ('bar', 'box') and bool(normalize_statistics(step.get('aggregate'))):
|
|
25
28
|
groups = list(dict.fromkeys(groups + x_columns))
|
|
26
|
-
if kind == 'scatter' and
|
|
29
|
+
if kind == 'scatter' and bool(normalize_statistics(step.get('aggregate'))):
|
|
27
30
|
values = list(dict.fromkeys(x_columns + values))
|
|
28
31
|
return kind, values, groups, x_columns
|
|
29
32
|
|
|
30
33
|
|
|
34
|
+
@traced('_plot_source', kind="data" if 'shiny/inherited_source.py' not in ("plot_configs.py", "shiny/matplotlib_renderer.py") else "service")
|
|
31
35
|
def _plot_source(current, step):
|
|
32
36
|
"""Reconstruct the data contributing to a plot, with its exact aggregation."""
|
|
33
37
|
kind, values, groups, x_columns = _plot_columns(current, step)
|
|
@@ -38,9 +42,9 @@ def _plot_source(current, step):
|
|
|
38
42
|
current = current.loc[current[values].notna().any(axis=1)].copy()
|
|
39
43
|
if current.empty:
|
|
40
44
|
return current
|
|
41
|
-
aggregate =
|
|
45
|
+
aggregate = normalize_statistics(step.get('aggregate'))
|
|
42
46
|
frequency = FREQUENCIES.get(step.get('frequency'))
|
|
43
|
-
if aggregate
|
|
47
|
+
if aggregate:
|
|
44
48
|
if kind == 'line' and x_columns and pd.api.types.is_timedelta64_dtype(current[x_columns[0]]):
|
|
45
49
|
return pd.DataFrame()
|
|
46
50
|
if frequency is None and not groups:
|
|
@@ -58,7 +62,7 @@ def _plot_source(current, step):
|
|
|
58
62
|
return current.loc[:, keep].copy()
|
|
59
63
|
|
|
60
64
|
|
|
61
|
-
def
|
|
65
|
+
def _legacy_source_frame(summary, steps):
|
|
62
66
|
"""Rebuild a table source from its owning Summary and ordered settings.
|
|
63
67
|
|
|
64
68
|
Each step is a configuration snapshot, never a DataFrame snapshot.
|
|
@@ -93,11 +97,13 @@ def source_frame(summary, steps):
|
|
|
93
97
|
if step.get('_source_kind') == 'plot':
|
|
94
98
|
current = _plot_source(current, step)
|
|
95
99
|
elif step.get('_source_kind') == 'table':
|
|
96
|
-
aggregate =
|
|
97
|
-
if aggregate
|
|
100
|
+
aggregate = normalize_statistics(step.get('aggregate'))
|
|
101
|
+
if aggregate and not current.empty:
|
|
98
102
|
frequency = FREQUENCIES.get(step.get('frequency'))
|
|
99
103
|
groups = list(step.get('group_by') or [])
|
|
100
|
-
numeric = [c for c in
|
|
104
|
+
numeric = [c for c in (step.get('aggregate_columns') if step.get('aggregate_columns') is not None
|
|
105
|
+
else _numeric_columns(current))
|
|
106
|
+
if c in current.columns and c not in groups and current[c].notna().any()]
|
|
101
107
|
if numeric:
|
|
102
108
|
current = _aggregate(current, numeric, frequency, groups, aggregate,
|
|
103
109
|
only_complete=bool(step.get('only_complete', True)) if frequency is not None else False,
|
|
@@ -110,3 +116,17 @@ def source_steps(parent_steps, parent_settings, kind):
|
|
|
110
116
|
step = copy.deepcopy(parent_settings)
|
|
111
117
|
step['_source_kind'] = kind
|
|
112
118
|
return copy.deepcopy(parent_steps or []) + [step]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@traced('source_frame', kind='data')
|
|
122
|
+
def source_frame(summary, steps):
|
|
123
|
+
"""Reconstruct derived table data with the unified executor by default.
|
|
124
|
+
|
|
125
|
+
Set ``APMTOOLS_TRANSFORM_ENGINE=0`` (also ``false``, ``no``, or
|
|
126
|
+
``legacy``) to temporarily retain the historical processing behavior.
|
|
127
|
+
Errors from the selected executor propagate without silent fallback.
|
|
128
|
+
"""
|
|
129
|
+
if os.getenv('APMTOOLS_TRANSFORM_ENGINE', '').strip().lower() not in ('0', 'false', 'no', 'legacy'):
|
|
130
|
+
from .transformations import execute_view, from_legacy_source_steps
|
|
131
|
+
return execute_view(summary, from_legacy_source_steps(steps))
|
|
132
|
+
return _legacy_source_frame(summary, steps)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Native and converted Matplotlib figures and panel layout."""
|
|
2
2
|
from __future__ import annotations
|
|
3
|
+
from .profiling import traced
|
|
3
4
|
import threading
|
|
4
5
|
from datetime import date, datetime, time, timedelta
|
|
5
6
|
from collections.abc import Sequence
|
|
@@ -293,21 +294,25 @@ def _place_outside_legend(fig, axes, position="right", font_size=12, ncol=1,
|
|
|
293
294
|
A single legend is shared across panels. Only one full canvas draw is used
|
|
294
295
|
for measuring; no automatic column search or placement fallback.
|
|
295
296
|
"""
|
|
297
|
+
# Entry identity is positional within a panel, not its editable label.
|
|
298
|
+
# This retains two distinct handles renamed to the same displayed text,
|
|
299
|
+
# while collapsing corresponding entries repeated across panels.
|
|
296
300
|
seen = {}
|
|
297
301
|
for ax in axes:
|
|
298
302
|
old = ax.get_legend()
|
|
299
303
|
if old is None:
|
|
300
304
|
continue
|
|
301
|
-
for handle, label in zip(old.legend_handles,
|
|
302
|
-
[t.get_text() for t in old.get_texts()]):
|
|
303
|
-
if label and
|
|
304
|
-
seen[
|
|
305
|
+
for index, (handle, label) in enumerate(zip(old.legend_handles,
|
|
306
|
+
[t.get_text() for t in old.get_texts()])):
|
|
307
|
+
if label and index not in seen:
|
|
308
|
+
seen[index] = (handle, label)
|
|
305
309
|
old.set_visible(False)
|
|
306
310
|
old.set_in_layout(False)
|
|
307
311
|
if not seen:
|
|
308
312
|
fig.tight_layout()
|
|
309
313
|
return
|
|
310
|
-
|
|
314
|
+
handles = [entry[0] for entry in seen.values()]
|
|
315
|
+
labels = [entry[1] for entry in seen.values()]
|
|
311
316
|
pos = position if position in ("right", "left", "top", "bottom") else "right"
|
|
312
317
|
ncol = max(1, min(int(ncol or 1), len(labels)))
|
|
313
318
|
legend = fig.legend(handles, labels, ncol=ncol, loc="center",
|
|
@@ -343,6 +348,7 @@ def _place_outside_legend(fig, axes, position="right", font_size=12, ncol=1,
|
|
|
343
348
|
# between facet axes. This changes only the internal subplot gaps.
|
|
344
349
|
fig.subplots_adjust(wspace=panel_wspace, hspace=panel_hspace)
|
|
345
350
|
|
|
351
|
+
@traced('_matplotlib_direct_figure', kind="data" if 'shiny/matplotlib_renderer.py' not in ("plot_configs.py", "shiny/matplotlib_renderer.py") else "service")
|
|
346
352
|
def _matplotlib_direct_figure(spec, *, title="", x_label="", y_label="",
|
|
347
353
|
legend_labels="", trace_colors="", palette="Default",
|
|
348
354
|
x_min="", x_max="", y_min="", y_max="",
|
|
@@ -767,9 +773,8 @@ def _matplotlib_direct_figure(spec, *, title="", x_label="", y_label="",
|
|
|
767
773
|
|
|
768
774
|
if kind not in ("box", "bar") and not spec.get("multi_axis"):
|
|
769
775
|
handles, texts = ax.get_legend_handles_labels()
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
ax.legend(unique.values(), unique.keys(), title="Legend")
|
|
776
|
+
if handles and not spec.get("_hide_ungrouped_legend", not bool(spec["groups"])):
|
|
777
|
+
ax.legend(handles, texts, title="Legend")
|
|
773
778
|
if multi_y:
|
|
774
779
|
from matplotlib.lines import Line2D
|
|
775
780
|
# Independent legend keys for the two dimensions, with explicit headings.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Convert multi-statistic wide results to renderer-compatible plot data."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def prepare_multi_stat_plot(frame, sources, statistics, *, split_statistics=False,
|
|
8
|
+
split_variables=False):
|
|
9
|
+
"""Return a plot frame and Y columns with optional virtual dimensions.
|
|
10
|
+
|
|
11
|
+
When split_statistics is disabled, every source/statistic combination is
|
|
12
|
+
displayed as a separate Y series. When enabled, the statistics dimension
|
|
13
|
+
becomes a categorical grouping key, while series remain source-specific.
|
|
14
|
+
Missing source/statistic combinations stay as all-NA columns.
|
|
15
|
+
"""
|
|
16
|
+
sources = list(dict.fromkeys(sources))
|
|
17
|
+
statistics = list(dict.fromkeys(statistics))
|
|
18
|
+
columns = [f"{source}_{stat}" for source in sources for stat in statistics]
|
|
19
|
+
work = pd.DataFrame(frame).copy()
|
|
20
|
+
for col in columns:
|
|
21
|
+
if col not in work:
|
|
22
|
+
work[col] = float('nan')
|
|
23
|
+
if not split_statistics and not split_variables:
|
|
24
|
+
return work, columns
|
|
25
|
+
|
|
26
|
+
# Always copy Y values before melting so that scatter X columns which
|
|
27
|
+
# overlap selected Y variables remain available to the renderer.
|
|
28
|
+
temporaries = []
|
|
29
|
+
names = {}
|
|
30
|
+
for i, source in enumerate(sources):
|
|
31
|
+
for stat in statistics:
|
|
32
|
+
temp = f"__multi_stat_y_{i}_{stat}"
|
|
33
|
+
while temp in work:
|
|
34
|
+
temp += "_"
|
|
35
|
+
work[temp] = work[f"{source}_{stat}"]
|
|
36
|
+
temporaries.append(temp)
|
|
37
|
+
names[temp] = (source, stat)
|
|
38
|
+
work = work.melt(id_vars=[c for c in work if c not in temporaries],
|
|
39
|
+
value_vars=temporaries, var_name="__stat_source", value_name="__stat_value")
|
|
40
|
+
work["variables_y"] = work["__stat_source"].map(lambda k: str(names[k][0]))
|
|
41
|
+
work["statistics"] = work["__stat_source"].map(lambda k: str(names[k][1]))
|
|
42
|
+
work = work.drop(columns=["__stat_source"])
|
|
43
|
+
# Restrict the groups if only one of the two virtual dimensions is selected
|
|
44
|
+
# by combining the other dimension into the plot series label.
|
|
45
|
+
if split_statistics and split_variables:
|
|
46
|
+
return work, ["__stat_value"]
|
|
47
|
+
if split_statistics:
|
|
48
|
+
if len(sources) == 1:
|
|
49
|
+
return work, ["__stat_value"]
|
|
50
|
+
# Multiple sources must remain distinct series under a statistic group.
|
|
51
|
+
# Group composition is handled by the caller.
|
|
52
|
+
return work, ["__stat_value"]
|
|
53
|
+
return work, ["__stat_value"]
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Summarize opt-in Shiny traces: python -m apmtools.shiny.profile_report trace.jsonl"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
import argparse
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def summarize(path):
|
|
10
|
+
groups = defaultdict(list)
|
|
11
|
+
failures = defaultdict(int)
|
|
12
|
+
for line in Path(path).read_text(encoding='utf-8').splitlines():
|
|
13
|
+
if not line.strip():
|
|
14
|
+
continue
|
|
15
|
+
try:
|
|
16
|
+
record = json.loads(line)
|
|
17
|
+
except json.JSONDecodeError:
|
|
18
|
+
continue
|
|
19
|
+
if record.get('event') != 'execution':
|
|
20
|
+
continue
|
|
21
|
+
key = (record.get('kind', 'unknown'), record.get('name', 'unknown'))
|
|
22
|
+
groups[key].append(float(record.get('duration_ms', 0)))
|
|
23
|
+
if record.get('error'):
|
|
24
|
+
failures[key] += 1
|
|
25
|
+
rows = []
|
|
26
|
+
for key, durations in groups.items():
|
|
27
|
+
ordered = sorted(durations)
|
|
28
|
+
rows.append({'kind': key[0], 'name': key[1], 'calls': len(ordered),
|
|
29
|
+
'total_ms': round(sum(ordered), 2),
|
|
30
|
+
'mean_ms': round(sum(ordered)/len(ordered), 2),
|
|
31
|
+
'p95_ms': ordered[min(len(ordered)-1, int(.95*(len(ordered)-1)))],
|
|
32
|
+
'max_ms': ordered[-1], 'errors': failures[key]})
|
|
33
|
+
return sorted(rows, key=lambda r: r['total_ms'], reverse=True)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def main():
|
|
37
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
38
|
+
parser.add_argument('trace', help='Path to APMTOOLS_TRACE_PATH JSONL file')
|
|
39
|
+
parser.add_argument('--top', type=int, default=35)
|
|
40
|
+
parser.add_argument('--json', action='store_true')
|
|
41
|
+
args = parser.parse_args()
|
|
42
|
+
rows = summarize(args.trace)
|
|
43
|
+
if args.json:
|
|
44
|
+
print(json.dumps(rows[:args.top], indent=2))
|
|
45
|
+
return
|
|
46
|
+
print(f"{'kind':17} {'name':37} {'calls':>6} {'total ms':>10} {'p95 ms':>10} {'errors':>6}")
|
|
47
|
+
for r in rows[:args.top]:
|
|
48
|
+
print(f"{r['kind'][:17]:17} {r['name'][:37]:37} {r['calls']:6} {r['total_ms']:10.2f} {r['p95_ms']:10.2f} {r['errors']:6}")
|
|
49
|
+
|
|
50
|
+
if __name__ == '__main__':
|
|
51
|
+
main()
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Opt-in, low-overhead JSONL tracing for Shiny callbacks and data operations.
|
|
2
|
+
|
|
3
|
+
Set APMTOOLS_TRACE_PATH to a writable .jsonl path before launching the app.
|
|
4
|
+
Records invocations (not Shiny internal invalidation notifications). Never logs
|
|
5
|
+
row values or input contents. Each record is one JSON object.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import contextvars
|
|
10
|
+
import functools
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
import uuid
|
|
17
|
+
|
|
18
|
+
_current_span = contextvars.ContextVar('apmtools_trace_span', default=None)
|
|
19
|
+
_lock = threading.RLock()
|
|
20
|
+
_session = uuid.uuid4().hex[:12]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def enabled():
|
|
24
|
+
return bool(os.environ.get('APMTOOLS_TRACE_PATH'))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _emit(event):
|
|
28
|
+
path = os.environ.get('APMTOOLS_TRACE_PATH')
|
|
29
|
+
if not path:
|
|
30
|
+
return
|
|
31
|
+
record = {'timestamp_ns': time.time_ns(), 'session': _session, **event}
|
|
32
|
+
try:
|
|
33
|
+
with _lock:
|
|
34
|
+
destination = Path(path).expanduser()
|
|
35
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
36
|
+
with destination.open('a', encoding='utf-8') as stream:
|
|
37
|
+
stream.write(json.dumps(record, default=str, separators=(',', ':')) + '\n')
|
|
38
|
+
except OSError:
|
|
39
|
+
# Profiling must never interrupt the Shiny application.
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def traced(label=None, *, kind='function', trigger=None):
|
|
44
|
+
"""Instrument a synchronous or asynchronous callable without altering its API."""
|
|
45
|
+
def decorate(func):
|
|
46
|
+
name = label or f'{func.__module__}.{func.__qualname__}'
|
|
47
|
+
if hasattr(func, '__wrapped_by_apmtools_trace__'):
|
|
48
|
+
return func
|
|
49
|
+
if __import__('inspect').iscoroutinefunction(func):
|
|
50
|
+
@functools.wraps(func)
|
|
51
|
+
async def wrapped(*args, **kwargs):
|
|
52
|
+
return await _run_async(func, name, kind, trigger, args, kwargs)
|
|
53
|
+
else:
|
|
54
|
+
@functools.wraps(func)
|
|
55
|
+
def wrapped(*args, **kwargs):
|
|
56
|
+
if not enabled():
|
|
57
|
+
return func(*args, **kwargs)
|
|
58
|
+
parent = _current_span.get()
|
|
59
|
+
span = uuid.uuid4().hex[:12]
|
|
60
|
+
token = _current_span.set(span)
|
|
61
|
+
start = time.perf_counter_ns()
|
|
62
|
+
error = None
|
|
63
|
+
try:
|
|
64
|
+
return func(*args, **kwargs)
|
|
65
|
+
except BaseException as exc:
|
|
66
|
+
error = type(exc).__name__
|
|
67
|
+
raise
|
|
68
|
+
finally:
|
|
69
|
+
duration = time.perf_counter_ns() - start
|
|
70
|
+
_current_span.reset(token)
|
|
71
|
+
_emit({'event': 'execution', 'kind': kind, 'name': name,
|
|
72
|
+
'trigger': trigger, 'span': span, 'parent': parent,
|
|
73
|
+
'duration_ms': round(duration / 1e6, 3), 'error': error})
|
|
74
|
+
wrapped.__wrapped_by_apmtools_trace__ = True
|
|
75
|
+
return wrapped
|
|
76
|
+
return decorate
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
async def _run_async(func, name, kind, trigger, args, kwargs):
|
|
80
|
+
if not enabled():
|
|
81
|
+
return await func(*args, **kwargs)
|
|
82
|
+
parent = _current_span.get()
|
|
83
|
+
span = uuid.uuid4().hex[:12]
|
|
84
|
+
token = _current_span.set(span)
|
|
85
|
+
start = time.perf_counter_ns()
|
|
86
|
+
error = None
|
|
87
|
+
try:
|
|
88
|
+
return await func(*args, **kwargs)
|
|
89
|
+
except BaseException as exc:
|
|
90
|
+
error = type(exc).__name__
|
|
91
|
+
raise
|
|
92
|
+
finally:
|
|
93
|
+
duration = time.perf_counter_ns() - start
|
|
94
|
+
_current_span.reset(token)
|
|
95
|
+
_emit({'event': 'execution', 'kind': kind, 'name': name,
|
|
96
|
+
'trigger': trigger, 'span': span, 'parent': parent,
|
|
97
|
+
'duration_ms': round(duration / 1e6, 3), 'error': error})
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def marker(name, **metadata):
|
|
101
|
+
"""Mark a user interaction or application lifecycle event (no raw data)."""
|
|
102
|
+
if enabled():
|
|
103
|
+
_emit({'event': 'marker', 'name': name, 'parent': _current_span.get(), **metadata})
|
|
@@ -51,3 +51,17 @@ def snapshot_inputs(current, input_values):
|
|
|
51
51
|
if key in SNAPSHOT_INPUT_KEYS and value is not None:
|
|
52
52
|
result[key] = list(value) if isinstance(value, (tuple, list)) else value
|
|
53
53
|
return result
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def restore_panel_selection(groups, current, saved, pending=False):
|
|
57
|
+
"""Reconcile facet choices without erasing a not-yet-mounted saved facet.
|
|
58
|
+
|
|
59
|
+
Returns (selection, pending). Once restored, user edits take precedence.
|
|
60
|
+
"""
|
|
61
|
+
groups = list(groups or [])
|
|
62
|
+
saved = list(saved or [])
|
|
63
|
+
if pending:
|
|
64
|
+
if not all(name in groups for name in saved):
|
|
65
|
+
return [], True
|
|
66
|
+
return [name for name in saved if name in groups], False
|
|
67
|
+
return [name for name in (current or []) if name in groups], False
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Headless preparation of new and restored Shiny tabs.
|
|
2
|
+
|
|
3
|
+
The UI still owns Shiny module creation, reactive counters and DOM mounting.
|
|
4
|
+
This module resolves the data and settings needed *before* mounting a tab.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import copy
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import Any
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
from .derived_views import prepare_table_source
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class PlotInitialization:
|
|
18
|
+
label: str
|
|
19
|
+
settings: dict
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class TableInitialization:
|
|
24
|
+
label: str
|
|
25
|
+
frame: pd.DataFrame
|
|
26
|
+
settings: dict | None
|
|
27
|
+
lineage: list
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class TabInitializer:
|
|
31
|
+
"""Resolve plot/table initialization without importing Shiny."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, workspace, views, summary):
|
|
34
|
+
self.workspace = workspace
|
|
35
|
+
self.views = views
|
|
36
|
+
self.summary = summary
|
|
37
|
+
|
|
38
|
+
def new_plot(self, selected, number):
|
|
39
|
+
# A partially initialized input can fail during a snapshot; preserve
|
|
40
|
+
# the existing 'start with defaults' behavior for this operation.
|
|
41
|
+
inherited = {}
|
|
42
|
+
if self.workspace.kind(selected) == 'plot':
|
|
43
|
+
try:
|
|
44
|
+
inherited = copy.deepcopy(self.workspace.plot_snapshots[selected]())
|
|
45
|
+
except Exception:
|
|
46
|
+
inherited = {}
|
|
47
|
+
return PlotInitialization(f'Plot {number}', inherited)
|
|
48
|
+
|
|
49
|
+
def restored_plot(self, preset_id, tab_id, number):
|
|
50
|
+
preset = self.views.preset(preset_id)
|
|
51
|
+
inherited = self.views.restore_plot(preset_id, tab_id)
|
|
52
|
+
return PlotInitialization(preset.get('name') or f'Plot {number}', inherited)
|
|
53
|
+
|
|
54
|
+
def new_table(self, selected, number):
|
|
55
|
+
kind = self.workspace.kind(selected)
|
|
56
|
+
if kind == 'table':
|
|
57
|
+
frame, steps = prepare_table_source(
|
|
58
|
+
self.summary, parent_kind='table',
|
|
59
|
+
parent_steps=self.workspace.table_lineage.get(selected, []),
|
|
60
|
+
parent_settings=self.workspace.table_settings[selected]())
|
|
61
|
+
elif kind == 'plot':
|
|
62
|
+
frame, steps = prepare_table_source(
|
|
63
|
+
self.summary, parent_kind='plot',
|
|
64
|
+
parent_settings=self.workspace.plot_snapshots[selected]())
|
|
65
|
+
else:
|
|
66
|
+
frame, steps = prepare_table_source(self.summary)
|
|
67
|
+
return TableInitialization(f'Table {number}', frame, None, steps)
|
|
68
|
+
|
|
69
|
+
def restored_table(self, preset_id, number):
|
|
70
|
+
preset = self.views.preset(preset_id)
|
|
71
|
+
_source, settings = self.views.restore_table(preset_id)
|
|
72
|
+
frame, steps = prepare_table_source(self.summary, restored_settings=settings)
|
|
73
|
+
return TableInitialization(preset.get('name') or f'Table {number}',
|
|
74
|
+
frame, settings, steps)
|