apmtools 0.2.56__tar.gz → 0.2.58__tar.gz

{apmtools-0.2.56 → apmtools-0.2.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: apmtools
-Version: 0.2.56
+Version: 0.2.58
 Summary: A collection of tools for processing air pollution monitoring data
 Project-URL: Homepage, https://github.com/federlorenz/apmtools
 Author-email: Federico Lorenzetti <lorenzetti.federico@gmail.com>

{apmtools-0.2.56 → apmtools-0.2.58}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apmtools"
-version = "0.2.56"
+version = "0.2.58"
 authors = [
   { name="Federico Lorenzetti", email="lorenzetti.federico@gmail.com" },
 ]

{apmtools-0.2.56 → apmtools-0.2.58}/src/apmtools/classes.py

@@ -9,6 +9,7 @@ from csv import writer
 from dateutil.parser import parse
 
 from .units import upas_units, lascar_units, purple_units, lcm_units
+from .completeness import analyse_completeness, complete_interval_mask
 
 class DictionaryPlus(dict):
     def __init__(self, *args, **kwargs):
@@ -1194,6 +1195,19 @@ class Summary(pd.DataFrame):
         if attribute in self.columns:
             return set(self[attribute])
 
+    def completeness_report(self, value_cols, frequency, *, time_col='datetime',
+                            id_col='identifier', min_completeness=1.0,
+                            include_empty_intervals=False):
+        """Report observed/expected samples by interval and measurement.
+
+        The result is an ordinary DataFrame with one row per identifier,
+        interval and variable. See :func:`apmtools.completeness.analyse_completeness`.
+        """
+        return analyse_completeness(
+            self, value_cols, frequency, time_col=time_col, id_col=id_col,
+            min_completeness=min_completeness,
+            include_empty_intervals=include_empty_intervals)
+
     def group_stat(self, value_cols, frequency=None, grouping_by=["identifier"], time_col='datetime', id_col='identifier',
                    min_completeness=1.0, only_complete=True, flooring_timestamp=True, aggregate=["mean"]):
         """
@@ -1224,179 +1238,21 @@ class Summary(pd.DataFrame):
                     print(f"{v} not in  Summary columns")
                     return None
 
-        df = self.dropna(subset=[time_col]).copy()
+        # Non-temporal grouping also works on previously aggregated frames.
+        # Only operations involving a frequency need the source time column.
+        if frequency is not None and time_col not in self.columns:
+            raise KeyError(f"Temporal aggregation requires column {time_col!r}")
+        df = (self.dropna(subset=[time_col]).copy()
+              if frequency is not None else self.copy())
 
         if frequency != None:
 
             if only_complete:
-                def median_interval(g):
-                    return g.sort_values().diff().dropna().median()
-
-                def sorting_frequency(x):
-                    if len(list(filter(lambda z: True if "week" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "week" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "weekday" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "weekday" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "day" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "day" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "hour" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "hour" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "minute" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "minute" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "second" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "second" in z else False, x))[0]
-
-                def freq_ranked():
-                    if isinstance(frequency, str):
-                        return frequency
-                    if isinstance(frequency, list):
-                        return sorting_frequency(frequency)
-
-                intervals = df.groupby(id_col)[time_col].apply(median_interval)
-                if isinstance(frequency, str):
-                    if len(frequency.split(" ")) == 1:
-                        match frequency:
-                            case "hour":
-                                expected = (pd.Timedelta(1, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("h")
-                            case "day":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("d")
-                            case "minute":
-                                expected = (pd.Timedelta(1, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("min")
-                            case "second":
-                                expected = (pd.Timedelta(1, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("s")
-                            case "week":
-                                expected = (pd.Timedelta(1, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("w")
-                            case "weekday":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday()))
-                    if len(frequency.split(" ")) == 2:
-                        mult = float(frequency.split(" ")[0])
-                        match frequency.split(" ")[1]:
-                            case "hour":
-                                expected = (pd.Timedelta(mult, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}h")
-                            case "day":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}d")
-                            case "minute":
-                                expected = (pd.Timedelta(mult, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}min")
-                            case "second":
-                                expected = (pd.Timedelta(mult, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}s")
-                            case "week":
-                                expected = (pd.Timedelta(mult, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}w")
-                            case "weekday":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday//mult))*mult
-
-                    counts = df.groupby([id_col, "xxxyyyxxx"])[
-                        value_cols].count()
-                    expe = counts.index.get_level_values(id_col).map(expected)
-                    completeness = counts.div(expe, axis=0)
-                    complete_hours = completeness[completeness >= min_completeness].dropna(
-                    ).index
-                    mask = pd.MultiIndex.from_arrays(
-                        [df[id_col], df["xxxyyyxxx"]]).isin(complete_hours)
-                    df = df.loc[mask]
-
-                if isinstance(frequency, list):
-                    freq = freq_ranked()
-                    if len(freq.split(" ")) == 1:
-                        match freq:
-                            case "hour":
-                                expected = (pd.Timedelta(1, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("h")
-                            case "day":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("d")
-                            case "minute":
-                                expected = (pd.Timedelta(1, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("min")
-                            case "second":
-                                expected = (pd.Timedelta(1, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("s")
-                            case "week":
-                                expected = (pd.Timedelta(1, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("w")
-                            case "weekday":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday()))
-                    if len(freq.split(" ")) == 2:
-                        mult = float(freq.split(" ")[0])
-                        match freq.split(" ")[1]:
-                            case "hour":
-                                expected = (pd.Timedelta(mult, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}h")
-                            case "day":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}d")
-                            case "minute":
-                                expected = (pd.Timedelta(mult, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}m")
-                            case "second":
-                                expected = (pd.Timedelta(mult, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}s")
-                            case "week":
-                                expected = (pd.Timedelta(mult, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}w")
-                            case "weekday":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday//mult))*mult
-
-                    counts = df.groupby([id_col, "xxxyyyxxx"])[
-                        value_cols].count()
-                    expe = counts.index.get_level_values(id_col).map(expected)
-                    completeness = counts.div(expe, axis=0)
-                    complete_hours = completeness[completeness >= min_completeness].dropna(
-                    ).index
-                    mask = pd.MultiIndex.from_arrays(
-                        [df[id_col], df["xxxyyyxxx"]]).isin(complete_hours)
-                    df = df.loc[mask]
+                # Stage 4A: one shared completeness implementation for analysis
+                # and aggregation. Retain existing all-value filtering semantics.
+                df = df.loc[complete_interval_mask(
+                    df, value_cols, frequency, time_col=time_col,
+                    id_col=id_col, min_completeness=min_completeness)].copy()
 
             if isinstance(frequency, str):
                 if len(frequency.split(" ")) == 1:
@@ -1555,38 +1411,20 @@ class Summary(pd.DataFrame):
                                 df[freq] = df["datetime"].map(
                                     lambda x: (x.weekday//mult))*mult
 
-        if isinstance(frequency, str):
-            if grouping_by != None:
-                output = df.groupby(grouping_by+[frequency], dropna=False)
-            else:
-                output = df.groupby([frequency], dropna=False)
-        if isinstance(frequency, list):
-            if grouping_by != None:
-                output = df.groupby(grouping_by+frequency, dropna=False)
-            else:
-                output = df.groupby(frequency, dropna=False)
-        if frequency == None:
-            if grouping_by != None:
-                output = df.groupby(grouping_by, dropna=False)
-            else:
-                output = df
+        # A frequency may also be an explicitly selected grouping variable.
+        # pandas allows repeated group keys but reset_index() then attempts to
+        # insert the same column twice (e.g. "cannot insert hour, already exists").
+        # Preserve the requested order while grouping each key only once.
+        grouping_keys = list(dict.fromkeys(
+            list(grouping_by or []) +
+            ([frequency] if isinstance(frequency, str) else
+             list(frequency) if isinstance(frequency, list) else [])
+        ))
+        if frequency is not None or grouping_by is not None:
+            output = df.groupby(grouping_keys, dropna=False) if aggregate is not None else df[grouping_keys]
+        else:
+            output = df
 
-        if isinstance(frequency, str) and aggregate is None:
-            if grouping_by != None:
-                output = df[grouping_by+[frequency]]
-            else:
-                output = df[[frequency]]
-        if isinstance(frequency, list) and aggregate is None:
-            if grouping_by != None:
-                output = df[grouping_by+frequency]
-            else:
-                output = df[frequency]
-        if frequency == None and aggregate is None:
-            if grouping_by != None:
-                output = df[grouping_by]
-            else:
-                output = df
-                
         if aggregate is not None:
             output = output[value_cols].agg(aggregate)
 

apmtools-0.2.58/src/apmtools/completeness.py

@@ -0,0 +1,148 @@
+"""Sampling completeness calculations shared by Summary and future quality views.
+
+Coverage is estimated from each identifier's median positive sampling gap.
+The first and last partially observed bins cannot be classified as complete
+without a known deployment schedule: results describe coverage inside bins
+within the observed recording span, not compliance with a planned deployment.
+"""
+from __future__ import annotations
+
+import re
+import pandas as pd
+
+_UNITS = {'second': 's', 'minute': 'min', 'hour': 'h', 'day': 'D', 'week': 'W'}
+_DURATION = {'second': pd.Timedelta(seconds=1),
+             'minute': pd.Timedelta(minutes=1),
+             'hour': pd.Timedelta(hours=1),
+             'day': pd.Timedelta(days=1),
+             'week': pd.Timedelta(weeks=1),
+             'weekday': pd.Timedelta(days=1)}
+
+
+def _primary_frequency(frequency):
+    if isinstance(frequency, (tuple, list)):
+        if not frequency:
+            raise ValueError('frequency cannot be empty')
+        # Match the historical group_stat ranking of coarsest first.
+        ranking = {'week': 0, 'weekday': 1, 'day': 2, 'hour': 3,
+                   'minute': 4, 'second': 5}
+        return min(frequency, key=lambda f: ranking[_parse_frequency(f)[1]])
+    return frequency
+
+
+def _parse_frequency(frequency):
+    if not isinstance(frequency, str):
+        raise ValueError('frequency must be a string or a sequence of strings')
+    match = re.fullmatch(r'(?:(\d+(?:\.\d+)?)\s+)?(second|minute|hour|day|week|weekday)', frequency.strip())
+    if not match:
+        raise ValueError(f'Unsupported completeness frequency: {frequency!r}')
+    count = float(match.group(1) or 1)
+    if not count.is_integer() or count <= 0:
+        raise ValueError('Completeness frequency multiplier must be a positive integer')
+    return int(count), match.group(2)
+
+
+def _interval_bins(timestamps, frequency):
+    multiplier, unit = _parse_frequency(frequency)
+    series = pd.to_datetime(timestamps)
+    if unit == 'weekday':
+        # Historical weekday grouping pools weekdays across calendar weeks.
+        return (series.dt.weekday // multiplier) * multiplier
+    if unit == 'week':
+        # Monday-anchored calendar weeks, including multiweek bins.
+        starts = series.dt.to_period('W-SUN').dt.start_time
+        if multiplier == 1:
+            return starts
+        epoch = pd.Timestamp('1970-01-05')
+        return epoch + ((starts - epoch) // pd.Timedelta(weeks=multiplier)) * pd.Timedelta(weeks=multiplier)
+    return series.dt.floor(f'{multiplier}{_UNITS[unit]}')
+
+
+def analyse_completeness(data, value_cols, frequency, *, time_col='datetime',
+                         id_col='identifier', min_completeness=1.0,
+                         include_empty_intervals=False):
+    """Return one row per (identifier, frequency bin, measurement).
+
+    Columns: identifier (or *id_col*), ``interval``, ``variable``,
+    ``observed``, ``expected``, ``missing``, ``completeness``, ``is_complete``,
+    ``sampling_interval``. ``observed`` counts non-null measurements, not
+    distinct timestamps; ``missing`` is clipped at zero. NaN expected counts
+    flag identifiers with insufficient timing information.
+
+    ``include_empty_intervals`` adds bins between each identifier's first and
+    last observed bin for ordinary temporal frequencies, except when a bin
+    is shorter than the identifier's sampling cadence (unscheduled bins are
+    not counted as missing); it does not infer
+    missing deployment time before the first or after the last observation.
+    """
+    if not 0 <= min_completeness <= 1:
+        raise ValueError('min_completeness must be between 0 and 1')
+    frequency = _primary_frequency(frequency)
+    multiplier, unit = _parse_frequency(frequency)
+    values = list(value_cols)
+    if not values:
+        raise ValueError('value_cols must not be empty')
+    required = [id_col, time_col, *values]
+    absent = [name for name in required if name not in data.columns]
+    if absent:
+        raise KeyError(f'Missing completeness columns: {absent}')
+    frame = pd.DataFrame(data)[required].copy()
+    frame[time_col] = pd.to_datetime(frame[time_col], errors='coerce')
+    frame = frame.dropna(subset=[id_col, time_col])
+    columns = [id_col, 'interval', 'variable', 'observed', 'expected',
+               'missing', 'completeness', 'is_complete', 'sampling_interval']
+    if frame.empty:
+        return pd.DataFrame(columns=columns)
+    frame['_interval'] = _interval_bins(frame[time_col], frequency)
+    # Nonpositive differences (duplicates) cannot describe a sampling cadence.
+    ordered = frame.sort_values([id_col, time_col])
+    diffs = ordered.groupby(id_col, sort=False)[time_col].diff()
+    cadence = (pd.DataFrame({id_col: ordered[id_col].to_numpy(),
+                             'gap': diffs.to_numpy()})
+               .loc[lambda x: x.gap > pd.Timedelta(0)]
+               .groupby(id_col)['gap'].median())
+    groups = frame.groupby([id_col, '_interval'], dropna=False, sort=True)[values].count()
+    if include_empty_intervals and unit != 'weekday':
+        duration = _DURATION[unit] * multiplier
+        new_parts = []
+        for identifier, portion in frame.groupby(id_col, sort=False):
+            first, last = portion['_interval'].min(), portion['_interval'].max()
+            # An interval shorter than the monitor cadence is not a scheduled
+            # observation interval. Do not manufacture missing sub-cadence bins.
+            monitor_cadence = cadence.get(identifier, pd.NaT)
+            if pd.notna(monitor_cadence) and duration < monitor_cadence:
+                bins = pd.DatetimeIndex(sorted(portion['_interval'].unique()))
+            else:
+                bins = pd.date_range(first, last, freq=duration)
+            new_parts.append(pd.MultiIndex.from_product([[identifier], bins], names=[id_col, '_interval']))
+        if new_parts:
+            all_bins = new_parts[0]
+            for additional in new_parts[1:]:
+                all_bins = all_bins.append(additional)
+            groups = groups.reindex(all_bins, fill_value=0)
+    long = (groups.rename_axis(index=[id_col, 'interval'])
+            .reset_index().melt(id_vars=[id_col, 'interval'],
+                                value_vars=values, var_name='variable',
+                                value_name='observed'))
+    long['sampling_interval'] = long[id_col].map(cadence)
+    expected = (_DURATION[unit] * multiplier) / long['sampling_interval']
+    expected = pd.to_numeric(expected, errors='coerce').round()
+    # A bin containing a recorded sample has one expected opportunity even
+    # when its width is smaller than the inferred sampling cadence.
+    # Such sub-cadence bins without records were excluded above.
+    long['expected'] = expected.clip(lower=1)
+    long['missing'] = (long['expected'] - long['observed']).clip(lower=0)
+    long['completeness'] = long['observed'] / long['expected']
+    long['is_complete'] = long['completeness'].ge(min_completeness) & long['expected'].notna()
+    return long[columns]
+
+
+def complete_interval_mask(data, value_cols, frequency, *, time_col='datetime',
+                           id_col='identifier', min_completeness=1.0):
+    """Row mask preserving legacy group_stat semantics (all values complete)."""
+    report = analyse_completeness(data, value_cols, frequency, time_col=time_col,
+                                  id_col=id_col, min_completeness=min_completeness)
+    complete = report.groupby([id_col, 'interval'])['is_complete'].all()
+    intervals = _interval_bins(pd.to_datetime(data[time_col]), _primary_frequency(frequency))
+    index = pd.MultiIndex.from_arrays([data[id_col], intervals], names=[id_col, 'interval'])
+    return pd.Series(index.isin(complete.index[complete]), index=data.index)

{apmtools-0.2.56 → apmtools-0.2.58}/src/apmtools/plot_configs.py

@@ -12,27 +12,10 @@ def _json_ready(value):
 
 
 def validate_saved_plots(plots):
-    """Validate the structure of retained presets without altering their values.
+    """Backward-compatible name for strict saved-view structural validation."""
+    from .view_schema import validate_saved_views
+    return validate_saved_views(plots)
 
-    Old snapshots remain supported: unknown settings are deliberately retained.
-    """
-    if plots is None:
-        return []
-    if not isinstance(plots, list):
-        raise TypeError("Summary.views must be a list or None")
-    seen = set()
-    for item in plots:
-        if not isinstance(item, dict):
-            raise ValueError("Saved plot entries must be dictionaries")
-        if item.get("schema_version") != 1 or item.get("type") not in ("plot", "table"):
-            raise ValueError("Unsupported saved plot configuration")
-        if not isinstance(item.get("settings"), dict):
-            raise ValueError("Saved plot settings must be a dictionary")
-        identifier = item.get("id")
-        if not isinstance(identifier, str) or not identifier or identifier in seen:
-            raise ValueError("Saved plot IDs must be unique nonempty strings")
-        seen.add(identifier)
-    return copy.deepcopy(plots)
 
 class PlotConfigManager:
     """Keep live tab drafts separate from explicitly retained Summary.plots.
@@ -177,6 +160,31 @@ class PlotConfigManager:
         self.revision += 1
         return preset_id
 
+    def retain_quality(self, name, settings, *, preset_id=None):
+        """Retain a quality dashboard configuration, never computed report data."""
+        name = str(name).strip()
+        if not name:
+            raise ValueError('Configuration name must not be empty')
+        preset_id = preset_id or uuid.uuid4().hex
+        item = {'schema_version': 1, 'id': preset_id, 'name': name,
+                'type': 'quality', 'settings': _json_ready(settings)}
+        items = self.saved()
+        for index, previous in enumerate(items):
+            if previous.get('id') == preset_id:
+                items[index] = item
+                break
+        else:
+            items.append(item)
+        self.summary.views = items
+        self.revision += 1
+        return preset_id
+
+    def restore_quality(self, preset_id):
+        item = next((p for p in self.saved() if p.get('id') == preset_id), None)
+        if item is None or item.get('type') != 'quality':
+            raise KeyError(preset_id)
+        return copy.deepcopy(item['settings'])
+
     def restore_table(self, preset_id):
         item = next((p for p in self.saved() if p.get('id') == preset_id), None)
         if item is None or item.get('type') != 'table':
@@ -188,18 +196,15 @@ class PlotConfigManager:
     def delete_saved(self, preset_id):
         """Delete one explicitly saved plot by stable ID, not its open draft."""
         items = copy.deepcopy(self.summary.views or [])
-        remaining = [p for p in items if not (p.get('id') == preset_id and p.get('type') in ('plot', 'table'))]
+        remaining = [p for p in items if not (p.get('id') == preset_id and p.get('type') in ('plot', 'table', 'quality'))]
         if len(remaining) == len(items):
             raise KeyError(f'Unknown saved plot: {preset_id}')
         self.summary.views = remaining
         self.revision += 1
 
     def saved(self):
-        items = validate_saved_plots(self.summary.views)
-        for item in items:
-            if item.get('type') == 'table':
-                item.pop('frame', None)  # migrate legacy snapshot records
-        return items
+        from .view_schema import migrate_saved_views
+        return migrate_saved_views(self.summary.views)
 
     def restore(self, preset_id, tab_id):
         item = next((p for p in self.saved() if p.get('id') == preset_id), None)

{apmtools-0.2.56 → apmtools-0.2.58}/src/apmtools/shiny/data.py

@@ -1,5 +1,6 @@
 """Filtering, aggregation and temporal segmentation."""
 from __future__ import annotations
+from .profiling import traced
 import threading
 from datetime import date, datetime, time, timedelta
 from collections.abc import Sequence
@@ -82,6 +83,7 @@ def _parse_datetime_filter(value):
 
 
 
+@traced('_apply_filters', kind="data" if 'shiny/data.py' not in ("plot_configs.py", "shiny/matplotlib_renderer.py") else "service")
 def _apply_filters(data, subset_filters, datetime_start=None, datetime_end=None):
     """Apply Summary filters, accepting dates, times and full datetimes.
 
@@ -123,6 +125,7 @@ def _apply_filters(data, subset_filters, datetime_start=None, datetime_end=None)
 
 
 
+@traced('_aggregate', kind="data" if 'shiny/data.py' not in ("plot_configs.py", "shiny/matplotlib_renderer.py") else "service")
 def _aggregate(data, value_columns, frequency, grouping_by, aggregate, *,
                only_complete=True, flooring_timestamp=True):
     """Aggregate through ``Summary.group_stat``.
@@ -138,13 +141,10 @@ def _aggregate(data, value_columns, frequency, grouping_by, aggregate, *,
         # Return empty rather than breaking Shiny's reactive graph.
         return pd.DataFrame()
 
-    result = data.group_stat(
-        value_cols=value_columns,
-        frequency=frequency,
-        grouping_by=grouping_by,
-        aggregate=[aggregate],
-        **({"only_complete": only_complete, "flooring_timestamp": flooring_timestamp}
-           if frequency is not None else {}),
+    from .view_aggregation import aggregate_view
+    result = aggregate_view(
+        data, value_columns, frequency, grouping_by, aggregate,
+        only_complete=only_complete, flooring_timestamp=flooring_timestamp,
     )
     if result is None:
         # group_stat() may return None when no rows survive completeness

apmtools-0.2.58/src/apmtools/shiny/derived_views.py

@@ -0,0 +1,35 @@
+"""Headless, configuration-only table source preparation for Shiny tab actions.
+
+The application creates tables from the owning Summary and ordered view
+settings, never from snapshots of already rendered plot/table data.
+"""
+from __future__ import annotations
+
+import pandas as pd
+
+from .inherited_source import source_frame, source_steps
+
+
+def prepare_table_source(summary, *, parent_kind=None, parent_settings=None,
+                         parent_steps=None, restored_settings=None):
+    """Return (display-ready frame, lineage steps) for new/loaded tables.
+
+    ``parent_kind`` is ``'plot'``, ``'table'`` or ``None`` (Info tab).
+    For restoration, pass ``restored_settings`` instead; its saved lineage is
+    replayed without modification.  No parent DataFrame is copied or cached.
+    """
+    if restored_settings is not None:
+        steps = restored_settings.get('source_steps', [])
+    elif parent_kind in ('plot', 'table'):
+        if parent_settings is None:
+            raise ValueError('Parent view settings must be supplied')
+        steps = source_steps(parent_steps if parent_kind == 'table' else [],
+                             parent_settings, parent_kind)
+    elif parent_kind is None:
+        steps = []
+    else:
+        raise ValueError(f'Unknown parent view kind: {parent_kind!r}')
+    frame = source_frame(summary, steps)
+    frame = frame.reset_index(drop=True)
+    frame.columns = [str(column) for column in frame.columns]
+    return frame, steps