apmtools 0.2.57__tar.gz → 0.2.59__tar.gz

{apmtools-0.2.57 → apmtools-0.2.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: apmtools
-Version: 0.2.57
+Version: 0.2.59
 Summary: A collection of tools for processing air pollution monitoring data
 Project-URL: Homepage, https://github.com/federlorenz/apmtools
 Author-email: Federico Lorenzetti <lorenzetti.federico@gmail.com>

{apmtools-0.2.57 → apmtools-0.2.59}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apmtools"
-version = "0.2.57"
+version = "0.2.59"
 authors = [
   { name="Federico Lorenzetti", email="lorenzetti.federico@gmail.com" },
 ]

{apmtools-0.2.57 → apmtools-0.2.59}/src/apmtools/classes.py

@@ -9,6 +9,7 @@ from csv import writer
 from dateutil.parser import parse
 
 from .units import upas_units, lascar_units, purple_units, lcm_units
+from .completeness import analyse_completeness, complete_interval_mask
 
 class DictionaryPlus(dict):
     def __init__(self, *args, **kwargs):
@@ -1194,6 +1195,19 @@ class Summary(pd.DataFrame):
         if attribute in self.columns:
             return set(self[attribute])
 
+    def completeness_report(self, value_cols, frequency, *, time_col='datetime',
+                            id_col='identifier', min_completeness=1.0,
+                            include_empty_intervals=False):
+        """Report observed/expected samples by interval and measurement.
+
+        The result is an ordinary DataFrame with one row per identifier,
+        interval and variable. See :func:`apmtools.completeness.analyse_completeness`.
+        """
+        return analyse_completeness(
+            self, value_cols, frequency, time_col=time_col, id_col=id_col,
+            min_completeness=min_completeness,
+            include_empty_intervals=include_empty_intervals)
+
     def group_stat(self, value_cols, frequency=None, grouping_by=["identifier"], time_col='datetime', id_col='identifier',
                    min_completeness=1.0, only_complete=True, flooring_timestamp=True, aggregate=["mean"]):
         """
@@ -1234,174 +1248,11 @@ class Summary(pd.DataFrame):
         if frequency != None:
 
             if only_complete:
-                def median_interval(g):
-                    return g.sort_values().diff().dropna().median()
-
-                def sorting_frequency(x):
-                    if len(list(filter(lambda z: True if "week" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "week" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "weekday" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "weekday" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "day" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "day" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "hour" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "hour" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "minute" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "minute" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "second" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "second" in z else False, x))[0]
-
-                def freq_ranked():
-                    if isinstance(frequency, str):
-                        return frequency
-                    if isinstance(frequency, list):
-                        return sorting_frequency(frequency)
-
-                intervals = df.groupby(id_col)[time_col].apply(median_interval)
-                if isinstance(frequency, str):
-                    if len(frequency.split(" ")) == 1:
-                        match frequency:
-                            case "hour":
-                                expected = (pd.Timedelta(1, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("h")
-                            case "day":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("d")
-                            case "minute":
-                                expected = (pd.Timedelta(1, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("min")
-                            case "second":
-                                expected = (pd.Timedelta(1, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("s")
-                            case "week":
-                                expected = (pd.Timedelta(1, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("w")
-                            case "weekday":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday()))
-                    if len(frequency.split(" ")) == 2:
-                        mult = float(frequency.split(" ")[0])
-                        match frequency.split(" ")[1]:
-                            case "hour":
-                                expected = (pd.Timedelta(mult, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}h")
-                            case "day":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}d")
-                            case "minute":
-                                expected = (pd.Timedelta(mult, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}min")
-                            case "second":
-                                expected = (pd.Timedelta(mult, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}s")
-                            case "week":
-                                expected = (pd.Timedelta(mult, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}w")
-                            case "weekday":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday//mult))*mult
-
-                    counts = df.groupby([id_col, "xxxyyyxxx"])[
-                        value_cols].count()
-                    expe = counts.index.get_level_values(id_col).map(expected)
-                    completeness = counts.div(expe, axis=0)
-                    complete_hours = completeness[completeness >= min_completeness].dropna(
-                    ).index
-                    mask = pd.MultiIndex.from_arrays(
-                        [df[id_col], df["xxxyyyxxx"]]).isin(complete_hours)
-                    df = df.loc[mask]
-
-                if isinstance(frequency, list):
-                    freq = freq_ranked()
-                    if len(freq.split(" ")) == 1:
-                        match freq:
-                            case "hour":
-                                expected = (pd.Timedelta(1, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("h")
-                            case "day":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("d")
-                            case "minute":
-                                expected = (pd.Timedelta(1, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("min")
-                            case "second":
-                                expected = (pd.Timedelta(1, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("s")
-                            case "week":
-                                expected = (pd.Timedelta(1, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("w")
-                            case "weekday":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday()))
-                    if len(freq.split(" ")) == 2:
-                        mult = float(freq.split(" ")[0])
-                        match freq.split(" ")[1]:
-                            case "hour":
-                                expected = (pd.Timedelta(mult, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}h")
-                            case "day":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}d")
-                            case "minute":
-                                expected = (pd.Timedelta(mult, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}m")
-                            case "second":
-                                expected = (pd.Timedelta(mult, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}s")
-                            case "week":
-                                expected = (pd.Timedelta(mult, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}w")
-                            case "weekday":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday//mult))*mult
-
-                    counts = df.groupby([id_col, "xxxyyyxxx"])[
-                        value_cols].count()
-                    expe = counts.index.get_level_values(id_col).map(expected)
-                    completeness = counts.div(expe, axis=0)
-                    complete_hours = completeness[completeness >= min_completeness].dropna(
-                    ).index
-                    mask = pd.MultiIndex.from_arrays(
-                        [df[id_col], df["xxxyyyxxx"]]).isin(complete_hours)
-                    df = df.loc[mask]
+                # Stage 4A: one shared completeness implementation for analysis
+                # and aggregation. Retain existing all-value filtering semantics.
+                df = df.loc[complete_interval_mask(
+                    df, value_cols, frequency, time_col=time_col,
+                    id_col=id_col, min_completeness=min_completeness)].copy()
 
             if isinstance(frequency, str):
                 if len(frequency.split(" ")) == 1:

apmtools-0.2.59/src/apmtools/completeness.py

@@ -0,0 +1,148 @@
+"""Sampling completeness calculations shared by Summary and future quality views.
+
+Coverage is estimated from each identifier's median positive sampling gap.
+The first and last partially observed bins cannot be classified as complete
+without a known deployment schedule: results describe coverage inside bins
+within the observed recording span, not compliance with a planned deployment.
+"""
+from __future__ import annotations
+
+import re
+import pandas as pd
+
+_UNITS = {'second': 's', 'minute': 'min', 'hour': 'h', 'day': 'D', 'week': 'W'}
+_DURATION = {'second': pd.Timedelta(seconds=1),
+             'minute': pd.Timedelta(minutes=1),
+             'hour': pd.Timedelta(hours=1),
+             'day': pd.Timedelta(days=1),
+             'week': pd.Timedelta(weeks=1),
+             'weekday': pd.Timedelta(days=1)}
+
+
+def _primary_frequency(frequency):
+    if isinstance(frequency, (tuple, list)):
+        if not frequency:
+            raise ValueError('frequency cannot be empty')
+        # Match the historical group_stat ranking of coarsest first.
+        ranking = {'week': 0, 'weekday': 1, 'day': 2, 'hour': 3,
+                   'minute': 4, 'second': 5}
+        return min(frequency, key=lambda f: ranking[_parse_frequency(f)[1]])
+    return frequency
+
+
+def _parse_frequency(frequency):
+    if not isinstance(frequency, str):
+        raise ValueError('frequency must be a string or a sequence of strings')
+    match = re.fullmatch(r'(?:(\d+(?:\.\d+)?)\s+)?(second|minute|hour|day|week|weekday)', frequency.strip())
+    if not match:
+        raise ValueError(f'Unsupported completeness frequency: {frequency!r}')
+    count = float(match.group(1) or 1)
+    if not count.is_integer() or count <= 0:
+        raise ValueError('Completeness frequency multiplier must be a positive integer')
+    return int(count), match.group(2)
+
+
+def _interval_bins(timestamps, frequency):
+    multiplier, unit = _parse_frequency(frequency)
+    series = pd.to_datetime(timestamps)
+    if unit == 'weekday':
+        # Historical weekday grouping pools weekdays across calendar weeks.
+        return (series.dt.weekday // multiplier) * multiplier
+    if unit == 'week':
+        # Monday-anchored calendar weeks, including multiweek bins.
+        starts = series.dt.to_period('W-SUN').dt.start_time
+        if multiplier == 1:
+            return starts
+        epoch = pd.Timestamp('1970-01-05')
+        return epoch + ((starts - epoch) // pd.Timedelta(weeks=multiplier)) * pd.Timedelta(weeks=multiplier)
+    return series.dt.floor(f'{multiplier}{_UNITS[unit]}')
+
+
+def analyse_completeness(data, value_cols, frequency, *, time_col='datetime',
+                         id_col='identifier', min_completeness=1.0,
+                         include_empty_intervals=False):
+    """Return one row per (identifier, frequency bin, measurement).
+
+    Columns: identifier (or *id_col*), ``interval``, ``variable``,
+    ``observed``, ``expected``, ``missing``, ``completeness``, ``is_complete``,
+    ``sampling_interval``. ``observed`` counts non-null measurements, not
+    distinct timestamps; ``missing`` is clipped at zero. NaN expected counts
+    flag identifiers with insufficient timing information.
+
+    ``include_empty_intervals`` adds bins between each identifier's first and
+    last observed bin for ordinary temporal frequencies, except when a bin
+    is shorter than the identifier's sampling cadence (unscheduled bins are
+    not counted as missing); it does not infer
+    missing deployment time before the first or after the last observation.
+    """
+    if not 0 <= min_completeness <= 1:
+        raise ValueError('min_completeness must be between 0 and 1')
+    frequency = _primary_frequency(frequency)
+    multiplier, unit = _parse_frequency(frequency)
+    values = list(value_cols)
+    if not values:
+        raise ValueError('value_cols must not be empty')
+    required = [id_col, time_col, *values]
+    absent = [name for name in required if name not in data.columns]
+    if absent:
+        raise KeyError(f'Missing completeness columns: {absent}')
+    frame = pd.DataFrame(data)[required].copy()
+    frame[time_col] = pd.to_datetime(frame[time_col], errors='coerce')
+    frame = frame.dropna(subset=[id_col, time_col])
+    columns = [id_col, 'interval', 'variable', 'observed', 'expected',
+               'missing', 'completeness', 'is_complete', 'sampling_interval']
+    if frame.empty:
+        return pd.DataFrame(columns=columns)
+    frame['_interval'] = _interval_bins(frame[time_col], frequency)
+    # Nonpositive differences (duplicates) cannot describe a sampling cadence.
+    ordered = frame.sort_values([id_col, time_col])
+    diffs = ordered.groupby(id_col, sort=False)[time_col].diff()
+    cadence = (pd.DataFrame({id_col: ordered[id_col].to_numpy(),
+                             'gap': diffs.to_numpy()})
+               .loc[lambda x: x.gap > pd.Timedelta(0)]
+               .groupby(id_col)['gap'].median())
+    groups = frame.groupby([id_col, '_interval'], dropna=False, sort=True)[values].count()
+    if include_empty_intervals and unit != 'weekday':
+        duration = _DURATION[unit] * multiplier
+        new_parts = []
+        for identifier, portion in frame.groupby(id_col, sort=False):
+            first, last = portion['_interval'].min(), portion['_interval'].max()
+            # An interval shorter than the monitor cadence is not a scheduled
+            # observation interval. Do not manufacture missing sub-cadence bins.
+            monitor_cadence = cadence.get(identifier, pd.NaT)
+            if pd.notna(monitor_cadence) and duration < monitor_cadence:
+                bins = pd.DatetimeIndex(sorted(portion['_interval'].unique()))
+            else:
+                bins = pd.date_range(first, last, freq=duration)
+            new_parts.append(pd.MultiIndex.from_product([[identifier], bins], names=[id_col, '_interval']))
+        if new_parts:
+            all_bins = new_parts[0]
+            for additional in new_parts[1:]:
+                all_bins = all_bins.append(additional)
+            groups = groups.reindex(all_bins, fill_value=0)
+    long = (groups.rename_axis(index=[id_col, 'interval'])
+            .reset_index().melt(id_vars=[id_col, 'interval'],
+                                value_vars=values, var_name='variable',
+                                value_name='observed'))
+    long['sampling_interval'] = long[id_col].map(cadence)
+    expected = (_DURATION[unit] * multiplier) / long['sampling_interval']
+    expected = pd.to_numeric(expected, errors='coerce').round()
+    # A bin containing a recorded sample has one expected opportunity even
+    # when its width is smaller than the inferred sampling cadence.
+    # Such sub-cadence bins without records were excluded above.
+    long['expected'] = expected.clip(lower=1)
+    long['missing'] = (long['expected'] - long['observed']).clip(lower=0)
+    long['completeness'] = long['observed'] / long['expected']
+    long['is_complete'] = long['completeness'].ge(min_completeness) & long['expected'].notna()
+    return long[columns]
+
+
+def complete_interval_mask(data, value_cols, frequency, *, time_col='datetime',
+                           id_col='identifier', min_completeness=1.0):
+    """Row mask preserving legacy group_stat semantics (all values complete)."""
+    report = analyse_completeness(data, value_cols, frequency, time_col=time_col,
+                                  id_col=id_col, min_completeness=min_completeness)
+    complete = report.groupby([id_col, 'interval'])['is_complete'].all()
+    intervals = _interval_bins(pd.to_datetime(data[time_col]), _primary_frequency(frequency))
+    index = pd.MultiIndex.from_arrays([data[id_col], intervals], names=[id_col, 'interval'])
+    return pd.Series(index.isin(complete.index[complete]), index=data.index)

{apmtools-0.2.57 → apmtools-0.2.59}/src/apmtools/plot_configs.py

@@ -1,6 +1,5 @@
 """Plot configuration lifecycle: reactive drafts and explicitly retained presets."""
 from __future__ import annotations
-from .shiny.profiling import traced
 import copy
 import json
 import uuid
@@ -161,6 +160,31 @@ class PlotConfigManager:
         self.revision += 1
         return preset_id
 
+    def retain_quality(self, name, settings, *, preset_id=None):
+        """Retain a quality dashboard configuration, never computed report data."""
+        name = str(name).strip()
+        if not name:
+            raise ValueError('Configuration name must not be empty')
+        preset_id = preset_id or uuid.uuid4().hex
+        item = {'schema_version': 1, 'id': preset_id, 'name': name,
+                'type': 'quality', 'settings': _json_ready(settings)}
+        items = self.saved()
+        for index, previous in enumerate(items):
+            if previous.get('id') == preset_id:
+                items[index] = item
+                break
+        else:
+            items.append(item)
+        self.summary.views = items
+        self.revision += 1
+        return preset_id
+
+    def restore_quality(self, preset_id):
+        item = next((p for p in self.saved() if p.get('id') == preset_id), None)
+        if item is None or item.get('type') != 'quality':
+            raise KeyError(preset_id)
+        return copy.deepcopy(item['settings'])
+
     def restore_table(self, preset_id):
         item = next((p for p in self.saved() if p.get('id') == preset_id), None)
         if item is None or item.get('type') != 'table':
@@ -172,7 +196,7 @@ class PlotConfigManager:
     def delete_saved(self, preset_id):
         """Delete one explicitly saved plot by stable ID, not its open draft."""
         items = copy.deepcopy(self.summary.views or [])
-        remaining = [p for p in items if not (p.get('id') == preset_id and p.get('type') in ('plot', 'table'))]
+        remaining = [p for p in items if not (p.get('id') == preset_id and p.get('type') in ('plot', 'table', 'quality'))]
         if len(remaining) == len(items):
             raise KeyError(f'Unknown saved plot: {preset_id}')
         self.summary.views = remaining

apmtools-0.2.59/src/apmtools/shiny/quality_filters.py

@@ -0,0 +1,76 @@
+"""Reusable, Shiny-independent subset filtering for quality dashboards."""
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from datetime import datetime, time, timedelta
+from .data import _parse_datetime_filter, _subset_choices
+
+
+def filter_quality_data(frame, settings):
+    """Apply the same staged categorical, numeric and date filters as tables.
+
+    The configured selected values are Shiny selector *keys*, resolved against
+    the source dataframe. Missing numeric values remain unchanged, consistent
+    with the existing table filtering policy.
+    """
+    current = frame.copy()
+    for step in settings.get('subset_chain', []):
+        column = step.get('column')
+        if column not in current.columns:
+            continue
+        _, value_map = _subset_choices(frame, column)
+        values = [value_map[key] for key in step.get('values', []) if key in value_map]
+        if not values:
+            continue
+        style = step.get('style', 'all')
+        if hasattr(current, 'subset'):
+            current = current.subset({column: values}, filter_style=style)
+        else:
+            matches = current[column].isin([v for v in values if v is not None])
+            if any(v is None for v in values):
+                matches |= current[column].isna()
+            if style in ('negative all', 'negative any'):
+                matches = ~matches
+            current = current.loc[matches].copy()
+    # Derived plot/table frames are often ordinary pandas DataFrames (without
+    # Summary.dtfilter). Filter their actual datetime column without assuming
+    # that the source column is literally named "datetime".
+    time_col = settings.get('time_col')
+    if time_col not in current.columns or not pd.api.types.is_datetime64_any_dtype(current[time_col]):
+        time_col = next((col for col in current.columns
+                         if pd.api.types.is_datetime64_any_dtype(current[col])), None)
+    if time_col is not None:
+        timestamps = pd.to_datetime(current[time_col], errors='coerce')
+        mask = pd.Series(True, index=current.index)
+        for field, lower in (('datetime_start', True), ('datetime_end', False)):
+            kind, value = _parse_datetime_filter(settings.get(field))
+            if kind == 'date':
+                bound = datetime.combine(value, time.min)
+                if lower:
+                    mask &= timestamps.ge(bound)
+                else:
+                    mask &= timestamps.lt(bound + timedelta(days=1))
+            elif kind == 'datetime':
+                mask &= timestamps.ge(value) if lower else timestamps.lt(value)
+            elif kind == 'time':
+                times = timestamps.dt.time
+                mask &= times.ge(value) if lower else times.lt(value)
+        current = current.loc[mask.fillna(False)].copy()
+    for column in settings.get('numeric_subset_columns', []):
+        if column not in current.columns or not pd.api.types.is_numeric_dtype(current[column]):
+            continue
+        values = pd.to_numeric(current[column], errors='coerce')
+        for comparison in ('gt', 'lt'):
+            raw = settings.get('numeric_subset_bounds', {}).get(column, {}).get(comparison, '')
+            try:
+                bound = float(str(raw).strip())
+            except (ValueError, TypeError):
+                continue
+            if not np.isfinite(bound):
+                continue
+            mask = values.gt(bound) if comparison == 'gt' else values.lt(bound)
+            current = current.loc[mask.fillna(False) | values.isna()].copy()
+            values = values.loc[current.index]
+    return current

apmtools-0.2.59/src/apmtools/shiny/quality_view.py

@@ -0,0 +1,118 @@
+"""Headless data-quality summaries and heatmap preparation.
+
+The dashboard never infers completeness independently of the Stage 4A engine.
+"""
+from __future__ import annotations
+
+import pandas as pd
+from ..completeness import analyse_completeness
+
+QUALITY_FREQUENCIES = ('minute', 'hour', 'day', 'week')
+
+
+def quality_report(data, settings):
+    """Recompute per-identifier, interval and variable coverage from source data."""
+    values = list(settings.get('value_cols', []))
+    if not values:
+        raise ValueError('Select at least one measurement variable')
+    freq = settings.get('frequency', 'hour')
+    if freq not in QUALITY_FREQUENCIES:
+        raise ValueError(f'Unsupported dashboard frequency: {freq}')
+    return analyse_completeness(
+        data, values, freq,
+        id_col=settings.get('id_col', 'identifier'),
+        time_col=settings.get('time_col', 'datetime'),
+        min_completeness=float(settings.get('min_completeness', 1.0)),
+        include_empty_intervals=bool(settings.get('include_empty_intervals', True)),
+    )
+
+
+def quality_summary(report):
+    """Per-monitor and variable coverage, including zero-observation bins."""
+    names = ['identifier', 'variable', 'intervals', 'complete_intervals',
+             'observed', 'expected', 'missing', 'coverage_percent',
+             'complete_percent']
+    if report.empty:
+        return pd.DataFrame(columns=names)
+    key = report.columns[0]
+    stats = (report.groupby([key, 'variable'], dropna=False, sort=True)
+             .agg(intervals=('is_complete', 'size'),
+                  complete_intervals=('is_complete', 'sum'),
+                  observed=('observed', 'sum'),
+                  expected=('expected', 'sum'),
+                  missing=('missing', 'sum')).reset_index())
+    stats['coverage_percent'] = (100 * stats['observed'] / stats['expected']).where(stats['expected'] > 0)
+    stats['complete_percent'] = 100 * stats['complete_intervals'] / stats['intervals']
+    return stats.rename(columns={key: 'identifier'})[names]
+
+
+def selected_heatmap_variable(value_cols, selected):
+    """Return a heatmap variable present in the coverage report.
+
+    Changing the selected measurements cannot leave the heatmap looking for a
+    numeric column that was not included in the report.
+    """
+    choices = list(value_cols or [])
+    return selected if selected in choices else (choices[0] if choices else None)
+
+
+def quality_heatmap_data(report, variable, *, mode='coverage'):
+    """Return interval-by-identifier heatmap values on a 0–1 scale.
+
+    ``mode='coverage'`` shows the measured observed/expected fraction, without
+    reference to the completeness threshold. ``mode='threshold'`` is the
+    historical binary classification (1=complete, 0=incomplete).
+    ``mode='threshold_coverage'`` shows 1 for bins meeting the threshold, and
+    the *actual* fraction for bins failing it. This retains the continuous
+    shades for incomplete bins without losing the threshold classification.
+    Undefined expected counts remain NaN in every mode.
+    """
+    if mode not in ('coverage', 'threshold', 'threshold_coverage'):
+        raise ValueError("heatmap mode must be 'coverage', 'threshold', or 'threshold_coverage'")
+    if report.empty:
+        return pd.DataFrame()
+    key = report.columns[0]
+    portion = report.loc[report['variable'].eq(variable)].copy()
+    if portion.empty:
+        return pd.DataFrame()
+    if mode == 'threshold':
+        portion['_heatmap_value'] = portion['is_complete'].astype(float).where(
+            portion['expected'].notna())
+    elif mode == 'threshold_coverage':
+        portion['_heatmap_value'] = portion['completeness'].where(
+            ~portion['is_complete'].astype(bool), 1.0)
+        portion['_heatmap_value'] = portion['_heatmap_value'].where(
+            portion['expected'].notna())
+    else:
+        portion['_heatmap_value'] = portion['completeness']
+    return portion.pivot_table(index=key, columns='interval',
+                               values='_heatmap_value', aggfunc='mean', dropna=False)
+
+
+def missing_interval_runs(report):
+    """Group consecutive zero-observation bins into gap episodes per monitor/variable.
+
+    A new episode begins whenever an observed bin separates missing bins.
+    The report must include empty intervals to expose fully absent time bins.
+    """
+    fields = ['identifier', 'variable', 'first_missing_interval', 'last_missing_interval',
+              'missing_intervals', 'expected_observations']
+    if report.empty:
+        return pd.DataFrame(columns=fields)
+    key = report.columns[0]
+    rows = []
+    for (identifier, variable), group in report.groupby([key, 'variable'], sort=False, dropna=False):
+        ordered = group.sort_values('interval')
+        run = []
+        def finish():
+            if run:
+                rows.append((identifier, variable, run[0]['interval'], run[-1]['interval'],
+                             len(run), sum(record['expected'] for record in run)))
+        for record in ordered.to_dict('records'):
+            if record['observed'] == 0:
+                run.append(record)
+            else:
+                finish()
+                run = []
+        finish()
+    return pd.DataFrame(rows, columns=fields)

{apmtools-0.2.57 → apmtools-0.2.59}/src/apmtools/shiny/tab_initialization.py

@@ -66,6 +66,26 @@ class TabInitializer:
             frame, steps = prepare_table_source(self.summary)
         return TableInitialization(f'Table {number}', frame, None, steps)
 
+    def new_quality(self, selected, number):
+        """Create a quality view from the current plot/table's transformed data.
+
+        The stored lineage replays the source transformation on restoration;
+        no potentially stale dataframe snapshots are retained in saved views.
+        """
+        kind = self.workspace.kind(selected)
+        if kind == 'table':
+            frame, steps = prepare_table_source(
+                self.summary, parent_kind='table',
+                parent_steps=self.workspace.table_lineage.get(selected, []),
+                parent_settings=self.workspace.table_settings[selected]())
+        elif kind == 'plot':
+            frame, steps = prepare_table_source(
+                self.summary, parent_kind='plot',
+                parent_settings=self.workspace.plot_snapshots[selected]())
+        else:
+            frame, steps = prepare_table_source(self.summary)
+        return TableInitialization(f'Quality {number}', frame, {'source_steps': steps}, steps)
+
     def restored_table(self, preset_id, number):
         preset = self.views.preset(preset_id)
         _source, settings = self.views.restore_table(preset_id)