apmtools 0.2.57__tar.gz → 0.2.58__tar.gz

{apmtools-0.2.57 → apmtools-0.2.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: apmtools
-Version: 0.2.57
+Version: 0.2.58
 Summary: A collection of tools for processing air pollution monitoring data
 Project-URL: Homepage, https://github.com/federlorenz/apmtools
 Author-email: Federico Lorenzetti <lorenzetti.federico@gmail.com>

{apmtools-0.2.57 → apmtools-0.2.58}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apmtools"
-version = "0.2.57"
+version = "0.2.58"
 authors = [
   { name="Federico Lorenzetti", email="lorenzetti.federico@gmail.com" },
 ]

{apmtools-0.2.57 → apmtools-0.2.58}/src/apmtools/classes.py

@@ -9,6 +9,7 @@ from csv import writer
 from dateutil.parser import parse
 
 from .units import upas_units, lascar_units, purple_units, lcm_units
+from .completeness import analyse_completeness, complete_interval_mask
 
 class DictionaryPlus(dict):
     def __init__(self, *args, **kwargs):
@@ -1194,6 +1195,19 @@ class Summary(pd.DataFrame):
         if attribute in self.columns:
             return set(self[attribute])
 
+    def completeness_report(self, value_cols, frequency, *, time_col='datetime',
+                            id_col='identifier', min_completeness=1.0,
+                            include_empty_intervals=False):
+        """Report observed/expected samples by interval and measurement.
+
+        The result is an ordinary DataFrame with one row per identifier,
+        interval and variable. See :func:`apmtools.completeness.analyse_completeness`.
+        """
+        return analyse_completeness(
+            self, value_cols, frequency, time_col=time_col, id_col=id_col,
+            min_completeness=min_completeness,
+            include_empty_intervals=include_empty_intervals)
+
     def group_stat(self, value_cols, frequency=None, grouping_by=["identifier"], time_col='datetime', id_col='identifier',
                    min_completeness=1.0, only_complete=True, flooring_timestamp=True, aggregate=["mean"]):
         """
@@ -1234,174 +1248,11 @@ class Summary(pd.DataFrame):
         if frequency != None:
 
             if only_complete:
-                def median_interval(g):
-                    return g.sort_values().diff().dropna().median()
-
-                def sorting_frequency(x):
-                    if len(list(filter(lambda z: True if "week" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "week" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "weekday" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "weekday" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "day" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "day" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "hour" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "hour" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "minute" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "minute" in z else False, x))[0]
-                    if len(list(filter(lambda z: True if "second" in z else False, x))) > 0:
-                        return list(filter(lambda z: True if "second" in z else False, x))[0]
-
-                def freq_ranked():
-                    if isinstance(frequency, str):
-                        return frequency
-                    if isinstance(frequency, list):
-                        return sorting_frequency(frequency)
-
-                intervals = df.groupby(id_col)[time_col].apply(median_interval)
-                if isinstance(frequency, str):
-                    if len(frequency.split(" ")) == 1:
-                        match frequency:
-                            case "hour":
-                                expected = (pd.Timedelta(1, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("h")
-                            case "day":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("d")
-                            case "minute":
-                                expected = (pd.Timedelta(1, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("min")
-                            case "second":
-                                expected = (pd.Timedelta(1, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("s")
-                            case "week":
-                                expected = (pd.Timedelta(1, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("w")
-                            case "weekday":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday()))
-                    if len(frequency.split(" ")) == 2:
-                        mult = float(frequency.split(" ")[0])
-                        match frequency.split(" ")[1]:
-                            case "hour":
-                                expected = (pd.Timedelta(mult, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}h")
-                            case "day":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}d")
-                            case "minute":
-                                expected = (pd.Timedelta(mult, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}min")
-                            case "second":
-                                expected = (pd.Timedelta(mult, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}s")
-                            case "week":
-                                expected = (pd.Timedelta(mult, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}w")
-                            case "weekday":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday//mult))*mult
-
-                    counts = df.groupby([id_col, "xxxyyyxxx"])[
-                        value_cols].count()
-                    expe = counts.index.get_level_values(id_col).map(expected)
-                    completeness = counts.div(expe, axis=0)
-                    complete_hours = completeness[completeness >= min_completeness].dropna(
-                    ).index
-                    mask = pd.MultiIndex.from_arrays(
-                        [df[id_col], df["xxxyyyxxx"]]).isin(complete_hours)
-                    df = df.loc[mask]
-
-                if isinstance(frequency, list):
-                    freq = freq_ranked()
-                    if len(freq.split(" ")) == 1:
-                        match freq:
-                            case "hour":
-                                expected = (pd.Timedelta(1, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("h")
-                            case "day":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("d")
-                            case "minute":
-                                expected = (pd.Timedelta(1, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("min")
-                            case "second":
-                                expected = (pd.Timedelta(1, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("s")
-                            case "week":
-                                expected = (pd.Timedelta(1, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor("w")
-                            case "weekday":
-                                expected = (pd.Timedelta(1, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday()))
-                    if len(freq.split(" ")) == 2:
-                        mult = float(freq.split(" ")[0])
-                        match freq.split(" ")[1]:
-                            case "hour":
-                                expected = (pd.Timedelta(mult, "hour") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}h")
-                            case "day":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}d")
-                            case "minute":
-                                expected = (pd.Timedelta(mult, "minute") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}m")
-                            case "second":
-                                expected = (pd.Timedelta(mult, "second") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}s")
-                            case "week":
-                                expected = (pd.Timedelta(mult, "W") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].dt.floor(
-                                    f"{mult}w")
-                            case "weekday":
-                                expected = (pd.Timedelta(mult, "day") /
-                                            intervals).round().astype(int)
-                                df["xxxyyyxxx"] = df["datetime"].map(
-                                    lambda x: (x.weekday//mult))*mult
-
-                    counts = df.groupby([id_col, "xxxyyyxxx"])[
-                        value_cols].count()
-                    expe = counts.index.get_level_values(id_col).map(expected)
-                    completeness = counts.div(expe, axis=0)
-                    complete_hours = completeness[completeness >= min_completeness].dropna(
-                    ).index
-                    mask = pd.MultiIndex.from_arrays(
-                        [df[id_col], df["xxxyyyxxx"]]).isin(complete_hours)
-                    df = df.loc[mask]
+                # Stage 4A: one shared completeness implementation for analysis
+                # and aggregation. Retain existing all-value filtering semantics.
+                df = df.loc[complete_interval_mask(
+                    df, value_cols, frequency, time_col=time_col,
+                    id_col=id_col, min_completeness=min_completeness)].copy()
 
             if isinstance(frequency, str):
                 if len(frequency.split(" ")) == 1:

apmtools-0.2.58/src/apmtools/completeness.py

@@ -0,0 +1,148 @@
+"""Sampling completeness calculations shared by Summary and future quality views.
+
+Coverage is estimated from each identifier's median positive sampling gap.
+The first and last partially observed bins cannot be classified as complete
+without a known deployment schedule: results describe coverage inside bins
+within the observed recording span, not compliance with a planned deployment.
+"""
+from __future__ import annotations
+
+import re
+import pandas as pd
+
+_UNITS = {'second': 's', 'minute': 'min', 'hour': 'h', 'day': 'D', 'week': 'W'}
+_DURATION = {'second': pd.Timedelta(seconds=1),
+             'minute': pd.Timedelta(minutes=1),
+             'hour': pd.Timedelta(hours=1),
+             'day': pd.Timedelta(days=1),
+             'week': pd.Timedelta(weeks=1),
+             'weekday': pd.Timedelta(days=1)}
+
+
+def _primary_frequency(frequency):
+    if isinstance(frequency, (tuple, list)):
+        if not frequency:
+            raise ValueError('frequency cannot be empty')
+        # Match the historical group_stat ranking of coarsest first.
+        ranking = {'week': 0, 'weekday': 1, 'day': 2, 'hour': 3,
+                   'minute': 4, 'second': 5}
+        return min(frequency, key=lambda f: ranking[_parse_frequency(f)[1]])
+    return frequency
+
+
+def _parse_frequency(frequency):
+    if not isinstance(frequency, str):
+        raise ValueError('frequency must be a string or a sequence of strings')
+    match = re.fullmatch(r'(?:(\d+(?:\.\d+)?)\s+)?(second|minute|hour|day|week|weekday)', frequency.strip())
+    if not match:
+        raise ValueError(f'Unsupported completeness frequency: {frequency!r}')
+    count = float(match.group(1) or 1)
+    if not count.is_integer() or count <= 0:
+        raise ValueError('Completeness frequency multiplier must be a positive integer')
+    return int(count), match.group(2)
+
+
+def _interval_bins(timestamps, frequency):
+    multiplier, unit = _parse_frequency(frequency)
+    series = pd.to_datetime(timestamps)
+    if unit == 'weekday':
+        # Historical weekday grouping pools weekdays across calendar weeks.
+        return (series.dt.weekday // multiplier) * multiplier
+    if unit == 'week':
+        # Monday-anchored calendar weeks, including multiweek bins.
+        starts = series.dt.to_period('W-SUN').dt.start_time
+        if multiplier == 1:
+            return starts
+        epoch = pd.Timestamp('1970-01-05')
+        return epoch + ((starts - epoch) // pd.Timedelta(weeks=multiplier)) * pd.Timedelta(weeks=multiplier)
+    return series.dt.floor(f'{multiplier}{_UNITS[unit]}')
+
+
+def analyse_completeness(data, value_cols, frequency, *, time_col='datetime',
+                         id_col='identifier', min_completeness=1.0,
+                         include_empty_intervals=False):
+    """Return one row per (identifier, frequency bin, measurement).
+
+    Columns: identifier (or *id_col*), ``interval``, ``variable``,
+    ``observed``, ``expected``, ``missing``, ``completeness``, ``is_complete``,
+    ``sampling_interval``. ``observed`` counts non-null measurements, not
+    distinct timestamps; ``missing`` is clipped at zero. NaN expected counts
+    flag identifiers with insufficient timing information.
+
+    ``include_empty_intervals`` adds bins between each identifier's first and
+    last observed bin for ordinary temporal frequencies, except when a bin
+    is shorter than the identifier's sampling cadence (unscheduled bins are
+    not counted as missing); it does not infer
+    missing deployment time before the first or after the last observation.
+    """
+    if not 0 <= min_completeness <= 1:
+        raise ValueError('min_completeness must be between 0 and 1')
+    frequency = _primary_frequency(frequency)
+    multiplier, unit = _parse_frequency(frequency)
+    values = list(value_cols)
+    if not values:
+        raise ValueError('value_cols must not be empty')
+    required = [id_col, time_col, *values]
+    absent = [name for name in required if name not in data.columns]
+    if absent:
+        raise KeyError(f'Missing completeness columns: {absent}')
+    frame = pd.DataFrame(data)[required].copy()
+    frame[time_col] = pd.to_datetime(frame[time_col], errors='coerce')
+    frame = frame.dropna(subset=[id_col, time_col])
+    columns = [id_col, 'interval', 'variable', 'observed', 'expected',
+               'missing', 'completeness', 'is_complete', 'sampling_interval']
+    if frame.empty:
+        return pd.DataFrame(columns=columns)
+    frame['_interval'] = _interval_bins(frame[time_col], frequency)
+    # Nonpositive differences (duplicates) cannot describe a sampling cadence.
+    ordered = frame.sort_values([id_col, time_col])
+    diffs = ordered.groupby(id_col, sort=False)[time_col].diff()
+    cadence = (pd.DataFrame({id_col: ordered[id_col].to_numpy(),
+                             'gap': diffs.to_numpy()})
+               .loc[lambda x: x.gap > pd.Timedelta(0)]
+               .groupby(id_col)['gap'].median())
+    groups = frame.groupby([id_col, '_interval'], dropna=False, sort=True)[values].count()
+    if include_empty_intervals and unit != 'weekday':
+        duration = _DURATION[unit] * multiplier
+        new_parts = []
+        for identifier, portion in frame.groupby(id_col, sort=False):
+            first, last = portion['_interval'].min(), portion['_interval'].max()
+            # An interval shorter than the monitor cadence is not a scheduled
+            # observation interval. Do not manufacture missing sub-cadence bins.
+            monitor_cadence = cadence.get(identifier, pd.NaT)
+            if pd.notna(monitor_cadence) and duration < monitor_cadence:
+                bins = pd.DatetimeIndex(sorted(portion['_interval'].unique()))
+            else:
+                bins = pd.date_range(first, last, freq=duration)
+            new_parts.append(pd.MultiIndex.from_product([[identifier], bins], names=[id_col, '_interval']))
+        if new_parts:
+            all_bins = new_parts[0]
+            for additional in new_parts[1:]:
+                all_bins = all_bins.append(additional)
+            groups = groups.reindex(all_bins, fill_value=0)
+    long = (groups.rename_axis(index=[id_col, 'interval'])
+            .reset_index().melt(id_vars=[id_col, 'interval'],
+                                value_vars=values, var_name='variable',
+                                value_name='observed'))
+    long['sampling_interval'] = long[id_col].map(cadence)
+    expected = (_DURATION[unit] * multiplier) / long['sampling_interval']
+    expected = pd.to_numeric(expected, errors='coerce').round()
+    # A bin containing a recorded sample has one expected opportunity even
+    # when its width is smaller than the inferred sampling cadence.
+    # Such sub-cadence bins without records were excluded above.
+    long['expected'] = expected.clip(lower=1)
+    long['missing'] = (long['expected'] - long['observed']).clip(lower=0)
+    long['completeness'] = long['observed'] / long['expected']
+    long['is_complete'] = long['completeness'].ge(min_completeness) & long['expected'].notna()
+    return long[columns]
+
+
+def complete_interval_mask(data, value_cols, frequency, *, time_col='datetime',
+                           id_col='identifier', min_completeness=1.0):
+    """Row mask preserving legacy group_stat semantics (all values complete)."""
+    report = analyse_completeness(data, value_cols, frequency, time_col=time_col,
+                                  id_col=id_col, min_completeness=min_completeness)
+    complete = report.groupby([id_col, 'interval'])['is_complete'].all()
+    intervals = _interval_bins(pd.to_datetime(data[time_col]), _primary_frequency(frequency))
+    index = pd.MultiIndex.from_arrays([data[id_col], intervals], names=[id_col, 'interval'])
+    return pd.Series(index.isin(complete.index[complete]), index=data.index)

{apmtools-0.2.57 → apmtools-0.2.58}/src/apmtools/plot_configs.py

@@ -1,6 +1,5 @@
 """Plot configuration lifecycle: reactive drafts and explicitly retained presets."""
 from __future__ import annotations
-from .shiny.profiling import traced
 import copy
 import json
 import uuid
@@ -161,6 +160,31 @@ class PlotConfigManager:
         self.revision += 1
         return preset_id
 
+    def retain_quality(self, name, settings, *, preset_id=None):
+        """Retain a quality dashboard configuration, never computed report data."""
+        name = str(name).strip()
+        if not name:
+            raise ValueError('Configuration name must not be empty')
+        preset_id = preset_id or uuid.uuid4().hex
+        item = {'schema_version': 1, 'id': preset_id, 'name': name,
+                'type': 'quality', 'settings': _json_ready(settings)}
+        items = self.saved()
+        for index, previous in enumerate(items):
+            if previous.get('id') == preset_id:
+                items[index] = item
+                break
+        else:
+            items.append(item)
+        self.summary.views = items
+        self.revision += 1
+        return preset_id
+
+    def restore_quality(self, preset_id):
+        item = next((p for p in self.saved() if p.get('id') == preset_id), None)
+        if item is None or item.get('type') != 'quality':
+            raise KeyError(preset_id)
+        return copy.deepcopy(item['settings'])
+
     def restore_table(self, preset_id):
         item = next((p for p in self.saved() if p.get('id') == preset_id), None)
         if item is None or item.get('type') != 'table':
@@ -172,7 +196,7 @@ class PlotConfigManager:
     def delete_saved(self, preset_id):
         """Delete one explicitly saved plot by stable ID, not its open draft."""
         items = copy.deepcopy(self.summary.views or [])
-        remaining = [p for p in items if not (p.get('id') == preset_id and p.get('type') in ('plot', 'table'))]
+        remaining = [p for p in items if not (p.get('id') == preset_id and p.get('type') in ('plot', 'table', 'quality'))]
         if len(remaining) == len(items):
             raise KeyError(f'Unknown saved plot: {preset_id}')
         self.summary.views = remaining

apmtools-0.2.58/src/apmtools/shiny/quality_view.py

@@ -0,0 +1,96 @@
+"""Headless data-quality summaries and heatmap preparation.
+
+The dashboard never infers completeness independently of the Stage 4A engine.
+"""
+from __future__ import annotations
+
+import pandas as pd
+from ..completeness import analyse_completeness
+
+QUALITY_FREQUENCIES = ('minute', 'hour', 'day', 'week')
+
+
+def quality_report(data, settings):
+    """Recompute per-identifier, interval and variable coverage from source data."""
+    values = list(settings.get('value_cols', []))
+    if not values:
+        raise ValueError('Select at least one measurement variable')
+    freq = settings.get('frequency', 'hour')
+    if freq not in QUALITY_FREQUENCIES:
+        raise ValueError(f'Unsupported dashboard frequency: {freq}')
+    return analyse_completeness(
+        data, values, freq,
+        id_col=settings.get('id_col', 'identifier'),
+        time_col=settings.get('time_col', 'datetime'),
+        min_completeness=float(settings.get('min_completeness', 1.0)),
+        include_empty_intervals=bool(settings.get('include_empty_intervals', True)),
+    )
+
+
+def quality_summary(report):
+    """Per-monitor and variable coverage, including zero-observation bins."""
+    names = ['identifier', 'variable', 'intervals', 'complete_intervals',
+             'observed', 'expected', 'missing', 'coverage_percent',
+             'complete_percent']
+    if report.empty:
+        return pd.DataFrame(columns=names)
+    key = report.columns[0]
+    stats = (report.groupby([key, 'variable'], dropna=False, sort=True)
+             .agg(intervals=('is_complete', 'size'),
+                  complete_intervals=('is_complete', 'sum'),
+                  observed=('observed', 'sum'),
+                  expected=('expected', 'sum'),
+                  missing=('missing', 'sum')).reset_index())
+    stats['coverage_percent'] = (100 * stats['observed'] / stats['expected']).where(stats['expected'] > 0)
+    stats['complete_percent'] = 100 * stats['complete_intervals'] / stats['intervals']
+    return stats.rename(columns={key: 'identifier'})[names]
+
+
+def selected_heatmap_variable(value_cols, selected):
+    """Return a heatmap variable present in the coverage report.
+
+    Changing the selected measurements cannot leave the heatmap looking for a
+    numeric column that was not included in the report.
+    """
+    choices = list(value_cols or [])
+    return selected if selected in choices else (choices[0] if choices else None)
+
+
+def quality_heatmap_data(report, variable):
+    """Interval coverage matrix (identifiers x bins); missing data remain NaN."""
+    if report.empty:
+        return pd.DataFrame()
+    key = report.columns[0]
+    portion = report.loc[report['variable'].eq(variable), [key, 'interval', 'completeness']].copy()
+    if portion.empty:
+        return pd.DataFrame()
+    return portion.pivot_table(index=key, columns='interval', values='completeness', aggfunc='mean', dropna=False)
+
+
+def missing_interval_runs(report):
+    """Group consecutive zero-observation bins into gap episodes per monitor/variable.
+
+    A new episode begins whenever an observed bin separates missing bins.
+    The report must include empty intervals to expose fully absent time bins.
+    """
+    fields = ['identifier', 'variable', 'first_missing_interval', 'last_missing_interval',
+              'missing_intervals', 'expected_observations']
+    if report.empty:
+        return pd.DataFrame(columns=fields)
+    key = report.columns[0]
+    rows = []
+    for (identifier, variable), group in report.groupby([key, 'variable'], sort=False, dropna=False):
+        ordered = group.sort_values('interval')
+        run = []
+        def finish():
+            if run:
+                rows.append((identifier, variable, run[0]['interval'], run[-1]['interval'],
+                             len(run), sum(record['expected'] for record in run)))
+        for record in ordered.to_dict('records'):
+            if record['observed'] == 0:
+                run.append(record)
+            else:
+                finish()
+                run = []
+        finish()
+    return pd.DataFrame(rows, columns=fields)

{apmtools-0.2.57 → apmtools-0.2.58}/src/apmtools/shiny/ui.py

@@ -15,6 +15,7 @@ from .config import VARIABLE_UNITS, PLOT_TYPES, FREQUENCIES, AGGREGATES, _MPL_PA
 from .columns import _alphabetical, _categorical_columns, _datetime_columns, _default_group, _default_x, _normalise_y, _numeric_columns, _timedelta_columns
 from .data import _aggregate, _apply_filters, _composite_labels, _subset_choices
 from .view_aggregation import normalize_statistics
+from .quality_view import quality_report, quality_summary, quality_heatmap_data, missing_interval_runs, QUALITY_FREQUENCIES, selected_heatmap_variable
 from .matplotlib_renderer import _axis_bounds_from_spec, _format_panel_bounds, _matplotlib_defaults, _matplotlib_direct_figure, _pad_axis_bounds, _panel_axis_bounds, _panel_partitions, _panel_trace_defaults
 from .elements import element_keys, filter_elements, default_visible_keys
 from .export import _safe_csv_filename, _standalone_matplotlib_script
@@ -168,8 +169,8 @@ def _build_summary_ui_server(
                            "plots and panel titles (Matplotlib backend)."),
             ),
             ui.h4("Plot tabs, export and application theme"),
-            ui.p("Use + New plot to add a plot tab, or Close selected plot to "
-                 "remove the current plot. The Info tab remains available."),
+            ui.p("Use New plot, New table, or New quality to open independent views. "
+                 "Close tab removes the selected view; the Info tab remains available."),
             ui.p("Export plotted data as CSV saves the data displayed in the plot. "
                  "Export plot saves the chart. With the Matplotlib backend, "
                  "Export as a Python file and associated data creates a "
@@ -2030,6 +2031,166 @@ if (!window.__apmColorPickerInstalled) {
             return snapshot
 
 
+    # Quality dashboards have independent controls and do not modify plot data.
+    @module.ui
+    def quality_tab_ui(settings=None):
+        settings = settings or {}
+        id_choices = _alphabetical([c for c in columns if c not in numeric_columns])
+        times = datetime_columns
+        selected_values = [v for v in settings.get("value_cols", numeric_columns[:1]) if v in numeric_columns]
+        selected_heatmap = selected_heatmap_variable(selected_values, settings.get("heatmap_variable"))
+        return ui.div(
+            ui.layout_sidebar(
+                ui.sidebar(
+                    ui.h4("Coverage analysis"),
+                    ui.input_select("id_col", "Monitor / identifier column",
+                                    choices=id_choices,
+                                    selected=settings.get("id_col", "identifier") if settings.get("id_col", "identifier") in id_choices else (id_choices[0] if id_choices else None)),
+                    ui.input_select("time_col", "Timestamp column",
+                                    choices=times,
+                                    selected=settings.get("time_col", "datetime") if settings.get("time_col", "datetime") in times else (times[0] if times else None)),
+                    ui.input_selectize("value_cols", "Measurements",
+                                       choices=numeric_columns, multiple=True,
+                                       selected=selected_values),
+                    ui.input_select("frequency", "Aggregation frequency",
+                                    choices={v: v.capitalize() for v in QUALITY_FREQUENCIES},
+                                    selected=settings.get("frequency", "hour")),
+                    ui.input_numeric("min_completeness", "Minimum completeness (0–1)",
+                                     value=settings.get("min_completeness", 1.0), min=0, max=1, step=0.05),
+                    ui.input_checkbox("include_empty_intervals", "Include entirely missing intervals",
+                                      value=settings.get("include_empty_intervals", True)),
+                    ui.input_select("preview_rows", "Interval table preview",
+                                    choices={"500": "500 rows", "1000": "1,000 rows", "5000": "5,000 rows", "all": "All rows"},
+                                    selected="1000"),
+                    ui.input_select("heatmap_variable", "Heatmap measurement",
+                                    choices={v: v for v in selected_values},
+                                    selected=selected_heatmap),
+                    ui.p("Coverage is estimated from each monitor's observed sampling cadence. "
+                         "Intervals outside the recording span cannot be inferred.",
+                         class_="small text-muted"),
+                    ui.download_button("download_quality_csv", "Export interval coverage CSV"),
+                    title="Data quality", position="left", open="open", width="320px",
+                ),
+                ui.div(
+                    ui.output_ui("quality_status"),
+                    ui.h4("Coverage by monitor and measurement"),
+                    ui.output_data_frame("quality_summary_grid"),
+                    ui.h4("Completeness heatmap"),
+                    ui.output_plot("quality_heatmap", height="420px"),
+                    ui.h4("Entirely missing interval sequences"),
+                    ui.output_data_frame("quality_gaps_grid"),
+                    ui.h4("Individual intervals"),
+                    ui.output_data_frame("quality_intervals_grid"),
+                    class_="p-3", style="min-width: 0; width: 100%;",
+                ),
+            ), class_="summary-quality-tab",
+        )
+
+    @module.server
+    def quality_tab_server(input, output, session, settings=None):
+        settings = settings or {}
+
+        @reactive.calc
+        def report_settings():
+            # Keep heatmap-only edits out of the expensive coverage calculation.
+            return {
+                "id_col": input.id_col(),
+                "time_col": input.time_col(),
+                "value_cols": list(input.value_cols() or []),
+                "frequency": input.frequency(),
+                "min_completeness": float(input.min_completeness() if input.min_completeness() is not None else 1.0),
+                "include_empty_intervals": bool(input.include_empty_intervals()),
+            }
+
+        @reactive.effect
+        @reactive.event(input.value_cols)
+        def sync_heatmap_choices():
+            # Only selected measurement variables appear in the report. Keep
+            # the heatmap selector in sync, but never recreate its input.
+            choices = list(input.value_cols() or [])
+            with reactive.isolate():
+                selected = input.heatmap_variable()
+            ui.update_select("heatmap_variable", choices={v: v for v in choices},
+                             selected=selected_heatmap_variable(choices, selected) or "")
+
+        @reactive.calc
+        def quality_settings():
+            conf = report_settings()
+            return {**conf, "heatmap_variable": selected_heatmap_variable(
+                conf["value_cols"], input.heatmap_variable())}
+
+        @reactive.calc
+        def report():
+            conf = report_settings()
+            if not conf["value_cols"] or not conf["id_col"] or not conf["time_col"]:
+                return pd.DataFrame()
+            return quality_report(summary, conf)
+
+        @render.ui
+        def quality_status():
+            try:
+                result = report()
+                if result.empty:
+                    return ui.p("Select a measurement to analyse coverage.", class_="text-muted")
+                missing_bins = int((result["observed"] == 0).sum())
+                return ui.p(f"{len(result):,} measurement intervals; "
+                            f"{int(result['is_complete'].sum()):,} complete; "
+                            f"{missing_bins:,} entirely missing.", class_="text-muted")
+            except (ValueError, KeyError) as exc:
+                return ui.p(str(exc), class_="text-danger")
+
+        @render.data_frame
+        def quality_summary_grid():
+            return render.DataGrid(quality_summary(report()), filters=True, width="100%", height="230px")
+
+        @render.data_frame
+        def quality_gaps_grid():
+            return render.DataGrid(missing_interval_runs(report()), filters=True, width="100%", height="190px")
+
+        @render.data_frame
+        def quality_intervals_grid():
+            data = report()
+            preview = input.preview_rows()
+            if preview != 'all':
+                data = data.head(int(preview))
+            data = data.copy()
+            if 'sampling_interval' in data:
+                data['sampling_interval'] = data['sampling_interval'].astype(str)
+            return render.DataGrid(data, filters=True, width="100%", height="320px")
+
+        @render.plot
+        def quality_heatmap():
+            import matplotlib.pyplot as plt
+            from matplotlib.colors import Normalize
+            conf = quality_settings()
+            matrix = quality_heatmap_data(report(), conf["heatmap_variable"])
+            fig, ax = plt.subplots(figsize=(10, 4))
+            if matrix.empty:
+                ax.text(0.5, 0.5, "No data for selected measurement", ha="center", va="center")
+                ax.set_axis_off()
+                return fig
+            colors = plt.get_cmap("viridis").copy()
+            colors.set_bad("lightgray")
+            image = ax.imshow(matrix.to_numpy(dtype=float), aspect="auto", interpolation="nearest",
+                              cmap=colors, norm=Normalize(vmin=0, vmax=1))
+            ax.set_yticks(range(len(matrix.index)))
+            ax.set_yticklabels([str(v) for v in matrix.index], fontsize=8)
+            ticks = sorted(set([0, len(matrix.columns) - 1] + list(range(0, len(matrix.columns), max(1, len(matrix.columns) // 8)))))
+            ax.set_xticks(ticks)
+            ax.set_xticklabels([str(matrix.columns[i]) for i in ticks], rotation=35, ha="right", fontsize=8)
+            ax.set_xlabel("Aggregation interval")
+            ax.set_ylabel(conf["id_col"])
+            ax.set_title(f"{conf['heatmap_variable']} coverage")
+            fig.colorbar(image, ax=ax, label="Observed / expected")
+            fig.tight_layout()
+            return fig
+
+        @render.download(filename="completeness_intervals.csv")
+        def download_quality_csv():
+            yield report().to_csv(index=False)
+
+        return quality_settings
+
     @module.ui
     def table_tab_ui(frame, settings=None):
         settings = settings or {}
@@ -2517,6 +2678,7 @@ if (!window.__apmColorPickerInstalled) {
             ui.div(
                 ui.input_action_button("add_tab", "New plot", class_="btn-primary"),
                 ui.input_action_button("add_table", "New table", class_="btn-primary"),
+                ui.input_action_button("add_quality", "New quality", class_="btn-primary"),
                 ui.input_action_button("retain_plot", "Save", class_="btn-outline-primary"),
                 ui.input_action_button("restore_plot", "Load", class_="btn-outline-primary"),
                 ui.input_action_button("undo_view", "Undo", class_="btn-outline-secondary"),
@@ -2557,6 +2719,7 @@ if (!window.__apmColorPickerInstalled) {
         table_settings_by_tab = workspace.table_settings
         table_source_steps = workspace.table_lineage
         next_table = reactive.value(0)
+        next_quality = reactive.value(0)
         from ..plot_configs import PlotConfigManager
         manager = config_manager or PlotConfigManager(data)
         views = ViewController(workspace, manager)
@@ -2596,6 +2759,22 @@ if (!window.__apmColorPickerInstalled) {
                 except (TypeError, ValueError, KeyError):
                     pass
 
+        def register_quality_history(tab_id):
+            first = True
+            @reactive.effect
+            def sync_quality_history():
+                nonlocal first
+                if workspace.kind(tab_id) != 'quality':
+                    return
+                try:
+                    state = dict(workspace.quality_settings[tab_id]())
+                    if first:
+                        first = False
+                    elif views.record_edit(tab_id, state):
+                        refresh_history()
+                except (TypeError, ValueError, KeyError):
+                    pass
+
         @render.ui
         def view_edit_status():
             history_revision.get()
@@ -2632,7 +2811,7 @@ if (!window.__apmColorPickerInstalled) {
         @traced('retain_selected', kind='reactive_effect', trigger='@reactive.event(input.retain_plot)')
         def retain_selected():
             selected = input.plot_tabs()
-            if selected not in tab_filters and selected not in table_frames:
+            if selected not in tab_filters and selected not in table_frames and selected not in workspace.quality_settings:
                 ui.notification_show("Select a plot or table tab to save", type="warning")
                 return
             ui.modal_show(ui.modal(
@@ -2827,6 +3006,21 @@ if (!window.__apmColorPickerInstalled) {
                         ui.insert_nav_panel("plot_tabs", panel, select=True)
                         ui.modal_remove()
                         return
+                    if preset.get("type") == "quality":
+                        number = next_quality.get() + 1
+                        tab_id = f"quality_{number}"
+                        settings = views.restore_quality(preset_id)
+                        panel = ui.nav_panel(preset.get('name') or f"Quality {number}",
+                                             quality_tab_ui(tab_id, settings), value=tab_id)
+                        snapshot = quality_tab_server(tab_id, settings)
+                        workspace.add_quality(tab_id, snapshot)
+                        begin_history(tab_id, settings, restored=True)
+                        register_quality_history(tab_id)
+                        next_quality.set(number)
+                        active_tabs.set(workspace.active_tabs)
+                        ui.insert_nav_panel("plot_tabs", panel, select=True)
+                        ui.modal_remove()
+                        return
                     number = next_tab.get() + 1
                     tab_id = f"plot_{number}"
                     initialized = initializer.restored_plot(preset_id, tab_id, number)
@@ -2860,6 +3054,20 @@ if (!window.__apmColorPickerInstalled) {
             active_tabs.set(workspace.active_tabs)
             ui.insert_nav_panel("plot_tabs", new_panel, select=True)
 
+        @reactive.effect
+        @reactive.event(input.add_quality)
+        def add_quality():
+            number = next_quality.get() + 1
+            tab_id = f"quality_{number}"
+            panel = ui.nav_panel(f"Quality {number}", quality_tab_ui(tab_id), value=tab_id)
+            settings = quality_tab_server(tab_id)
+            workspace.add_quality(tab_id, settings)
+            begin_history(tab_id, {})
+            register_quality_history(tab_id)
+            next_quality.set(number)
+            active_tabs.set(workspace.active_tabs)
+            ui.insert_nav_panel("plot_tabs", panel, select=True)
+
         @reactive.effect
         @reactive.event(input.add_table)
         @traced('add_table', kind='reactive_effect', trigger='@reactive.event(input.add_table)')
@@ -2895,7 +3103,15 @@ if (!window.__apmColorPickerInstalled) {
                 # initialized consistently, rather than patching only inputs
                 # that happen to be visible. Preserve the same history stack.
                 kind = workspace.kind(selected)
-                if kind == "plot":
+                if kind == "quality":
+                    number = next_quality.get() + 1
+                    next_quality.set(number)
+                    replacement = f"quality_{number}"
+                    panel = ui.nav_panel(f"Quality {number}", quality_tab_ui(replacement, settings), value=replacement)
+                    quality_snapshot = quality_tab_server(replacement, settings)
+                    workspace.add_quality(replacement, quality_snapshot)
+                    register_quality_history(replacement)
+                elif kind == "plot":
                     number = next_tab.get() + 1
                     next_tab.set(number)
                     replacement = f"plot_{number}"

{apmtools-0.2.57 → apmtools-0.2.58}/src/apmtools/shiny/view_controller.py

@@ -42,11 +42,11 @@ class ViewController:
 
     def saved(self):
         """Return independently mutable copies of both saved view types."""
-        return [p for p in self.manager.saved() if p.get('type', 'plot') in ('plot', 'table')]
+        return [p for p in self.manager.saved() if p.get('type', 'plot') in ('plot', 'table', 'quality')]
 
     def choices(self):
         """Stable ID -> human-readable label, independent of the active tab."""
-        return {p['id']: f"{'Table' if p.get('type') == 'table' else 'Plot'}: {p.get('name', p['id'])}"
+        return {p['id']: f"{ {'table': 'Table', 'plot': 'Plot', 'quality': 'Quality'}.get(p.get('type', 'plot'), 'Plot')}: {p.get('name', p['id'])}"
                 for p in self.saved()}
 
     def preset(self, preset_id):
@@ -72,7 +72,18 @@ class ViewController:
             if tab_id in self.history._tabs:
                 self.history.mark_saved(tab_id, settings)
             return preset_id
-        raise ValueError('No active plot or table tab')
+        if kind == 'quality':
+            settings = copy.deepcopy(self.workspace.quality_settings[tab_id]())
+            preset_id = self.manager.retain_quality(name, settings)
+            if tab_id in self.history._tabs:
+                self.history.mark_saved(tab_id, settings)
+            return preset_id
+        raise ValueError('No active plot, table or quality tab')
+
+    def restore_quality(self, preset_id):
+        if self.preset(preset_id).get('type') != 'quality':
+            raise ValueError('Saved configuration is not a quality dashboard')
+        return self.manager.restore_quality(preset_id)
 
     def delete(self, preset_id):
         """Delete a retained view without affecting any open tab."""

{apmtools-0.2.57 → apmtools-0.2.58}/src/apmtools/shiny/workspace_controller.py

@@ -11,6 +11,7 @@ class WorkspaceController:
 
     def __init__(self):
         self.plot_snapshots = {}
+        self.quality_settings = {}
         self.table_frames = {}
         self.table_settings = {}
         self.table_lineage = {}
@@ -19,6 +20,8 @@ class WorkspaceController:
     def kind(self, tab_id):
         if tab_id in self.plot_snapshots:
             return 'plot'
+        if tab_id in self.quality_settings:
+            return 'quality'
         if tab_id in self.table_frames:
             return 'table'
         return None
@@ -41,9 +44,18 @@ class WorkspaceController:
         self.active_tabs += (tab_id,)
         return self.active_tabs
 
+    def add_quality(self, tab_id, settings):
+        self._check_new(tab_id)
+        if not callable(settings):
+            raise TypeError('Quality settings must be callable')
+        self.quality_settings[tab_id] = settings
+        self.active_tabs += (tab_id,)
+        return self.active_tabs
+
     def remove(self, tab_id):
         if tab_id not in self.active_tabs:
             return False
+        self.quality_settings.pop(tab_id, None)
         self.plot_snapshots.pop(tab_id, None)
         self.table_frames.pop(tab_id, None)
         self.table_settings.pop(tab_id, None)

{apmtools-0.2.57 → apmtools-0.2.58}/src/apmtools/view_schema.py

@@ -38,7 +38,7 @@ def validate_saved_views(views):
     for index, item in enumerate(views):
         if not isinstance(item, dict):
             raise ValueError(f'Saved view {index} must be a dictionary')
-        if item.get('schema_version') != CURRENT_SCHEMA_VERSION or item.get('type') not in ('plot', 'table'):
+        if item.get('schema_version') != CURRENT_SCHEMA_VERSION or item.get('type') not in ('plot', 'table', 'quality'):
             raise ValueError(f'Unsupported saved view configuration at index {index}')
         if not isinstance(item.get('settings'), dict):
             raise ValueError(f'Saved view settings must be a dictionary at index {index}')