apmtools 0.2.46__tar.gz → 0.2.48__tar.gz

{apmtools-0.2.46 → apmtools-0.2.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: apmtools
-Version: 0.2.46
+Version: 0.2.48
 Summary: A collection of tools for processing air pollution monitoring data
 Project-URL: Homepage, https://github.com/federlorenz/apmtools
 Author-email: Federico Lorenzetti <lorenzetti.federico@gmail.com>

{apmtools-0.2.46 → apmtools-0.2.48}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apmtools"
-version = "0.2.46"
+version = "0.2.48"
 authors = [
   { name="Federico Lorenzetti", email="lorenzetti.federico@gmail.com" },
 ]

{apmtools-0.2.46 → apmtools-0.2.48}/src/apmtools/classes.py

@@ -574,7 +574,7 @@ class Dataset(DictionaryPlus):
                     self.scan_folder(directory=f"{directory}{j}/", levels=levels,
                                      level=level+1, monitor=monitor, levels_dict=levels_dict, utc_timezone_shift=utc_timezone_shift, interpolate=interpolate, add_m=add_m)
 
-    def save_summary(self, save_csv=True, filename="./", columns={}, merged=True):
+    def save_summary(self, save_csv=True, filename="./", all_variables=False, merged=True):
         types_in = list(set(type(v) for v in self.values()))
 
         def match_class(x):
@@ -619,13 +619,9 @@ class Dataset(DictionaryPlus):
                         "start", "end", "length", "datetime","timedelta"]
             columns2 = list(set(d.meta()).difference(
                 set(["filter", "header", "parameters"])))
-            if "upas" not in columns.keys():
-                columns3 = ["pm25"]
-            else:
-                columns3 = ["pm25"]+columns["upas"] if columns["upas"] != "all" else ["pm25"]+list(
-                    d.set_attrib("columns"))
-            columns4 = ["pm25correctedrh",
-                        "pm25correctedrhandgrav", "filterid", "grav_not", "grav"]
+            columns3 = ["upas_pm25"] if not all_variables else ["upas_"+i for i in list(d.set_attrib("columns"))]+["upas_pm25"]
+            columns4 = ["upas_pm25_RHcorrected",
+                        "upas_pm25_RHgravcorrected", "upas_filterid", "upas_gravimetric", "upas_gravimetric_blankcorrected"]
             buffer = StringIO()
             with buffer as file:
                 df = writer(file, delimiter=",")
@@ -658,15 +654,13 @@ class Dataset(DictionaryPlus):
                                 app.append(v.m[m] if v.m[m] != None else "")
                             else:
                                 app.append("")
-                        if "upas" not in columns.keys():
-                            app.append(v["PM2_5MC"].iloc[i])
-                        else:
-                            app.append(v["PM2_5MC"].iloc[i])
-                            for col in columns3[1:]:
-                                if col in v.columns:
-                                    app.append(v[col].iloc[i])
-                                else:
-                                    app.append(None)
+
+                        for col in columns3[:-1]:
+                            if col[5:] in v.columns:
+                                app.append(v[col[5:]].iloc[i])
+                            else:
+                                app.append(None)
+                        app.append(v["PM2_5MC"].iloc[i])
                         xx = (a*v["PM2_5MC"].iloc[i])+(b*v["AtmoRH"].iloc[i])+c
                         app.append(xx)
                         if grav != "":
@@ -695,7 +689,7 @@ class Dataset(DictionaryPlus):
             columns1 = ["identifier", "monitor",
                         "start", "end", "length", "datetime", "timedelta"]
             columns2 = list(set(d.meta()))
-            columns3 = ["CO(ppm)"]
+            columns3 = ["lascar_CO(ppm)"]
 
             buffer = StringIO()
             with buffer as file:
@@ -743,9 +737,9 @@ class Dataset(DictionaryPlus):
             columns1 = ["identifier", "monitor",
                         "start", "end", "length", "datetime", "timedelta"]
             columns2 = list(set(d.meta()))
-            columns3 = ["sensorName", "deviceId", "resolution",
-                         "PM2.5", "PM10", "CO2", "NO2", "Temperature", "Humidity"]
-            columns4 = ["pm25correctedrh"]
+            columns3 = ["lcm_sensorName", "lcm_deviceId", "lcm_resolution",
+                        "lcm_PM2.5", "lcm_PM10", "lcm_CO2", "lcm_NO2", "lcm_Temperature", "lcm_Humidity"] if all_variables else ["lcm_PM2.5"]
+            columns4 = ["lcm_PM2.5correctedrh"]
 
             buffer = StringIO()
             with buffer as file:
@@ -774,7 +768,7 @@ class Dataset(DictionaryPlus):
                             else:
                                 app.append("")
                         for m in columns3:
-                            app.append(v[m].iloc[i])
+                            app.append(v[m[4:]].iloc[i])
                         xx = (a*v["PM2.5"].iloc[i])+(b*v["Humidity"].iloc[i])+c
                         app.append(xx)
                         df.writerow(list(map(str, app)))
@@ -793,13 +787,8 @@ class Dataset(DictionaryPlus):
             columns1 = ["identifier", "monitor",
                         "start", "end", "length", "datetime", "timedelta"]
             columns2 = list(set(d.meta()))
-            columns3 = ["pm2_5", "pm2_5_adj"]
-            columns4 = []
-            if "purple" not in columns.keys():
-                columns4 = []
-            else:
-                columns4 = columns["purple"] if columns["purple"] != "all" else list(
-                    d.set_attrib("columns"))
+            columns3 = ["purple_pm25", "purple_pm25_RHcorrected"]
+            columns4 = [] if not all_variables else ["purple_"+i for i in list(d.set_attrib("columns"))]
             buffer = StringIO()
             with buffer as file:
                 df = writer(file, delimiter=",")
@@ -829,17 +818,9 @@ class Dataset(DictionaryPlus):
                         app.append(
                             ((v["pm2_5_cf_1"]+v["pm2_5_cf_1_b"])/2).iloc[i])
                         app.append(v["pm_adj"].iloc[i])
-                        if "purple" in columns.keys():
-                            if columns["purple"] == "all":
-                                for col in list(d.set_attrib("columns")):
-                                    if col in v.columns:
-                                        app.append(v[col].iloc[i])
-                                    else:
-                                        app.append(None)
-                            else:
-                                for col in columns["purple"]:
-                                    if col in v.columns:
-                                        app.append(v[col].iloc[i])
+                        for col in columns4:
+                                if col[6:] in v.columns:
+                                        app.append(v[col[6:]].iloc[i])
                                     else:
                                         app.append(None)
                         df.writerow(list(map(str, app)))

{apmtools-0.2.46 → apmtools-0.2.48}/src/apmtools/shiny/appearance.py

@@ -1,5 +1,6 @@
 """Figure labels and appearance controls."""
 from __future__ import annotations
+import re
 import threading
 from datetime import date, datetime, time, timedelta
 from collections.abc import Sequence
@@ -38,7 +39,7 @@ def _label_plot_axes(fig, plot_type, x_column, y_columns, *,
     elif plot_type in {"box", "bar"}:
         categories = category_columns or [x_column]
         fig.update_xaxes(title_text=" | ".join(_axis_label(c) for c in categories))
-        fig.update_yaxes(title_text=_axis_label(y_columns[0]))
+        fig.update_yaxes(title_text="<br>".join(_axis_label(y) for y in y_columns))
     elif plot_type == "line" and len(y_columns) > 1:
         fig.update_xaxes(title_text=_axis_label(x_column))
         fig.update_yaxes(title_text="<br>".join(_axis_label(y) for y in y_columns))
@@ -59,7 +60,7 @@ def _label_plot_axes(fig, plot_type, x_column, y_columns, *,
 def _parse_label_map(value):
     """Parse semicolon-separated original=replacement entries."""
     result = {}
-    for entry in (value or "").split(";"):
+    for entry in re.split(r"[;\r\n]+", value or ""):
         if "=" in entry:
             key, replacement = entry.split("=", 1)
             if key.strip() and replacement.strip():

apmtools-0.2.48/src/apmtools/shiny/color_editor.py

@@ -0,0 +1,64 @@
+"""Stable colour-editor keys for the actual prepared plotting data."""
+from __future__ import annotations
+import re
+import pandas as pd
+
+_HEX = re.compile(r"#[0-9a-fA-F]{6}\Z")
+
+def valid_hex_color(value):
+    return isinstance(value, str) and _HEX.fullmatch(value.strip()) is not None
+
+
+def active_color_keys(spec, *, mode="group", panel_by=()):
+    """Return colour dimensions with nonempty plotted observations.
+
+    Faceting removes its grouping columns. For multi-Y plots, 'both' uses
+    combinations, while group/variable/none use the corresponding colour
+    dimension (the same keys accepted by multi_y_styles).
+    """
+    if "error" in spec or spec["data"].empty:
+        return ()
+    if spec.get("hist_variables"):
+        return tuple(dict.fromkeys(spec["data"].loc[
+            spec["data"][spec["x"]].notna(), "__hist_series"].astype(str)))
+    from .matplotlib_renderer import _panel_partitions
+    from .multi_axis import records
+    parts = (_panel_partitions(spec, panel_by) if panel_by
+             else [(None, spec["data"], spec["groups"])])
+    keys = []
+    for _, frame, remaining in parts:
+        if frame.empty:
+            continue
+        sub = {**spec, "data": frame, "groups": remaining,
+               "_original_groups": spec["groups"]} if panel_by else {**spec, "data": frame, "groups": remaining}
+        if spec.get("multi_axis"):
+            pairs = [(str(y), str(g) if remaining else None)
+                     for _, g, y, _, _, _ in records(sub)]
+        else:
+            ys = list(sub["ys"])
+            groups = (list(pd.unique(frame["__plot_group"].astype(str)))
+                      if remaining and "__plot_group" in frame else [None])
+            pairs = []
+            for g in groups:
+                rows = (frame.loc[frame["__plot_group"].astype(str) == g]
+                        if g is not None else frame)
+                for y in ys:
+                    if y in rows and rows[y].notna().any():
+                        pairs.append((str(y), g))
+        multi = bool(spec.get("multi_axis") or len(sub["ys"]) > 1)
+        for y, g in pairs:
+            if not multi or mode in ("group", "none"):
+                key = g if g is not None else y
+            elif mode == "variable":
+                key = y
+            else:
+                key = f"{g}, {y}" if g is not None else y
+            if key not in keys and key != "All observations":
+                keys.append(key)
+    return tuple(keys)
+
+
+def color_defaults(spec, keys, palette="Default"):
+    from .config import _MPL_PALETTES
+    values = _MPL_PALETTES.get(palette, _MPL_PALETTES["Default"])
+    return {key: values[i % len(values)] for i, key in enumerate(keys)}

apmtools-0.2.48/src/apmtools/shiny/elements.py

@@ -0,0 +1,96 @@
+"""Selectable plotted elements, independent of the rendering backend."""
+import pandas as pd
+
+
+def _group_labels(spec):
+    frame = spec['data']
+    if spec.get('groups') and '__plot_group' in frame:
+        return frame['__plot_group'].astype(str)
+    return pd.Series('All observations', index=frame.index)
+
+
+def element_keys(spec):
+    """Selectable element identifiers with at least one valid plotted value.
+
+    Elements with exclusively missing observations are not selectable, rather
+    than merely being unchecked by default.
+    """
+    return default_visible_keys(spec)
+
+
+def filter_elements(spec, selected):
+    """Remove hidden elements without changing the underlying Summary or filters.
+
+    For multi-Y lines, individual group/variable combinations are removed by
+    masking only their Y values. Panels with no remaining values disappear.
+    """
+    if 'error' in spec or selected is None:
+        return spec
+    allowed = set(selected)
+    if spec.get('hist_variables'):
+        frame = spec['data'].loc[spec['data']['__hist_series'].isin(allowed)].copy()
+        return {**spec, 'data': frame, '_visible_elements': allowed}
+    if spec.get('multi_axis'):
+        from .multi_axis import records
+        filtered = {**spec, '_visible_elements': allowed}
+        keep = pd.Index([])
+        for _, _, _, _, _, rows in records(filtered):
+            keep = keep.union(rows.index, sort=False)
+        return {**filtered, 'data': spec['data'].loc[spec['data'].index.isin(keep)].copy()}
+    frame = spec['data'].copy()
+    kind = spec['kind']
+    groups = _group_labels(spec)
+    if kind == 'line':
+        keep_any = pd.Series(False, index=frame.index)
+        for y in spec['ys']:
+            keys = groups.map(lambda g: f'{g} | {y}' if len(spec['ys']) > 1 else g)
+            permitted = keys.isin(allowed)
+            frame.loc[~permitted, y] = float('nan')
+            keep_any |= permitted & frame[y].notna()
+        frame = frame.loc[keep_any].copy()
+    elif kind in ('box', 'bar'):
+        keys = [f'{c} | {g}' if spec['groups'] else c
+                for c, g in zip(frame[spec['x']].astype(str), groups)]
+        frame = frame.loc[pd.Series(keys, index=frame.index).isin(allowed)].copy()
+    else:
+        frame = frame.loc[groups.isin(allowed)].copy()
+    return {**spec, 'data': frame, '_visible_elements': allowed}
+
+
+def default_visible_keys(spec):
+    """Elements with at least one nonmissing plotted value (not merely rows)."""
+    if 'error' in spec:
+        return []
+    if spec.get('multi_axis'):
+        from .multi_axis import all_keys
+        return all_keys(spec, nonmissing=True)
+    frame, kind, x = spec['data'], spec['kind'], spec['x']
+    if spec.get('hist_variables'):
+        return list(dict.fromkeys(frame.loc[frame[x].notna(), '__hist_series'].astype(str)))
+    groups = _group_labels(spec)
+    if kind == 'line':
+        return list(dict.fromkeys(
+            f'{group} | {y}' if len(spec['ys']) > 1 else group
+            for y in spec['ys'] if y in frame
+            for group in groups[frame[y].notna()]
+        ))
+    if kind in ('box', 'bar'):
+        y = next((y for y in spec['ys'] if y in frame), None)
+        if y is None:
+            return []
+        valid = frame[y].notna()
+        return list(dict.fromkeys(
+            f'{category} | {group}' if spec['groups'] else str(category)
+            for category, group in zip(frame.loc[valid, x].astype(str), groups.loc[valid])
+        ))
+    # Scatter requires both coordinates; histogram requires its numeric x.
+    if kind == 'scatter':
+        valid = frame[x].notna()
+        for y in spec['ys']:
+            if y in frame:
+                valid &= frame[y].notna()
+    else:
+        valid = frame[x].notna()
+    return list(dict.fromkeys(groups.loc[valid]))
+
+

apmtools-0.2.48/src/apmtools/shiny/histogram_multi.py

@@ -0,0 +1,32 @@
+"""Long-format data preparation for overlaid multi-variable histograms."""
+import pandas as pd
+from .data import _composite_labels
+
+
+def histogram_long_data(frame, variables, *, group_columns=()):
+    """Create one histogram series per variable/group using a shared numeric axis.
+
+    Non-histogram columns are retained so faceting, export, and filtering work.
+    """
+    variables = list(dict.fromkeys(variables))
+    if not variables:
+        raise ValueError("At least one histogram variable is required")
+    missing = [name for name in variables if name not in frame]
+    if missing:
+        raise KeyError(f"Missing histogram variables: {missing}")
+    groups = list(group_columns)
+    for col in groups:
+        if col not in frame:
+            raise KeyError(col)
+    chunks = []
+    for name in variables:
+        part = frame.copy()
+        part["__hist_value"] = pd.to_numeric(part[name], errors="coerce")
+        part["__hist_variable"] = name
+        part["__hist_series"] = (name + " | " + _composite_labels(part, groups)
+                                 if groups else name)
+        part = part.loc[part["__hist_value"].notna()]
+        chunks.append(part)
+    if not chunks:
+        return frame.iloc[0:0].copy()
+    return pd.concat(chunks, ignore_index=True)