avoca 0.14.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

avoca/bindings/qa_tool.py

@@ -37,10 +37,11 @@ def export_EmpaQATool(
     station: str = "XXX",
     revision_date: datetime | None = None,
     dataset: datetime | str | None = None,
-    export_names: dict[str, str] = {},
+    export_names: dict[str, str] | None = None,
     datetime_offsets: tuple[timedelta, timedelta] | None = None,
     substances: list[str] = [],
     rounding_decimals: int = 4,
+    df_substances: pd.DataFrame | None = None,
 ) -> Path:
     """Export to the EmpaQATool format.
 
@@ -64,7 +65,17 @@ def export_EmpaQATool(
     :arg datetime_offsets: Tuple of two timedelta to use for the start and end datetime
     :arg substances: List of substances to export. You can also specify group names.
         If not specified, this will use the substances from `df_substances`.
+        If a substance is present here and not in `df_substances`, it will still be exported.
     :arg rounding_decimals: Number of decimals to round the values to.
+    :arg df_substances: DataFrame with substance information.
+        If provided, the substances to export will be taken from this dataframe.
+        Columns:
+        - index: substance name
+        - export: bool, whether to export the substance
+        - export_name: str, name to use in the export file
+
+
+    :returns: Path to the exported file.
 
     """
 
@@ -113,12 +124,42 @@ def export_EmpaQATool(
     logger.debug(f"df_out: {df_out.head()}")
     if not substances:
         substances = compounds_from_df(df)
+        if df_substances is not None and "export" in df_substances.columns:
+            # Remove the substances that should not be exported
+            substances = [
+                s
+                for s in substances
+                if s not in df_substances.index or df_substances.loc[s, "export"]
+            ]
 
     remove_infs = lambda x: x.replace([np.inf, -np.inf], np.nan)
     is_invalid = lambda x: x.isin([np.inf, -np.inf]) | pd.isna(x)
     clean_col = lambda x: remove_infs(x).round(rounding_decimals).astype(str)
 
+    if export_names is None:
+        export_names = {}
+
+    if df_substances is not None and "export_name" in df_substances.columns:
+        # Read export names from the dataframe if provided
+        for substance in substances:
+            if not substance or substance not in df_substances.index:
+                continue
+            export_name_df = df_substances.loc[substance, "export_name"]
+            if not export_name_df or pd.isna(export_name_df):
+                continue
+            if substance in export_names and export_names[substance] != export_name_df:
+                logger.warning(
+                    f"Substance {substance} found in both df_substances and"
+                    " export_names. Using the name from export_names.\n"
+                    f"  - export_names (used): {export_names[substance]}\n"
+                    f"  - df_substances: {export_name_df}"
+                )
+                continue
+            export_names[substance] = export_name_df
+
     for substance in substances:
+        if not substance:
+            continue
 
         export_name = export_names.get(substance, substance)
 
@@ -234,12 +275,12 @@ def read_empaqatool(file_path: Path, shift: timedelta | None = None) -> pd.DataF
         dt += shift
     columns[("-", "datetime")] = dt
 
-     # Last column is empty
-    compounds = [ '-'.join(s[:-1]) for col in df.columns if len(s:=col.split("-")) >= 2]
-
-
-    for compound in compounds: 
+    # Last column is empty
+    compounds = [
+        "-".join(s[:-1]) for col in df.columns if len(s := col.split("-")) >= 2
+    ]
 
+    for compound in compounds:
 
         flag_col = f"{compound}-flag"
         value_col = f"{compound}-value"
@@ -248,8 +289,8 @@ def read_empaqatool(file_path: Path, shift: timedelta | None = None) -> pd.DataF
 
         mapping = {
             "conc": value_col,
-            "u_expanded":acc_col, 
-            "u_precision":precision_col, 
+            "u_expanded": acc_col,
+            "u_precision": precision_col,
         }
 
         flag_values = (pd.to_numeric(df[flag_col]) * 1e3).astype(int).mod(1000)
@@ -263,10 +304,10 @@ def read_empaqatool(file_path: Path, shift: timedelta | None = None) -> pd.DataF
             serie = pd.to_numeric(df[value])
             mask_nan = flags == QA_Flag.MISSING.value
             serie[mask_nan] = np.nan
-            columns[(compound, key)] = serie 
+            columns[(compound, key)] = serie
 
         columns[(compound, "flag")] = flags
-        
+
         mask_nan = columns[(compound, "conc")].isna()
         columns[(compound, "flag")][mask_nan] |= QA_Flag.MISSING.value
 

avoca/manager.py

@@ -20,6 +20,7 @@ class AssignerManager:
     _assigners_importpath = {
         "RetentionTimeChecker": "avoca.qa_class.rt",
         "ExtremeValues": "avoca.qa_class.zscore",
+        "RollingWindow": "avoca.qa_class.rolling",
         "ExtremeConcentrations": "avoca.qa_class.concs",
         "XY_Correlations": "avoca.qa_class.zscore",
         "TestAssigner": "avoca.qa_class.test",

avoca/plots.py

@@ -77,28 +77,17 @@ def plot_yearly_plotly(
     import plotly.graph_objects as go
 
     dt_column = ("-", "datetime")
-    serie = df[(compound, "conc")]
-    dt = df[dt_column]
+
     if ("-", "type") in df.columns:
         mask_air = df[("-", "type")] == "air"
-        serie = serie[mask_air]
-        dt = dt[mask_air]
-    if ("-", "type") in df_new.columns:
+        df = df[mask_air]
+    if df_new is not None and ("-", "type") in df_new.columns:
         mask_air_new = df_new[("-", "type")] == "air"
         df_new = df_new[mask_air_new]
 
+    dt = df[dt_column]
     x = dt.dt.day_of_year + dt.dt.hour / 24.0
-    df_to_plot = pd.DataFrame(
-        {
-            "conc": serie.values,
-            "year": dt.dt.year.values,
-        },
-        index=x.values,
-    )
-    # Break down by year, to have year as columns and conc as values
-    df_to_plot = df_to_plot.pivot_table(
-        index=df_to_plot.index, columns="year", values="conc"
-    )
+
     fig = go.Figure()
 
     hover_template = "Timestamp: %{text}<br>Conc: %{y:.2f} ppt"
@@ -110,29 +99,44 @@ def plot_yearly_plotly(
         "hovertemplate": hover_template,
     }
 
-    for year in df_to_plot.columns:
+    if (compound, "conc") in df:
+        serie = df[(compound, "conc")]
+        df_to_plot = pd.DataFrame(
+            {
+                "conc": serie.values,
+                "year": dt.dt.year.values,
+            },
+            index=x.values,
+        )
+        # Break down by year, to have year as columns and conc as values
+        df_to_plot = df_to_plot.pivot_table(
+            index=df_to_plot.index, columns="year", values="conc"
+        )
+        for year in df_to_plot.columns:
+            fig.add_trace(
+                go.Scatter(
+                    x=df_to_plot.index,
+                    y=df_to_plot[year],
+                    name=str(year),
+                    zorder=-year,
+                    text=dt[dt.dt.year == year].dt.strftime("%y%m%d.%H%M"),
+                    **kwargs,
+                )
+            )
+
+    x_values = pd.date_range(start="2024-01-01", end="2024-12-31", freq="MS")
+
+    if df_new is not None and (compound, "conc") in df_new:
+        dt_new = df_new[dt_column]
         fig.add_trace(
             go.Scatter(
-                x=df_to_plot.index,
-                y=df_to_plot[year],
-                name=str(year),
-                zorder=-year,
-                text=dt[dt.dt.year == year].dt.strftime("%y%m%d.%H%M"),
+                x=dt_new.dt.dayofyear + dt_new.dt.hour / 24.0,
+                y=df_new[(compound, "conc")],
+                name="New Data",
+                text=dt_new.dt.strftime("%y%m%d.%H%M"),
                 **kwargs,
             )
         )
-    x_values = pd.date_range(start="2024-01-01", end="2024-12-31", freq="MS")
-
-    dt_new = df_new[dt_column]
-    fig.add_trace(
-        go.Scatter(
-            x=dt_new.dt.dayofyear + dt_new.dt.hour / 24.0,
-            y=df_new[(compound, "conc")],
-            name="New Data",
-            text=dt_new.dt.strftime("%y%m%d.%H%M"),
-            **kwargs,
-        )
-    )
     fig.update_layout(
         xaxis_title="Time of Year",
         yaxis_title=f"{compound} (ppt)",

avoca/qa_class/rolling.py

@@ -28,6 +28,9 @@ class RollingWindow(ExtremeValues):
     :param only_greater: If True, only values greater than the threshold will be flagged.
         The values lower than the negative threshold will not be flagged.
         By default, this is True if use_log_normal is True, and False otherwise.
+    :param rolling_window: The size of the rolling window as a `timedelta` object.
+        See `window` parameters in pandas documentation for more details.
+        https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.rolling.html#pandas-dataframe-rolling
     """
 
     require_datetime_index = True

avoca/utils/__init__.py

@@ -13,7 +13,7 @@ def compounds_from_df(df: pd.DataFrame) -> list[str]:
     Returns:
         The compounds in the dataframe.
     """
-    return [c for c in df.columns.get_level_values(0).unique() if c != "-"]
+    return [c for c in df.columns.get_level_values(0).unique() if c not in ["-", ""]]
 
 
 def runtypes_from_df(df: pd.DataFrame) -> list[str]:

avoca/utils/flags_doc.py

@@ -56,6 +56,9 @@ def parse_enum_comments(filepath: Path, enum_class_name: str) -> dict[Enum, str]
         exec(code, module)
         enum_cls = module[enum_class_name]
         for name, comment in comment_dict.items():
+            if not hasattr(enum_cls, name):
+                # Probably somehwere else in the file
+                continue
             enum_member = getattr(enum_cls, name)
             enum_obj[enum_member] = comment
 

{avoca-0.14.0.dist-info → avoca-0.15.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: avoca
-Version: 0.14.0
+Version: 0.15.0
 Summary: @voc@: Quality assessement of measurement data
 Project-URL: Homepage, https://gitlab.com/empa503/atmospheric-measurements/avoca
 Project-URL: Bug Tracker, https://gitlab.com/empa503/atmospheric-measurements/avoca/-/issues

{avoca-0.14.0.dist-info → avoca-0.15.0.dist-info}/RECORD

@@ -4,8 +4,8 @@ avoca/flagging.py,sha256=tg6k_TVHRXiMJCAij_kUS-S2gSshYt7FKvQ0nJdljYs,2328
 avoca/flags.py,sha256=wobuZoIJh6dFsdiqqYJLZ_AHe4pcFE9tjuoimNXLjIQ,1428
 avoca/io.py,sha256=67D5x1qkLqWC7wWehyOfX96L4H3-tn9x2V4jMCoIRqA,729
 avoca/logging.py,sha256=BrxgZQRfnkPSoQ0ZXhOzzhIsmbyjKvaJNG55MdM9jmA,86
-avoca/manager.py,sha256=ET-ATrSLi2rSV7PjBzwpjj0V_60MFxSIZqQ03aEIbdA,5284
-avoca/plots.py,sha256=UjfUgbfxd2veMOGHtSvJycru-w3gWsGjOVO__I-zqzQ,4205
+avoca/manager.py,sha256=Faf3UyaCV58TMCZz6tWrLcY-W1WUtuh1aMP85yUVlmQ,5336
+avoca/plots.py,sha256=zzoOJystasrKF2ikJLqcT8mlc-f-tu57vksXs-xRXv8,4424
 avoca/requirements.py,sha256=q4z6bJ6iW5jSy10Y0elfE9BoEcAZC2-kUqYi4zA6TGE,563
 avoca/settings.py,sha256=Px-sCGIlRyWI2RBJaGUY0K1V60kOZY9n41eft92bjN4,2112
 avoca/bindings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -15,24 +15,24 @@ avoca/bindings/gcwerks-report.conf,sha256=jO0I62DfgzrXXS1FuiW8ds-oc1_j8kpFCO61Fk
 avoca/bindings/gcwerks.py,sha256=a5n9Iot3r_ejnCEdILk4hE2uioONB75Soq5fvSLlDoo,14879
 avoca/bindings/gcwerks_gui.py,sha256=Fj3p8obFq3lWrWW0LlA8WBALP8-U70hvps5vZEt4NaM,9458
 avoca/bindings/nabel.py,sha256=VbC_ARvtso8onILAD8gROt5Y2URdx6NfAqMn4p1mUWU,3020
-avoca/bindings/qa_tool.py,sha256=ninHe3mrJ8GULxRCkRTZixw-vmNhqu4zwwONd5aXd1Q,9735
+avoca/bindings/qa_tool.py,sha256=hqsWUU99mYpkKfeULBoox4M2x7Bk0aYO4Q_8WGvt2og,11628
 avoca/bindings/synspec.py,sha256=W5RnBu-6eetmwjM8iMBe4wNwVNIaVpNW3bwa2ykGM2U,1733
 avoca/qa_class/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 avoca/qa_class/abstract.py,sha256=CLt-6WFhZhrvKTLVHpdbJYMFM50VPOGiO-GG6IRPWzA,6011
 avoca/qa_class/concs.py,sha256=TcQic69I1Kr40RJgCILTtyjVLn0K6_q6I5Y1Vi3dKwk,813
 avoca/qa_class/generate_classes_doc.py,sha256=osz01SRZ5SrwJXVlmbcainVwVjmealSSIdbzXzUEGKQ,1915
 avoca/qa_class/invalid.py,sha256=PDZHN0RZ8jND3QY09UcbwJYjjT6VqS4a0klO3QYiFig,2650
-avoca/qa_class/rolling.py,sha256=CQ2E0qJ7FxDT4TucItkJRmkqhzMoNSnwtVQQ_HzX9Jk,4059
+avoca/qa_class/rolling.py,sha256=m6KbfMdwSIDcXNTZqdth_I-YgZPnnUf8WAPSQGKBH6w,4324
 avoca/qa_class/rt.py,sha256=Bgv0DSSR-hIJ9kI6AdUkV6sXVS65gBxbASkk4TUHbnQ,5293
 avoca/qa_class/test.py,sha256=Xc88_Vwf3hvPiKKl4ILxZ2N985SY8eujUdnAoQu4mbo,591
 avoca/qa_class/zscore.py,sha256=jDw2UBmf7KBkskGOD5bgFy3RgNYUjc-9tYjSU-3L1ws,16714
 avoca/testing/__init__.py,sha256=CzkugadVit48-eMoMVtojZLHeSKgnmMMen6sGu6Q42Y,108
 avoca/testing/df.py,sha256=UQm6TdTDVRWvRNM5WnSWh6vdvDR1lqLNg0ti-B1L760,1865
 avoca/testing/utils.py,sha256=jVV0mIwLIpr0UBLMk8RjZH5J_dV_b6Gugxzo_WRgWU0,308
-avoca/utils/__init__.py,sha256=LEA2jJsqwSK2DBzXg00DbPhM1fXXREJ0XxLeuJtKapY,1398
-avoca/utils/flags_doc.py,sha256=cS7yKpxVh_SA6EdH3lSy9UpcIvhGwzAELRbkXN3CxO8,4168
+avoca/utils/__init__.py,sha256=SZc1bHrQyg1DIYnbdUmANtUhnQWlJaMhPrDSWS8oVRY,1408
+avoca/utils/flags_doc.py,sha256=jT1E0GN-B8ws_FyKGE20nlrKrgTHtoyjdo2r8RgYhwU,4294
 avoca/utils/torch_models.py,sha256=53TgOgSPMOOSGYy2cm1EGSK7qQkYMGEOq319KKM_Ir0,1015
-avoca-0.14.0.dist-info/METADATA,sha256=CayW94kozHUxF8sbKxE0pnWZnS0W5cjkEUKU7_QfgEc,1570
-avoca-0.14.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-avoca-0.14.0.dist-info/licenses/LICENCE.txt,sha256=4MY53j3v7tEKwjyuriVz9YjB4Dscm2nDMB2CcG9lOmk,1059
-avoca-0.14.0.dist-info/RECORD,,
+avoca-0.15.0.dist-info/METADATA,sha256=aJUaStu4uKLKeTarDTVyPTIflFiZ90OrcoktZKGRXWs,1570
+avoca-0.15.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+avoca-0.15.0.dist-info/licenses/LICENCE.txt,sha256=4MY53j3v7tEKwjyuriVz9YjB4Dscm2nDMB2CcG9lOmk,1059
+avoca-0.15.0.dist-info/RECORD,,