avoca 0.15.1__py3-none-any.whl → 0.17.0__py3-none-any.whl

avoca/bindings/ebas.py

@@ -19,6 +19,7 @@ from nilutility.datetime_helper import DatetimeInterval
 
 from avoca.bindings.ebas_flags import ebas_flag_to_avoca, flags_to_ebas, nan_flags
 from avoca.flags import QA_Flag
+from avoca.utils import compounds_from_df
 
 logger = logging.getLogger(__name__)
 
@@ -31,15 +32,29 @@ ebas_compname_of_var = {
 ebas_compname_to_var = {v: k for k, v in ebas_compname_of_var.items()}
 
 
-# Additional variables that can be in the dataset (not compound dependant)
-additional_vars = [
-    "temperature",
-    "pressure",
-]
-
 titles = {
     "temperature": "T_inlet",
     "pressure": "P_inlet",
+    "volume_sample": "svol",
+}
+
+unit_of_var = {
+    "C": "pmol/mol",
+    "conc_calib": "pmol/mol",
+    "rt": "s",
+    "w": "s",
+    "area": "area_unit",
+    "temperature": "K",
+    "pressure": "hPa",
+    "volume_sample": "mL",
+}
+
+ebas_varname_of_var = {
+    "rt": "rt",
+    "w": "pw",
+    "area": "pa",
+    "conc_calib": "cal",
+    "volume_sample": "sample_volume",
 }
 
 
@@ -51,6 +66,17 @@ class DataLevel(IntEnum):
     QA_CONCS = 2
 
 
+vars_to_export = {
+    DataLevel.AREAS: ["area", "rt", "w", "conc_calib"],
+    DataLevel.CONCS: ["C"],
+    DataLevel.QA_CONCS: ["C"],
+}
+
+# Additional variables that can be in the dataset (not compound dependant)
+additional_vars = {
+    DataLevel.AREAS: ["volume_sample", "temperature", "pressure"],
+}
+valid_additional_vars = sum(additional_vars.values(), [])
 concs_data_levels = [DataLevel.CONCS, DataLevel.QA_CONCS]
 
 
@@ -64,8 +90,8 @@ def data_level_after_qa(data_level: DataLevel) -> DataLevel:
 def set_dataframe(
     nas,
     df_export: pd.DataFrame,
-    compounds: dict[str, str],
-    data_level: DataLevel,
+    compounds: dict[str, str] | None = None,
+    data_level: DataLevel = DataLevel.CONCS,
     start_offset: timedelta | None = None,
     end_offset: timedelta | None = None,
     flag_all: list[int] = [],
@@ -108,28 +134,8 @@ def set_dataframe(
         )
     ]
 
-    vars_to_export = {
-        DataLevel.AREAS: ["area", "rt", "w", "conc_calib"],
-        DataLevel.CONCS: ["C"],
-        DataLevel.QA_CONCS: ["C"],
-    }
-
-    unit_of_var = {
-        "C": "pmol/mol",
-        "conc_calib": "pmol/mol",
-        "rt": "s",
-        "w": "s",
-        "area": "area_unit",
-        "temperature": "K",
-        "pressure": "hPa",
-    }
-
-    ebas_varname_of_var = {
-        "rt": "rt",
-        "w": "pw",
-        "area": "pa",
-        "conc_calib": "cal",
-    }
+    if compounds is None:
+        compounds = {c: c for c in compounds_from_df(df_export)}
 
     dict_flags_to_ebas = flags_to_ebas.copy()
 
@@ -151,12 +157,12 @@ def set_dataframe(
             )
         )
 
-    for var in additional_vars:
+    for var in additional_vars.get(data_level, []):
         var_col = ("-", var)
         if var_col not in df_export.columns:
             continue
         metadata = DataObject()
-        metadata.comp_name = var
+        metadata.comp_name = ebas_varname_of_var.get(var, var)
         metadata.title = titles.get(var, var)
         metadata.matrix = "instrument"
         metadata.unit = unit_of_var[var]
@@ -233,9 +239,9 @@ def set_dataframe(
             if var == "conc_calib":
                 # Set Nominal/measured=Calibration gas concentration
                 vnum = len(nas.variables) - 1
-                nas.add_var_characteristics(
-                    vnum, "Nominal/measured", "Calibration gas concentration"
-                )
+                # nas.add_var_characteristics(
+                #    vnum, "Nominal/measured", "Calibration gas concentration"
+                # )
 
             metadatas[sub] = metadata
     return metadatas
@@ -327,7 +333,10 @@ def nas_to_avoca(nas: EbasNasaAmes) -> pd.DataFrame:
             clean_for_df[("-", "status")] = calib_ids.astype(int)
             continue
 
-        if comp_name in additional_vars:
+        if comp_name == "sample_volume":
+            comp_name = "volume_sample"
+
+        if comp_name in valid_additional_vars:
             clean_for_df[("-", comp_name)] = np.array(values, dtype=float)
             continue
 
@@ -335,27 +344,27 @@ def nas_to_avoca(nas: EbasNasaAmes) -> pd.DataFrame:
         comp_name = comp_name.split("_")
         if len(comp_name) == 1:
             # Can be either concentration measured or calibration
-            compund = comp_name[0]
+            compound = comp_name[0]
             title: str = metadata["title"]
             if title.endswith("_cal"):
                 variable = "cal"
             else:
                 variable = "C"
         elif len(comp_name) == 2:
-            compund, variable = comp_name
+            compound, variable = comp_name
         elif len(comp_name) == 3:
-            compund, var_first, var_second = comp_name
+            compound, var_first, var_second = comp_name
             variable = f"{var_first}_{var_second}"
         elif len(comp_name) == 4 and comp_name[-1] == "compounds":
             # Concentration of merged compounds
-            compund = "_".join(comp_name)
+            compound = "_".join(comp_name)
             variable = "C"
         else:
             logger.warning(f"passing {comp_name}, could not be understood. Skipping.")
             continue
 
-        if compund not in compounds:
-            compounds.append(compund)
+        if compound not in compounds:
+            compounds.append(compound)
 
         # Convert the variable name to the avoca format
         if variable == "cal":
@@ -363,10 +372,12 @@ def nas_to_avoca(nas: EbasNasaAmes) -> pd.DataFrame:
             variable = "conc_calib"
         elif variable != "C":
             if variable not in ebas_compname_to_var:
-                raise ValueError(f"Variable {variable} not recognized")
+                raise ValueError(
+                    f"Variable {variable} from {comp_name=} not recognized"
+                )
             variable = ebas_compname_to_var[variable]
 
-        clean_for_df[(compund, variable)] = np.array(values, dtype=float)
+        clean_for_df[(compound, variable)] = np.array(values, dtype=float)
 
         flag_serie = pd.Series(
             [
@@ -375,7 +386,7 @@ def nas_to_avoca(nas: EbasNasaAmes) -> pd.DataFrame:
             ],
             dtype=int,
         )
-        flag_col = (compund, "flag")
+        flag_col = (compound, "flag")
         if variable == "conc_calib":
             # Calibration will have missing values for air smaples
             # so we need to remove the missing values

avoca/bindings/ebas_flags.py

@@ -9,6 +9,12 @@ flags_to_ebas: dict[QA_Flag, int] = {
     QA_Flag.EXTREME_VALUE: 458,
     QA_Flag.CALIBRATION: 683,  # 	I 	Invalid due to calibration. Used for Level 0.
     QA_Flag.BLANK: 684,  #  	Invalid due to zero/span check. Used for Level 0.
+    # Invalid due to laboratory standard measurement. Level 0.
+    QA_Flag.LABORATORY_STANDARD: 688,
+    # Invalid due to working standard measurement. Level 0.
+    QA_Flag.WORKING_STANDARD: 689,
+    # Invalid due to target standard measurement. Level 0.
+    QA_Flag.TARGET_MEASUREMENT: 690,
     QA_Flag.HEIGHT_INTEGRATION: 0,  # 	Valid
     QA_Flag.UNCORRELATED: 0,  # 	Valid
     QA_Flag.MET_OFFICE_BASELINE: 0,  # 	Valid

avoca/bindings/gcwerks.py

@@ -90,6 +90,8 @@ flag_values = {
     "F": QA_Flag.INVALIDATED_EXT,
     # X: An X flag is an 'un-do' the flag. If there is an automatic flag by GCWerks, but I decide I want that data point still included, I have the option to set an X flag.
     "X": ValidFlag,
+    # Nans read from pandas 
+    pd.NA: QA_Flag.MISSING,
 }
 
 # Show the flags and the columns they are applied to
@@ -203,6 +205,11 @@ def read_gcwerks(
         format=datetime_format,
     )
 
+    if not df[("-", "volume")].isna().all():
+        df[("-", "volume_sample")] = df[("-", "volume")]
+    # Drop useless columns
+    df = df.drop(columns=[("-", "date"), ("-", "time"), ("-", "volume")])
+
     substances = []
 
     for col in df.columns:
@@ -224,9 +231,9 @@ def read_gcwerks(
         flags: pd.Series = serie_str.str[-1]
         if col[1] in cols_float:
             # Remove the flag value when given
-            serie_str = serie_str.apply(
-                lambda x: x[:-1] if x[-1] in flags_allowed else x
-            )
+            mask_flag_allowed = flags.isin(flags_allowed)
+            serie_str = serie_str.where(~mask_flag_allowed, serie_str.str[:-1])
+
             # Convert the serie to numeric
             df[col] = pd.to_numeric(serie_str, errors="coerce")
 
@@ -359,6 +366,7 @@ def export(
             "time",
             "type",
             "sample",
+            "volume",
             f"{variables_str}",
             f"> {out_file}",
         )

avoca/flags.py

@@ -46,6 +46,14 @@ class QA_Flag(Flag):
     # Invalid Values
     INVALID_VALUES = auto()
 
+    # Target measurement
+    TARGET_MEASUREMENT = auto()
+    # Laboratory standard
+    LABORATORY_STANDARD = auto()
+    # Working standard
+    WORKING_STANDARD = auto()
+
+
 # Flags that are considered to have missing values
 nan_flags = [
     QA_Flag.MISSING,

avoca/io.py

@@ -8,6 +8,11 @@ date_format = "%Y-%m-%d %H:%M:%S"
 
 def to_csv(df: pd.DataFrame, path: Path, **kwargs) -> None:
     """Export a dataframe to a csv file."""
+
+    # Put the columsn with "-" first
+    cols = df.columns.tolist()
+    cols_sorted = sorted(cols, key=lambda x: (x[0] != "-", x))
+    df = df[cols_sorted]
     df.to_csv(path, index=False, date_format=date_format, **kwargs)
 
 

avoca/testing/utils.py

@@ -3,7 +3,7 @@ import pandas as pd
 
 def make_dt_index(df: pd.DataFrame | pd.Index) -> pd.DataFrame | pd.Index:
     """Create a datetime index for the dataframe."""
-    index = pd.date_range(start="2023-01-01", periods=len(df), freq="h")
+    index = pd.date_range(start="2023-01-01", periods=len(df), freq="h", unit="s")
     if isinstance(df, pd.Index):
         return index
     return df.set_index(index)

{avoca-0.15.1.dist-info → avoca-0.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: avoca
-Version: 0.15.1
+Version: 0.17.0
 Summary: @voc@: Quality assessement of measurement data
 Project-URL: Homepage, https://gitlab.com/empa503/atmospheric-measurements/avoca
 Project-URL: Bug Tracker, https://gitlab.com/empa503/atmospheric-measurements/avoca/-/issues

{avoca-0.15.1.dist-info → avoca-0.17.0.dist-info}/RECORD

@@ -1,18 +1,18 @@
 avoca/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 avoca/export_nas.py,sha256=B9B2iFSzB3f83nCfe2_vzouRblthK0_dGF8W3o0Kt5Y,155
 avoca/flagging.py,sha256=tg6k_TVHRXiMJCAij_kUS-S2gSshYt7FKvQ0nJdljYs,2328
-avoca/flags.py,sha256=wobuZoIJh6dFsdiqqYJLZ_AHe4pcFE9tjuoimNXLjIQ,1428
-avoca/io.py,sha256=67D5x1qkLqWC7wWehyOfX96L4H3-tn9x2V4jMCoIRqA,729
+avoca/flags.py,sha256=9LF-e8bcUdBQmxtoXU3ysx7KRzZiU6bU6nhDLhZtowU,1599
+avoca/io.py,sha256=VMGqSPdtPM5Xu4kugMbr6TaMS9-U6pnObMu2ERKhNxE,891
 avoca/logging.py,sha256=BrxgZQRfnkPSoQ0ZXhOzzhIsmbyjKvaJNG55MdM9jmA,86
 avoca/manager.py,sha256=Faf3UyaCV58TMCZz6tWrLcY-W1WUtuh1aMP85yUVlmQ,5336
 avoca/plots.py,sha256=zzoOJystasrKF2ikJLqcT8mlc-f-tu57vksXs-xRXv8,4424
 avoca/requirements.py,sha256=q4z6bJ6iW5jSy10Y0elfE9BoEcAZC2-kUqYi4zA6TGE,563
 avoca/settings.py,sha256=Px-sCGIlRyWI2RBJaGUY0K1V60kOZY9n41eft92bjN4,2112
 avoca/bindings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-avoca/bindings/ebas.py,sha256=vil4u4G6jGJrE12Z7nBvGpJuTAT9QyvbNNyWsWr5UaM,19306
-avoca/bindings/ebas_flags.py,sha256=N-JpmA6WCFjcYhvt7XjyOZMbR7vCdyPV6uHBlF45UJU,2397
+avoca/bindings/ebas.py,sha256=3oWgN3teyEvQ9acpD2767A18IbgxjtbOdmXD79PLVrE,19793
+avoca/bindings/ebas_flags.py,sha256=TEkmOI9Bia0C2KFO5GqLFXNIvcLak5yedlBFCKY5Gqg,2695
 avoca/bindings/gcwerks-report.conf,sha256=jO0I62DfgzrXXS1FuiW8ds-oc1_j8kpFCO61Fk-erBw,230
-avoca/bindings/gcwerks.py,sha256=a5n9Iot3r_ejnCEdILk4hE2uioONB75Soq5fvSLlDoo,14879
+avoca/bindings/gcwerks.py,sha256=2Keff174EUjRwbBRDNKwGFEMr6TxJ2mpsjIN71DjUsQ,15188
 avoca/bindings/gcwerks_gui.py,sha256=Fj3p8obFq3lWrWW0LlA8WBALP8-U70hvps5vZEt4NaM,9458
 avoca/bindings/nabel.py,sha256=6OzaG1imFhOCVDQTO7YXvPQjbTfo4063w74yEuAVCEk,2991
 avoca/bindings/qa_tool.py,sha256=hqsWUU99mYpkKfeULBoox4M2x7Bk0aYO4Q_8WGvt2og,11628
@@ -28,11 +28,11 @@ avoca/qa_class/test.py,sha256=Xc88_Vwf3hvPiKKl4ILxZ2N985SY8eujUdnAoQu4mbo,591
 avoca/qa_class/zscore.py,sha256=jDw2UBmf7KBkskGOD5bgFy3RgNYUjc-9tYjSU-3L1ws,16714
 avoca/testing/__init__.py,sha256=CzkugadVit48-eMoMVtojZLHeSKgnmMMen6sGu6Q42Y,108
 avoca/testing/df.py,sha256=UQm6TdTDVRWvRNM5WnSWh6vdvDR1lqLNg0ti-B1L760,1865
-avoca/testing/utils.py,sha256=jVV0mIwLIpr0UBLMk8RjZH5J_dV_b6Gugxzo_WRgWU0,308
+avoca/testing/utils.py,sha256=w0i-x3xG40JMxhoV_odVV7995VMKpUsHRYVJk2XYJ7I,318
 avoca/utils/__init__.py,sha256=SZc1bHrQyg1DIYnbdUmANtUhnQWlJaMhPrDSWS8oVRY,1408
 avoca/utils/flags_doc.py,sha256=jT1E0GN-B8ws_FyKGE20nlrKrgTHtoyjdo2r8RgYhwU,4294
 avoca/utils/torch_models.py,sha256=53TgOgSPMOOSGYy2cm1EGSK7qQkYMGEOq319KKM_Ir0,1015
-avoca-0.15.1.dist-info/METADATA,sha256=_KHPT1eKCbb2WV9ujy1-wLEDVhFuQKqUHIHm_ylHh4w,1570
-avoca-0.15.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-avoca-0.15.1.dist-info/licenses/LICENCE.txt,sha256=4MY53j3v7tEKwjyuriVz9YjB4Dscm2nDMB2CcG9lOmk,1059
-avoca-0.15.1.dist-info/RECORD,,
+avoca-0.17.0.dist-info/METADATA,sha256=1V1osu38cBEsVtCqP4wIbe37PT4LlKv56jqJZk1OI38,1570
+avoca-0.17.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+avoca-0.17.0.dist-info/licenses/LICENCE.txt,sha256=4MY53j3v7tEKwjyuriVz9YjB4Dscm2nDMB2CcG9lOmk,1059
+avoca-0.17.0.dist-info/RECORD,,