avoca 0.15.1__tar.gz → 0.16.0__tar.gz

{avoca-0.15.1 → avoca-0.16.0}/.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 cases
 
+.env/
 .mypy_cache
 docs/build
 
@@ -14,3 +15,4 @@ dist/
 # Generated by pytests
 simple_df.csv
 data/tests/export_empa_qa_tool/*.csv
+data/tests/new_NMHC_lev0/

{avoca-0.15.1 → avoca-0.16.0}/.gitlab-ci.yml

@@ -42,7 +42,7 @@ unit-test-job:   # This job runs in the test stage.
   script:
     - echo "Running unit tests..."
     - pip install .
-    - pip install pytest
+    - pip install pytest ebas-io
     - python -m pytest
     - echo "Test done"
 

{avoca-0.15.1 → avoca-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: avoca
-Version: 0.15.1
+Version: 0.16.0
 Summary: @voc@: Quality assessement of measurement data
 Project-URL: Homepage, https://gitlab.com/empa503/atmospheric-measurements/avoca
 Project-URL: Bug Tracker, https://gitlab.com/empa503/atmospheric-measurements/avoca/-/issues

{avoca-0.15.1 → avoca-0.16.0}/avoca/bindings/ebas.py

@@ -19,6 +19,7 @@ from nilutility.datetime_helper import DatetimeInterval
 
 from avoca.bindings.ebas_flags import ebas_flag_to_avoca, flags_to_ebas, nan_flags
 from avoca.flags import QA_Flag
+from avoca.utils import compounds_from_df
 
 logger = logging.getLogger(__name__)
 
@@ -31,15 +32,29 @@ ebas_compname_of_var = {
 ebas_compname_to_var = {v: k for k, v in ebas_compname_of_var.items()}
 
 
-# Additional variables that can be in the dataset (not compound dependant)
-additional_vars = [
-    "temperature",
-    "pressure",
-]
-
 titles = {
     "temperature": "T_inlet",
     "pressure": "P_inlet",
+    "volume_sample": "svol",
+}
+
+unit_of_var = {
+    "C": "pmol/mol",
+    "conc_calib": "pmol/mol",
+    "rt": "s",
+    "w": "s",
+    "area": "area_unit",
+    "temperature": "K",
+    "pressure": "hPa",
+    "volume_sample": "mL",
+}
+
+ebas_varname_of_var = {
+    "rt": "rt",
+    "w": "pw",
+    "area": "pa",
+    "conc_calib": "cal",
+    "volume_sample": "sample_volume",
 }
 
 
@@ -51,6 +66,17 @@ class DataLevel(IntEnum):
     QA_CONCS = 2
 
 
+vars_to_export = {
+    DataLevel.AREAS: ["area", "rt", "w", "conc_calib"],
+    DataLevel.CONCS: ["C"],
+    DataLevel.QA_CONCS: ["C"],
+}
+
+# Additional variables that can be in the dataset (not compound dependant)
+additional_vars = {
+    DataLevel.AREAS: ["volume_sample", "temperature", "pressure"],
+}
+valid_additional_vars = sum(additional_vars.values(), [])
 concs_data_levels = [DataLevel.CONCS, DataLevel.QA_CONCS]
 
 
@@ -64,8 +90,8 @@ def data_level_after_qa(data_level: DataLevel) -> DataLevel:
 def set_dataframe(
     nas,
     df_export: pd.DataFrame,
-    compounds: dict[str, str],
-    data_level: DataLevel,
+    compounds: dict[str, str] | None = None,
+    data_level: DataLevel = DataLevel.CONCS,
     start_offset: timedelta | None = None,
     end_offset: timedelta | None = None,
     flag_all: list[int] = [],
@@ -108,28 +134,8 @@ def set_dataframe(
         )
     ]
 
-    vars_to_export = {
-        DataLevel.AREAS: ["area", "rt", "w", "conc_calib"],
-        DataLevel.CONCS: ["C"],
-        DataLevel.QA_CONCS: ["C"],
-    }
-
-    unit_of_var = {
-        "C": "pmol/mol",
-        "conc_calib": "pmol/mol",
-        "rt": "s",
-        "w": "s",
-        "area": "area_unit",
-        "temperature": "K",
-        "pressure": "hPa",
-    }
-
-    ebas_varname_of_var = {
-        "rt": "rt",
-        "w": "pw",
-        "area": "pa",
-        "conc_calib": "cal",
-    }
+    if compounds is None:
+        compounds = {c: c for c in compounds_from_df(df_export)}
 
     dict_flags_to_ebas = flags_to_ebas.copy()
 
@@ -151,12 +157,12 @@ def set_dataframe(
             )
         )
 
-    for var in additional_vars:
+    for var in additional_vars.get(data_level, []):
         var_col = ("-", var)
         if var_col not in df_export.columns:
             continue
         metadata = DataObject()
-        metadata.comp_name = var
+        metadata.comp_name = ebas_varname_of_var.get(var, var)
         metadata.title = titles.get(var, var)
         metadata.matrix = "instrument"
         metadata.unit = unit_of_var[var]
@@ -233,9 +239,9 @@ def set_dataframe(
             if var == "conc_calib":
                 # Set Nominal/measured=Calibration gas concentration
                 vnum = len(nas.variables) - 1
-                nas.add_var_characteristics(
-                    vnum, "Nominal/measured", "Calibration gas concentration"
-                )
+                # nas.add_var_characteristics(
+                #    vnum, "Nominal/measured", "Calibration gas concentration"
+                # )
 
             metadatas[sub] = metadata
     return metadatas
@@ -327,7 +333,10 @@ def nas_to_avoca(nas: EbasNasaAmes) -> pd.DataFrame:
             clean_for_df[("-", "status")] = calib_ids.astype(int)
             continue
 
-        if comp_name in additional_vars:
+        if comp_name == "sample_volume":
+            comp_name = "volume_sample"
+
+        if comp_name in valid_additional_vars:
             clean_for_df[("-", comp_name)] = np.array(values, dtype=float)
             continue
 
@@ -335,27 +344,27 @@ def nas_to_avoca(nas: EbasNasaAmes) -> pd.DataFrame:
         comp_name = comp_name.split("_")
         if len(comp_name) == 1:
             # Can be either concentration measured or calibration
-            compund = comp_name[0]
+            compound = comp_name[0]
             title: str = metadata["title"]
             if title.endswith("_cal"):
                 variable = "cal"
             else:
                 variable = "C"
         elif len(comp_name) == 2:
-            compund, variable = comp_name
+            compound, variable = comp_name
         elif len(comp_name) == 3:
-            compund, var_first, var_second = comp_name
+            compound, var_first, var_second = comp_name
             variable = f"{var_first}_{var_second}"
         elif len(comp_name) == 4 and comp_name[-1] == "compounds":
             # Concentration of merged compounds
-            compund = "_".join(comp_name)
+            compound = "_".join(comp_name)
             variable = "C"
         else:
             logger.warning(f"passing {comp_name}, could not be understood. Skipping.")
             continue
 
-        if compund not in compounds:
-            compounds.append(compund)
+        if compound not in compounds:
+            compounds.append(compound)
 
         # Convert the variable name to the avoca format
         if variable == "cal":
@@ -363,10 +372,12 @@ def nas_to_avoca(nas: EbasNasaAmes) -> pd.DataFrame:
             variable = "conc_calib"
         elif variable != "C":
             if variable not in ebas_compname_to_var:
-                raise ValueError(f"Variable {variable} not recognized")
+                raise ValueError(
+                    f"Variable {variable} from {comp_name=} not recognized"
+                )
             variable = ebas_compname_to_var[variable]
 
-        clean_for_df[(compund, variable)] = np.array(values, dtype=float)
+        clean_for_df[(compound, variable)] = np.array(values, dtype=float)
 
         flag_serie = pd.Series(
             [
@@ -375,7 +386,7 @@ def nas_to_avoca(nas: EbasNasaAmes) -> pd.DataFrame:
             ],
             dtype=int,
         )
-        flag_col = (compund, "flag")
+        flag_col = (compound, "flag")
         if variable == "conc_calib":
             # Calibration will have missing values for air smaples
             # so we need to remove the missing values

{avoca-0.15.1 → avoca-0.16.0}/avoca/bindings/ebas_flags.py

@@ -9,6 +9,12 @@ flags_to_ebas: dict[QA_Flag, int] = {
     QA_Flag.EXTREME_VALUE: 458,
     QA_Flag.CALIBRATION: 683,  # 	I 	Invalid due to calibration. Used for Level 0.
     QA_Flag.BLANK: 684,  #  	Invalid due to zero/span check. Used for Level 0.
+    # Invalid due to laboratory standard measurement. Level 0.
+    QA_Flag.LABORATORY_STANDARD: 688,
+    # Invalid due to working standard measurement. Level 0.
+    QA_Flag.WORKING_STANDARD: 689,
+    # Invalid due to target standard measurement. Level 0.
+    QA_Flag.TARGET_MEASUREMENT: 690,
     QA_Flag.HEIGHT_INTEGRATION: 0,  # 	Valid
     QA_Flag.UNCORRELATED: 0,  # 	Valid
     QA_Flag.MET_OFFICE_BASELINE: 0,  # 	Valid

{avoca-0.15.1 → avoca-0.16.0}/avoca/bindings/gcwerks.py

@@ -203,6 +203,11 @@ def read_gcwerks(
         format=datetime_format,
     )
 
+    if not df[("-", "volume")].isna().all():
+        df[("-", "volume_sample")] = df[("-", "volume")]
+    # Drop useless columns
+    df = df.drop(columns=[("-", "date"), ("-", "time"), ("-", "volume")])
+
     substances = []
 
     for col in df.columns:
@@ -359,6 +364,7 @@ def export(
             "time",
             "type",
             "sample",
+            "volume",
             f"{variables_str}",
             f"> {out_file}",
         )

{avoca-0.15.1 → avoca-0.16.0}/avoca/flags.py

@@ -46,6 +46,14 @@ class QA_Flag(Flag):
     # Invalid Values
     INVALID_VALUES = auto()
 
+    # Target measurement
+    TARGET_MEASUREMENT = auto()
+    # Laboratory standard
+    LABORATORY_STANDARD = auto()
+    # Working standard
+    WORKING_STANDARD = auto()
+
+
 # Flags that are considered to have missing values
 nan_flags = [
     QA_Flag.MISSING,

avoca-0.16.0/data/tests/new_NMHC_lev0.nas

@@ -0,0 +1,116 @@
+95 1001
+Constantin, Lionel; Vollmer, Martin K.; Reimann, Stefan
+CH01L, Swiss Federal Laboratories for Materials Science and Technology, EMPA, Laboratory for Air Pollution, Überlandstrasse 129, , 8600, Dübendorf, Switzerland
+Constantin, Lionel
+ACTRIS
+1 1
+2024 01 01 2024 02 20
+0
+days from file reference point
+19
+1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
+99.999999 99 99.9 9.999 999999 99.99 99999.9 9999.9999 9.999 999999 99.99 99999.9 9999.99999 9.999 9999999 99.99 99999.9 999.99999 9.999
+end_time of measurement, days from the file reference point
+status, no unit, Status type=calibration standard, Matrix=instrument, Comment=See metadata elements "Calibration standard ID" and "Secondary standard ID"
+sample_volume, ml, QA1 measure ID=, QA1 date=, QA1 document URL=
+numflag, no unit
+2-methylbutane_peak_area, area_unit
+2-methylbutane_peak_width, s
+2-methylbutane_retention_time, s
+2-methylbutane, pmol/mol, Nominal/measured=Calibration gas concentration
+numflag, no unit
+2-methylpropane_peak_area, area_unit
+2-methylpropane_peak_width, s
+2-methylpropane_retention_time, s
+2-methylpropane, pmol/mol, Nominal/measured=Calibration gas concentration
+numflag, no unit
+benzene_peak_area, area_unit
+benzene_peak_width, s
+benzene_retention_time, s
+benzene, pmol/mol, Nominal/measured=Calibration gas concentration
+numflag, no unit
+0
+62
+Data definition:              EBAS_1.1
+Set type code:                TI
+Timezone:                     UTC
+File name:                    CH0001G.20240219123300.20240220160444.online_gc.NMHC.air.1402mn.61mn.CH01L_Agilent_GC-MS-MEDUSA_Medusa-12_JFJ.CH01L_gc_ms.lev0.nas
+Startdate:                    20240219123300
+Revision date:                20240220160444
+Version:                      1
+Version description:          initial revision, automatically generated, manually inspected
+Statistics:                   arithmetic mean
+Data level:                   0
+Period code:                  1y
+Resolution code:              61mn
+Sample duration:              20mn
+Orig. time res.:              30mn
+Station code:                 CH0001G
+Platform code:                CH0001S
+Station name:                 Jungfraujoch
+Station WDCA-ID:              GAWACH__JFJ
+Station GAW-ID:               JFJ
+Station GAW-Name:             Jungfraujoch
+Station other IDs:            
+Station land use:             Gravel and stone
+Station setting:              Mountain
+Station GAW type:             G
+Station WMO region:           6
+Station latitude:             46.547500
+Station longitude:             7.985000
+Station altitude:             3578.0 m
+Measurement altitude:         3566.0 m
+Measurement height:           7.00 m
+Regime:                       IMG
+Component:                    NMHC
+Unit:                         
+Matrix:                       air
+Laboratory code:              CH01L
+Instrument type:              online_gc
+Instrument name:              Agilent_GC-MS-MEDUSA_Medusa-12_JFJ
+Instrument manufacturer:      AGAGE+Agilent+Agilent
+Instrument model:             MEDUSA+6890N+5975B
+Instrument serial number:     Medusa-12
+Method ref:                   CH01L_gc_ms
+Standard method:              SOP=ACTRIS_VOC_2014
+Calibration scale:              
+Calibration standard ID:      "Status calibration standard: 1, Manufacturer: NPL (passivated aluminum cylinder), Batch: E-223"
+Secondary standard ID:        "Status calibration standard: 2, Manufacturer: In House Aluminium cylinder (Luxfer), Batch: Referenzluft_15; Status calibration standard: 3, Manufacturer: In House Aluminium cylinder (Luxfer), Batch: Referenzluft_04"
+Inlet type:                   Hat or hood
+Inlet description:
+Flow rate:                    10.0 l/min
+Detection limit expl.:        Detection limit equals 3 times the precision
+Measurement uncertainty expl.: "Includes reproducibility (precision) + systematic errors (accuracy). Ref:http://www.atmos-meas-tech.net/8/2715/2015/"
+Zero/negative values code:    Zero/negative impossible
+Zero/negative values:         Values below 1 pmol/mol are not reported by the instrument.
+QA1 measure ID:               ACTRIS-VOC-1
+QA1 date:                     20120101000000
+QA1 document URL:             "https://doi.org/10.5194/amt-8-2715-2015"
+Originator:                   Constantin, Lionel, lionel.constantin@empa.ch, Swiss Federal Laboratories for Materials Science and Technology, EMPA, Laboratory for Air Pollution, Überlandstrasse 129, , 8600, Dübendorf, Switzerland
+Originator:                   Vollmer, Martin K., martin.vollmer@empa.ch, Swiss Federal Laboratories for Materials Science and Technology, EMPA, Laboratory for Air Pollution, Überlandstrasse 129, , 8600, Dübendorf, Switzerland
+Originator:                   Reimann, Stefan, stefan.reimann@empa.ch, Swiss Federal Laboratories for Materials Science and Technology, EMPA, Laboratory for Air Pollution, Überlandstrasse 129, , 8600, Dübendorf, Switzerland
+Submitter:                    Constantin, Lionel, lionel.constantin@empa.ch, Swiss Federal Laboratories for Materials Science and Technology, EMPA, Laboratory for Air Pollution, Überlandstrasse 129, , 8600, Dübendorf, Switzerland
+Acknowledgement:              Request acknowledgement details from data originator
+Comment:                       
+ starttime    endtime cal  svol   flag C5H12_pa C5H12_pw C5H12_rt C5H12_cal  flag C4H10_pa C4H10_pw C4H10_rt  C4H10_cal  flag C6H6_pa C6H6_pw C6H6_rt  C6H6_cal  flag
+ 49.522917  49.536806   1   1.0  0.688    88741     1.83   1384.3  150.6571 0.688    40606     2.16   1286.5  114.34055 0.688  431786    1.85   1481.1  81.37975  0.688
+ 49.565278  49.579167   0   1.0  0.000    61563     1.82   1384.3 9999.9999 0.000    70360     2.11   1286.5 9999.99999 0.000  454651    1.96   1481.1 999.99999  0.000
+ 49.607639  49.621528   0   1.0  0.000    67381     1.84   1384.3 9999.9999 0.000    77391     2.15   1286.4 9999.99999 0.000  461898    1.95   1481.1 999.99999  0.000
+ 49.687500  49.701389   2   1.0  0.689    88678     1.79   1384.3   25.2632 0.689    40694     2.15   1286.5   40.11231 0.689  430971    1.91   1481.1  42.32401  0.689
+ 49.729861  49.743750   0   1.0  0.000    60136     1.78   1384.3 9999.9999 0.000    72347     2.12   1286.5 9999.99999 0.000  443753    1.95   1481.1 999.99999  0.000
+ 49.772222  49.786111   0   1.0  0.000    51128     1.81   1384.3 9999.9999 0.000    62677     2.15   1286.5 9999.99999 0.000  414173    1.94   1481.1 999.99999  0.000
+ 49.814583  49.828472   2   1.0  0.689    88719     1.80   1384.3   25.2632 0.689    40407     2.16   1286.5    1.11231 0.689  431761    1.92   1481.2  42.32401  0.689
+ 49.856944  49.870833   0   1.0  0.000    62053     1.86   1384.3 9999.9999 0.000    69984     2.14   1286.5 9999.99999 0.000  452174    1.89   1481.1 999.99999  0.000
+ 49.899306  49.913194   0   1.0  0.000    54817     1.82   1384.3 9999.9999 0.000    64974     2.20   1286.5 9999.99999 0.000  422200    1.92   1481.1 999.99999  0.000
+ 49.979167  49.993056   2   1.0  0.689    87896     1.76   1384.3   25.2632 0.689    40750     2.19   1286.5   40.11231 0.689  430206    1.90   1481.2  42.32401  0.689
+ 50.021528  50.035417   0   1.0  0.000    53367     1.80   1384.3 9999.9999 0.000    62509     2.14   1286.5 9999.99999 0.000  444104    1.92   1481.2 999.99999  0.000
+ 50.063889  50.077778   0   1.0  0.000    50587     1.83   1384.3 9999.9999 0.000    61325     2.18   1286.5 9999.99999 0.000  428892    1.90   1481.1 999.99999  0.000
+ 50.106250  50.120139   3   1.0  0.689    88477     1.84   1384.3   22.2100 0.689    40254     2.14   1286.5   51.11830 0.689  426318    1.92   1481.2  45.77210  0.689
+ 50.148611  50.162500   0   1.0  0.000    52307     1.82   1384.3 9999.9999 0.000    62443     2.17   1286.5 9999.99999 0.000  434506    1.91   1481.1 999.99999  0.000
+ 50.190972  50.204861   0   1.0  0.000    57199     1.83   1384.4 9999.9999 0.000    66991     2.23   1286.5 9999.99999 0.000  434542    1.93   1481.2 999.99999  0.000
+ 50.270833  50.284722   3   1.0  0.689    88520     1.81   1384.3   22.2100 0.689    40429     2.17   1286.5   51.11830 0.689  428859    1.85   1481.1  45.77210  0.689
+ 50.313194  50.327083   0   1.0  0.000    69188     1.89   1384.3 9999.9999 0.000    76958     2.11   1286.5 9999.99999 0.000  457871    1.95   1481.1 999.99999  0.000
+ 50.355556  50.369444   0   1.0  0.000    66581     1.82   1384.4 9999.9999 0.000    76772     2.16   1286.5 9999.99999 0.000  455124    1.91   1481.2 999.99999  0.000
+ 50.397917  50.411806   3   1.0  0.690    86968     1.83   1384.3   22.2100 0.690    40311     2.16   1286.5   51.11830 0.690  425819    1.90   1481.1  45.77210  0.690
+ 50.440278  50.454167   0   1.0  0.000    55852     1.76   1384.3 9999.9999 0.000    68588     2.17   1286.5 9999.99999 0.000  416956    1.94   1481.2 999.99999  0.000
+ 50.482639  50.496528  99   1.0  0.000   999999    99.99  99999.9 9999.9999 0.999   999999    99.99  99999.9 9999.99999 0.999 9999999   99.99  99999.9 999.99999  0.999

{avoca-0.15.1 → avoca-0.16.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "avoca"
-version = "0.15.1"
+version = "0.16.0"
 authors = [
   { name="Lionel Constantin", email="lionel.constantin@empa.ch" },
 ]

avoca-0.16.0/tests/bindings/gcwerks.dat

@@ -0,0 +1,7 @@
+Created:  6 May 21 07:53 GMT
+     -    -      -       -   methane    ethane    ethene   propane   propene
+  date time volume    type      area      area      area      area      area
+210101 0025    1.0    air   1506527    187333     54226    124023      9004
+210101 0126    1.1    air   1532954    191969     72289    124942     14789
+210101 0226    1.0    air   1445250    175597     39373    104478      6643
+210101 0327    0.9    air   1434459    171817     38493     99796      5926

avoca-0.16.0/tests/bindings/test_ebas.py

@@ -0,0 +1,59 @@
+from ebas.io.file.nasa_ames import EbasNasaAmes
+
+from avoca.bindings.ebas import DataLevel, nas_to_avoca, set_dataframe
+from avoca.testing import testdata_dir
+from avoca.utils import compounds_from_df
+
+
+def test_read_lev0_template():
+
+    filepath = testdata_dir / "new_NMHC_lev0.nas"
+
+    nas = EbasNasaAmes()
+    nas.read(filepath)
+
+    df = nas_to_avoca(nas)
+
+    print(df.columns)
+
+    assert ("-", "start_datetime") in df.columns
+    assert ("-", "end_datetime") in df.columns
+
+    compounds = compounds_from_df(df)
+    expected_compounds = [
+        "2-methylbutane",
+        "2-methylpropane",
+        "benzene",
+    ]
+    assert sorted(compounds) == sorted(expected_compounds)
+
+
+def test_given_nas_is_valid_to_write():
+    filepath = testdata_dir / "new_NMHC_lev0.nas"
+
+    nas = EbasNasaAmes()
+    nas.read(filepath)
+    outputs_dir = filepath.with_suffix("")
+    outputs_dir.mkdir(exist_ok=True)
+    nas.write(createfiles=True, destdir=outputs_dir)
+
+
+def test_read_and_write():
+    filepath = testdata_dir / "new_NMHC_lev0.nas"
+
+    nas = EbasNasaAmes()
+    nas.read(filepath)
+
+    df = nas_to_avoca(nas)
+
+    new_nas = EbasNasaAmes()
+
+    metadatas = set_dataframe(
+        new_nas,
+        df,
+        data_level=DataLevel.AREAS,
+    )
+
+    outputs_dir = filepath.with_suffix("")
+    outputs_dir.mkdir(exist_ok=True)
+    new_nas.write(createfiles=True, destdir=outputs_dir)

avoca-0.15.1/tests/bindings/gcwerks.dat

@@ -1,7 +0,0 @@
-Created:  6 May 21 07:53 GMT
-     -    -             -   methane    ethane    ethene   propane   propene
-  date time          type      area      area      area      area      area
-210101 0025           air   1506527    187333     54226    124023      9004
-210101 0126           air   1532954    191969     72289    124942     14789
-210101 0226           air   1445250    175597     39373    104478      6643
-210101 0327           air   1434459    171817     38493     99796      5926