apmtools 0.2.31__tar.gz → 0.2.33__tar.gz

{apmtools-0.2.31 → apmtools-0.2.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: apmtools
-Version: 0.2.31
+Version: 0.2.33
 Summary: A collection of tools for processing air pollution monitoring data
 Project-URL: Homepage, https://github.com/federlorenz/apmtools
 Author-email: Federico Lorenzetti <lorenzetti.federico@gmail.com>

{apmtools-0.2.31 → apmtools-0.2.33}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apmtools"
-version = "0.2.31"
+version = "0.2.33"
 authors = [
   { name="Federico Lorenzetti", email="lorenzetti.federico@gmail.com" },
 ]

{apmtools-0.2.31 → apmtools-0.2.33}/src/apmtools/classes.py

@@ -8,7 +8,7 @@ from io import StringIO
 from csv import writer
 from dateutil.parser import parse
 
-from .units import upas_units,lascar_units,purple_units
+from .units import upas_units, lascar_units, purple_units, lcm_units
 
 class DictionaryPlus(dict):
     def __init__(self, *args, **kwargs):
@@ -451,11 +451,11 @@ class Dataset(DictionaryPlus):
 
     def scan_folder(self, directory="./", levels=[], level=0, monitor=None, levels_dict=None, gmt_timezone_shift=0, interpolate=False, add_m={}):
 
-        from .data_processing import upas_processing, lascar_processing, purple_processing
+        from .data_processing import upas_processing, lascar_processing, purple_processing, lcm_processing
 
-        if len(set(levels).intersection(set(["identifier", "start", "end", "length", "time", "pm25", "pm25correctedrh", "pm25correctedrhandgrav", "filterid", "grav_not", "grav"]))) > 0:
+        if len(set(levels).intersection(set(["identifier", "start", "end", "length", "time", "pm25", "pm25correctedrh", "pm25correctedrhandgrav", "filterid", "grav_not", "grav","PM2.5","PM10","C02","NO2"]))) > 0:
             print(
-                f"the following levels names are not allowed: {set(levels).intersection(set(["identifier", "start", "end", "length", "time", "pm25", "pm25correctedrh", "pm25correctedrhandgrav", "filterid", "grav_not", "grav"]))}. Please choose different level names")
+                f"the following levels names are not allowed: {set(levels).intersection(set(["identifier", "start", "end", "length", "time", "pm25", "pm25correctedrh", "pm25correctedrhandgrav", "filterid", "grav_not", "grav","PM2.5","PM10","C02","NO2"]))}. Please choose different level names")
             return
         if levels_dict == None:
             levels_dict = dict(
@@ -473,6 +473,8 @@ class Dataset(DictionaryPlus):
                         return "purple"
                     case ['Time', 'CO(ppm)']:
                         return "lascar"
+                    case ['sensorName', 'deviceId']:
+                        return "lcm"
                     case _:
                         return None
             except:
@@ -518,6 +520,21 @@ class Dataset(DictionaryPlus):
                             except:
                                 print(
                                     f"processing of file {directory}{j} failed. Is this a {monitor} file?")
+                        case "lcm":
+                            try:
+                                processed = lcm_processing(
+                                    directory, file=j, interpolate_data=interpolate)
+                                for k, v in levels_dict.items():
+                                    if (v not in ["None", "_", "-", None]) and (k not in ["monitor", "", None, "-", "_"]):
+                                        processed.m[k] = v
+                                processed.m["rejected"] = False
+                                processed.m["filename"] = j
+                                for key, value in add_m.items():
+                                    processed.m[key] = value
+                                self[str(uuid.uuid4())] = processed
+                            except:
+                                print(
+                                    f"processing of file {directory}{j} failed. Is this a {monitor} file?")
                         case "purple":
                             try:
                                 processed = purple_processing(
@@ -559,6 +576,8 @@ class Dataset(DictionaryPlus):
                 return ("lascar", run_lascar())
             elif x is Purple:
                 return ("purple", run_purple())
+            elif x is Lcm:
+                return ("lcm", run_lcm())
             # elif x is Apm:
             #     return "Apm"
             # elif x is PolarH10:
@@ -572,6 +591,8 @@ class Dataset(DictionaryPlus):
                     return "Upas"
                 case Lascar():
                     return "Lascar"
+                case Lcm():
+                    return "Lcm"
                 case Purple():
                     return "Purple"
                 case Apm():
@@ -702,6 +723,58 @@ class Dataset(DictionaryPlus):
                 out["time"] = pd.to_datetime(out["time"])
                 return (Summary(out, metadata=set(columns2)))
 
+        def run_lcm():
+            a = 0.524
+            b = -0.0862
+            c = 5.75
+            d = self.subset(condition=lambda x: match_monitor(x) == "Lcm")
+            columns1 = ["identifier", "monitor",
+                        "start", "end", "length", "time"]
+            columns2 = list(set(d.meta()))
+            columns3 = ["sensorName", "deviceId", "resolution",
+                         "PM2.5", "PM10", "CO2", "NO2", "Temperature", "Humidity"]
+            columns4 = ["pm25correctedrh"]
+
+            buffer = StringIO()
+            with buffer as file:
+                df = writer(file, delimiter=",")
+                df.writerow(columns1+columns2+columns3+columns4)
+                count = 0
+                for k, v in d.items():
+                    print(f"processing Lcm {count+1} out of {len(d)}")
+                    v = v.dropna(subset="PM2.5")
+                    identifier = k
+                    start = v.start
+                    end = v.end
+                    length = v.length
+                    for i in range(len(v)):
+                        app = []
+                        app.append(identifier)
+                        app.append("lcm")
+                        app.append(start)
+                        app.append(end)
+                        app.append(length)
+                        app.append(v.index[i])
+                        for m in columns2:
+                            if m in v.m.keys():
+                                app.append(v.m[m] if v.m[m] != None else "")
+                            else:
+                                app.append("")
+                        for m in columns3:
+                            app.append(v[m].iloc[i])
+                        xx = (a*v["PM2.5"].iloc[i])+(b*v["Humidity"].iloc[i])+c
+                        app.append(xx)
+                        df.writerow(list(map(str, app)))
+                    count += 1
+                file.seek(0)
+                out = pd.read_csv(file)
+                out["start"] = pd.to_datetime(out["start"])
+                out["end"] = pd.to_datetime(out["end"])
+                out["length"] = pd.to_timedelta(out["length"])
+                out["time"] = pd.to_datetime(out["time"])
+                return (Summary(out, metadata=set(columns2)))
+
+
         def run_purple():
             d = self.subset(condition=lambda x: match_monitor(x) == "Purple")
             columns1 = ["identifier", "monitor",
@@ -819,6 +892,8 @@ class Dataset(DictionaryPlus):
                     return "Upas"
                 case Lascar():
                     return "Lascar"
+                case Lcm():
+                    return "Lcm"
                 case Purple():
                     return "Purple"
                 case Apm():
@@ -1454,9 +1529,26 @@ class Summary(pd.DataFrame):
             else:
                 output = df
 
-        output = output[value_cols].agg(aggregate)
+        if isinstance(frequency, str) and aggregate is None:
+            if grouping_by != None:
+                output = df[grouping_by+[frequency]]
+            else:
+                output = df[[frequency]]
+        if isinstance(frequency, list) and aggregate is None:
+            if grouping_by != None:
+                output = df[grouping_by+frequency]
+            else:
+                output = df[frequency]
+        if frequency == None and aggregate is None:
+            if grouping_by != None:
+                output = df[grouping_by]
+            else:
+                output = df
+                
+        if aggregate is not None:
+            output = output[value_cols].agg(aggregate)
 
-        if frequency != None or grouping_by != None:
+        if aggregate != None and (frequency != None or grouping_by != None):
             output.columns = [
                 "_".join(str(x) for x in col if pd.notna(x) and x != "")
                 for col in output.columns.to_flat_index()
@@ -1472,8 +1564,8 @@ class Summary(pd.DataFrame):
         output.m["aggregate"] = aggregate
 
         return output
-
-
+    
+    
     def get_monitor(self, value):
         
         out = self.subset({"monitor": [value]})
@@ -1936,6 +2028,37 @@ class PurpleSeries(ApmSeries):
     def _constructor(self):
         return PurpleSeries
 
+class Lcm(Apm):
+    monitor = "lcm"
+
+    def __init__(self, *args, **kwargs):
+        Apm.__init__(self, *args, **kwargs)
+        self.m = {}
+        self.variable = "PM2.5"
+
+    @property
+    def _constructor(self):
+        return Lcm
+
+    @property
+    def _constructor_sliced(self):
+        return LcmSeries
+
+    @property
+    def units(self):
+        return lcm_units
+
+class LcmSeries(ApmSeries):
+    def __init__(self, *args, **kwargs):
+        LcmSeries.__init__(self, *args, **kwargs)
+    _metadata = ['m']
+
+    @property
+    def _constructor(self):
+        return LcmSeries
+
+
+
 class PolarH10(dict):
     monitor = "polarh10"
     def __init__(self, *args, **kwargs):

{apmtools-0.2.31 → apmtools-0.2.33}/src/apmtools/data_processing.py

@@ -4,7 +4,7 @@ import pandas as pd
 import numpy as np
 import datetime as dt
 import os as os
-from .classes import Apm, Sum, PolarH10, DictionaryPlus, Grav_Filter, Upas, Lascar, Purple
+from .classes import Apm, Sum, PolarH10, DictionaryPlus, Grav_Filter, Upas, Lascar, Purple, Lcm
 from zipfile import ZipFile
 from copy import deepcopy
 from pandas.errors import EmptyDataError
@@ -841,6 +841,19 @@ def lascar_processing(directory, file, interpolation=1,interval="30 seconds", in
     out = Lascar(df)
     return out
 
+
+def lcm_processing(directory, file, interpolation=1, interval="300 seconds", interpolate_data=True):
+    numeric = ["PM2.5",	"PM10",	"CO2",	"NO2",	"Temperature",	"Humidity"]
+    dtformat = '%Y-%m-%dT%H:%M:%S'
+    df = pd.read_csv(directory+file,  index_col="timestamp",
+                     date_format={'Time': dtformat})
+    if interpolate_data:
+        df = interpolate(df, 300, interpolation, pd.Timedelta(
+            '00:06:00'), numeric_columns=numeric, add_binary_counter=False)
+        df = keep_interval(df, interval)
+    out = Lcm(df)
+    return out
+
 def sum_interpolation(file, interpolation=1, interval="5 minutes", timing=False):
     if timing:
         start = time.process_time()