tsp 1.8.1__py3-none-any.whl → 1.10.2__py3-none-any.whl

tsp/dataloggers/Vemco.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+*DESCRIPTION*
+
+Author: rparker
+Created: 2024-01-10
+"""
+import os
+import pathlib
+import pandas as pd
+import datetime as dt
+
+from .AbstractReader import AbstractReader
+
+
+class Vemco(AbstractReader):
+    def __init__(self):
+        self.META = {}
+
+    def read(self, file_path: str):
+        file_extention = pathlib.Path(file_path).suffix.lower()
+        if file_extention == ".000":
+            try:
+                with open(file_path, "r", encoding="cp1252") as file:
+                    first_line = file.readline()
+                if first_line.startswith("*"):
+                    self._read_old_000_logger_file(file_path)
+                else:
+                    raise IOError(f"{os.path.basename(file_path)} unreadable. Try opening this file in Logger VUE and "
+                                  f"exporting it as a .csv")
+            except UnicodeDecodeError as e:
+                raise IOError(f"{os.path.basename(file_path)} unreadable. Try opening this file in Logger VUE and "
+                              f"exporting it as a .csv")
+        elif file_extention == ".csv":
+            self._read_logger_vue_csv(file_path)
+        else:
+            raise ValueError("File is not a .csv or .000")
+        return self.DATA
+
+    def _read_old_000_logger_file(self, file_path):
+        with open(file_path, "r", encoding="cp1252") as file:
+            lines = file.readlines()
+        header_lines = lines[:6]
+        data_lines = [l.strip().split(",") for l in lines[6:]]
+        self.META["logger_model"] = header_lines[0].split("=")[-1].strip()
+        self.META["logger_sn"] = header_lines[1].split("=")[-1].strip()
+        self.META["study_id"] = header_lines[2].split("=")[-1].strip()
+        self.META["logging_start"] = dt.datetime.strptime(header_lines[3].split("=")[-1].strip(), "%d/%m/%Y %H:%M:%S")
+        self.META["download_date"] = dt.datetime.strptime(header_lines[4].split("=")[-1].strip(), "%d/%m/%Y %H:%M:%S")
+        sample_interval = dt.datetime.strptime(header_lines[5].split("=")[-1].strip(), "%H:%M:%S")
+        self.META["sample_interval"] = dt.timedelta(hours=sample_interval.hour, minutes=sample_interval.minute,
+                                                    seconds=sample_interval.second)
+        self.META['raw'] = "".join(header_lines)
+        self.DATA = pd.DataFrame(data_lines[1:], columns=["TIME", "TEMPERATURE"])
+        self.DATA["TIME"] = pd.to_datetime(self.DATA["TIME"], format="%d/%m/%Y %H:%M:%S")
+        self.DATA["TEMPERATURE"] = pd.to_numeric(self.DATA["TEMPERATURE"], errors='coerce')
+        return
+
+    def _read_logger_vue_csv(self, file_path):
+        with open(file_path, "r", encoding="cp1252") as file:
+            lines = file.readlines()
+
+        header_lines = lines[:7]
+        data_lines = [l.strip().split(",") for l in lines[7:]]
+        model_and_sn = header_lines[1][15:-1]
+        self.META["logger_sn"] = model_and_sn.split("-")[-1]
+        self.META["logger_model"] = "-".join(model_and_sn.split("-")[:-1])
+        self.META["study_id"] = header_lines[2].split(":")[-1].strip()
+        self.META["logging_start"] = dt.datetime.strptime(header_lines[4][18:-1], "%Y-%m-%d %H:%M:%S")
+        self.META["download_date"] = dt.datetime.strptime(header_lines[5][17:-1], "%Y-%m-%d %H:%M:%S")
+        if header_lines[3].endswith(")"):
+            data_tz = dt.timezone(dt.timedelta(hours=int(header_lines[3][45:-2])))
+            self.META["logging_start"] = self.META["logging_start"].replace(tzinfo=data_tz)
+            self.META["download_date"] = self.META["download_date"].replace(tzinfo=data_tz)
+            self.META["utc_offset"] = data_tz
+        sample_interval = dt.datetime.strptime(header_lines[6][17:-1], "%H:%M:%S")
+        self.META["sample_interval"] = dt.timedelta(hours=sample_interval.hour, minutes=sample_interval.minute,
+                                                    seconds=sample_interval.second)
+        self.META['raw'] = "".join(header_lines)
+        self.DATA = pd.DataFrame(data_lines[1:], columns=["date", "time", "TEMPERATURE"])
+        self.DATA["TIME"] = pd.to_datetime(self.DATA["date"] + " " + self.DATA["time"], format="%Y-%m-%d %H:%M:%S")
+        self.DATA["TEMPERATURE"] = pd.to_numeric(self.DATA["TEMPERATURE"], errors='coerce')
+        self.DATA.drop(columns=["date", "time"], inplace=True)
+        return

tsp/dataloggers/__init__.py

@@ -1,15 +1,15 @@
-from .HOBO import HOBO, HOBOProperties
-from .FG2 import FG2
-from .GP5W import GP5W
-from .Geoprecision import detect_geoprecision_type
-from .logr import LogR
-
-HOBO.__module__ = __name__
-HOBOProperties.__module__ = __name__
-FG2.__module__ =__name__
-GP5W.__module__ = __name__
-LogR.__module__ = __name__
-
-__all__ = ['HOBO','HOBOProperties',
-           'FG2','GP5W', 'detect_geoprecision_type',
-           'LogR']
+from .HOBO import HOBO, HOBOProperties
+from .FG2 import FG2
+from .GP5W import GP5W
+from .Geoprecision import detect_geoprecision_type
+from .logr import LogR
+
+HOBO.__module__ = __name__
+HOBOProperties.__module__ = __name__
+FG2.__module__ =__name__
+GP5W.__module__ = __name__
+LogR.__module__ = __name__
+
+__all__ = ['HOBO','HOBOProperties',
+           'FG2','GP5W', 'detect_geoprecision_type',
+           'LogR']

tsp/dataloggers/logr.py

@@ -1,115 +1,196 @@
-import pandas as pd
-import regex as re
-import numpy as np
-
-
-class LogR:
-    
-    SEP = ","
-
-    def __init__(self):
-        pass
-
-    def read(self, file):
-        headers = read_logr_header(file)
-        
-        columns = [line.strip().split(',') for line in headers if is_columns_row(line)][0]
-        labels = [line.strip().split(',') for line in headers if is_label_row(line)][0]
-        data = pd.read_csv(file, skiprows=len(headers))
-        
-        data.columns = ["TIME" if c == 'timestamp' else c for c in columns]
-        data['TIME'] = pd.to_datetime(data['TIME'], format=dateformat())
-        
-        channels = pd.Series(data.columns).str.match("^CH")
-        
-        self.DATA = data
-        self.META = {
-            'label': labels,
-            'guessed_depths': guess_depths(labels)[-sum(channels):]
-        }
-
-        return self.DATA
-
-
-def read_logr_header(file: str) -> "list":
-    """ Read metadata / header lines from LogR file 
-
-    Parameters
-    ----------
-    file : str
-        path to a LogR file
-
-    Returns
-    -------
-    list
-        list of lines in the header block
-
-    Raises
-    ------
-    ValueError
-        _description_
-    """
-    found_data = False
-    max_rows = 50
-    header_lines = []
-
-    with open(file) as f:
-        while not found_data and max_rows:
-            max_rows -= 1
-            
-            line = f.readline()
-
-            if is_data_row(line):
-                found_data = True
-                break
-
-            else: 
-                header_lines.append(line)
-
-    if not found_data:
-        raise ValueError("Could not find start of data")
-
-    return header_lines 
-
-
-def guess_depths(labels: "list[str]") -> "list[float]":
-    pattern = re.compile(r"(-?[\d\.]+)")
-
-    matches = [pattern.search(l) for l in labels]
-    depths = [float(d.group(1)) if d else None for d in matches]
-
-    return depths
-
-
-def guessed_depths_ok(depths, n_channel) -> bool:
-    """ Evaluate whether the guessed depths are valid """
-    d = np.array(depths, dtype='float64')
-
-    # monotonic (by convention)
-    if not (np.diff(d) > 0).all() or (np.diff(d) < 0).all():
-        return False
-
-    # equal to number of channels
-    if not sum(~np.isnan(d)) == n_channel:
-        return False
-
-    return True
-
-
-def dateformat():
-    return "%Y/%m/%d %H:%M:%S"
-
-
-def is_data_row(line: str) -> bool:
-    pattern = re.compile(r"^,\d{4}/\d{2}/\d{2}\s\d{2}:")
-    return bool(pattern.match(line))
-
-
-def is_columns_row(line:str) -> bool:
-    pattern = re.compile(r"^SensorId")
-    return bool(pattern.match(line))
-
-
-def is_label_row(line: str) -> bool:
-    pattern = re.compile(r"^Label")
-    return bool(pattern.match(line))
+import pandas as pd
+import regex as re
+import numpy as np
+import datetime as dt
+
+
+class LogR:
+    SEP = ","
+
+    def __init__(self):
+        self.DATA = None
+        self.META = None
+        pass
+
+    def read(self, file: str, cfg_txt: str = None):
+        raw = is_raw_data(file)
+        if raw and cfg_txt is None:
+            raise ValueError("cfg_txt must be specified if providing raw data.")
+
+        if cfg_txt is not None:
+            config_params = read_cfg_file(cfg_txt)
+        else:
+            config_params = None
+
+        header_rows = read_logr_header(file)
+        columns = [line.strip().split(',') for line in header_rows if is_columns_row(line)][0]
+        labels = [line.strip().split(',') for line in header_rows if is_label_row(line)][0]
+        data = pd.read_csv(file, header=len(header_rows) - 1,
+                           names=["TIME" if c == 'timestamp' else c for c in columns])
+        if raw:
+            data = convert_raw_to_temperatures(data=data, channel_metadata=config_params["channel_metadata"])
+        else:
+            data['TIME'] = pd.to_datetime(data['TIME'], format=dateformat())
+            if config_params is not None and "UTC Offset" in config_params.keys():
+                pattern = re.compile(r"-?\+?\d{2}:?\d{0,2}")
+                match = pattern.search(config_params["UTC Offset"])
+                if match is None:
+                    raise ValueError("Could not parse UTC offset")
+                offset = [int(ele) for ele in match.group().split(":")]
+                if len(offset) == 1:
+                    tz = dt.timezone(dt.timedelta(hours=offset[0]))
+                elif len(offset) == 2:
+                    tz = dt.timezone(dt.timedelta(hours=offset[0], minutes=offset[1]))
+                else:
+                    raise ValueError("Could not parse UTC offset")
+                data['TIME'] = data['TIME'].dt.tz_localize(tz)
+
+        channels = pd.Series(data.columns).str.match("^CH")
+
+        self.DATA = data
+        self.META = {'label': labels,
+                     'guessed_depths': guess_depths(labels)[-sum(channels):]}
+        if config_params is not None:
+            self.META = self.META | config_params
+
+        return self.DATA
+
+
+def read_cfg_file(file_path: str):
+    metadata = dict()
+    with open(file_path, "r") as f:
+        for i in range(50):
+            line = f.readline()
+            if line.startswith("ChannelID"):
+                break
+            if line != "\n":
+                line = line.split(":")
+                metadata[line[0].strip()] = line[1].strip()
+    if "Serial Number" in metadata.keys():
+        metadata["logger_sn"] = metadata["Serial Number"]
+        del metadata["Serial Number"]
+    metadata["channel_metadata"] = pd.read_csv(file_path, delimiter="\t", header=len(metadata.keys()),
+                                               index_col="ChannelID")
+    return metadata
+
+
+def convert_raw_to_temperatures(data: pd.DataFrame, channel_metadata: pd.DataFrame):
+    data["TIME"] = pd.to_datetime(data["TIME"], unit="s", utc=True)
+    voltage_ref = 2.5
+    r_fixed = 7500
+    r_correction = 100
+    for channel in channel_metadata.index:
+        if channel in data.columns:
+            coefficents = {letter: channel_metadata.loc[channel, letter] for letter in ["A", "B", "C", "D", "E"]}
+            resistances = r_fixed / (voltage_ref / data[channel] - 1) - r_correction
+            data[channel] = 1 / (coefficents["A"] + coefficents["B"] * np.log(resistances)
+                                 + coefficents["C"] * np.power(np.log(resistances), 3)
+                                 + coefficents["D"] * np.power(np.log(resistances), 5)) - 273.15
+    return data
+
+
+def read_logr_header(file: str) -> list:
+    """ Read metadata / header lines from LogR file 
+
+    Parameters
+    ----------
+    file : str
+        path to a LogR file
+
+    Returns
+    -------
+    list
+        list of lines in the header block
+
+    Raises
+    ------
+    ValueError
+        _description_
+    """
+    found_data = False
+    max_rows = 50
+    header_lines = list()
+
+    with open(file) as f:
+        while not found_data and max_rows:
+            max_rows -= 1
+            line = f.readline()
+            if is_data_row(line):
+                found_data = True
+                break
+            else:
+                header_lines.append(line)
+    if not found_data:
+        raise ValueError("Could not find start of data")
+    return header_lines
+
+
+def guess_depths(labels: list[str]) -> list[float]:
+    pattern = re.compile(r"(-?[\d\.]+)")
+
+    matches = [pattern.search(l) for l in labels]
+    depths = [float(d.group(1)) if d else None for d in matches]
+
+    return depths
+
+
+def guessed_depths_ok(depths, n_channel) -> bool:
+    """ Evaluate whether the guessed depths are valid """
+    d = np.array(depths, dtype='float64')
+
+    # monotonic (by convention)
+    if not (np.diff(d) > 0).all() or (np.diff(d) < 0).all():
+        return False
+
+    # equal to number of channels
+    if not sum(~np.isnan(d)) == n_channel:
+        return False
+
+    return True
+
+
+def dateformat():
+    return "%Y/%m/%d %H:%M:%S"
+
+
+def is_data_row(line: str) -> bool:
+    if line == "":
+        return False
+    second_element = line.split(",")[1]
+    try:
+        if second_element.isnumeric():
+            dt.datetime.fromtimestamp(float(second_element))
+        else:
+            dt.datetime.strptime(second_element, dateformat())
+        return True
+    except:
+        return False
+
+
+def is_raw_data(file: str) -> bool:
+    with open(file) as f:
+        for i in range(50):
+            line = f.readline()
+            if line == "":
+                continue
+            second_element = line.split(",")[1]
+            try:
+                if second_element.isnumeric():
+                    dt.datetime.fromtimestamp(float(second_element))
+                    return True
+                else:
+                    dt.datetime.strptime(second_element, dateformat())
+                    return False
+            except:
+                pass
+    raise RuntimeError("Could not determine if raw data")
+
+
+def is_columns_row(line: str) -> bool:
+    pattern = re.compile(r"^SensorId")
+    return bool(pattern.match(line))
+
+
+def is_label_row(line: str) -> bool:
+    pattern = re.compile(r"^Label")
+    return bool(pattern.match(line))