seabirdfilehandler 0.4.2__py3-none-any.whl → 0.5.0__py3-none-any.whl

seabirdfilehandler/datafiles.py

@@ -0,0 +1,259 @@
+from pathlib import Path
+import xmltodict
+import pandas as pd
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class DataFile:
+    """Collection of methods for the SeaBird files that feature some kind of
+    data table that is represented in a pandas dataframe.
+
+    Parameters
+    ----------
+
+    Returns
+    -------
+
+    """
+
+    def __init__(
+        self,
+        path_to_file: Path | str,
+        only_header: bool = False,
+    ):
+        self.path_to_file = Path(path_to_file)
+        self.file_name = self.path_to_file.stem
+        self.file_dir = self.path_to_file.parent
+        self.only_header = only_header
+        self.raw_file_data = []  # the text file input
+        self.header = []  # the full file header
+        self.sbe9_data = []  # device specific information
+        self.metadata = {}  # non-SeaBird metadata
+        self.metadata_list = []  # unstructured metadata for easier export
+        self.data_table_description = []  # the column names and other info
+        self.sensor_data = []
+        self.sensors = {}  # xml-parsed sensor data
+        self.processing_info = []  # everything after the sensor data
+        self.data = []  # the data table
+        self.file_data = self.raw_file_data  # variable file information
+        self.read_file()
+        self.metadata = self.structure_metadata(self.metadata_list)
+        if len(self.sensor_data) > 0:
+            self.sensors = self.sensor_xml_to_flattened_dict(
+                "".join(self.sensor_data)
+            )
+
+    def __str__(self) -> str:
+        return "/n".join(self.file_data)
+
+    def __repr__(self) -> str:
+        return str(self.path_to_file.absolute())
+
+    def __eq__(self, other) -> bool:
+        return self.file_data == other.file_data
+
+    def read_file(self):
+        """Reads and structures all the different information present in the
+        file. Lists and Dictionaries are the data structures of choice. Uses
+        basic prefix checking to distinguish different header information.
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+
+        """
+        past_sensors = False
+        with self.path_to_file.open("r", encoding="latin-1") as file:
+            for line in file:
+                self.raw_file_data.append(line)
+                line_prefix = line[:2]
+                if line_prefix == "* ":
+                    self.header.append(line)
+                    self.sbe9_data.append(line[2:])
+                elif line_prefix == "**":
+                    self.header.append(line)
+                    self.metadata_list.append(line[3:])
+                elif line_prefix == "# ":
+                    self.header.append(line)
+                    if line[2:].strip()[0] == "<":
+                        self.sensor_data.append(line[2:])
+                        past_sensors = True
+                    else:
+                        if past_sensors:
+                            self.processing_info.append(line[2:])
+                        else:
+                            self.data_table_description.append(line[2:])
+                else:
+                    if line.startswith("*END*"):
+                        self.header.append(line)
+                        if self.only_header:
+                            break
+                    else:
+                        self.data.append(line)
+
+    def sensor_xml_to_flattened_dict(
+        self, sensor_data: str
+    ) -> list[dict] | dict:
+        """Reads the pure xml sensor input and creates a multilevel dictionary,
+        dropping the first two dictionaries, as they are single entry only
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+
+        """
+        full_sensor_dict = xmltodict.parse(sensor_data, process_comments=True)
+        try:
+            sensors = full_sensor_dict["Sensors"]["sensor"]
+        except KeyError as error:
+            logger.error(f"XML is not formatted as expected: {error}")
+            return full_sensor_dict
+        else:
+            # create a tidied version of the xml-parsed sensor dict
+            tidied_sensor_list = []
+            for entry in sensors:
+                # use comment value as type descriptor
+                comment = entry["#comment"]
+                split_comment = comment.split(",")
+                new_entry = split_comment[1].strip()
+                if split_comment[-1] == " 2":
+                    new_entry += " 2"
+                # remove second-level dict
+                calibration_info = list(entry.values())[-1]
+                try:
+                    new_dict = {
+                        "Channel": entry["@Channel"],
+                        "SensorName": new_entry,
+                        **calibration_info,
+                    }
+                except TypeError:
+                    new_dict = {
+                        "Channel": entry["@Channel"],
+                        "SensorName": new_entry,
+                        "Info": calibration_info,
+                    }
+                tidied_sensor_list.append(new_dict)
+            return tidied_sensor_list
+
+    def structure_metadata(self, metadata_list: list) -> dict:
+        """Creates a dictionary to store the metadata that is added by using
+        werums dship API.
+
+        Parameters
+        ----------
+        metadata_list: list :
+            a list of the individual lines of metadata found in the file
+
+        Returns
+        -------
+        a dictionary of the lines of metadata divided into key-value pairs
+        """
+        out_dict = {}
+        for line in metadata_list:
+            try:
+                (key, val) = line.split("=")
+            except ValueError:
+                out_dict["text"] = line
+            else:
+                out_dict[key.strip()] = val.strip()
+        return out_dict
+
+    def define_output_path(
+        self,
+        file_path: Path | str | None = None,
+        file_name: str | None = None,
+        file_type: str = ".csv",
+    ) -> Path:
+        """Creates a Path object holding the desired output path.
+
+        Parameters
+        ----------
+        file_path : Path :
+            directory the file sits in (Default value = self.file_dir)
+        file_name : str :
+            the original file name (Default value = self.file_name)
+        file_type : str :
+            the output file type (Default = '.csv')
+        Returns
+        -------
+        a Path object consisting of the full path of the new file
+
+        """
+        file_path = self.file_dir if file_path is None else file_path
+        file_name = self.file_name if file_name is None else file_name
+        if file_type[0] != ".":
+            file_type = "." + file_type
+        return Path(file_path).joinpath(file_name).with_suffix(file_type)
+
+    def to_csv(
+        self,
+        selected_columns: list | None = None,
+        with_header: bool = True,
+        output_file_path: Path | str | None = None,
+        output_file_name: str | None = None,
+    ):
+        """Writes a csv from the current dataframe. Takes a list of columns to
+        use, a boolean for writing the header and the output file parameters.
+
+        Parameters
+        ----------
+        selected_columns : list :
+            a list of columns to include in the csv
+            (Default value = self.df.columns)
+        with_header : boolean :
+            indicating whether the header shall appear in the output
+             (Default value = True)
+        output_file_path : Path :
+            file directory (Default value = None)
+        output_file_name : str :
+            original file name (Default value = None)
+
+        Returns
+        -------
+
+        """
+        selected_columns = (
+            self.df.columns if selected_columns is None else selected_columns
+        )
+        df = self.df[selected_columns].reset_index(drop=True)
+        new_file_path = self.define_output_path(
+            output_file_path, output_file_name
+        )
+        if with_header:
+            with open(new_file_path, "w") as file:
+                for line in self.header:
+                    file.write(line)
+            df.to_csv(new_file_path, index=False, mode="a")
+        else:
+            df.to_csv(new_file_path, index=False, mode="w")
+        logger.info(f"Wrote file {self.path_to_file} to {new_file_path}.")
+
+    def selecting_columns(
+        self,
+        list_of_columns: list | str,
+        df: pd.DataFrame | None = None,
+    ):
+        """Alters the dataframe to only hold the given columns.
+
+        Parameters
+        ----------
+        list_of_columns: list or str : a collection of columns
+        df : pandas.Dataframe :
+            Dataframe (Default value = None)
+
+        Returns
+        -------
+
+        """
+        df = self.df if df is None else df
+        # ensure that the input is a list, so that isin() can do its job
+        if isinstance(list_of_columns, str):
+            list_of_columns = [list_of_columns]
+        if isinstance(df, pd.DataFrame):
+            self.df = df[list_of_columns].reset_index(drop=True)

seabirdfilehandler/file_collection.py

@@ -1,11 +1,16 @@
 from pathlib import Path
 import logging
 from collections import UserList
-from typing import Type
+from typing import Callable, Type
 import pandas as pd
 import numpy as np
-from seabirdfilehandler import SeaBirdFile, CnvFile, BottleFile, BottleLogFile
-from seabirdfilehandler.datatablefiles import DataTableFile
+from seabirdfilehandler import (
+    CnvFile,
+    BottleFile,
+    BottleLogFile,
+)
+from seabirdfilehandler import DataFile
+from seabirdfilehandler.utils import get_unique_sensor_data
 
 logger = logging.getLogger(__name__)
 
@@ -28,28 +33,26 @@ class FileCollection(UserList):
         self,
         path_to_files: str | Path,
         file_suffix: str,
-        pattern: str | None = None,
         only_metadata: bool = False,
+        sorting_key: Callable | None = None,
     ):
         super().__init__()
         self.path_to_files = Path(path_to_files)
         self.file_suffix = file_suffix.strip(".")
-        self.file_type: Type[SeaBirdFile]
+        self.file_type: Type[DataFile]
         self.extract_file_type()
         self.individual_file_paths = []
-        self.collect_files()
-        if pattern:
-            # TODO: implement pattern handling
-            self.pattern = pattern
-        else:
-            self.load_files(only_metadata)
+        self.collect_files(sorting_key=sorting_key)
+        self.load_files(only_metadata)
         if not only_metadata:
-            if self.file_type == DataTableFile:
+            if self.file_type == DataFile:
                 self.df_list = self.get_dataframes()
                 self.df = self.get_collection_dataframe(self.df_list)
             if self.file_type == CnvFile:
                 self.data_meta_info = self.get_data_table_meta_info()
-            self.sensor_data = self.get_sensor_data()
+            self.sensor_data = get_unique_sensor_data(
+                [file.sensors for file in self.data]
+            )
 
     def __str__(self):
         return "/n".join(self.data)
@@ -66,13 +69,19 @@ class FileCollection(UserList):
                 self.file_type = value
                 break
             else:
-                self.file_type = SeaBirdFile
+                self.file_type = DataFile
 
-    def collect_files(self):
+    def collect_files(
+        self,
+        sorting_key: Callable | None = lambda file: int(
+            file.stem.split("_")[4]
+        ),
+    ):
         """ """
-        for path in self.path_to_files.rglob(f"*{self.file_suffix}"):
-            self.individual_file_paths.append(path)
-        self.individual_file_paths.sort()
+        self.individual_file_paths = sorted(
+            self.path_to_files.rglob(f"*{self.file_suffix}"),
+            key=sorting_key,
+        )
 
     def load_files(self, only_metadata: bool = False):
         """ """
@@ -246,25 +255,4 @@ class FileCollection(UserList):
 
     def get_data_table_meta_info(self) -> list[list[dict]]:
         """ """
-        return [file.data_header_meta_info for file in self.data]
-
-    def get_sensor_data(self) -> list[tuple[list[dict]]]:
-        """ """
-        unique = []
-        last_unique = None
-        for file in [file for file in self.data]:
-            cast_sensors = file.sensors
-            if last_unique is None:
-                unique.append((file.file_name, cast_sensors))
-            else:
-                differing_dicts = [
-                    current_dict
-                    for last_dict, current_dict in zip(
-                        last_unique, cast_sensors
-                    )
-                    if current_dict != last_dict
-                ]
-                if differing_dicts:
-                    unique.append((file.file_name, differing_dicts))
-            last_unique = cast_sensors
-        return unique
+        return [file.parameters.metadata for file in self.data]

seabirdfilehandler/parameter.py

@@ -113,6 +113,20 @@ class Parameters(UserDict):
             )
         return parameter_dict
 
+    def _form_data_table_info(self) -> list:
+        """Recreates the data table descriptions, like column names and spans
+        from the structured dictionaries these values were stored in."""
+        new_table_info = []
+        for key, value in self.data_table_stats.items():
+            new_table_info.append(f"{key} = {value}\n")
+        for index, (name, _) in enumerate(self.data_table_names_and_spans):
+            new_table_info.append(f"name {index} = {name}\n")
+        for index, (_, span) in enumerate(self.data_table_names_and_spans):
+            new_table_info.append(f"span {index} = {span}\n")
+        for key, value in self.data_table_misc.items():
+            new_table_info.append(f"{key} = {value}\n")
+        return new_table_info
+
     def differentiate_table_description(self):
         """
         The original method that structures data table metadata.
@@ -144,7 +158,10 @@ class Parameters(UserDict):
             (name, span)
             for name, span in zip(column_names, column_value_spans)
         ]
-        self.data_table_misc = post
+        self.data_table_misc = {
+            line.split("=")[0].strip(): line.split("=")[1].strip()
+            for line in post
+        }
 
     def add_parameter(self, parameter: Parameter):
         """
@@ -201,7 +218,6 @@ class Parameters(UserDict):
             data = np.full(
                 fill_value=data,
                 shape=self.full_data_array.shape[0],
-                dtype=type(data),
             )
         parameter = Parameter(data=data, metadata=metadata)
         self.add_parameter(parameter)
@@ -263,7 +279,17 @@ class Parameters(UserDict):
         ).T
         columns = [parameter.name for parameter in self.get_parameter_list()]
         assert data.shape[1] == len(columns)
-        return pd.DataFrame(data=data, columns=columns, dtype=float)
+        df = pd.DataFrame(data=data, columns=columns)
+        for column in df.columns:
+            if column.lower() not in [
+                "latitude",
+                "longitude",
+                "event",
+                "cast",
+                "flag",
+            ]:
+                df[column].astype("float64")
+        return df
 
     def with_name_type(self, name_type: str = "shortname"):
         """

seabirdfilehandler/utils.py

@@ -0,0 +1,53 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_unique_sensor_data(
+    sensor_data: list[list[dict]],
+) -> list[tuple[list[dict]]]:
+    """
+    Returns all the unique sensors and their configuration used in the given
+    collection of sensor data. These will typically be parsed from xml inside
+    .cnv or .xmlcon files.
+    If for example, the first oxygen sensor has been replaced after the 8 cast,
+    then we will see that in the output structure by a seconde tuple, with the
+    number 8 and the individual sensor information for that new oxygen sensor.
+
+    Parameters
+    ----------
+    sensor_data:
+        The structure of xml-parsed dicts inside two organizing lists.
+
+    Returns
+    -------
+    The input structure stripped down to unique sensor data and appended by
+    the index, at which this new sensor appeared the first time.
+
+    """
+    unique = []
+    last_unique = None
+    for index, individual_sensor_data in enumerate(
+        [file for file in sensor_data]
+    ):
+        if last_unique is None:
+            unique.append((index, individual_sensor_data))
+        else:
+            differing_dicts = [
+                current_dict
+                for last_dict, current_dict in zip(
+                    last_unique, individual_sensor_data
+                )
+                if current_dict != last_dict
+            ]
+            if differing_dicts:
+                unique.append((index, differing_dicts))
+        last_unique = individual_sensor_data
+    return unique
+
+
+class UnexpectedFileFormat(Exception):
+    def __init__(self, file_type: str, error: str) -> None:
+        message = f"{file_type} is not formatted as expected: {error}"
+        logger.error(message)
+        super().__init__(message)

seabirdfilehandler/xmlfiles.py

@@ -4,6 +4,8 @@ import xml.etree.ElementTree as ET
 import json
 import xmltodict
 
+from seabirdfilehandler.utils import UnexpectedFileFormat
+
 
 class XMLFile(UserDict):
     """
@@ -78,6 +80,58 @@ class XMLCONFile(XMLFile):
 
     def __init__(self, path_to_file):
         super().__init__(path_to_file)
+        self.sensor_info = self.get_sensor_info()
+
+    def get_sensor_info(self) -> list[dict]:
+        """
+        Creates a multilevel dictionary, dropping the first four dictionaries,
+        to retrieve pure sensor information.
+
+        Returns
+        -------
+        A list of all the individual sensor information, stored in dictionaries
+
+        """
+        try:
+            sensors = self.data["SBE_InstrumentConfiguration"]["Instrument"][
+                "SensorArray"
+            ]["Sensor"]
+        except KeyError as error:
+            raise UnexpectedFileFormat("XMLCON", error)
+        else:
+            # create a tidied version of the xml-parsed sensor dict
+            sensor_names = []
+            tidied_sensor_list = []
+            for entry in sensors:
+                sensor_key = list(entry.keys())[-1]
+                if not sensor_key.endswith(("Sensor", "Meter")):
+                    continue
+                sensor_name = sensor_key.removesuffix("Sensor")
+                # the wetlab sensors feature a suffix _Sensor
+                sensor_name = sensor_name.removesuffix("_")
+                # assuming, that the first sensor in the xmlcon is also on the
+                # first sensor strand, the second occurence of the name is
+                # suffixed with '2'
+                if sensor_name in sensor_names:
+                    sensor_name += "2"
+                sensor_names.append(sensor_name)
+                # move the calibration info one dictionary level up
+                calibration_info = entry[sensor_key]
+                # build the new dictionary
+                try:
+                    new_dict = {
+                        "Channel": str(int(entry["@index"]) + 1),
+                        "SensorName": sensor_name,
+                        **calibration_info,
+                    }
+                except TypeError:
+                    new_dict = {
+                        "Channel": entry["@Channel"],
+                        "SensorName": sensor_name,
+                        "Info": calibration_info,
+                    }
+                tidied_sensor_list.append(new_dict)
+            return tidied_sensor_list
 
 
 class PsaFile(XMLFile):

{seabirdfilehandler-0.4.2.dist-info → seabirdfilehandler-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: seabirdfilehandler
-Version: 0.4.2
+Version: 0.5.0
 Summary: Library of parsers to interact with SeaBird CTD files.
 Keywords: CTD,parser,seabird,data
 Author: Emil Michels

seabirdfilehandler-0.5.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+seabirdfilehandler/__init__.py,sha256=5JTzYE3oRdrxkC9_etAnFQ1cy10PHtpmesdR6n5PoPQ,192
+seabirdfilehandler/bottlefile.py,sha256=QBUqtKhD-dUPbgc_sz8NOGEFFkAjL3g5r1oAsqQnUcQ,6063
+seabirdfilehandler/bottlelogfile.py,sha256=CjBeITQS27Ar80bfxguoAnVkYxI1ioAiVTnlvwevw7E,4448
+seabirdfilehandler/cnvfile.py,sha256=-mCuQX76uuWIETljem1DjzDbZ5eNIUucjoHejTQu_sU,9685
+seabirdfilehandler/datafiles.py,sha256=z27PZJBvEbl1aFXpDQ0QTM_yR5NTaNK-HC9_z3t7zuM,8991
+seabirdfilehandler/dataframe_meta_accessor.py,sha256=x4mSEN49us6Ezzjdt41fl5Ry8IJR09ORrZ1roOIJbyc,6439
+seabirdfilehandler/file_collection.py,sha256=qkEdlI-hcoyuOdmgTr8wdAr1mXXkVuKkF9J4j2-v3kY,6882
+seabirdfilehandler/logging.yaml,sha256=mXxbhJPio3OGaukTpc3rLGA8Ywq1DNqp0Vn5YCbH6jY,459
+seabirdfilehandler/parameter.py,sha256=UuwFzege94sqPt0kOjEqtMGGol4hjuFjj2_EH7o0pzA,14374
+seabirdfilehandler/utils.py,sha256=5KXdB8Hdv65dv5tPyXxNMct1mCEOyA3S8XP54AFAnx0,1745
+seabirdfilehandler/validation_modules.py,sha256=eZ6x0giftUtlxnRMOnK_vCkgccdwUXPrDjajFa-E6n0,4698
+seabirdfilehandler/xmlfiles.py,sha256=L_puQf8eg0ojv85AyEMID4jnwkOlV_fgZP3W5yeSUBY,4668
+seabirdfilehandler-0.5.0.dist-info/LICENSE,sha256=Ifd1VPmYv32oJd2QVh3wIQP9X05vYJlcY6kONz360ws,34603
+seabirdfilehandler-0.5.0.dist-info/METADATA,sha256=jPHzHpxz9OY48QHifHT9crI9a5pxYae1aCw6jAhqtYM,1289
+seabirdfilehandler-0.5.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+seabirdfilehandler-0.5.0.dist-info/RECORD,,

{seabirdfilehandler-0.4.2.dist-info → seabirdfilehandler-0.5.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.1
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any