seabirdfilehandler 0.4.3__py3-none-any.whl → 0.5.1__py3-none-any.whl

seabirdfilehandler/datafiles.py

@@ -0,0 +1,265 @@
+from pathlib import Path
+import xmltodict
+import pandas as pd
+import numpy as np
+import logging
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - [%(levelname)s] - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    handlers=[
+        logging.FileHandler("filehandler.log"),
+        logging.StreamHandler(),
+    ],
+)
+
+logger = logging.getLogger(__name__)
+
+
+class DataFile:
+    """Collection of methods for the SeaBird files that feature some kind of
+    data table that is represented in a pandas dataframe.
+
+    Parameters
+    ----------
+
+    Returns
+    -------
+
+    """
+
+    def __init__(
+        self,
+        path_to_file: Path | str,
+        only_header: bool = False,
+    ):
+        self.path_to_file = Path(path_to_file)
+        self.file_name = self.path_to_file.stem
+        self.file_dir = self.path_to_file.parent
+        self.only_header = only_header
+        self.raw_file_data = []  # the text file input
+        self.header = []  # the full file header
+        self.sbe9_data = []  # device specific information
+        self.metadata = {}  # non-SeaBird metadata
+        self.metadata_list = []  # unstructured metadata for easier export
+        self.data_table_description = []  # the column names and other info
+        self.sensor_data = []
+        self.sensors = {}  # xml-parsed sensor data
+        self.processing_info = []  # everything after the sensor data
+        self.data = []  # the data table
+        self.file_data = self.raw_file_data  # variable file information
+        self.read_file()
+        self.metadata = self.structure_metadata(self.metadata_list)
+        if len(self.sensor_data) > 0:
+            self.sensors = self.sensor_xml_to_flattened_dict(
+                "".join(self.sensor_data)
+            )
+
+    def __str__(self) -> str:
+        return "/n".join(self.file_data)
+
+    def __repr__(self) -> str:
+        return str(self.path_to_file.absolute())
+
+    def __eq__(self, other) -> bool:
+        return self.file_data == other.file_data
+
+    def read_file(self):
+        """Reads and structures all the different information present in the
+        file. Lists and Dictionaries are the data structures of choice. Uses
+        basic prefix checking to distinguish different header information.
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+
+        """
+        past_sensors = False
+        with self.path_to_file.open("r", encoding="latin-1") as file:
+            for line in file:
+                self.raw_file_data.append(line)
+                line_prefix = line[:2]
+                if line_prefix == "* ":
+                    self.header.append(line)
+                    self.sbe9_data.append(line[2:])
+                elif line_prefix == "**":
+                    self.header.append(line)
+                    self.metadata_list.append(line[3:])
+                elif line_prefix == "# ":
+                    self.header.append(line)
+                    if line[2:].strip()[0] == "<":
+                        self.sensor_data.append(line[2:])
+                        past_sensors = True
+                    else:
+                        if past_sensors:
+                            self.processing_info.append(line[2:])
+                        else:
+                            self.data_table_description.append(line[2:])
+                else:
+                    if line.startswith("*END*"):
+                        self.header.append(line)
+                        if self.only_header:
+                            break
+                    else:
+                        self.data.append(line)
+
+    def sensor_xml_to_flattened_dict(
+        self, sensor_data: str
+    ) -> list[dict] | dict:
+        """Reads the pure xml sensor input and creates a multilevel dictionary,
+        dropping the first two dictionaries, as they are single entry only
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+
+        """
+        full_sensor_dict = xmltodict.parse(sensor_data, process_comments=True)
+        try:
+            sensors = full_sensor_dict["Sensors"]["sensor"]
+        except KeyError as error:
+            logger.error(f"XML is not formatted as expected: {error}")
+            return full_sensor_dict
+        else:
+            # create a tidied version of the xml-parsed sensor dict
+            tidied_sensor_list = []
+            for entry in sensors:
+                # use comment value as type descriptor
+                comment = entry["#comment"]
+                split_comment = comment.split(",")
+                new_entry = split_comment[1].strip()
+                if split_comment[-1] == " 2":
+                    new_entry += " 2"
+                # remove second-level dict
+                calibration_info = list(entry.values())[-1]
+                try:
+                    new_dict = {
+                        "Channel": entry["@Channel"],
+                        "SensorName": new_entry,
+                        **calibration_info,
+                    }
+                except TypeError:
+                    new_dict = {
+                        "Channel": entry["@Channel"],
+                        "SensorName": new_entry,
+                        "Info": calibration_info,
+                    }
+                tidied_sensor_list.append(new_dict)
+            return tidied_sensor_list
+
+    def structure_metadata(self, metadata_list: list) -> dict:
+        """Creates a dictionary to store the metadata that is added by using
+        werums dship API.
+
+        Parameters
+        ----------
+        metadata_list: list :
+            a list of the individual lines of metadata found in the file
+
+        Returns
+        -------
+        a dictionary of the lines of metadata divided into key-value pairs
+        """
+        out_dict = {}
+        for line in metadata_list:
+            try:
+                (key, val) = line.split("=")
+            except ValueError:
+                out_dict["text"] = line
+            else:
+                out_dict[key.strip()] = val.strip()
+        return out_dict
+
+    def define_output_path(
+        self,
+        file_path: Path | str | None = None,
+        file_name: str | None = None,
+        file_type: str = ".csv",
+    ) -> Path:
+        """Creates a Path object holding the desired output path.
+
+        Parameters
+        ----------
+        file_path : Path :
+            directory the file sits in (Default value = self.file_dir)
+        file_name : str :
+            the original file name (Default value = self.file_name)
+        file_type : str :
+            the output file type (Default = '.csv')
+        Returns
+        -------
+        a Path object consisting of the full path of the new file
+
+        """
+        file_path = self.file_dir if file_path is None else file_path
+        file_name = self.file_name if file_name is None else file_name
+        if file_type[0] != ".":
+            file_type = "." + file_type
+        return Path(file_path).joinpath(file_name).with_suffix(file_type)
+
+    def to_csv(
+        self,
+        data: pd.DataFrame | np.ndarray,
+        with_header: bool = True,
+        output_file_path: Path | str | None = None,
+        output_file_name: str | None = None,
+    ):
+        """Writes a csv from the current dataframe. Takes a list of columns to
+        use, a boolean for writing the header and the output file parameters.
+
+        Parameters
+        ----------
+        selected_columns : list :
+            a list of columns to include in the csv
+            (Default value = self.df.columns)
+        with_header : boolean :
+            indicating whether the header shall appear in the output
+             (Default value = True)
+        output_file_path : Path :
+            file directory (Default value = None)
+        output_file_name : str :
+            original file name (Default value = None)
+
+        Returns
+        -------
+
+        """
+        new_file_path = self.define_output_path(
+            output_file_path, output_file_name
+        )
+        if with_header:
+            with open(new_file_path, "w") as file:
+                for line in self.header:
+                    file.write(line)
+        if isinstance(data, pd.DataFrame):
+            data.to_csv(new_file_path, index=False, mode="a")
+        else:
+            np.savetxt(new_file_path, data, delimiter=",")
+
+    def selecting_columns(
+        self,
+        list_of_columns: list | str,
+        df: pd.DataFrame,
+    ):
+        """Alters the dataframe to only hold the given columns.
+
+        Parameters
+        ----------
+        list_of_columns: list or str : a collection of columns
+        df : pandas.Dataframe :
+            Dataframe (Default value = None)
+
+        Returns
+        -------
+
+        """
+        # ensure that the input is a list, so that isin() can do its job
+        if isinstance(list_of_columns, str):
+            list_of_columns = [list_of_columns]
+        if isinstance(df, pd.DataFrame):
+            self.df = df[list_of_columns].reset_index(drop=True)

seabirdfilehandler/file_collection.py

@@ -1,16 +1,15 @@
 from pathlib import Path
 import logging
 from collections import UserList
-from typing import Type
+from typing import Callable, Type
 import pandas as pd
 import numpy as np
 from seabirdfilehandler import (
-    SeaBirdFile,
     CnvFile,
     BottleFile,
     BottleLogFile,
 )
-from seabirdfilehandler.datatablefiles import DataTableFile
+from seabirdfilehandler import DataFile
 from seabirdfilehandler.utils import get_unique_sensor_data
 
 logger = logging.getLogger(__name__)
@@ -34,23 +33,19 @@ class FileCollection(UserList):
         self,
         path_to_files: str | Path,
         file_suffix: str,
-        pattern: str | None = None,
         only_metadata: bool = False,
+        sorting_key: Callable | None = None,
     ):
         super().__init__()
         self.path_to_files = Path(path_to_files)
         self.file_suffix = file_suffix.strip(".")
-        self.file_type: Type[SeaBirdFile]
+        self.file_type: Type[DataFile]
         self.extract_file_type()
         self.individual_file_paths = []
-        self.collect_files()
-        if pattern:
-            # TODO: implement pattern handling
-            self.pattern = pattern
-        else:
-            self.load_files(only_metadata)
+        self.collect_files(sorting_key=sorting_key)
+        self.load_files(only_metadata)
         if not only_metadata:
-            if self.file_type == DataTableFile:
+            if self.file_type == DataFile:
                 self.df_list = self.get_dataframes()
                 self.df = self.get_collection_dataframe(self.df_list)
             if self.file_type == CnvFile:
@@ -74,13 +69,19 @@ class FileCollection(UserList):
                 self.file_type = value
                 break
             else:
-                self.file_type = SeaBirdFile
+                self.file_type = DataFile
 
-    def collect_files(self):
+    def collect_files(
+        self,
+        sorting_key: Callable | None = lambda file: int(
+            file.stem.split("_")[3]
+        ),
+    ):
         """ """
-        for path in self.path_to_files.rglob(f"*{self.file_suffix}"):
-            self.individual_file_paths.append(path)
-        self.individual_file_paths.sort()
+        self.individual_file_paths = sorted(
+            self.path_to_files.rglob(f"*{self.file_suffix}"),
+            key=sorting_key,
+        )
 
     def load_files(self, only_metadata: bool = False):
         """ """
@@ -254,4 +255,4 @@ class FileCollection(UserList):
 
     def get_data_table_meta_info(self) -> list[list[dict]]:
         """ """
-        return [file.data_header_meta_info for file in self.data]
+        return [file.parameters.metadata for file in self.data]

seabirdfilehandler/parameter.py

@@ -113,6 +113,20 @@ class Parameters(UserDict):
             )
         return parameter_dict
 
+    def _form_data_table_info(self) -> list:
+        """Recreates the data table descriptions, like column names and spans
+        from the structured dictionaries these values were stored in."""
+        new_table_info = []
+        for key, value in self.data_table_stats.items():
+            new_table_info.append(f"{key} = {value}\n")
+        for index, (name, _) in enumerate(self.data_table_names_and_spans):
+            new_table_info.append(f"name {index} = {name}\n")
+        for index, (_, span) in enumerate(self.data_table_names_and_spans):
+            new_table_info.append(f"span {index} = {span}\n")
+        for key, value in self.data_table_misc.items():
+            new_table_info.append(f"{key} = {value}\n")
+        return new_table_info
+
     def differentiate_table_description(self):
         """
         The original method that structures data table metadata.
@@ -144,7 +158,10 @@ class Parameters(UserDict):
             (name, span)
             for name, span in zip(column_names, column_value_spans)
         ]
-        self.data_table_misc = post
+        self.data_table_misc = {
+            line.split("=")[0].strip(): line.split("=")[1].strip()
+            for line in post
+        }
 
     def add_parameter(self, parameter: Parameter):
         """
@@ -201,7 +218,6 @@ class Parameters(UserDict):
             data = np.full(
                 fill_value=data,
                 shape=self.full_data_array.shape[0],
-                dtype=type(data),
             )
         parameter = Parameter(data=data, metadata=metadata)
         self.add_parameter(parameter)
@@ -263,7 +279,17 @@ class Parameters(UserDict):
         ).T
         columns = [parameter.name for parameter in self.get_parameter_list()]
         assert data.shape[1] == len(columns)
-        return pd.DataFrame(data=data, columns=columns, dtype=float)
+        df = pd.DataFrame(data=data, columns=columns)
+        for column in df.columns:
+            if column.lower() not in [
+                "latitude",
+                "longitude",
+                "event",
+                "cast",
+                "flag",
+            ]:
+                df[column].astype("float64")
+        return df
 
     def with_name_type(self, name_type: str = "shortname"):
         """

{seabirdfilehandler-0.4.3.dist-info → seabirdfilehandler-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: seabirdfilehandler
-Version: 0.4.3
+Version: 0.5.1
 Summary: Library of parsers to interact with SeaBird CTD files.
 Keywords: CTD,parser,seabird,data
 Author: Emil Michels

seabirdfilehandler-0.5.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+seabirdfilehandler/__init__.py,sha256=5JTzYE3oRdrxkC9_etAnFQ1cy10PHtpmesdR6n5PoPQ,192
+seabirdfilehandler/bottlefile.py,sha256=nnfoDczPMG_ge40dT2rHNhifR7-NRgnZNFrfPM_9OSQ,5925
+seabirdfilehandler/bottlelogfile.py,sha256=MtMmEebdAktO3mk6KbmJC7dfx9sRLbV5qqDQt2qtpJE,4310
+seabirdfilehandler/cnvfile.py,sha256=LXpJcC3ukiD-2b5vy4aKESCbIvwV12TwQy1G6Y25_GE,9709
+seabirdfilehandler/datafiles.py,sha256=lqENvdGSwRKT6PyNFN2etaWKMA-4OONG0x-up1W5ezo,8991
+seabirdfilehandler/file_collection.py,sha256=b5iJaP4F34Vq7-FiJOlPvfS4IePGWsYx20XwWbZQw1A,6882
+seabirdfilehandler/parameter.py,sha256=UuwFzege94sqPt0kOjEqtMGGol4hjuFjj2_EH7o0pzA,14374
+seabirdfilehandler/utils.py,sha256=5KXdB8Hdv65dv5tPyXxNMct1mCEOyA3S8XP54AFAnx0,1745
+seabirdfilehandler/validation_modules.py,sha256=eZ6x0giftUtlxnRMOnK_vCkgccdwUXPrDjajFa-E6n0,4698
+seabirdfilehandler/xmlfiles.py,sha256=L_puQf8eg0ojv85AyEMID4jnwkOlV_fgZP3W5yeSUBY,4668
+seabirdfilehandler-0.5.1.dist-info/LICENSE,sha256=Ifd1VPmYv32oJd2QVh3wIQP9X05vYJlcY6kONz360ws,34603
+seabirdfilehandler-0.5.1.dist-info/METADATA,sha256=2VrJmgeRr-Par2zU5A--xDS5r_7VsKzi-HLi8SlPUX4,1289
+seabirdfilehandler-0.5.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+seabirdfilehandler-0.5.1.dist-info/RECORD,,

{seabirdfilehandler-0.4.3.dist-info → seabirdfilehandler-0.5.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.1
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any

seabirdfilehandler/dataframe_meta_accessor.py

@@ -1,184 +0,0 @@
-import pandas as pd
-import logging
-from pandas.api.extensions import register_series_accessor
-from pandas.api.extensions import register_dataframe_accessor
-import warnings
-
-
-logger = logging.getLogger(__name__)
-
-
-class MetadataHandler:
-    """
-    The base class for the pandas series and dataframe accessors.
-    Offers a very basic metadata handling, by using a dictionary as metadata
-    store. The accessors then allow to access this metadata store and
-    corresponding methods by calling 'df.meta' or 'series.meta', respectively.
-    Mainly targeted for usage with dataframes featuring data from CNV files,
-    it for example allows the attachement of parameter metadata found in the
-    CNV header to individual dataframe columns.
-
-    This approach was chosen over others, like directly subclassing the pandas
-    dataframe or series class, or a seperate metadata storage, due to its
-    simplicity and ability to keep using the full powerfull pandas library
-    without the need to implement each and every transformation. Of course,
-    the 'attrs' attribute does offer a similar metadata storage. But at the
-    time of writing this, it is still in a very experimental condition and does
-    not propagate reliably.
-    """
-
-    def __init__(self, pandas_obj):
-        self._obj = pandas_obj
-        if not hasattr(self._obj, "_metadata_store"):
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore")
-                self._obj._metadata_store = {}
-
-    @property
-    def metadata(self):
-        return self._obj._metadata_store
-
-    @metadata.setter
-    def metadata(self, value):
-        self._obj._metadata_store = value
-
-    def get(self, key, default=None):
-        return self._obj._metadata_store.get(key, default)
-
-    def set(self, key, value):
-        self._obj._metadata_store[key] = value
-
-    def clear(self):
-        self._obj._metadata_store.clear()
-
-
-@register_series_accessor("meta")
-class SeriesMetaAccessor(MetadataHandler):
-    """
-    Series implementation of the Metadata Accessor.
-    Does not offer anything more than the base class at the moment.
-    """
-
-    def __init__(self, pandas_obj):
-        super().__init__(pandas_obj)
-
-
-@register_dataframe_accessor("meta")
-class DataFrameMetaAccessor(MetadataHandler):
-    """
-    DataFrame implementation of the Metadata Accessor.
-    Introduces another attribute, '_header_level_detail', that stores the
-    currently displayed metadata as column names. Additionally offers methods
-    to sync metadata between the dataframe and its series, and the handling of
-    common operations, like renaming or the addition of new columns.
-    """
-
-    def __init__(self, pandas_obj):
-        super().__init__(pandas_obj)
-        if not hasattr(self._obj, "_header_level_detail"):
-            self._obj._header_level_detail = "shortname"
-        # Initialize DataFrame metadata
-        self.aggregate_series_metadata()
-
-    @property
-    def header_detail(self):
-        return self._obj._header_level_detail
-
-    @header_detail.setter
-    def header_detail(self, value):
-        self._obj._header_level_detail = value
-
-    @property
-    def metadata(self):
-        return self._obj._metadata_store
-
-    @metadata.setter
-    def metadata(self, value):
-        meta_dict = {
-            shortname: self.add_default_metadata(shortname, metainfo)
-            for shortname, metainfo in value.items()
-        }
-        self._obj._metadata_store = meta_dict
-        self.propagate_metadata_to_series()
-
-    def aggregate_series_metadata(self):
-        """Aggregate metadata from Series within the DataFrame."""
-        for column in self._obj.columns:
-            if isinstance(self._obj[column], pd.Series) and hasattr(
-                self._obj[column], "meta"
-            ):
-                self.metadata[column] = self._obj[column].meta.metadata
-
-    def propagate_metadata_to_series(self):
-        """Propagate DataFrame-level metadata back to Series."""
-        for column in self._obj.columns:
-            if isinstance(self._obj[column], pd.Series) and hasattr(
-                self._obj[column], "meta"
-            ):
-                for key, value in self.metadata.items():
-                    if key == column:
-                        try:
-                            self._obj[column].meta.metadata = value
-                        except TypeError:
-                            logger.error(f"{column}: {value}")
-
-    def update_metadata_on_rename(self, rename_dict):
-        """Update metadata when columns are renamed."""
-        new_metadata = {}
-        for old_name, new_name in rename_dict.items():
-            for key, value in self.metadata.items():
-                if key == old_name:
-                    new_metadata[new_name] = value
-        self.metadata = new_metadata
-        self.propagate_metadata_to_series()
-
-    def rename(self, rename_key):
-        """Rename the column names by using a metadata point."""
-        rename_dict = {
-            column: (
-                self._obj[column].meta.get(rename_key)
-                if rename_key in list(self._obj[column].meta.metadata.keys())
-                else column
-            )
-            for column in self._obj.columns
-        }
-        self._obj.rename(columns=rename_dict, inplace=True)
-        self.header_detail = rename_key
-        self.update_metadata_on_rename(rename_dict)
-
-    def add_column(
-        self,
-        name: str,
-        data: pd.Series | list,
-        location: int | None = None,
-        metadata: dict = {},
-    ):
-        """Add a column and use or generate metadata for it."""
-        location = len(self._obj.columns) if location is None else location
-        self._obj.insert(
-            loc=location,
-            column=name,
-            value=data,
-            allow_duplicates=False,
-        )
-        self.metadata[name] = self.add_default_metadata(name, metadata)
-        self.propagate_metadata_to_series()
-
-    def add_default_metadata(
-        self,
-        name: str,
-        metadata: dict = {},
-        list_of_keys: list = [
-            "shortname",
-            "longinfo",
-            "name",
-            "metainfo",
-            "unit",
-        ],
-    ) -> dict:
-        """Fill up missing metadata points with a default value."""
-        default = {}
-        for key in list_of_keys:
-            if key not in list(metadata.keys()):
-                default[key] = name
-        return {**metadata, **default}