seabirdfilehandler 0.4.0__py3-none-any.whl

seabirdfilehandler/file_collection.py

@@ -0,0 +1,269 @@
+from pathlib import Path
+import logging
+from collections import UserList
+from typing import Type
+import pandas as pd
+import numpy as np
+from seabirdfilehandler import SeaBirdFile, CnvFile, BottleFile, BottleLogFile
+from seabirdfilehandler.datatablefiles import DataTableFile
+
+logger = logging.getLogger(__name__)
+
+
+class FileCollection(UserList):
+    """A representation of multiple files of the same kind. These files share
+    the same suffix and are otherwise closely connected to each other. A common
+    use case would be the collection of CNVs to allow for easier processing or
+    integration of field calibration measurements.
+
+    Parameters
+    ----------
+
+    Returns
+    -------
+
+    """
+
+    def __init__(
+        self,
+        path_to_files: str | Path,
+        file_suffix: str,
+        pattern: str | None = None,
+        only_metadata: bool = False,
+    ):
+        super().__init__()
+        self.path_to_files = Path(path_to_files)
+        self.file_suffix = file_suffix.strip(".")
+        self.file_type: Type[SeaBirdFile]
+        self.extract_file_type()
+        self.individual_file_paths = []
+        self.collect_files()
+        if pattern:
+            # TODO: implement pattern handling
+            self.pattern = pattern
+        else:
+            self.load_files(only_metadata)
+        if not only_metadata:
+            if self.file_type == DataTableFile:
+                self.df_list = self.get_dataframes()
+                self.df = self.get_collection_dataframe(self.df_list)
+            if self.file_type == CnvFile:
+                self.data_meta_info = self.get_data_table_meta_info()
+            self.sensor_data = self.get_sensor_data()
+
+    def __str__(self):
+        return "/n".join(self.data)
+
+    def extract_file_type(self):
+        """ """
+        mapping_suffix_to_type = {
+            "cnv": CnvFile,
+            "btl": BottleFile,
+            "bl": BottleLogFile,
+        }
+        for key, value in mapping_suffix_to_type.items():
+            if key == self.file_suffix:
+                self.file_type = value
+                break
+            else:
+                self.file_type = SeaBirdFile
+
+    def collect_files(self):
+        """ """
+        for path in self.path_to_files.rglob(f"*{self.file_suffix}"):
+            self.individual_file_paths.append(path)
+        self.individual_file_paths.sort()
+
+    def load_files(self, only_metadata: bool = False):
+        """ """
+        for file in self.individual_file_paths:
+            try:
+                self.data.append(self.file_type(file))
+            except TypeError:
+                logger.error(
+                    f"Could not open file {
+                        file} with the type "
+                    f"{self.file_type}."
+                )
+                continue
+
+    def get_dataframes(
+        self,
+        event_log: bool = False,
+        coordinates: bool = False,
+        time_correction: bool = False,
+        cast_identifier: bool = False,
+        long_header_names: bool = False,
+        full_data_header: bool = True,
+    ) -> list[pd.DataFrame]:
+        """
+
+        Parameters
+        ----------
+        event_log: bool :
+             (Default value = False)
+        coordinates: bool :
+             (Default value = False)
+        time_correction: bool :
+             (Default value = False)
+        cast_identifier: bool :
+             (Default value = False)
+        long_header_names: bool :
+             (Default value = False)
+        full_data_header: bool :
+             (Default value = True)
+
+        Returns
+        -------
+
+        """
+        for index, file in enumerate(self.data):
+            if full_data_header:
+                file.rename_dataframe_header(header_detail_level="longinfo")
+            elif long_header_names:
+                file.rename_dataframe_header(header_detail_level="name")
+            if event_log:
+                file.add_station_and_event_column()
+            if coordinates:
+                file.add_position_columns()
+            if time_correction:
+                file.absolute_time_calculation()
+                file.add_start_time()
+            if cast_identifier:
+                file.add_cast_number(index + 1)
+        return [file.df for file in self.data]
+
+    def get_collection_dataframe(
+        self, list_of_dfs: list[pd.DataFrame] | None = None
+    ) -> pd.DataFrame:
+        """
+
+        Parameters
+        ----------
+        list_of_dfs: list[pd.DataFrame] | None :
+             (Default value = None)
+
+        Returns
+        -------
+
+        """
+        if not list_of_dfs:
+            list_of_dfs = self.get_dataframes()
+        df = pd.concat(list_of_dfs, ignore_index=True)
+        # df.meta.metadata = list_of_dfs[0].meta.metadata
+        return df
+
+    def tidy_collection_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+
+        Parameters
+        ----------
+        df: pd.DataFrame :
+
+
+        Returns
+        -------
+
+        """
+        df = self.use_bad_flag_for_nan(df)
+        df = self.set_dtype_to_float(df)
+        return self.select_real_scan_data(df)
+
+    def use_bad_flag_for_nan(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+
+        Parameters
+        ----------
+        df: pd.DataFrame :
+
+
+        Returns
+        -------
+
+        """
+        bad_flags = set()
+        for file in self.data:
+            for line in file.data_table_description:
+                if line.startswith("bad_flag"):
+                    flag = line.split("=")[1].strip()
+                    bad_flags.add(flag)
+        for flag in bad_flags:
+            df.replace(to_replace=flag, value=np.nan, inplace=True)
+        return df
+
+    def set_dtype_to_float(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+
+        Parameters
+        ----------
+        df: pd.DataFrame :
+
+
+        Returns
+        -------
+
+        """
+        for parameter in df.columns:
+            if parameter in ["datetime"]:
+                continue
+            try:
+                df[parameter] = df[parameter].astype("float")
+            finally:
+                continue
+        return df
+
+    def select_real_scan_data(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+
+        Parameters
+        ----------
+        df: pd.DataFrame :
+
+
+        Returns
+        -------
+
+        """
+        # TODO: fix this hardcoded name
+        try:
+            df = df.loc[df["Scan Count"].notna()]
+        finally:
+            pass
+        return df
+
+    def to_csv(self, file_name):
+        """
+
+        Parameters
+        ----------
+        file_name :
+
+
+        Returns
+        -------
+
+        """
+        self.get_collection_dataframe().to_csv(file_name)
+
+    def get_data_table_meta_info(self) -> list[list[dict]]:
+        """ """
+        return [file.data_header_meta_info for file in self.data]
+
+    def get_sensor_data(self) -> list[tuple[list[dict]]]:
+        """ """
+        unique = []
+        last_unique = None
+        for file in [file for file in self.data]:
+            cast_sensors = file.sensors
+            if last_unique is None:
+                unique.append((file.file_name, cast_sensors))
+            else:
+                differing_dicts = [
+                    current_dict
+                    for last_dict, current_dict in zip(last_unique, cast_sensors)
+                    if current_dict != last_dict
+                ]
+                if differing_dicts:
+                    unique.append((file.file_name, differing_dicts))
+            last_unique = cast_sensors
+        return unique

seabirdfilehandler/logging.yaml

@@ -0,0 +1,23 @@
+version: 1
+formatters:
+  simple:
+    format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    formatter: simple
+    stream: ext://sys.stdout
+  file:
+    class: logging.FileHandler
+    filename: seabirdfiles.log
+    level: DEBUG
+    formatter: simple
+loggers:
+  simpleExample:
+    level: DEBUG
+    handlers: [console]
+    propagate: no
+root:
+  level: DEBUG
+  handlers: [console]

seabirdfilehandler/parameter.py

@@ -0,0 +1,410 @@
+from __future__ import annotations
+from typing import Tuple
+import re
+from collections import UserDict
+import numpy as np
+import pandas as pd
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class Parameters(UserDict):
+    """
+    A collection of all the parameters in a CnvFile.
+
+    Allows for a much cleaner handling of parameter data and their metadata.
+    Will be heavily expanded.
+
+    Parameters
+    ----------
+    data: list:
+        The raw data as extraced by SeaBirdFile
+    metadata: list,
+        The raw metadata as extraced by SeaBirdFile
+
+    Returns
+    -------
+
+    """
+
+    def __init__(
+        self,
+        data: list,
+        metadata: list,
+    ):
+        self.raw_input_data = data
+        self.raw_metadata = metadata
+        self.full_data_array = self.create_full_ndarray()
+        self.differentiate_table_description()
+        self.metadata, self.duplicate_columns = self.reading_data_header(metadata)
+        self.data = self.create_parameter_instances()
+
+    def get_parameter_list(self) -> list[Parameter]:
+        """ """
+        return list(self.data.values())
+
+    def create_full_ndarray(self, data_table: list = []) -> np.ndarray:
+        """
+        Builds a numpy array representing the data table in a cnv file.
+
+        Parameters
+        ----------
+        data_table: list :
+            The data to work with
+             (Default value = [])
+
+        Returns
+        -------
+        A numpy array of the same shape as the cnv files data table
+
+        """
+        data_table = self.raw_input_data if len(data_table) == 0 else data_table
+        n = 11
+        row_list = []
+        for line in data_table:
+            row_list.append(
+                [line[i : i + n].split()[0] for i in range(0, len(line) - n, n)]
+            )
+        return np.array(row_list, dtype=float)
+
+    def create_parameter_instances(
+        self,
+        metadata: dict[str, dict] = {},
+    ) -> dict[str, Parameter]:
+        """
+        Differentiates the individual parameter columns into separate parameter
+        instances.
+
+        Parameters
+        ----------
+        metadata: dict[str, dict] :
+            The structured metadata dictionary
+             (Default value = {})
+
+        Returns
+        -------
+        A dictionary of parameter instances
+
+        """
+        metadata = self.metadata if len(list(metadata.keys())) == 0 else metadata
+        parameter_dict = {}
+        list_of_metadata_shortnames = list(metadata.keys())
+        if self.full_data_array.shape[1] != len(list_of_metadata_shortnames):
+            raise AssertionError(
+                f"{self.full_data_array.shape} and metadata:{metadata}"
+            )
+        for i in range(self.full_data_array.shape[1]):
+            column_data = self.full_data_array[:, i]
+            key = list_of_metadata_shortnames[i]
+            parameter_dict[key] = Parameter(data=column_data, metadata=metadata[key])
+        return parameter_dict
+
+    def differentiate_table_description(self):
+        """
+        The original method that structures data table metadata.
+
+        Needs heavy refactoring.
+        """
+        past_spans = False
+        pre = []
+        column_names = []
+        column_value_spans = []
+        post = []
+        for line in self.raw_metadata:
+            if line.startswith("name"):
+                column_names.append(line.split("=")[1].strip())
+            elif line.startswith("span"):
+                past_spans = True
+                column_value_spans.append(line.split("=")[1].strip())
+            else:
+                if not past_spans:
+                    pre.append(line)
+                else:
+                    post.append(line)
+        assert len(column_names) == len(column_value_spans)
+        self.data_table_stats = {
+            line.split("=")[0].strip(): line.split("=")[1].strip() for line in pre
+        }
+        self.data_table_names_and_spans = [
+            (name, span) for name, span in zip(column_names, column_value_spans)
+        ]
+        self.data_table_misc = post
+
+    def add_parameter(self, parameter: Parameter):
+        """
+        Adds one parameter instance to the collection.
+
+        Parameters
+        ----------
+        parameter: Parameter :
+            The new parameter
+
+        """
+        self.data[parameter.name] = parameter
+
+    def create_parameter(
+        self,
+        data: np.ndarray | int | float | str,
+        metadata: dict = {},
+        name: str = "",
+    ) -> Parameter:
+        """
+        Creates a new parameter instance with the given data and metadata.
+
+        The input data is either a numpy array or a single value. The single
+        value will be broadcasted to the shape of the data table. A use-case
+        would be the addition of an 'event' or 'cast' column.
+
+        Parameters
+        ----------
+        data: np.ndarray | int | float | str :
+            Data to use or expand
+
+        metadata: dict :
+            Metadata for the new parameter
+             (Default value = {})
+        name: str :
+            Name to use for missing metadata values
+             (Default value = "")
+
+        Returns
+        -------
+        The new parameter instance
+
+        """
+        if len(metadata) < 5:
+            if len(name) > 0:
+                metadata = self.add_default_metadata(name=name, metadata=metadata)
+            else:
+                raise ValueError("Please specify either a name or sufficient metadata")
+        if not isinstance(data, np.ndarray):
+            data = np.full(
+                fill_value=data,
+                shape=self.full_data_array.shape[0],
+                dtype=type(data),
+            )
+        parameter = Parameter(data=data, metadata=metadata)
+        self.add_parameter(parameter)
+        return parameter
+
+    def add_default_metadata(
+        self,
+        name: str,
+        metadata: dict = {},
+        list_of_keys: list = [
+            "shortname",
+            "longinfo",
+            "name",
+            "metainfo",
+            "unit",
+        ],
+    ) -> dict:
+        """
+        Fills up missing metadata points with a default value.
+
+        Parameters
+        ----------
+        name: str :
+            The value to use as default
+        metadata: dict :
+            The present metadata
+             (Default value = {})
+        list_of_keys: list :
+             The expected metadata keys
+
+        Returns
+        -------
+        The full metadata dictionary
+
+        """
+        default = {}
+        for key in list_of_keys:
+            if key not in list(metadata.keys()):
+                if key in ["metainfo", "unit"]:
+                    default[key] = ""
+                default[key] = name
+        return {**metadata, **default}
+
+    def update_spans(self):
+        """Updates all spans of the parameters."""
+        for parameter in self.get_parameter_list():
+            parameter.update_span()
+
+    def get_spans(self) -> list[tuple[int, int]]:
+        """Returns all span tuples of the parameters."""
+        # update spans first
+        self.update_spans()
+        return [parameter.span for parameter in self.get_parameter_list()]
+
+    def get_pandas_dataframe(self) -> pd.DataFrame:
+        """Returns a pandas DataFrame of the current parameter data."""
+        data = np.array([parameter.data for parameter in self.get_parameter_list()]).T
+        columns = [parameter.name for parameter in self.get_parameter_list()]
+        assert data.shape[1] == len(columns)
+        return pd.DataFrame(data=data, columns=columns, dtype=float)
+
+    def with_name_type(self, name_type: str = "shortname"):
+        """
+        Uses the given name_type as column descriptors.
+
+        Parameters
+        ----------
+        name_type: str :
+            The metadata name to use
+             (Default value = "shortname")
+
+        """
+        for parameter in self.get_parameter_list():
+            parameter.use_name(name_type)
+
+    def reading_data_header(
+        self, header_info: list = []
+    ) -> Tuple[dict[str, dict], list[int]]:
+        """Reads the tables header data from the header.
+
+        Parameters
+        ----------
+        header_info : list:
+            the header values from the file
+        header_info: list :
+             (Default value = [])
+
+        Returns
+        -------
+
+
+        """
+        if len(header_info) == 0:
+            header_info = self.raw_metadata
+        table_header = {}
+        duplicate_columns = []
+        for line in header_info:
+            if line.startswith("name"):
+                header_meta_info = {}
+                # get basic shortname and the full, non-differentiated info
+                shortname = longinfo = line_info = line.split("=")[1].strip()
+                try:
+                    shortname, longinfo = line_info.split(":")
+                except IndexError:
+                    pass
+                finally:
+                    shortname = shortname.strip()
+                    if shortname in list(table_header.keys()):
+                        try:
+                            duplicate_columns.append(
+                                int(line.split("=")[0].strip().split()[1])
+                            )
+                        except IndexError as error:
+                            logger.error(
+                                f"Could not resolve duplicate column: {
+                                    shortname}, {error}"
+                            )
+                    else:
+                        header_meta_info["shortname"] = shortname
+                        header_meta_info["longinfo"] = longinfo.strip()
+                        metainfo = self._extract_data_header_meta_info(longinfo.strip())
+                        header_meta_info = {**header_meta_info, **metainfo}
+                        table_header[shortname.strip()] = header_meta_info
+        return table_header, duplicate_columns
+
+    def _extract_data_header_meta_info(self, line: str) -> dict:
+        """Extracts the individual information bits inside of the header lines
+
+        Parameters
+        ----------
+        line : str:
+            one header line, trimmed by the 'name =' prefix and the shortname
+        line: str :
+
+
+        Returns
+        -------
+
+
+        """
+        regex_string = r"(?:(?P<name0>.+),\s(?P<metainfo0>.+)\s\[(?P<unit0>.+)\]|(?P<name2>.+)\s\[(?P<unit2>.+)\]|(?P<name3>.+),\s(?P<metainfo2>.[^\s]+)|(?P<name4>.+))"
+        regex_check = re.search(regex_string, line, flags=re.IGNORECASE)
+        if regex_check:
+            regex_info = dict(regex_check.groupdict())
+            regex_info = {
+                key[:-1]: value
+                for key, value in regex_info.items()
+                if value is not None
+            }
+            if len(regex_info) > 2:
+                # check for second sensors and adjust their names
+                if regex_info["metainfo"][-1] == "2":
+                    regex_info["name"] = regex_info["name"] + " 2"
+                    regex_info["metainfo"] = regex_info["metainfo"][:-1]
+                    if len(regex_info["metainfo"]) == 0:
+                        regex_info.pop("metainfo")
+            if regex_info["name"] == "flag":
+                regex_info["metainfo"] = regex_info["name"]
+                regex_info["unit"] = regex_info["name"]
+            return regex_info
+        return {}
+
+
+class Parameter:
+    """A representation of one parameter in a cnv file.
+
+    Consists of the values of the parameter as well as the metadata.
+
+    Parameters
+    ----------
+
+    Returns
+    -------
+
+    """
+
+    def __init__(
+        self,
+        data: np.ndarray,
+        metadata: dict,
+    ) -> None:
+        self.data = data
+        self.metadata = metadata
+        self.name = metadata["shortname"]
+        self.update_span()
+
+    def __str__(self) -> str:
+        return str(self.metadata["longinfo"])
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    def __eq__(self, other) -> bool:
+        return self.data == other.data
+
+    def get_pandas_series(self) -> pd.Series:
+        """Returns a pandas Series of the current parameter data."""
+        return pd.Series(data=self.data, name=self.name)
+
+    def use_name(self, name_type: str = "shortname"):
+        """
+        Uses the given name as parameter descriptor.
+
+        Parameters
+        ----------
+        name_type: str :
+            The metadata name to use
+             (Default value = "shortname")
+
+        """
+        try:
+            self.name = self.metadata[name_type]
+        except KeyError:
+            return
+
+    def update_span(self):
+        """
+        Updates the data span.
+
+        Uses the first value if dtype is not numeric.
+        """
+        if self.data.dtype in ["float64", "int"]:
+            self.span = (self.data.min(), self.data.max())
+        else:
+            self.span = (self.data[0], self.data[0])