seabirdfilehandler 0.5.3__tar.gz → 0.5.4__tar.gz

{seabirdfilehandler-0.5.3 → seabirdfilehandler-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: seabirdfilehandler
-Version: 0.5.3
+Version: 0.5.4
 Summary: Library of parsers to interact with SeaBird CTD files.
 Keywords: CTD,parser,seabird,data
 Author: Emil Michels
@@ -16,7 +16,8 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: pandas (>=2.2.1)
 Requires-Dist: xmltodict (>=0.13.0)
-Project-URL: Homepage, https://git.io-warnemuende.de/CTD-Software/SeabirdFileHandler
+Project-URL: Documentation, https://ctd-software.pages.io-warnemuende.de/seabirdfilehandler
+Project-URL: Homepage, https://ctd-software.pages.io-warnemuende.de/seabirdfilehandler
 Project-URL: Repository, https://git.io-warnemuende.de/CTD-Software/SeabirdFileHandler
 Description-Content-Type: text/markdown
 

{seabirdfilehandler-0.5.3 → seabirdfilehandler-0.5.4}/pyproject.toml

@@ -16,10 +16,11 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
 ]
-urls.homepage = "https://git.io-warnemuende.de/CTD-Software/SeabirdFileHandler"
+urls.homepage = "https://ctd-software.pages.io-warnemuende.de/seabirdfilehandler"
 urls.repository = "https://git.io-warnemuende.de/CTD-Software/SeabirdFileHandler"
+urls.documentation = "https://ctd-software.pages.io-warnemuende.de/seabirdfilehandler"
 dynamic = []
-version = "0.5.3"
+version = "0.5.4"
 
 [tool.poetry]
 

{seabirdfilehandler-0.5.3 → seabirdfilehandler-0.5.4}/src/seabirdfilehandler/bottlefile.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Union
 from datetime import datetime, time
 import pandas as pd
@@ -22,12 +23,13 @@ class BottleFile(DataFile):
 
     """
 
-    def __init__(self, path_to_file):
-        super().__init__(path_to_file)
-        self.original_df = self.create_dataframe()
-        self.df = self.original_df
-        self.setting_dataframe_dtypes()
-        self.adding_timestamp_column()
+    def __init__(self, path_to_file: Path | str, only_header: bool = False):
+        super().__init__(path_to_file, only_header)
+        if not only_header:
+            self.original_df = self.create_dataframe()
+            self.df = self.original_df
+            self.setting_dataframe_dtypes()
+            self.adding_timestamp_column()
 
     def create_dataframe(self):
         """Creates a dataframe out of the btl file. Manages the double data

{seabirdfilehandler-0.5.3 → seabirdfilehandler-0.5.4}/src/seabirdfilehandler/cnvfile.py

@@ -60,9 +60,11 @@ class CnvFile(DataFile):
         super().__init__(path_to_file, only_header)
         self.validation_modules = self.obtaining_validation_modules()
         self.start_time = self.reading_start_time()
-        self.parameters = Parameters(self.data, self.data_table_description)
+        self.parameters = Parameters(
+            self.data, self.data_table_description, only_header
+        )
         if create_dataframe:
-            self.df = self.parameters.get_pandas_dataframe()
+            self.df = self.create_dataframe()
         if absolute_time_calculation:
             self.absolute_time_calculation()
         if event_log_column:
@@ -70,6 +72,13 @@ class CnvFile(DataFile):
         if coordinate_columns:
             self.add_position_columns()
 
+    def create_dataframe(self) -> pd.DataFrame:
+        """
+        Plain dataframe creator.
+        """
+        self.df = self.parameters.get_pandas_dataframe()
+        return self.df
+
     def reading_start_time(
         self,
         time_source: str = "System UTC",

seabirdfilehandler-0.5.4/src/seabirdfilehandler/file_collection.py

@@ -0,0 +1,411 @@
+from __future__ import annotations
+from pathlib import Path
+import logging
+from collections import UserList
+from typing import Callable, Type
+import pandas as pd
+import numpy as np
+from seabirdfilehandler import (
+    CnvFile,
+    BottleFile,
+    BottleLogFile,
+)
+from seabirdfilehandler import DataFile
+from seabirdfilehandler.utils import get_unique_sensor_data
+
+logger = logging.getLogger(__name__)
+
+
+def get_collection(
+    path_to_files: Path | str,
+    file_suffix: str = "cnv",
+    only_metadata: bool = False,
+    sorting_key: Callable | None = None,
+) -> Type[FileCollection]:
+    """
+    Factory to create instances of FileCollection, depending on input type.
+
+    Parameters
+    ----------
+    path_to_files : Path | str :
+        The path to the directory to search for files.
+    file_suffix : str :
+        The suffix to search for. (Default value = "cnv")
+    only_metadata : bool :
+        Whether to read only metadata. (Default value = False)
+    sorting_key : Callable | None :
+        A callable that returns the filename-part to use to sort the collection. (Default value = None)
+    Returns
+    -------
+    An instance of FileCollection or one of its children.
+
+    """
+    mapping_suffix_to_type = {
+        "cnv": CnvCollection,
+        "btl": FileCollection,
+        "bl": FileCollection,
+    }
+    file_suffix = file_suffix.strip(".")
+    try:
+        collection = mapping_suffix_to_type[file_suffix](
+            path_to_files, file_suffix, only_metadata, sorting_key
+        )
+    except ValueError:
+        raise ValueError(f"Unknown input file type: {file_suffix}, aborting.")
+    else:
+        return collection
+
+
+class FileCollection(UserList):
+    """
+    A representation of multiple files of the same kind. These files share
+    the same suffix and are otherwise closely connected to each other. A common
+    use case would be the collection of CNVs to allow for easier processing or
+    integration of field calibration measurements.
+
+    Parameters
+    ----------
+    path_to_files : Path | str :
+        The path to the directory to search for files.
+    file_suffix : str :
+        The suffix to search for. (Default value = "cnv")
+    only_metadata : bool :
+        Whether to read only metadata. (Default value = False)
+    sorting_key : Callable | None :
+        A callable that returns the filename-part to use to sort the collection. (Default value = None)
+    """
+
+    def __init__(
+        self,
+        path_to_files: str | Path,
+        file_suffix: str,
+        only_metadata: bool = False,
+        sorting_key: Callable | None = None,
+    ):
+        super().__init__()
+        self.path_to_files = Path(path_to_files)
+        self.file_suffix = file_suffix.strip(".")
+        self.file_type = self.extract_file_type(self.file_suffix)
+        self.individual_file_paths = self.collect_files(
+            sorting_key=sorting_key
+        )
+        self.data = self.load_files(only_metadata)
+        if not only_metadata:
+            self.df_list = self.get_dataframes()
+            self.df = self.get_collection_dataframe(self.df_list)
+
+    def __str__(self):
+        return "/n".join(self.data)
+
+    def extract_file_type(self, suffix: str) -> Type[DataFile]:
+        """
+        Determines the file type using the input suffix.
+
+        Parameters
+        ----------
+        suffix : str :
+            The file suffix.
+        Returns
+        -------
+        An object corresponding to the given suffix.
+        """
+        mapping_suffix_to_type = {
+            "cnv": CnvFile,
+            "btl": BottleFile,
+            "bl": BottleLogFile,
+        }
+        file_type = DataFile
+        for key, value in mapping_suffix_to_type.items():
+            if key == suffix:
+                file_type = value
+                break
+        return file_type
+
+    def collect_files(
+        self,
+        sorting_key: Callable | None = lambda file: int(
+            file.stem.split("_")[3]
+        ),
+    ) -> list[Path]:
+        """
+        Creates a list of target files, recursively from the given directory.
+        These can be sorted with the help of the sorting_key parameter, which
+        is a Callable that identifies the part of the filename that shall be
+        used for sorting.
+
+        Parameters
+        ----------
+        sorting_key : Callable | None :
+            The part of the filename to use in sorting. (Default value = lambda file: int(file.stem.split("_")[3]))
+        Returns
+        -------
+        A list of all paths found.
+        """
+        return sorted(
+            self.path_to_files.rglob(f"*{self.file_suffix}"),
+            key=sorting_key,
+        )
+
+    def load_files(self, only_metadata: bool = False) -> list[DataFile]:
+        """
+        Creates python instances of each file.
+
+        Parameters
+        ----------
+        only_metadata : bool :
+            Whether to load only file metadata. (Default value = False)
+        Returns
+        -------
+        A list of all instances.
+        """
+        data = []
+        for file in self.individual_file_paths:
+            try:
+                data.append(self.file_type(file, only_metadata))
+            except TypeError:
+                logger.error(
+                    f"Could not open file {file} with the type "
+                    f"{self.file_type}."
+                )
+                continue
+        return data
+
+    def get_dataframes(
+        self,
+        event_log: bool = False,
+        coordinates: bool = False,
+        time_correction: bool = False,
+        cast_identifier: bool = False,
+    ) -> list[pd.DataFrame]:
+        """
+        Collects all individual dataframes and allows additional column
+        creation.
+
+        Parameters
+        ----------
+        event_log : bool :
+            (Default value = False)
+        coordinates : bool :
+            (Default value = False)
+        time_correction : bool :
+            (Default value = False)
+        cast_identifier : bool :
+            (Default value = False)
+
+        Returns
+        -------
+        A list of the individual pandas DataFrames.
+        """
+        for index, file in enumerate(self.data):
+            if event_log:
+                file.add_station_and_event_column()
+            if coordinates:
+                file.add_position_columns()
+            if time_correction:
+                file.absolute_time_calculation()
+                file.add_start_time()
+            if cast_identifier:
+                file.add_cast_number(index + 1)
+        return [file.df for file in self.data]
+
+    def get_collection_dataframe(
+        self, list_of_dfs: list[pd.DataFrame] | None = None
+    ) -> pd.DataFrame:
+        """
+        Creates one DataFrame from the individual ones, by concatenation.
+
+        Parameters
+        ----------
+        list_of_dfs : list[pd.DataFrame] | None :
+            A list of the individual DataFrames. (Default value = None)
+        Returns
+        -------
+        A pandas DataFrame representing the whole dataset.
+        """
+        if not list_of_dfs:
+            list_of_dfs = self.get_dataframes()
+        if not list_of_dfs:
+            raise ValueError("No dataframes to concatenate.")
+        df = pd.concat(list_of_dfs, ignore_index=True)
+        self.df = df
+        return df
+
+    def tidy_collection_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Apply the different dataframe edits to the given dataframe.
+
+        Parameters
+        ----------
+        df : pd.DataFrame :
+            A DataFrame to edit.
+        Returns
+        -------
+        The tidied dataframe.
+        """
+        df = self.use_bad_flag_for_nan(df)
+        df = self.set_dtype_to_float(df)
+        return self.select_real_scan_data(df)
+
+    def use_bad_flag_for_nan(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Replace all Nan values by the bad flag value, defined inside the files.
+
+        Parameters
+        ----------
+        df : pd.DataFrame :
+            The dataframe to edit.
+        Returns
+        -------
+        The edited DataFrame.
+        """
+        bad_flags = set()
+        for file in self.data:
+            for line in file.data_table_description:
+                if line.startswith("bad_flag"):
+                    flag = line.split("=")[1].strip()
+                    bad_flags.add(flag)
+        for flag in bad_flags:
+            df.replace(to_replace=flag, value=np.nan, inplace=True)
+        return df
+
+    def set_dtype_to_float(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Use the float-dtype for all DataFrame columns.
+
+        Parameters
+        ----------
+        df : pd.DataFrame :
+            The dataframe to edit.
+        Returns
+        -------
+        The edited DataFrame.
+        """
+        for parameter in df.columns:
+            if parameter in ["datetime"]:
+                continue
+            try:
+                df[parameter] = df[parameter].astype("float")
+            finally:
+                continue
+        return df
+
+    def select_real_scan_data(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Drop data rows have no 'Scan' value, if that column exists.
+
+        Parameters
+        ----------
+        df : pd.DataFrame :
+            The dataframe to edit.
+        Returns
+        -------
+        The edited DataFrame.
+        """
+        try:
+            scan_column = [
+                c for c in df.columns if c.lower().startswith("scan")
+            ][0]
+        except IndexError:
+            return df
+        else:
+            df = df.loc[df[scan_column].notna()]
+        return df
+
+    def to_csv(self, file_name):
+        """
+        Writes a csv file with the given filename.
+
+        Parameters
+        ----------
+        file_name :
+            The new csv file name.
+        """
+        self.df.to_csv(file_name)
+
+
+class CnvCollection(FileCollection):
+    """
+    Specific methods to work with collections of .cnv files.
+    """
+
+    def __init__(
+        self,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.data_meta_info = self.get_data_table_meta_info()
+        self.sensor_data = get_unique_sensor_data(
+            [file.sensors for file in self.data]
+        )
+        self.array = self.get_array()
+
+    def get_dataframes(
+        self,
+        event_log: bool = False,
+        coordinates: bool = False,
+        time_correction: bool = False,
+        cast_identifier: bool = False,
+    ) -> list[pd.DataFrame]:
+        """
+        Collects all individual dataframes and allows additional column
+        creation.
+
+        Parameters
+        ----------
+        event_log : bool :
+            (Default value = False)
+        coordinates : bool :
+            (Default value = False)
+        time_correction : bool :
+            (Default value = False)
+        cast_identifier : bool :
+            (Default value = False)
+        Returns
+        -------
+        A list of the individual pandas DataFrames.
+        """
+        for index, file in enumerate(self.data):
+            if event_log:
+                file.add_station_and_event_column()
+            if coordinates:
+                file.add_position_columns()
+            if time_correction:
+                file.absolute_time_calculation()
+                file.add_start_time()
+            if cast_identifier:
+                file.add_cast_number(index + 1)
+        return [file.create_dataframe() for file in self.data]
+
+    def get_data_table_meta_info(self) -> list[dict]:
+        """
+        Ensures the same data description in all input cnv files and returns
+        it.
+        Acts as an early alarm when working on different kinds of files, which
+        cannot be concatenated together.
+
+        Returns
+        -------
+        A list of dictionaries that represent the data column information.
+        """
+        all_column_descriptions = [
+            file.parameters.metadata for file in self.data
+        ]
+        for info in all_column_descriptions:
+            if all_column_descriptions[0] != info:
+                raise AssertionError(
+                    "Acting on differently formed data files, aborting"
+                )
+        return all_column_descriptions[0]
+
+    def get_array(self) -> np.ndarray:
+        """
+        Creates a collection array of all individual file arrays.
+
+        Returns
+        -------
+        A numpy array, representing the data of all input files.
+        """
+        return np.concatenate(
+            [file.parameters.create_full_ndarray() for file in self.data]
+        )

{seabirdfilehandler-0.5.3 → seabirdfilehandler-0.5.4}/src/seabirdfilehandler/parameter.py

@@ -32,15 +32,17 @@ class Parameters(UserDict):
         self,
         data: list,
         metadata: list,
+        only_header: bool = False,
     ):
         self.raw_input_data = data
         self.raw_metadata = metadata
-        self.full_data_array = self.create_full_ndarray()
         self.differentiate_table_description()
         self.metadata, self.duplicate_columns = self.reading_data_header(
             metadata
         )
-        self.data = self.create_parameter_instances()
+        if not only_header:
+            self.full_data_array = self.create_full_ndarray()
+            self.data = self.create_parameter_instances()
 
     def get_parameter_names(self) -> list[str]:
         return [parameter["name"] for parameter in self.metadata.values()]

seabirdfilehandler-0.5.3/src/seabirdfilehandler/file_collection.py

@@ -1,258 +0,0 @@
-from pathlib import Path
-import logging
-from collections import UserList
-from typing import Callable, Type
-import pandas as pd
-import numpy as np
-from seabirdfilehandler import (
-    CnvFile,
-    BottleFile,
-    BottleLogFile,
-)
-from seabirdfilehandler import DataFile
-from seabirdfilehandler.utils import get_unique_sensor_data
-
-logger = logging.getLogger(__name__)
-
-
-class FileCollection(UserList):
-    """A representation of multiple files of the same kind. These files share
-    the same suffix and are otherwise closely connected to each other. A common
-    use case would be the collection of CNVs to allow for easier processing or
-    integration of field calibration measurements.
-
-    Parameters
-    ----------
-
-    Returns
-    -------
-
-    """
-
-    def __init__(
-        self,
-        path_to_files: str | Path,
-        file_suffix: str,
-        only_metadata: bool = False,
-        sorting_key: Callable | None = None,
-    ):
-        super().__init__()
-        self.path_to_files = Path(path_to_files)
-        self.file_suffix = file_suffix.strip(".")
-        self.file_type: Type[DataFile]
-        self.extract_file_type()
-        self.individual_file_paths = []
-        self.collect_files(sorting_key=sorting_key)
-        self.load_files(only_metadata)
-        if not only_metadata:
-            if self.file_type == DataFile:
-                self.df_list = self.get_dataframes()
-                self.df = self.get_collection_dataframe(self.df_list)
-            if self.file_type == CnvFile:
-                self.data_meta_info = self.get_data_table_meta_info()
-            self.sensor_data = get_unique_sensor_data(
-                [file.sensors for file in self.data]
-            )
-
-    def __str__(self):
-        return "/n".join(self.data)
-
-    def extract_file_type(self):
-        """ """
-        mapping_suffix_to_type = {
-            "cnv": CnvFile,
-            "btl": BottleFile,
-            "bl": BottleLogFile,
-        }
-        for key, value in mapping_suffix_to_type.items():
-            if key == self.file_suffix:
-                self.file_type = value
-                break
-            else:
-                self.file_type = DataFile
-
-    def collect_files(
-        self,
-        sorting_key: Callable | None = lambda file: int(
-            file.stem.split("_")[3]
-        ),
-    ):
-        """ """
-        self.individual_file_paths = sorted(
-            self.path_to_files.rglob(f"*{self.file_suffix}"),
-            key=sorting_key,
-        )
-
-    def load_files(self, only_metadata: bool = False):
-        """ """
-        for file in self.individual_file_paths:
-            try:
-                self.data.append(self.file_type(file))
-            except TypeError:
-                logger.error(
-                    f"Could not open file {file} with the type "
-                    f"{self.file_type}."
-                )
-                continue
-
-    def get_dataframes(
-        self,
-        event_log: bool = False,
-        coordinates: bool = False,
-        time_correction: bool = False,
-        cast_identifier: bool = False,
-        long_header_names: bool = False,
-        full_data_header: bool = True,
-    ) -> list[pd.DataFrame]:
-        """
-
-        Parameters
-        ----------
-        event_log: bool :
-             (Default value = False)
-        coordinates: bool :
-             (Default value = False)
-        time_correction: bool :
-             (Default value = False)
-        cast_identifier: bool :
-             (Default value = False)
-        long_header_names: bool :
-             (Default value = False)
-        full_data_header: bool :
-             (Default value = True)
-
-        Returns
-        -------
-
-        """
-        for index, file in enumerate(self.data):
-            if full_data_header:
-                file.rename_dataframe_header(header_detail_level="longinfo")
-            elif long_header_names:
-                file.rename_dataframe_header(header_detail_level="name")
-            if event_log:
-                file.add_station_and_event_column()
-            if coordinates:
-                file.add_position_columns()
-            if time_correction:
-                file.absolute_time_calculation()
-                file.add_start_time()
-            if cast_identifier:
-                file.add_cast_number(index + 1)
-        return [file.df for file in self.data]
-
-    def get_collection_dataframe(
-        self, list_of_dfs: list[pd.DataFrame] | None = None
-    ) -> pd.DataFrame:
-        """
-
-        Parameters
-        ----------
-        list_of_dfs: list[pd.DataFrame] | None :
-             (Default value = None)
-
-        Returns
-        -------
-
-        """
-        if not list_of_dfs:
-            list_of_dfs = self.get_dataframes()
-        df = pd.concat(list_of_dfs, ignore_index=True)
-        # df.meta.metadata = list_of_dfs[0].meta.metadata
-        return df
-
-    def tidy_collection_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
-        """
-
-        Parameters
-        ----------
-        df: pd.DataFrame :
-
-
-        Returns
-        -------
-
-        """
-        df = self.use_bad_flag_for_nan(df)
-        df = self.set_dtype_to_float(df)
-        return self.select_real_scan_data(df)
-
-    def use_bad_flag_for_nan(self, df: pd.DataFrame) -> pd.DataFrame:
-        """
-
-        Parameters
-        ----------
-        df: pd.DataFrame :
-
-
-        Returns
-        -------
-
-        """
-        bad_flags = set()
-        for file in self.data:
-            for line in file.data_table_description:
-                if line.startswith("bad_flag"):
-                    flag = line.split("=")[1].strip()
-                    bad_flags.add(flag)
-        for flag in bad_flags:
-            df.replace(to_replace=flag, value=np.nan, inplace=True)
-        return df
-
-    def set_dtype_to_float(self, df: pd.DataFrame) -> pd.DataFrame:
-        """
-
-        Parameters
-        ----------
-        df: pd.DataFrame :
-
-
-        Returns
-        -------
-
-        """
-        for parameter in df.columns:
-            if parameter in ["datetime"]:
-                continue
-            try:
-                df[parameter] = df[parameter].astype("float")
-            finally:
-                continue
-        return df
-
-    def select_real_scan_data(self, df: pd.DataFrame) -> pd.DataFrame:
-        """
-
-        Parameters
-        ----------
-        df: pd.DataFrame :
-
-
-        Returns
-        -------
-
-        """
-        # TODO: fix this hardcoded name
-        try:
-            df = df.loc[df["Scan Count"].notna()]
-        finally:
-            pass
-        return df
-
-    def to_csv(self, file_name):
-        """
-
-        Parameters
-        ----------
-        file_name :
-
-
-        Returns
-        -------
-
-        """
-        self.get_collection_dataframe().to_csv(file_name)
-
-    def get_data_table_meta_info(self) -> list[list[dict]]:
-        """ """
-        return [file.parameters.metadata for file in self.data]