seabirdfilehandler 0.5.3__py3-none-any.whl → 0.5.4__py3-none-any.whl

seabirdfilehandler/bottlefile.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Union
 from datetime import datetime, time
 import pandas as pd
@@ -22,12 +23,13 @@ class BottleFile(DataFile):
 
     """
 
-    def __init__(self, path_to_file):
-        super().__init__(path_to_file)
-        self.original_df = self.create_dataframe()
-        self.df = self.original_df
-        self.setting_dataframe_dtypes()
-        self.adding_timestamp_column()
+    def __init__(self, path_to_file: Path | str, only_header: bool = False):
+        super().__init__(path_to_file, only_header)
+        if not only_header:
+            self.original_df = self.create_dataframe()
+            self.df = self.original_df
+            self.setting_dataframe_dtypes()
+            self.adding_timestamp_column()
 
     def create_dataframe(self):
         """Creates a dataframe out of the btl file. Manages the double data

seabirdfilehandler/cnvfile.py

@@ -60,9 +60,11 @@ class CnvFile(DataFile):
         super().__init__(path_to_file, only_header)
         self.validation_modules = self.obtaining_validation_modules()
         self.start_time = self.reading_start_time()
-        self.parameters = Parameters(self.data, self.data_table_description)
+        self.parameters = Parameters(
+            self.data, self.data_table_description, only_header
+        )
         if create_dataframe:
-            self.df = self.parameters.get_pandas_dataframe()
+            self.df = self.create_dataframe()
         if absolute_time_calculation:
             self.absolute_time_calculation()
         if event_log_column:
@@ -70,6 +72,13 @@ class CnvFile(DataFile):
         if coordinate_columns:
             self.add_position_columns()
 
+    def create_dataframe(self) -> pd.DataFrame:
+        """
+        Plain dataframe creator.
+        """
+        self.df = self.parameters.get_pandas_dataframe()
+        return self.df
+
     def reading_start_time(
         self,
         time_source: str = "System UTC",

seabirdfilehandler/file_collection.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 from pathlib import Path
 import logging
 from collections import UserList
@@ -15,18 +16,63 @@ from seabirdfilehandler.utils import get_unique_sensor_data
 logger = logging.getLogger(__name__)
 
 
+def get_collection(
+    path_to_files: Path | str,
+    file_suffix: str = "cnv",
+    only_metadata: bool = False,
+    sorting_key: Callable | None = None,
+) -> Type[FileCollection]:
+    """
+    Factory to create instances of FileCollection, depending on input type.
+
+    Parameters
+    ----------
+    path_to_files : Path | str :
+        The path to the directory to search for files.
+    file_suffix : str :
+        The suffix to search for. (Default value = "cnv")
+    only_metadata : bool :
+        Whether to read only metadata. (Default value = False)
+    sorting_key : Callable | None :
+        A callable that returns the filename-part to use to sort the collection. (Default value = None)
+    Returns
+    -------
+    An instance of FileCollection or one of its children.
+
+    """
+    mapping_suffix_to_type = {
+        "cnv": CnvCollection,
+        "btl": FileCollection,
+        "bl": FileCollection,
+    }
+    file_suffix = file_suffix.strip(".")
+    try:
+        collection = mapping_suffix_to_type[file_suffix](
+            path_to_files, file_suffix, only_metadata, sorting_key
+        )
+    except ValueError:
+        raise ValueError(f"Unknown input file type: {file_suffix}, aborting.")
+    else:
+        return collection
+
+
 class FileCollection(UserList):
-    """A representation of multiple files of the same kind. These files share
+    """
+    A representation of multiple files of the same kind. These files share
     the same suffix and are otherwise closely connected to each other. A common
     use case would be the collection of CNVs to allow for easier processing or
     integration of field calibration measurements.
 
     Parameters
     ----------
-
-    Returns
-    -------
-
+    path_to_files : Path | str :
+        The path to the directory to search for files.
+    file_suffix : str :
+        The suffix to search for. (Default value = "cnv")
+    only_metadata : bool :
+        Whether to read only metadata. (Default value = False)
+    sorting_key : Callable | None :
+        A callable that returns the filename-part to use to sort the collection. (Default value = None)
     """
 
     def __init__(
@@ -39,61 +85,90 @@ class FileCollection(UserList):
         super().__init__()
         self.path_to_files = Path(path_to_files)
         self.file_suffix = file_suffix.strip(".")
-        self.file_type: Type[DataFile]
-        self.extract_file_type()
-        self.individual_file_paths = []
-        self.collect_files(sorting_key=sorting_key)
-        self.load_files(only_metadata)
+        self.file_type = self.extract_file_type(self.file_suffix)
+        self.individual_file_paths = self.collect_files(
+            sorting_key=sorting_key
+        )
+        self.data = self.load_files(only_metadata)
         if not only_metadata:
-            if self.file_type == DataFile:
-                self.df_list = self.get_dataframes()
-                self.df = self.get_collection_dataframe(self.df_list)
-            if self.file_type == CnvFile:
-                self.data_meta_info = self.get_data_table_meta_info()
-            self.sensor_data = get_unique_sensor_data(
-                [file.sensors for file in self.data]
-            )
+            self.df_list = self.get_dataframes()
+            self.df = self.get_collection_dataframe(self.df_list)
 
     def __str__(self):
         return "/n".join(self.data)
 
-    def extract_file_type(self):
-        """ """
+    def extract_file_type(self, suffix: str) -> Type[DataFile]:
+        """
+        Determines the file type using the input suffix.
+
+        Parameters
+        ----------
+        suffix : str :
+            The file suffix.
+        Returns
+        -------
+        An object corresponding to the given suffix.
+        """
         mapping_suffix_to_type = {
             "cnv": CnvFile,
             "btl": BottleFile,
             "bl": BottleLogFile,
         }
+        file_type = DataFile
         for key, value in mapping_suffix_to_type.items():
-            if key == self.file_suffix:
-                self.file_type = value
+            if key == suffix:
+                file_type = value
                 break
-            else:
-                self.file_type = DataFile
+        return file_type
 
     def collect_files(
         self,
         sorting_key: Callable | None = lambda file: int(
             file.stem.split("_")[3]
         ),
-    ):
-        """ """
-        self.individual_file_paths = sorted(
+    ) -> list[Path]:
+        """
+        Creates a list of target files, recursively from the given directory.
+        These can be sorted with the help of the sorting_key parameter, which
+        is a Callable that identifies the part of the filename that shall be
+        used for sorting.
+
+        Parameters
+        ----------
+        sorting_key : Callable | None :
+            The part of the filename to use in sorting. (Default value = lambda file: int(file.stem.split("_")[3]))
+        Returns
+        -------
+        A list of all paths found.
+        """
+        return sorted(
             self.path_to_files.rglob(f"*{self.file_suffix}"),
             key=sorting_key,
         )
 
-    def load_files(self, only_metadata: bool = False):
-        """ """
+    def load_files(self, only_metadata: bool = False) -> list[DataFile]:
+        """
+        Creates python instances of each file.
+
+        Parameters
+        ----------
+        only_metadata : bool :
+            Whether to load only file metadata. (Default value = False)
+        Returns
+        -------
+        A list of all instances.
+        """
+        data = []
         for file in self.individual_file_paths:
             try:
-                self.data.append(self.file_type(file))
+                data.append(self.file_type(file, only_metadata))
             except TypeError:
                 logger.error(
                     f"Could not open file {file} with the type "
                     f"{self.file_type}."
                 )
                 continue
+        return data
 
     def get_dataframes(
         self,
@@ -101,35 +176,27 @@ class FileCollection(UserList):
         coordinates: bool = False,
         time_correction: bool = False,
         cast_identifier: bool = False,
-        long_header_names: bool = False,
-        full_data_header: bool = True,
     ) -> list[pd.DataFrame]:
         """
+        Collects all individual dataframes and allows additional column
+        creation.
 
         Parameters
         ----------
-        event_log: bool :
-             (Default value = False)
-        coordinates: bool :
-             (Default value = False)
-        time_correction: bool :
-             (Default value = False)
-        cast_identifier: bool :
-             (Default value = False)
-        long_header_names: bool :
-             (Default value = False)
-        full_data_header: bool :
-             (Default value = True)
+        event_log : bool :
+            (Default value = False)
+        coordinates : bool :
+            (Default value = False)
+        time_correction : bool :
+            (Default value = False)
+        cast_identifier : bool :
+            (Default value = False)
 
         Returns
         -------
-
+        A list of the individual pandas DataFrames.
         """
         for index, file in enumerate(self.data):
-            if full_data_header:
-                file.rename_dataframe_header(header_detail_level="longinfo")
-            elif long_header_names:
-                file.rename_dataframe_header(header_detail_level="name")
             if event_log:
                 file.add_station_and_event_column()
             if coordinates:
@@ -145,33 +212,35 @@ class FileCollection(UserList):
         self, list_of_dfs: list[pd.DataFrame] | None = None
     ) -> pd.DataFrame:
         """
+        Creates one DataFrame from the individual ones, by concatenation.
 
         Parameters
         ----------
-        list_of_dfs: list[pd.DataFrame] | None :
-             (Default value = None)
-
+        list_of_dfs : list[pd.DataFrame] | None :
+            A list of the individual DataFrames. (Default value = None)
         Returns
         -------
-
+        A pandas DataFrame representing the whole dataset.
         """
         if not list_of_dfs:
             list_of_dfs = self.get_dataframes()
+        if not list_of_dfs:
+            raise ValueError("No dataframes to concatenate.")
         df = pd.concat(list_of_dfs, ignore_index=True)
-        # df.meta.metadata = list_of_dfs[0].meta.metadata
+        self.df = df
         return df
 
     def tidy_collection_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
         """
+        Apply the different dataframe edits to the given dataframe.
 
         Parameters
         ----------
-        df: pd.DataFrame :
-
-
+        df : pd.DataFrame :
+            A DataFrame to edit.
         Returns
         -------
-
+        The tidied dataframe.
         """
         df = self.use_bad_flag_for_nan(df)
         df = self.set_dtype_to_float(df)
@@ -179,15 +248,15 @@ class FileCollection(UserList):
 
     def use_bad_flag_for_nan(self, df: pd.DataFrame) -> pd.DataFrame:
         """
+        Replace all Nan values by the bad flag value, defined inside the files.
 
         Parameters
         ----------
-        df: pd.DataFrame :
-
-
+        df : pd.DataFrame :
+            The dataframe to edit.
         Returns
         -------
-
+        The edited DataFrame.
         """
         bad_flags = set()
         for file in self.data:
@@ -201,15 +270,15 @@ class FileCollection(UserList):
 
     def set_dtype_to_float(self, df: pd.DataFrame) -> pd.DataFrame:
         """
+        Use the float-dtype for all DataFrame columns.
 
         Parameters
         ----------
-        df: pd.DataFrame :
-
-
+        df : pd.DataFrame :
+            The dataframe to edit.
         Returns
         -------
-
+        The edited DataFrame.
         """
         for parameter in df.columns:
             if parameter in ["datetime"]:
@@ -222,37 +291,121 @@ class FileCollection(UserList):
 
     def select_real_scan_data(self, df: pd.DataFrame) -> pd.DataFrame:
         """
+        Drop data rows have no 'Scan' value, if that column exists.
 
         Parameters
         ----------
-        df: pd.DataFrame :
-
-
+        df : pd.DataFrame :
+            The dataframe to edit.
         Returns
         -------
-
+        The edited DataFrame.
         """
-        # TODO: fix this hardcoded name
         try:
-            df = df.loc[df["Scan Count"].notna()]
-        finally:
-            pass
+            scan_column = [
+                c for c in df.columns if c.lower().startswith("scan")
+            ][0]
+        except IndexError:
+            return df
+        else:
+            df = df.loc[df[scan_column].notna()]
         return df
 
     def to_csv(self, file_name):
         """
+        Writes a csv file with the given filename.
 
         Parameters
         ----------
         file_name :
+            The new csv file name.
+        """
+        self.df.to_csv(file_name)
+
+
+class CnvCollection(FileCollection):
+    """
+    Specific methods to work with collections of .cnv files.
+    """
+
+    def __init__(
+        self,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.data_meta_info = self.get_data_table_meta_info()
+        self.sensor_data = get_unique_sensor_data(
+            [file.sensors for file in self.data]
+        )
+        self.array = self.get_array()
+
+    def get_dataframes(
+        self,
+        event_log: bool = False,
+        coordinates: bool = False,
+        time_correction: bool = False,
+        cast_identifier: bool = False,
+    ) -> list[pd.DataFrame]:
+        """
+        Collects all individual dataframes and allows additional column
+        creation.
+
+        Parameters
+        ----------
+        event_log : bool :
+            (Default value = False)
+        coordinates : bool :
+            (Default value = False)
+        time_correction : bool :
+            (Default value = False)
+        cast_identifier : bool :
+            (Default value = False)
+        Returns
+        -------
+        A list of the individual pandas DataFrames.
+        """
+        for index, file in enumerate(self.data):
+            if event_log:
+                file.add_station_and_event_column()
+            if coordinates:
+                file.add_position_columns()
+            if time_correction:
+                file.absolute_time_calculation()
+                file.add_start_time()
+            if cast_identifier:
+                file.add_cast_number(index + 1)
+        return [file.create_dataframe() for file in self.data]
 
+    def get_data_table_meta_info(self) -> list[dict]:
+        """
+        Ensures the same data description in all input cnv files and returns
+        it.
+        Acts as an early alarm when working on different kinds of files, which
+        cannot be concatenated together.
 
         Returns
         -------
+        A list of dictionaries that represent the data column information.
+        """
+        all_column_descriptions = [
+            file.parameters.metadata for file in self.data
+        ]
+        for info in all_column_descriptions:
+            if all_column_descriptions[0] != info:
+                raise AssertionError(
+                    "Acting on differently formed data files, aborting"
+                )
+        return all_column_descriptions[0]
 
+    def get_array(self) -> np.ndarray:
         """
-        self.get_collection_dataframe().to_csv(file_name)
+        Creates a collection array of all individual file arrays.
 
-    def get_data_table_meta_info(self) -> list[list[dict]]:
-        """ """
-        return [file.parameters.metadata for file in self.data]
+        Returns
+        -------
+        A numpy array, representing the data of all input files.
+        """
+        return np.concatenate(
+            [file.parameters.create_full_ndarray() for file in self.data]
+        )

seabirdfilehandler/parameter.py

@@ -32,15 +32,17 @@ class Parameters(UserDict):
         self,
         data: list,
         metadata: list,
+        only_header: bool = False,
     ):
         self.raw_input_data = data
         self.raw_metadata = metadata
-        self.full_data_array = self.create_full_ndarray()
         self.differentiate_table_description()
         self.metadata, self.duplicate_columns = self.reading_data_header(
             metadata
         )
-        self.data = self.create_parameter_instances()
+        if not only_header:
+            self.full_data_array = self.create_full_ndarray()
+            self.data = self.create_parameter_instances()
 
     def get_parameter_names(self) -> list[str]:
         return [parameter["name"] for parameter in self.metadata.values()]

{seabirdfilehandler-0.5.3.dist-info → seabirdfilehandler-0.5.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: seabirdfilehandler
-Version: 0.5.3
+Version: 0.5.4
 Summary: Library of parsers to interact with SeaBird CTD files.
 Keywords: CTD,parser,seabird,data
 Author: Emil Michels
@@ -16,7 +16,8 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: pandas (>=2.2.1)
 Requires-Dist: xmltodict (>=0.13.0)
-Project-URL: Homepage, https://git.io-warnemuende.de/CTD-Software/SeabirdFileHandler
+Project-URL: Documentation, https://ctd-software.pages.io-warnemuende.de/seabirdfilehandler
+Project-URL: Homepage, https://ctd-software.pages.io-warnemuende.de/seabirdfilehandler
 Project-URL: Repository, https://git.io-warnemuende.de/CTD-Software/SeabirdFileHandler
 Description-Content-Type: text/markdown
 

{seabirdfilehandler-0.5.3.dist-info → seabirdfilehandler-0.5.4.dist-info}/RECORD

@@ -1,15 +1,15 @@
 seabirdfilehandler/__init__.py,sha256=rS1NfxKVWgOntk5NL-ndZyDt3LHW_tKr_F3iW_QbYvg,230
-seabirdfilehandler/bottlefile.py,sha256=nnfoDczPMG_ge40dT2rHNhifR7-NRgnZNFrfPM_9OSQ,5925
+seabirdfilehandler/bottlefile.py,sha256=qCh506J3MWZXM11243aw_oJRocVB0ZIipXQLEgkD5M0,6046
 seabirdfilehandler/bottlelogfile.py,sha256=MtMmEebdAktO3mk6KbmJC7dfx9sRLbV5qqDQt2qtpJE,4310
-seabirdfilehandler/cnvfile.py,sha256=0fDovJ6WJsxkbtonbuKIPDY2E5Ex1k-1WD5nkA2WRyg,9990
+seabirdfilehandler/cnvfile.py,sha256=j7IR3EgCrGD3riKOzFSKFNW6lkuzZYGYkxI_czcD8XU,10196
 seabirdfilehandler/datafiles.py,sha256=9r0Mh3zPYJJ3CoybgOBH4Dsq43kLDnca9m8s_V0cYU8,9378
-seabirdfilehandler/file_collection.py,sha256=b5iJaP4F34Vq7-FiJOlPvfS4IePGWsYx20XwWbZQw1A,6882
+seabirdfilehandler/file_collection.py,sha256=IXbNTpplF-BQmjDSPh6Cj_f5-mS5C-biBLDRnqaFhOo,12531
 seabirdfilehandler/geomar_ctd_file_parser.py,sha256=4eCnkE0mvPKC8Dic8sXP4xpfwnk3K2MQcGFBf6loT8k,2655
-seabirdfilehandler/parameter.py,sha256=wBBSe85-Eqc3YmntuZmwvny1l2exXATri-oak8xEKX0,14490
+seabirdfilehandler/parameter.py,sha256=CjC8T5wfbThryqVNjTcAolug9gi_BnInfZzkEfmKm5E,14561
 seabirdfilehandler/utils.py,sha256=5KXdB8Hdv65dv5tPyXxNMct1mCEOyA3S8XP54AFAnx0,1745
 seabirdfilehandler/validation_modules.py,sha256=eZ6x0giftUtlxnRMOnK_vCkgccdwUXPrDjajFa-E6n0,4698
 seabirdfilehandler/xmlfiles.py,sha256=L_puQf8eg0ojv85AyEMID4jnwkOlV_fgZP3W5yeSUBY,4668
-seabirdfilehandler-0.5.3.dist-info/LICENSE,sha256=Ifd1VPmYv32oJd2QVh3wIQP9X05vYJlcY6kONz360ws,34603
-seabirdfilehandler-0.5.3.dist-info/METADATA,sha256=7LNvhfKbt5_sjuLNhqO8t1h2iKKZyS7sJZhT0nh2c4w,2213
-seabirdfilehandler-0.5.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-seabirdfilehandler-0.5.3.dist-info/RECORD,,
+seabirdfilehandler-0.5.4.dist-info/LICENSE,sha256=Ifd1VPmYv32oJd2QVh3wIQP9X05vYJlcY6kONz360ws,34603
+seabirdfilehandler-0.5.4.dist-info/METADATA,sha256=yt3BkhGRUOlsQ8tLPjziKPmobuYTDNUybjAARMJrcZw,2307
+seabirdfilehandler-0.5.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+seabirdfilehandler-0.5.4.dist-info/RECORD,,