masster 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl

masster/sample/h5.py

@@ -1,1297 +1,1321 @@
-import json
-import os
-
-import h5py
-import numpy as np
-import polars as pl
-
-from typing import Any, Dict, List, Optional, Tuple
-
-from masster.chromatogram import Chromatogram
-from masster.spectrum import Spectrum
-
-
-def _save_sample5(self, filename=None, include_ms1=True, include_scans=True):
-    """
-    Save the instance data to a sample5 HDF5 file with optimized compression.
-
-    This optimized version uses context-aware compression settings for better
-    performance and smaller file sizes. Different compression algorithms are 
-    selected based on data type and usage patterns.
-
-    Args:
-        filename (str, optional): Target file name. If None, uses default based on file_path.
-        include_ms1 (bool, optional): Whether to include MS1 data. Defaults to True.
-        include_scans (bool, optional): Whether to include scan data. Defaults to True.
-
-    Stores:
-        - metadata/format (str): Data format identifier (master-sample-1)
-        - metadata/file_path (str): Source file path
-        - metadata/file_type (str): Source file type
-        - metadata/label (str): Sample label
-        - metadata/parameters (str): Parameters as JSON string with optimized compression
-        - scans/: Scan DataFrame data with fast-access compression for IDs, standard for others
-        - features/: Feature DataFrame data with JSON compression for objects, fast compression for core data
-        - ms1/: MS1-level data with numeric compression
-
-    Compression Strategy:
-        - LZF + shuffle: Fast access data (feature_uid, rt, mz, intensity, scan_id)
-        - GZIP level 6: JSON objects (chromatograms, spectra) and string data
-        - GZIP level 9: Bulk storage data (large MS2 spectrum collections)
-        - LZF: Standard numeric arrays
-
-    Performance Improvements:
-        - 8-15% smaller file sizes
-        - 20-50% faster save operations for large files
-        - Context-aware compression selection
-    """
-    if filename is None:
-        # save to default file name
-        if self.file_path is not None:
-            filename = os.path.splitext(self.file_path)[0] + ".sample5"
-        else:
-            self.logger.error("either filename or file_path must be provided")
-            return
-
-    # if no extension is given, add .sample5
-    if not filename.endswith(".sample5"):
-        filename += ".sample5"
-
-    self.logger.debug(f"Saving sample to {filename} with optimized LZF+shuffle compression")
-
-    # delete existing file if it exists
-    if os.path.exists(filename):
-        os.remove(filename)
-
-    with h5py.File(filename, "w") as f:
-        # Create groups for organization
-        metadata_group = f.create_group("metadata")
-        features_group = f.create_group("features")
-        scans_group = f.create_group("scans")
-        ms1_group = f.create_group("ms1")
-
-        # Store metadata
-        metadata_group.attrs["format"] = "master-sample-1"
-        if self.file_path is not None:
-            metadata_group.attrs["file_path"] = str(self.file_path)
-        else:
-            metadata_group.attrs["file_path"] = ""
-        if self.file_type is not None:
-            metadata_group.attrs["file_type"] = str(self.file_type)
-        else:
-            metadata_group.attrs["file_type"] = ""
-        if self.label is not None:
-            metadata_group.attrs["label"] = str(self.label)
-        else:
-            metadata_group.attrs["label"] = ""
-
-        # Store DataFrames
-        if self.scans_df is not None and include_scans:
-            scans_df = self.scans_df.clone()
-            for col in scans_df.columns:
-                data = scans_df[col].to_numpy()
-                # Handle different data types safely
-                if data.dtype == object:
-                    try:
-                        str_data = np.array(
-                            ["" if x is None else str(x) for x in data],
-                            dtype="S",
-                        )
-                        scans_group.create_dataset(
-                            col,
-                            data=str_data,
-                            compression="gzip",
-                        )
-                        scans_group[col].attrs["dtype"] = "string_converted"
-                    except Exception:
-                        try:
-                            # Try to convert to numeric using numpy
-                            numeric_data = np.array([float(x) if x is not None and str(x).replace('.', '').replace('-', '').isdigit() else np.nan for x in data])
-                            if not np.isnan(numeric_data).all():
-                                scans_group.create_dataset(
-                                    col,
-                                    data=numeric_data,
-                                    compression="gzip",
-                                )
-                                scans_group[col].attrs["dtype"] = "numeric_converted"
-                            else:
-                                json_data = np.array(
-                                    [json.dumps(x, default=str) for x in data],
-                                    dtype="S",
-                                )
-                                scans_group.create_dataset(
-                                    col,
-                                    data=json_data,
-                                    compression="gzip",
-                                )
-                                scans_group[col].attrs["dtype"] = "json_serialized"
-                        except Exception:
-                            str_repr_data = np.array([str(x) for x in data], dtype="S")
-                            scans_group.create_dataset(
-                                col,
-                                data=str_repr_data,
-                                compression="gzip",
-                            )
-                            scans_group[col].attrs["dtype"] = "string_repr"
-                else:
-                    scans_group.create_dataset(col, data=data, compression="lzf", shuffle=True)
-                    scans_group[col].attrs["dtype"] = "native"
-            scans_group.attrs["columns"] = list(scans_df.columns)
-
-        if self.features_df is not None:
-            features = self.features_df.clone()
-            for col in features.columns:
-                # get column dtype
-                dtype = str(features[col].dtype).lower()
-                if dtype == "object":
-                    if col == "chrom":
-                        # this column contains either None or Chromatogram objects
-                        # convert to json with to_json() and store them as compressed strings
-                        data = features[col]
-                        data_as_str = []
-                        for i in range(len(data)):
-                            if data[i] is not None:
-                                data_as_str.append(data[i].to_json())
-                            else:
-                                data_as_str.append("None")
-                        features_group.create_dataset(
-                            col,
-                            data=data_as_str,
-                            compression="gzip",
-                        )
-                    elif col == "ms2_scans":
-                        # this column contains either None or lists of integers (scan indices)
-                        # convert each to JSON string for storage (HDF5 can't handle inhomogeneous arrays)
-                        data = features[col]
-                        data_as_json_strings = []
-                        for i in range(len(data)):
-                            if data[i] is not None:
-                                data_as_json_strings.append(json.dumps(list(data[i])))
-                            else:
-                                data_as_json_strings.append("None")
-                        features_group.create_dataset(
-                            col,
-                            data=data_as_json_strings,
-                            compression="gzip",
-                        )
-                    elif col == "ms2_specs":
-                        # this column contains either None or lists of Spectrum objects
-                        # convert each spectrum to json and store as list of json strings
-                        data = features[col]
-                        data_as_lists_of_strings = []
-                        for i in range(len(data)):
-                            if data[i] is not None:
-                                # Convert list of Spectrum objects to list of JSON strings
-                                spectrum_list = data[i]
-                                json_strings = []
-                                for spectrum in spectrum_list:
-                                    if spectrum is not None:
-                                        json_strings.append(spectrum.to_json())
-                                    else:
-                                        json_strings.append("None")
-                                data_as_lists_of_strings.append(json_strings)
-                            else:
-                                data_as_lists_of_strings.append(["None"])
-                        # Convert to numpy array for HDF5 storage
-                        serialized_data = []
-                        for item in data_as_lists_of_strings:
-                            serialized_data.append(json.dumps(item))
-                        features_group.create_dataset(
-                            col,
-                            data=serialized_data,
-                            compression="gzip",
-                        )
-
-                    else:
-                        self.logger.warning(
-                            f"Unexpectedly, column '{col}' has dtype 'object'. Implement serialization for this column.",
-                        )
-                    continue
-                elif dtype == "string":
-                    data = features[col].to_list()
-                    # convert None to 'None' strings
-                    data = ["None" if x is None else x for x in data]
-                    features_group.create_dataset(col, data=data, compression="lzf", shuffle=True)
-                else:
-                    try:
-                        data = features[col].to_numpy()
-                        features_group.create_dataset(col, data=data)
-                    except Exception:
-                        self.logger.warning(
-                            f"Failed to save column '{col}' with dtype '{dtype}'. It may contain unsupported data types.",
-                        )
-            features_group.attrs["columns"] = list(features.columns)
-
-        # Store arrays
-        if self.ms1_df is not None and include_ms1:
-            # the df is a polars DataFrame
-            for col in self.ms1_df.columns:
-                ms1_group.create_dataset(
-                    col,
-                    data=self.ms1_df[col].to_numpy(),
-                    compression="gzip",
-                )
-
-        # Store parameters as JSON
-        if self.parameters is not None:
-            # Convert parameters dict to JSON string
-            params_json = json.dumps(self.parameters, default=str)
-            metadata_group.attrs["parameters"] = params_json
-
-        # Store lib and lib_match - removed (no longer saving lib data)
-
-    self.logger.info(f"Sample saved successfully to {filename}")
-    if self.features is not None:
-        # save the features as a separate file
-        self._save_featureXML(filename=filename.replace(".sample5", ".featureXML"))
-
-
-def _load_sample5(self, filename: str, map: bool = True):
-    """
-    Load instance data from a sample5 HDF5 file.
-
-    Restores all attributes that were saved with save_sample5() method using the
-    schema defined in sample5_schema.json for proper Polars DataFrame reconstruction.
-
-    Args:
-        filename (str): Path to the sample5 HDF5 file to load.
-        map (bool, optional): Whether to map featureXML file if available. Defaults to True.
-
-    Returns:
-        None (modifies self in place)
-
-    Notes:
-        - Restores DataFrames with proper schema typing from sample5_schema.json
-        - Handles Chromatogram and Spectrum object reconstruction
-        - Properly handles MS2 scan lists and spectrum lists
-    """
-    # Load schema for proper DataFrame reconstruction
-    schema_path = os.path.join(os.path.dirname(__file__), "sample5_schema.json")
-    try:
-        with open(schema_path) as f:
-            schema = json.load(f)
-    except FileNotFoundError:
-        self.logger.warning(
-            f"Schema file {schema_path} not found. Using default types.",
-        )
-        schema = {}
-
-    with h5py.File(filename, "r") as f:
-        # Load metadata
-        if "metadata" in f:
-            metadata_group = f["metadata"]
-            self.file_path = decode_metadata_attr(metadata_group.attrs.get("file_path", ""))
-            self.file_type = decode_metadata_attr(metadata_group.attrs.get("file_type", ""))
-            self.label = decode_metadata_attr(metadata_group.attrs.get("label", ""))
-            
-            # Load parameters from JSON in metadata
-            loaded_data = load_parameters_from_metadata(metadata_group)
-            
-            # Always create a fresh sample_defaults object
-            from masster.sample.defaults.sample_def import sample_defaults
-            self.parameters = sample_defaults()
-            
-            # Initialize history and populate from loaded data
-            self.history = {}
-            if loaded_data is not None and isinstance(loaded_data, dict):
-                # Store the loaded data in history
-                self.history = loaded_data
-                # If there are sample parameters in the history, use them to update defaults
-                if "sample" in loaded_data:
-                    sample_params = loaded_data["sample"]
-                    if isinstance(sample_params, dict):
-                        self.parameters.set_from_dict(sample_params, validate=False)
-
-        # Load scans_df
-        if "scans" in f:
-            scans_group = f["scans"]
-            data: dict[str, Any] = {}
-            missing_columns = []
-            for col in schema.get("scans_df", {}).get("columns", []):
-                if col not in scans_group:
-                    self.logger.debug(f"Column '{col}' not found in sample5/scans.")
-                    data[col] = None
-                    missing_columns.append(col)
-                    continue
-
-                dtype = schema["scans_df"]["columns"][col].get("dtype", "native")
-                match dtype:
-                    case "pl.Object":
-                        self.logger.debug(f"Unexpected Object column '{col}'")
-                        data[col] = None
-                        missing_columns.append(col)
-
-                    case _:
-                        data[col] = scans_group[col][:]
-
-            # create polars DataFrame from data
-            if data:
-                self.scans_df = pl.DataFrame(data)
-
-                # Convert "None" strings and NaN values to proper null values
-                for col in self.scans_df.columns:
-                    if self.scans_df[col].dtype == pl.Utf8:  # String columns
-                        self.scans_df = self.scans_df.with_columns([
-                            pl.when(pl.col(col).is_in(["None", "", "null", "NULL"]))
-                            .then(None)
-                            .otherwise(pl.col(col))
-                            .alias(col),
-                        ])
-                    elif self.scans_df[col].dtype in [
-                        pl.Float64,
-                        pl.Float32,
-                    ]:  # Float columns
-                        self.scans_df = self.scans_df.with_columns([
-                            pl.col(col).fill_nan(None).alias(col),
-                        ])
-
-                # update all columns with schema types
-                for col in self.scans_df.columns:
-                    if col in schema.get("scans_df", {}).get("columns", {}):
-                        try:
-                            dtype_str = schema["scans_df"]["columns"][col]["dtype"]
-                            # Convert dtype string to actual polars dtype
-                            if dtype_str.startswith("pl."):
-                                # Skip Object columns - they're already properly reconstructed
-                                if "Object" in dtype_str:
-                                    continue
-                                # Handle different polars data types
-                                if "Int" in dtype_str:
-                                    # Convert to numeric first, handling different input types
-                                    if self.scans_df[col].dtype == pl.Utf8:
-                                        # String data - convert to integer
-                                        self.scans_df = self.scans_df.with_columns(
-                                            pl.col(col)
-                                            .str.to_integer()
-                                            .cast(eval(dtype_str)),
-                                        )
-                                    elif self.scans_df[col].dtype in [
-                                        pl.Float64,
-                                        pl.Float32,
-                                    ]:
-                                        # Float data - cast to integer
-                                        self.scans_df = self.scans_df.with_columns(
-                                            pl.col(col).cast(eval(dtype_str)),
-                                        )
-                                    else:
-                                        # Try direct casting
-                                        self.scans_df = self.scans_df.with_columns(
-                                            pl.col(col).cast(eval(dtype_str)),
-                                        )
-                                elif "Float" in dtype_str:
-                                    # Convert to float, handling different input types
-                                    if self.scans_df[col].dtype == pl.Utf8:
-                                        # String data - convert to float
-                                        self.scans_df = self.scans_df.with_columns(
-                                            pl.col(col)
-                                            .str.to_decimal()
-                                            .cast(eval(dtype_str)),
-                                        )
-                                    else:
-                                        # Try direct casting
-                                        self.scans_df = self.scans_df.with_columns(
-                                            pl.col(col).cast(eval(dtype_str)),
-                                        )
-                                elif "Utf8" in dtype_str:
-                                    # Ensure it's string type
-                                    self.scans_df = self.scans_df.with_columns(
-                                        pl.col(col).cast(pl.Utf8),
-                                    )
-                                else:
-                                    # Handle special cases and try direct casting for other types
-                                    current_dtype = self.scans_df[col].dtype
-                                    target_dtype = eval(dtype_str)
-
-                                    # Handle binary data that might need string conversion first
-                                    if "Binary" in str(current_dtype):
-                                        # Convert binary to string first, then to target type
-                                        if target_dtype == pl.Utf8:
-                                            self.scans_df = self.scans_df.with_columns(
-                                                pl.col(col)
-                                                .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                .cast(target_dtype),
-                                            )
-                                        elif "Int" in str(target_dtype):
-                                            self.scans_df = self.scans_df.with_columns(
-                                                pl.col(col)
-                                                .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                .str.to_integer()
-                                                .cast(target_dtype),
-                                            )
-                                        elif "Float" in str(target_dtype):
-                                            self.scans_df = self.scans_df.with_columns(
-                                                pl.col(col)
-                                                .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                .str.to_decimal()
-                                                .cast(target_dtype),
-                                            )
-                                        else:
-                                            # Try direct casting
-                                            self.scans_df = self.scans_df.with_columns(
-                                                pl.col(col).cast(target_dtype),
-                                            )
-                                    else:
-                                        # Try direct casting for non-binary types
-                                        self.scans_df = self.scans_df.with_columns(
-                                            pl.col(col).cast(target_dtype),
-                                        )
-                        except Exception as e:
-                            self.logger.warning(
-                                f"Failed to cast column '{col}' in scans_df: {e}",
-                            )
-                    else:
-                        self.logger.warning(
-                            f"Column '{col}' in scans_df not found in schema, keeping original type.",
-                        )
-
-            # Ensure column order matches schema order
-            if "scans_df" in schema and "columns" in schema["scans_df"]:
-                schema_column_order = list(schema["scans_df"]["columns"].keys())
-                # Only reorder columns that exist in both schema and DataFrame
-                existing_columns = [col for col in schema_column_order if col in self.scans_df.columns]
-                if existing_columns:
-                    self.scans_df = self.scans_df.select(existing_columns)
-
-            else:
-                self.scans_df = None
-        else:
-            self.scans_df = None
-
-        # Load features_df
-        if "features" in f:
-            features_group = f["features"]
-            # columns = list(features_group.attrs.get('columns', []))
-            data = {}
-            missing_columns = []
-            for col in schema.get("features_df", {}).get("columns", []):
-                if col not in features_group:
-                    self.logger.debug(
-                        f"Column '{col}' not found in sample5/features.",
-                    )
-                    data[col] = None
-                    missing_columns.append(col)
-                    continue
-
-                dtype = schema["features_df"]["columns"][col].get("dtype", "native")
-                match dtype:
-                    case "pl.Object":
-                        match col:
-                            case "chrom":
-                                data_col = features_group[col][:]
-                                # Convert JSON strings back to Chromatogram objects
-                                reconstructed_data: list[Any] = []
-                                for item in data_col:
-                                    if isinstance(item, bytes):
-                                        item = item.decode("utf-8")
-
-                                    if item == "None" or item == "":
-                                        reconstructed_data.append(None)
-                                    else:
-                                        try:
-                                            reconstructed_data.append(
-                                                Chromatogram.from_json(item),
-                                            )
-                                        except (json.JSONDecodeError, ValueError):
-                                            reconstructed_data.append(None)
-
-                                data[col] = reconstructed_data
-                            case "ms2_scans":
-                                data_col = features_group[col][:]
-                                # Convert JSON strings back to lists of integers
-                                reconstructed_data = []
-                                for item in data_col:
-                                    if isinstance(item, bytes):
-                                        item = item.decode("utf-8")
-
-                                    if item == "None":
-                                        reconstructed_data.append(None)
-                                    else:
-                                        try:
-                                            # Parse JSON string to get list of integers
-                                            scan_list = json.loads(item)
-                                            reconstructed_data.append(scan_list)
-                                        except (json.JSONDecodeError, ValueError):
-                                            reconstructed_data.append(None)
-
-                                data[col] = reconstructed_data
-                            case "ms2_specs":
-                                data_col = features_group[col][:]
-                                # Convert JSON strings back to lists of Spectrum objects
-                                reconstructed_data = []
-                                for item in data_col:
-                                    if isinstance(item, bytes):
-                                        item = item.decode("utf-8")
-
-                                    # Parse the outer JSON (list of JSON strings)
-                                    json_list = json.loads(item)
-
-                                    if json_list == ["None"]:
-                                        # This was originally None
-                                        reconstructed_data.append(None)
-                                    else:
-                                        # This was originally a list of Spectrum objects
-                                        spectrum_list: list[Any] = []
-                                        for json_str in json_list:
-                                            if json_str == "None":
-                                                spectrum_list.append(None)
-                                            else:
-                                                spectrum_list.append(
-                                                    Spectrum.from_json(json_str),
-                                                )
-                                        reconstructed_data.append(spectrum_list)
-
-                                data[col] = reconstructed_data
-                            case _:
-                                self.logger.debug(f"Unexpected Object column '{col}'")
-                                data[col] = None
-                                missing_columns.append(col)
-
-                    case _:
-                        data[col] = features_group[col][:]
-
-            # create polars DataFrame from data
-            if data:
-                # Build schema for DataFrame creation to handle Object columns properly
-                df_schema = {}
-                for col, values in data.items():
-                    if col in schema.get("features_df", {}).get("columns", {}):
-                        dtype_str = schema["features_df"]["columns"][col]["dtype"]
-                        if dtype_str == "pl.Object":
-                            df_schema[col] = pl.Object
-                        else:
-                            # Let Polars infer the type initially, we'll cast later
-                            df_schema[col] = None
-                    else:
-                        df_schema[col] = None
-
-                # Create DataFrame with explicit Object types where needed
-                try:
-                    self.features_df = pl.DataFrame(data, schema=df_schema)
-                except Exception:
-                    # Fallback: create without schema and handle Object columns manually
-                    object_columns = {
-                        k: v
-                        for k, v in data.items()
-                        if k in schema.get("features_df", {}).get("columns", {})
-                        and schema["features_df"]["columns"][k]["dtype"] == "pl.Object"
-                    }
-                    regular_columns = {
-                        k: v for k, v in data.items() if k not in object_columns
-                    }
-
-                    # Create DataFrame with regular columns first
-                    if regular_columns:
-                        self.features_df = pl.DataFrame(regular_columns)
-                        # Add Object columns one by one
-                        for col, values in object_columns.items():
-                            self.features_df = self.features_df.with_columns([
-                                pl.Series(col, values, dtype=pl.Object),
-                            ])
-                    else:
-                        # Only Object columns
-                        self.features_df = pl.DataFrame()
-                        for col, values in object_columns.items():
-                            self.features_df = self.features_df.with_columns([
-                                pl.Series(col, values, dtype=pl.Object),
-                            ])
-
-                # update all columns with schema types (skip Object columns)
-                for col in self.features_df.columns:
-                    if col in schema.get("features_df", {}).get("columns", {}):
-                        try:
-                            dtype_str = schema["features_df"]["columns"][col]["dtype"]
-                            # Convert dtype string to actual polars dtype
-                            if dtype_str.startswith("pl."):
-                                # Skip Object columns - they're already properly reconstructed
-                                if "Object" in dtype_str:
-                                    continue
-                                # Handle different polars data types
-                                if "Int" in dtype_str:
-                                    # Convert to numeric first, handling different input types
-                                    if self.features_df[col].dtype == pl.Utf8:
-                                        # String data - convert to integer
-                                        self.features_df = (
-                                            self.features_df.with_columns(
-                                                pl.col(col)
-                                                .str.to_integer()
-                                                .cast(eval(dtype_str)),
-                                            )
-                                        )
-                                    elif self.features_df[col].dtype in [
-                                        pl.Float64,
-                                        pl.Float32,
-                                    ]:
-                                        # Float data - cast to integer
-                                        self.features_df = (
-                                            self.features_df.with_columns(
-                                                pl.col(col).cast(eval(dtype_str)),
-                                            )
-                                        )
-                                    else:
-                                        # Handle special cases and try direct casting for other types
-                                        current_dtype = self.features_df[col].dtype
-                                        target_dtype = eval(dtype_str)
-
-                                        # Handle binary data that might need string conversion first
-                                        if "Binary" in str(current_dtype):
-                                            # Convert binary to string first, then to target type
-                                            if target_dtype == pl.Utf8:
-                                                self.features_df = (
-                                                    self.features_df.with_columns(
-                                                        pl.col(col)
-                                                        .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                        .cast(target_dtype),
-                                                    )
-                                                )
-                                            elif "Int" in str(target_dtype):
-                                                self.features_df = (
-                                                    self.features_df.with_columns(
-                                                        pl.col(col)
-                                                        .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                        .str.to_integer()
-                                                        .cast(target_dtype),
-                                                    )
-                                                )
-                                            elif "Float" in str(target_dtype):
-                                                self.features_df = (
-                                                    self.features_df.with_columns(
-                                                        pl.col(col)
-                                                        .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                        .str.to_decimal()
-                                                        .cast(target_dtype),
-                                                    )
-                                                )
-                                            else:
-                                                # Try direct casting
-                                                self.features_df = (
-                                                    self.features_df.with_columns(
-                                                        pl.col(col).cast(target_dtype),
-                                                    )
-                                                )
-                                        else:
-                                            # Try direct casting for non-binary types
-                                            self.features_df = (
-                                                self.features_df.with_columns(
-                                                    pl.col(col).cast(target_dtype),
-                                                )
-                                            )
-                                elif "Float" in dtype_str:
-                                    # Convert to float, handling different input types
-                                    if self.features_df[col].dtype == pl.Utf8:
-                                        # String data - convert to float
-                                        self.features_df = (
-                                            self.features_df.with_columns(
-                                                pl.col(col)
-                                                .str.to_decimal()
-                                                .cast(eval(dtype_str)),
-                                            )
-                                        )
-                                    else:
-                                        # Handle special cases and try direct casting for other types
-                                        current_dtype = self.features_df[col].dtype
-                                        target_dtype = eval(dtype_str)
-
-                                        # Handle binary data that might need string conversion first
-                                        if "Binary" in str(current_dtype):
-                                            # Convert binary to string first, then to target type
-                                            if target_dtype == pl.Utf8:
-                                                self.features_df = (
-                                                    self.features_df.with_columns(
-                                                        pl.col(col)
-                                                        .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                        .cast(target_dtype),
-                                                    )
-                                                )
-                                            elif "Int" in str(target_dtype):
-                                                self.features_df = (
-                                                    self.features_df.with_columns(
-                                                        pl.col(col)
-                                                        .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                        .str.to_integer()
-                                                        .cast(target_dtype),
-                                                    )
-                                                )
-                                            elif "Float" in str(target_dtype):
-                                                self.features_df = (
-                                                    self.features_df.with_columns(
-                                                        pl.col(col)
-                                                        .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                        .str.to_decimal()
-                                                        .cast(target_dtype),
-                                                    )
-                                                )
-                                            else:
-                                                # Try direct casting
-                                                self.features_df = (
-                                                    self.features_df.with_columns(
-                                                        pl.col(col).cast(target_dtype),
-                                                    )
-                                                )
-                                        else:
-                                            # Try direct casting for non-binary types
-                                            self.features_df = (
-                                                self.features_df.with_columns(
-                                                    pl.col(col).cast(target_dtype),
-                                                )
-                                            )
-                                elif "Utf8" in dtype_str:
-                                    # Ensure it's string type
-                                    self.features_df = self.features_df.with_columns(
-                                        pl.col(col).cast(pl.Utf8),
-                                    )
-                                else:
-                                    # Handle special cases and try direct casting for other types
-                                    current_dtype = self.features_df[col].dtype
-                                    target_dtype = eval(dtype_str)
-
-                                    # Handle binary data that might need string conversion first
-                                    if "Binary" in str(current_dtype):
-                                        # Convert binary to string first, then to target type
-                                        if target_dtype == pl.Utf8:
-                                            self.features_df = (
-                                                self.features_df.with_columns(
-                                                    pl.col(col)
-                                                    .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                    .cast(target_dtype),
-                                                )
-                                            )
-                                        elif "Int" in str(target_dtype):
-                                            self.features_df = (
-                                                self.features_df.with_columns(
-                                                    pl.col(col)
-                                                    .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                    .str.to_integer()
-                                                    .cast(target_dtype),
-                                                )
-                                            )
-                                        elif "Float" in str(target_dtype):
-                                            self.features_df = (
-                                                self.features_df.with_columns(
-                                                    pl.col(col)
-                                                    .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                                                    .str.to_decimal()
-                                                    .cast(target_dtype),
-                                                )
-                                            )
-                                        else:
-                                            # Try direct casting
-                                            self.features_df = (
-                                                self.features_df.with_columns(
-                                                    pl.col(col).cast(target_dtype),
-                                                )
-                                            )
-                                    else:
-                                        # Try direct casting for non-binary types
-                                        self.features_df = (
-                                            self.features_df.with_columns(
-                                                pl.col(col).cast(target_dtype),
-                                            )
-                                        )
-                        except Exception as e:
-                            self.logger.warning(
-                                f"Failed to cast column '{col}' in features_df: {e}",
-                            )
-                    else:
-                        self.logger.warning(
-                            f"Column '{col}' in features_df not found in schema, keeping original type.",
-                        )
-
-                # FINAL null conversion pass - after all type casting is done
-                # This ensures "None" strings introduced by failed conversions are properly handled
-                for col in self.features_df.columns:
-                    if self.features_df[col].dtype == pl.Utf8:  # String columns
-                        self.features_df = self.features_df.with_columns([
-                            pl.when(pl.col(col).is_in(["None", "", "null", "NULL"]))
-                            .then(None)
-                            .otherwise(pl.col(col))
-                            .alias(col),
-                        ])
-                    # Float columns
-                    elif self.features_df[col].dtype in [pl.Float64, pl.Float32]:
-                        self.features_df = self.features_df.with_columns([
-                            pl.col(col).fill_nan(None).alias(col),
-                        ])
-
-                # Ensure column order matches schema order
-                if "features_df" in schema and "columns" in schema["features_df"]:
-                    schema_column_order = list(schema["features_df"]["columns"].keys())
-                    # Only reorder columns that exist in both schema and DataFrame
-                    existing_columns = [col for col in schema_column_order if col in self.features_df.columns]
-                    if existing_columns:
-                        self.features_df = self.features_df.select(existing_columns)
-
-            else:
-                self.features_df = None
-        else:
-            self.features_df = None
-
-        # Load ms1_df
-        if "ms1" in f:
-            ms1_group = f["ms1"]
-            data = {}
-
-            # Get all datasets in the ms1 group
-            for col in ms1_group.keys():
-                data[col] = ms1_group[col][:]
-
-            if data:
-                # Create DataFrame directly with Polars
-                self.ms1_df = pl.DataFrame(data)
-
-                # Apply schema if available
-                if "ms1_df" in schema and "columns" in schema["ms1_df"]:
-                    schema_columns = schema["ms1_df"]["columns"]
-                    for col in self.ms1_df.columns:
-                        if col in schema_columns:
-                            dtype_str = schema_columns[col]["dtype"]
-                            try:
-                                if "Int" in dtype_str:
-                                    self.ms1_df = self.ms1_df.with_columns([
-                                        pl.col(col).cast(pl.Int64, strict=False)
-                                    ])
-                                elif "Float" in dtype_str:
-                                    self.ms1_df = self.ms1_df.with_columns([
-                                        pl.col(col).cast(pl.Float64, strict=False)
-                                    ])
-                            except Exception as e:
-                                self.logger.warning(
-                                    f"Failed to apply schema type {dtype_str} to column {col}: {e}",
-                                )
-
-                # Convert "None" strings and NaN values to proper null values
-                self.ms1_df = clean_null_values_polars(self.ms1_df)
-            else:
-                self.ms1_df = None
-        else:
-            self.ms1_df = None
-
-        # Parameters are now loaded from metadata JSON (see above)
-        # Lib and lib_match are no longer saved/loaded
-
-    if map:
-        featureXML = filename.replace(".sample5", ".featureXML")
-        if os.path.exists(featureXML):
-            self._load_featureXML(featureXML)
-        else:
-            self.logger.warning(
-                f"Feature XML file {featureXML} not found, skipping loading.",
-            )
-
-    self.logger.info(f"Sample loaded successfully from {filename}")
-
-
-def load_schema(schema_path: str) -> Dict[str, Any]:
-    """
-    Load schema from JSON file with error handling.
-    
-    Args:
-        schema_path: Path to the schema JSON file
-        
-    Returns:
-        Dictionary containing the schema, empty dict if not found
-    """
-    try:
-        with open(schema_path) as f:
-            return json.load(f)  # type: ignore
-    except FileNotFoundError:
-        return {}
-
-
-def decode_metadata_attr(attr_value: Any) -> str:
-    """
-    Decode metadata attribute, handling both bytes and string types.
-    
-    Args:
-        attr_value: The attribute value to decode
-        
-    Returns:
-        String representation of the attribute
-    """
-    if isinstance(attr_value, bytes):
-        return attr_value.decode()
-    return str(attr_value) if attr_value is not None else ""
-
-
-def clean_null_values_polars(df: pl.DataFrame) -> pl.DataFrame:
-    """
-    Clean null values in a Polars DataFrame by converting string nulls to proper nulls.
-    
-    Args:
-        df: The Polars DataFrame to clean
-        
-    Returns:
-        Cleaned DataFrame
-    """
-    cleaned_df = df
-    for col in df.columns:
-        if df[col].dtype == pl.Utf8:  # String columns
-            cleaned_df = cleaned_df.with_columns([
-                pl.when(pl.col(col).is_in(["None", "", "null", "NULL"]))
-                .then(None)
-                .otherwise(pl.col(col))
-                .alias(col),
-            ])
-        elif df[col].dtype in [pl.Float64, pl.Float32]:  # Float columns
-            cleaned_df = cleaned_df.with_columns([
-                pl.col(col).fill_nan(None).alias(col),
-            ])
-    return cleaned_df
-
-
-def cast_column_by_dtype(df: pl.DataFrame, col: str, dtype_str: str) -> pl.DataFrame:
-    """
-    Cast a Polars DataFrame column to the specified dtype with appropriate handling.
-    
-    Args:
-        df: The Polars DataFrame
-        col: Column name to cast
-        dtype_str: Target dtype as string (e.g., 'pl.Int64')
-        
-    Returns:
-        DataFrame with the column cast to the new type
-    """
-    if not dtype_str.startswith("pl.") or "Object" in dtype_str:
-        return df
-        
-    try:
-        target_dtype = eval(dtype_str)
-        current_dtype = df[col].dtype
-        
-        if "Int" in dtype_str:
-            return _cast_to_int(df, col, current_dtype, target_dtype)
-        elif "Float" in dtype_str:
-            return _cast_to_float(df, col, current_dtype, target_dtype)
-        elif "Utf8" in dtype_str:
-            return df.with_columns(pl.col(col).cast(pl.Utf8))
-        else:
-            return _cast_with_binary_handling(df, col, current_dtype, target_dtype)
-            
-    except Exception:
-        return df
-
-
-def _cast_to_int(df: pl.DataFrame, col: str, current_dtype: pl.DataType, target_dtype: pl.DataType) -> pl.DataFrame:
-    """Helper function to cast column to integer type."""
-    if current_dtype == pl.Utf8:
-        return df.with_columns(
-            pl.col(col).str.to_integer().cast(target_dtype)
-        )
-    elif current_dtype in [pl.Float64, pl.Float32]:
-        return df.with_columns(pl.col(col).cast(target_dtype))
-    else:
-        return _cast_with_binary_handling(df, col, current_dtype, target_dtype)
-
-
-def _cast_to_float(df: pl.DataFrame, col: str, current_dtype: pl.DataType, target_dtype: pl.DataType) -> pl.DataFrame:
-    """Helper function to cast column to float type."""
-    if current_dtype == pl.Utf8:
-        return df.with_columns(
-            pl.col(col).str.to_decimal().cast(target_dtype)
-        )
-    else:
-        return _cast_with_binary_handling(df, col, current_dtype, target_dtype)
-
-
-def _cast_with_binary_handling(df: pl.DataFrame, col: str, current_dtype: pl.DataType, target_dtype: pl.DataType) -> pl.DataFrame:
-    """Helper function to handle binary data conversion."""
-    if "Binary" in str(current_dtype):
-        if target_dtype == pl.Utf8:
-            return df.with_columns(
-                pl.col(col).map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8).cast(target_dtype)
-            )
-        elif "Int" in str(target_dtype):
-            return df.with_columns(
-                pl.col(col)
-                .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                .str.to_integer()
-                .cast(target_dtype)
-            )
-        elif "Float" in str(target_dtype):
-            return df.with_columns(
-                pl.col(col)
-                .map_elements(lambda x: x.decode('utf-8') if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
-                .str.to_decimal()
-                .cast(target_dtype)
-            )
-    
-    # Fallback: try direct casting
-    return df.with_columns(pl.col(col).cast(target_dtype))
-
-
-def apply_schema_to_dataframe(df: pl.DataFrame, schema: Dict[str, Any], df_name: str) -> pl.DataFrame:
-    """
-    Apply schema type casting to a Polars DataFrame.
-    
-    Args:
-        df: The DataFrame to modify
-        schema: The schema dictionary
-        df_name: Name of the DataFrame in the schema (e.g., 'scans_df', 'features_df')
-        
-    Returns:
-        DataFrame with schema types applied
-    """
-    df_schema = schema.get(df_name, {}).get("columns", {})
-    
-    for col in df.columns:
-        if col in df_schema:
-            dtype_str = df_schema[col]["dtype"]
-            df = cast_column_by_dtype(df, col, dtype_str)
-    
-    return df
-
-
-def reconstruct_object_column(data_col: np.ndarray, col_name: str) -> List[Any]:
-    """
-    Reconstruct object columns from serialized data.
-    
-    Args:
-        data_col: Array containing serialized data
-        col_name: Name of the column for type-specific reconstruction
-        
-    Returns:
-        List of reconstructed objects
-    """
-    reconstructed_data: list[Any] = []
-    
-    for item in data_col:
-        if isinstance(item, bytes):
-            item = item.decode("utf-8")
-        
-        if item == "None" or item == "":
-            reconstructed_data.append(None)
-            continue
-            
-        try:
-            if col_name == "chrom":
-                reconstructed_data.append(Chromatogram.from_json(item))
-            elif col_name == "ms2_scans":
-                scan_list = json.loads(item)
-                reconstructed_data.append(scan_list)
-            elif col_name == "ms2_specs":
-                json_list = json.loads(item)
-                if json_list == ["None"]:
-                    reconstructed_data.append(None)
-                else:
-                    spectrum_list: list[Any] = []
-                    for json_str in json_list:
-                        if json_str == "None":
-                            spectrum_list.append(None)
-                        else:
-                            spectrum_list.append(Spectrum.from_json(json_str))
-                    reconstructed_data.append(spectrum_list)
-            else:
-                # Unknown object column
-                reconstructed_data.append(None)
-        except (json.JSONDecodeError, ValueError):
-            reconstructed_data.append(None)
-    
-    return reconstructed_data
-
-
-def load_dataframe_from_h5_group(
-    group: h5py.Group, 
-    schema: Dict[str, Any], 
-    df_name: str,
-    logger: Optional[Any] = None
-) -> Tuple[Optional[pl.DataFrame], List[str]]:
-    """
-    Load a Polars DataFrame from an HDF5 group using schema.
-    
-    Args:
-        group: The HDF5 group containing the DataFrame data
-        schema: The schema dictionary
-        df_name: Name of the DataFrame in the schema
-        logger: Optional logger for warnings
-        
-    Returns:
-        Tuple of (DataFrame or None, list of missing columns)
-    """
-    data: dict[str, Any] = {}
-    missing_columns = []
-    
-    # Load columns according to schema
-    schema_columns = schema.get(df_name, {}).get("columns", [])
-    
-    for col in schema_columns:
-        if col not in group:
-            if logger:
-                logger.warning(f"Column '{col}' not found in {df_name}.")
-            data[col] = None
-            missing_columns.append(col)
-            continue
-        
-        dtype = schema[df_name]["columns"][col].get("dtype", "native")
-        
-        if dtype == "pl.Object":
-            # Handle object columns specially
-            data[col] = reconstruct_object_column(group[col][:], col)
-        else:
-            data[col] = group[col][:]
-    
-    if not data:
-        return None, missing_columns
-    
-    # Create DataFrame with proper schema for Object columns
-    df_schema = {}
-    for col, values in data.items():
-        if col in schema_columns:
-            dtype_str = schema[df_name]["columns"][col]["dtype"]
-            if dtype_str == "pl.Object":
-                df_schema[col] = pl.Object
-    
-    try:
-        if df_schema:
-            df = pl.DataFrame(data, schema=df_schema)
-        else:
-            df = pl.DataFrame(data)
-    except Exception:
-        # Fallback: handle Object columns manually
-        df = _create_dataframe_with_object_columns(data, schema, df_name)
-    
-    # Clean null values
-    df = clean_null_values_polars(df)
-    
-    # Apply schema type casting
-    df = apply_schema_to_dataframe(df, schema, df_name)
-    
-    return df, missing_columns
-
-
-def _create_dataframe_with_object_columns(
-    data: Dict[str, Any], 
-    schema: Dict[str, Any], 
-    df_name: str
-) -> pl.DataFrame:
-    """
-    Create DataFrame handling Object columns manually when schema creation fails.
-    
-    Args:
-        data: Dictionary of column data
-        schema: The schema dictionary
-        df_name: Name of the DataFrame in the schema
-        
-    Returns:
-        Polars DataFrame with Object columns properly handled
-    """
-    schema_columns = schema.get(df_name, {}).get("columns", {})
-    
-    object_columns = {
-        k: v for k, v in data.items() 
-        if k in schema_columns and schema_columns[k]["dtype"] == "pl.Object"
-    }
-    regular_columns = {
-        k: v for k, v in data.items() if k not in object_columns
-    }
-    
-    # Create DataFrame with regular columns first
-    if regular_columns:
-        df = pl.DataFrame(regular_columns)
-        # Add Object columns one by one
-        for col, values in object_columns.items():
-            df = df.with_columns([pl.Series(col, values, dtype=pl.Object)])
-    else:
-        # Only Object columns
-        df = pl.DataFrame()
-        for col, values in object_columns.items():
-            df = df.with_columns([pl.Series(col, values, dtype=pl.Object)])
-    
-    return df
-
-
-def load_ms1_dataframe_from_h5_group(
-    group: h5py.Group, 
-    schema: Dict[str, Any],
-    logger: Optional[Any] = None
-) -> Optional[pl.DataFrame]:
-    """
-    Load MS1 DataFrame from HDF5 group.
-    
-    Args:
-        group: The HDF5 group containing MS1 data
-        schema: The schema dictionary
-        logger: Optional logger for warnings
-        
-    Returns:
-        Polars DataFrame or None
-    """
-    data = {}
-    
-    # Get all datasets in the ms1 group
-    for col in group.keys():
-        data[col] = group[col][:]
-    
-    if not data:
-        return None
-    
-    # Create DataFrame directly with Polars
-    ms1_df = pl.DataFrame(data)
-    
-    # Apply schema if available
-    if "ms1_df" in schema and "columns" in schema["ms1_df"]:
-        schema_columns = schema["ms1_df"]["columns"]
-        for col in ms1_df.columns:
-            if col in schema_columns:
-                dtype_str = schema_columns[col]["dtype"]
-                try:
-                    if "Int" in dtype_str:
-                        ms1_df = ms1_df.with_columns([
-                            pl.col(col).cast(pl.Int64, strict=False)
-                        ])
-                    elif "Float" in dtype_str:
-                        ms1_df = ms1_df.with_columns([
-                            pl.col(col).cast(pl.Float64, strict=False)
-                        ])
-                except Exception as e:
-                    if logger:
-                        logger.warning(
-                            f"Failed to apply schema type {dtype_str} to column {col}: {e}"
-                        )
-    
-    # Convert "None" strings and NaN values to proper null values
-    return clean_null_values_polars(ms1_df)
-
-
-def load_parameters_from_metadata(metadata_group: h5py.Group) -> Optional[Dict[str, Any]]:
-    """
-    Load parameters from HDF5 metadata group.
-    
-    Args:
-        metadata_group: The HDF5 metadata group containing parameters
-        
-    Returns:
-        Dictionary of parameters or None if not found
-    """
-    if "parameters" in metadata_group.attrs:
-        try:
-            params_json = decode_metadata_attr(metadata_group.attrs["parameters"])
-            # Ensure params_json is a string before attempting JSON decode
-            if isinstance(params_json, str) and params_json.strip():
-                result = json.loads(params_json)
-                # Ensure the result is a dictionary
-                if isinstance(result, dict):
-                    return result
-        except (json.JSONDecodeError, ValueError, TypeError) as e:
-            # Log the error for debugging
-            print(f"Warning: Failed to parse parameters JSON: {e}")
-            print(f"Raw parameter data type: {type(params_json)}")
-            print(f"Raw parameter data: {repr(params_json)}")
-    return None
-
-
-def create_h5_metadata_group(f: h5py.File, file_path: Optional[str], file_type: Optional[str], label: Optional[str]) -> None:
-    """
-    Create and populate metadata group in HDF5 file.
-    
-    Args:
-        f: The HDF5 file object
-        file_path: Source file path
-        file_type: Source file type  
-        label: Sample label
-    """
-    metadata_group = f.create_group("metadata")
-    metadata_group.attrs["format"] = "master-sample5-1"
-    metadata_group.attrs["file_path"] = str(file_path) if file_path is not None else ""
-    metadata_group.attrs["file_type"] = str(file_type) if file_type is not None else ""
-    metadata_group.attrs["label"] = str(label) if label is not None else ""
-
-
+import json
+import os
+
+import h5py
+import numpy as np
+import polars as pl
+
+from typing import Any, Dict, List, Optional, Tuple
+
+from masster.chromatogram import Chromatogram
+from masster.spectrum import Spectrum
+
+
+def _save_sample5(self, filename=None, include_ms1=True, include_scans=True):
+    """
+    Save the instance data to a sample5 HDF5 file with optimized compression.
+
+    This optimized version uses context-aware compression settings for better
+    performance and smaller file sizes. Different compression algorithms are
+    selected based on data type and usage patterns.
+
+    Args:
+        filename (str, optional): Target file name. If None, uses default based on file_path.
+        include_ms1 (bool, optional): Whether to include MS1 data. Defaults to True.
+        include_scans (bool, optional): Whether to include scan data. Defaults to True.
+
+    Stores:
+        - metadata/format (str): Data format identifier (master-sample-1)
+        - metadata/file_path (str): Source file path
+        - metadata/file_type (str): Source file type
+        - metadata/label (str): Sample label
+        - metadata/parameters (str): Parameters as JSON string with optimized compression
+        - scans/: Scan DataFrame data with fast-access compression for IDs, standard for others
+        - features/: Feature DataFrame data with JSON compression for objects, fast compression for core data
+        - ms1/: MS1-level data with numeric compression
+
+    Compression Strategy:
+        - LZF + shuffle: Fast access data (feature_uid, rt, mz, intensity, scan_id)
+        - GZIP level 6: JSON objects (chromatograms, spectra) and string data
+        - GZIP level 9: Bulk storage data (large MS2 spectrum collections)
+        - LZF: Standard numeric arrays
+
+    Performance Improvements:
+        - 8-15% smaller file sizes
+        - 20-50% faster save operations for large files
+        - Context-aware compression selection
+    """
+    if filename is None:
+        # save to default file name
+        if self.file_path is not None:
+            filename = os.path.splitext(self.file_path)[0] + ".sample5"
+        else:
+            self.logger.error("either filename or file_path must be provided")
+            return
+
+    # synchronize feature_map
+    if self.features is not None:
+        self._features_sync()
+
+    # if no extension is given, add .sample5
+    if not filename.endswith(".sample5"):
+        filename += ".sample5"
+
+    self.logger.debug(f"Saving sample to {filename} with optimized LZF+shuffle compression")
+
+    # delete existing file if it exists
+    if os.path.exists(filename):
+        os.remove(filename)
+
+    with h5py.File(filename, "w") as f:
+        # Create groups for organization
+        metadata_group = f.create_group("metadata")
+        features_group = f.create_group("features")
+        scans_group = f.create_group("scans")
+        ms1_group = f.create_group("ms1")
+
+        # Store metadata
+        metadata_group.attrs["format"] = "master-sample-1"
+        if self.file_path is not None:
+            metadata_group.attrs["file_path"] = str(self.file_path)
+        else:
+            metadata_group.attrs["file_path"] = ""
+        if self.file_source is not None:
+            metadata_group.attrs["file_source"] = str(self.file_source)
+        else:
+            metadata_group.attrs["file_source"] = ""
+        if self.file_type is not None:
+            metadata_group.attrs["file_type"] = str(self.file_type)
+        else:
+            metadata_group.attrs["file_type"] = ""
+        if self.label is not None:
+            metadata_group.attrs["label"] = str(self.label)
+        else:
+            metadata_group.attrs["label"] = ""
+
+        # Store DataFrames
+        if self.scans_df is not None and include_scans:
+            scans_df = self.scans_df.clone()
+            for col in scans_df.columns:
+                data = scans_df[col].to_numpy()
+                # Handle different data types safely
+                if data.dtype == object:
+                    try:
+                        str_data = np.array(
+                            ["" if x is None else str(x) for x in data],
+                            dtype="S",
+                        )
+                        scans_group.create_dataset(
+                            col,
+                            data=str_data,
+                            compression="gzip",
+                        )
+                        scans_group[col].attrs["dtype"] = "string_converted"
+                    except Exception:
+                        try:
+                            # Try to convert to numeric using numpy
+                            numeric_data = np.array([
+                                float(x)
+                                if x is not None and str(x).replace(".", "").replace("-", "").isdigit()
+                                else np.nan
+                                for x in data
+                            ])
+                            if not np.isnan(numeric_data).all():
+                                scans_group.create_dataset(
+                                    col,
+                                    data=numeric_data,
+                                    compression="gzip",
+                                )
+                                scans_group[col].attrs["dtype"] = "numeric_converted"
+                            else:
+                                json_data = np.array(
+                                    [json.dumps(x, default=str) for x in data],
+                                    dtype="S",
+                                )
+                                scans_group.create_dataset(
+                                    col,
+                                    data=json_data,
+                                    compression="gzip",
+                                )
+                                scans_group[col].attrs["dtype"] = "json_serialized"
+                        except Exception:
+                            str_repr_data = np.array([str(x) for x in data], dtype="S")
+                            scans_group.create_dataset(
+                                col,
+                                data=str_repr_data,
+                                compression="gzip",
+                            )
+                            scans_group[col].attrs["dtype"] = "string_repr"
+                else:
+                    scans_group.create_dataset(col, data=data, compression="lzf", shuffle=True)
+                    scans_group[col].attrs["dtype"] = "native"
+            scans_group.attrs["columns"] = list(scans_df.columns)
+
+        if self.features_df is not None:
+            features = self.features_df.clone()
+            for col in features.columns:
+                # get column dtype
+                dtype = str(features[col].dtype).lower()
+                if dtype == "object":
+                    if col == "chrom":
+                        # this column contains either None or Chromatogram objects
+                        # convert to json with to_json() and store them as compressed strings
+                        data = features[col]
+                        data_as_str = []
+                        for i in range(len(data)):
+                            if data[i] is not None:
+                                data_as_str.append(data[i].to_json())
+                            else:
+                                data_as_str.append("None")
+                        features_group.create_dataset(
+                            col,
+                            data=data_as_str,
+                            compression="gzip",
+                        )
+                    elif col == "ms2_scans":
+                        # this column contains either None or lists of integers (scan indices)
+                        # convert each to JSON string for storage (HDF5 can't handle inhomogeneous arrays)
+                        data = features[col]
+                        data_as_json_strings = []
+                        for i in range(len(data)):
+                            if data[i] is not None:
+                                data_as_json_strings.append(json.dumps(list(data[i])))
+                            else:
+                                data_as_json_strings.append("None")
+                        features_group.create_dataset(
+                            col,
+                            data=data_as_json_strings,
+                            compression="gzip",
+                        )
+                    elif col == "ms2_specs":
+                        # this column contains either None or lists of Spectrum objects
+                        # convert each spectrum to json and store as list of json strings
+                        data = features[col]
+                        data_as_lists_of_strings = []
+                        for i in range(len(data)):
+                            if data[i] is not None:
+                                # Convert list of Spectrum objects to list of JSON strings
+                                spectrum_list = data[i]
+                                json_strings = []
+                                for spectrum in spectrum_list:
+                                    if spectrum is not None:
+                                        json_strings.append(spectrum.to_json())
+                                    else:
+                                        json_strings.append("None")
+                                data_as_lists_of_strings.append(json_strings)
+                            else:
+                                data_as_lists_of_strings.append(["None"])
+                        # Convert to numpy array for HDF5 storage
+                        serialized_data = []
+                        for item in data_as_lists_of_strings:
+                            serialized_data.append(json.dumps(item))
+                        features_group.create_dataset(
+                            col,
+                            data=serialized_data,
+                            compression="gzip",
+                        )
+
+                    else:
+                        self.logger.warning(
+                            f"Unexpectedly, column '{col}' has dtype 'object'. Implement serialization for this column.",
+                        )
+                    continue
+                elif dtype == "string":
+                    data = features[col].to_list()
+                    # convert None to 'None' strings
+                    data = ["None" if x is None else x for x in data]
+                    features_group.create_dataset(col, data=data, compression="lzf", shuffle=True)
+                else:
+                    try:
+                        data = features[col].to_numpy()
+                        features_group.create_dataset(col, data=data)
+                    except Exception:
+                        self.logger.warning(
+                            f"Failed to save column '{col}' with dtype '{dtype}'. It may contain unsupported data types.",
+                        )
+            features_group.attrs["columns"] = list(features.columns)
+
+        # Store arrays
+        if self.ms1_df is not None and include_ms1:
+            # the df is a polars DataFrame
+            for col in self.ms1_df.columns:
+                ms1_group.create_dataset(
+                    col,
+                    data=self.ms1_df[col].to_numpy(),
+                    compression="gzip",
+                )
+
+        # Store parameters as JSON
+        if self.parameters is not None:
+            # Convert parameters dict to JSON string
+            params_json = json.dumps(self.parameters, default=str)
+            metadata_group.attrs["parameters"] = params_json
+
+        # Store lib and lib_match - removed (no longer saving lib data)
+
+    self.logger.info(f"Sample saved successfully to {filename}")
+    if self.features is not None:
+        # save the features as a separate file
+        self._save_featureXML(filename=filename.replace(".sample5", ".featureXML"))
+
+
+def _load_sample5(self, filename: str, map: bool = True):
+    """
+    Load instance data from a sample5 HDF5 file.
+
+    Restores all attributes that were saved with save_sample5() method using the
+    schema defined in sample5_schema.json for proper Polars DataFrame reconstruction.
+
+    Args:
+        filename (str): Path to the sample5 HDF5 file to load.
+        map (bool, optional): Whether to map featureXML file if available. Defaults to True.
+
+    Returns:
+        None (modifies self in place)
+
+    Notes:
+        - Restores DataFrames with proper schema typing from sample5_schema.json
+        - Handles Chromatogram and Spectrum object reconstruction
+        - Properly handles MS2 scan lists and spectrum lists
+    """
+    # Load schema for proper DataFrame reconstruction
+    schema_path = os.path.join(os.path.dirname(__file__), "sample5_schema.json")
+    try:
+        with open(schema_path) as f:
+            schema = json.load(f)
+    except FileNotFoundError:
+        self.logger.warning(
+            f"Schema file {schema_path} not found. Using default types.",
+        )
+        schema = {}
+
+    with h5py.File(filename, "r") as f:
+        # Load metadata
+        if "metadata" in f:
+            metadata_group = f["metadata"]
+            self.file_path = decode_metadata_attr(metadata_group.attrs.get("file_path", ""))
+            
+            # Load file_source if it exists, otherwise set it equal to file_path
+            if "file_source" in metadata_group.attrs:
+                self.file_source = decode_metadata_attr(metadata_group.attrs.get("file_source", ""))
+            else:
+                self.file_source = self.file_path
+            
+            self.file_type = decode_metadata_attr(metadata_group.attrs.get("file_type", ""))
+            self.label = decode_metadata_attr(metadata_group.attrs.get("label", ""))
+
+            # Load parameters from JSON in metadata
+            loaded_data = load_parameters_from_metadata(metadata_group)
+
+            # Always create a fresh sample_defaults object
+            from masster.sample.defaults.sample_def import sample_defaults
+
+            self.parameters = sample_defaults()
+
+            # Initialize history and populate from loaded data
+            self.history = {}
+            if loaded_data is not None and isinstance(loaded_data, dict):
+                # Store the loaded data in history
+                self.history = loaded_data
+                # If there are sample parameters in the history, use them to update defaults
+                if "sample" in loaded_data:
+                    sample_params = loaded_data["sample"]
+                    if isinstance(sample_params, dict):
+                        self.parameters.set_from_dict(sample_params, validate=False)
+
+        # Load scans_df
+        if "scans" in f:
+            scans_group = f["scans"]
+            data: dict[str, Any] = {}
+            missing_columns = []
+            for col in schema.get("scans_df", {}).get("columns", []):
+                if col not in scans_group:
+                    self.logger.debug(f"Column '{col}' not found in sample5/scans.")
+                    data[col] = None
+                    missing_columns.append(col)
+                    continue
+
+                dtype = schema["scans_df"]["columns"][col].get("dtype", "native")
+                match dtype:
+                    case "pl.Object":
+                        self.logger.debug(f"Unexpected Object column '{col}'")
+                        data[col] = None
+                        missing_columns.append(col)
+
+                    case _:
+                        data[col] = scans_group[col][:]
+
+            # create polars DataFrame from data
+            if data:
+                self.scans_df = pl.DataFrame(data)
+
+                # Convert "None" strings and NaN values to proper null values
+                for col in self.scans_df.columns:
+                    if self.scans_df[col].dtype == pl.Utf8:  # String columns
+                        self.scans_df = self.scans_df.with_columns([
+                            pl.when(pl.col(col).is_in(["None", "", "null", "NULL"]))
+                            .then(None)
+                            .otherwise(pl.col(col))
+                            .alias(col),
+                        ])
+                    elif self.scans_df[col].dtype in [
+                        pl.Float64,
+                        pl.Float32,
+                    ]:  # Float columns
+                        self.scans_df = self.scans_df.with_columns([
+                            pl.col(col).fill_nan(None).alias(col),
+                        ])
+
+                # update all columns with schema types
+                for col in self.scans_df.columns:
+                    if col in schema.get("scans_df", {}).get("columns", {}):
+                        try:
+                            dtype_str = schema["scans_df"]["columns"][col]["dtype"]
+                            # Convert dtype string to actual polars dtype
+                            if dtype_str.startswith("pl."):
+                                # Skip Object columns - they're already properly reconstructed
+                                if "Object" in dtype_str:
+                                    continue
+                                # Handle different polars data types
+                                if "Int" in dtype_str:
+                                    # Convert to numeric first, handling different input types
+                                    if self.scans_df[col].dtype == pl.Utf8:
+                                        # String data - convert to integer
+                                        self.scans_df = self.scans_df.with_columns(
+                                            pl.col(col).str.to_integer().cast(eval(dtype_str)),
+                                        )
+                                    elif self.scans_df[col].dtype in [
+                                        pl.Float64,
+                                        pl.Float32,
+                                    ]:
+                                        # Float data - cast to integer
+                                        self.scans_df = self.scans_df.with_columns(
+                                            pl.col(col).cast(eval(dtype_str)),
+                                        )
+                                    else:
+                                        # Try direct casting
+                                        self.scans_df = self.scans_df.with_columns(
+                                            pl.col(col).cast(eval(dtype_str)),
+                                        )
+                                elif "Float" in dtype_str:
+                                    # Convert to float, handling different input types
+                                    if self.scans_df[col].dtype == pl.Utf8:
+                                        # String data - convert to float
+                                        self.scans_df = self.scans_df.with_columns(
+                                            pl.col(col).str.to_decimal().cast(eval(dtype_str)),
+                                        )
+                                    else:
+                                        # Try direct casting
+                                        self.scans_df = self.scans_df.with_columns(
+                                            pl.col(col).cast(eval(dtype_str)),
+                                        )
+                                elif "Utf8" in dtype_str:
+                                    # Ensure it's string type
+                                    self.scans_df = self.scans_df.with_columns(
+                                        pl.col(col).cast(pl.Utf8),
+                                    )
+                                else:
+                                    # Handle special cases and try direct casting for other types
+                                    current_dtype = self.scans_df[col].dtype
+                                    target_dtype = eval(dtype_str)
+
+                                    # Handle binary data that might need string conversion first
+                                    if "Binary" in str(current_dtype):
+                                        # Convert binary to string first, then to target type
+                                        if target_dtype == pl.Utf8:
+                                            self.scans_df = self.scans_df.with_columns(
+                                                pl.col(col)
+                                                .map_elements(
+                                                    lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                    return_dtype=pl.Utf8,
+                                                )
+                                                .cast(target_dtype),
+                                            )
+                                        elif "Int" in str(target_dtype):
+                                            self.scans_df = self.scans_df.with_columns(
+                                                pl.col(col)
+                                                .map_elements(
+                                                    lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                    return_dtype=pl.Utf8,
+                                                )
+                                                .str.to_integer()
+                                                .cast(target_dtype),
+                                            )
+                                        elif "Float" in str(target_dtype):
+                                            self.scans_df = self.scans_df.with_columns(
+                                                pl.col(col)
+                                                .map_elements(
+                                                    lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                    return_dtype=pl.Utf8,
+                                                )
+                                                .str.to_decimal()
+                                                .cast(target_dtype),
+                                            )
+                                        else:
+                                            # Try direct casting
+                                            self.scans_df = self.scans_df.with_columns(
+                                                pl.col(col).cast(target_dtype),
+                                            )
+                                    else:
+                                        # Try direct casting for non-binary types
+                                        self.scans_df = self.scans_df.with_columns(
+                                            pl.col(col).cast(target_dtype),
+                                        )
+                        except Exception as e:
+                            self.logger.warning(
+                                f"Failed to cast column '{col}' in scans_df: {e}",
+                            )
+                    else:
+                        self.logger.warning(
+                            f"Column '{col}' in scans_df not found in schema, keeping original type.",
+                        )
+
+            # Ensure column order matches schema order
+            if "scans_df" in schema and "columns" in schema["scans_df"]:
+                schema_column_order = list(schema["scans_df"]["columns"].keys())
+                # Only reorder columns that exist in both schema and DataFrame
+                existing_columns = [col for col in schema_column_order if col in self.scans_df.columns]
+                if existing_columns:
+                    self.scans_df = self.scans_df.select(existing_columns)
+
+            else:
+                self.scans_df = None
+        else:
+            self.scans_df = None
+
+        # Load features_df
+        if "features" in f:
+            features_group = f["features"]
+            # columns = list(features_group.attrs.get('columns', []))
+            data = {}
+            missing_columns = []
+            for col in schema.get("features_df", {}).get("columns", []):
+                if col not in features_group:
+                    self.logger.debug(
+                        f"Column '{col}' not found in sample5/features.",
+                    )
+                    data[col] = None
+                    missing_columns.append(col)
+                    continue
+
+                dtype = schema["features_df"]["columns"][col].get("dtype", "native")
+                match dtype:
+                    case "pl.Object":
+                        match col:
+                            case "chrom":
+                                data_col = features_group[col][:]
+                                # Convert JSON strings back to Chromatogram objects
+                                reconstructed_data: list[Any] = []
+                                for item in data_col:
+                                    if isinstance(item, bytes):
+                                        item = item.decode("utf-8")
+
+                                    if item == "None" or item == "":
+                                        reconstructed_data.append(None)
+                                    else:
+                                        try:
+                                            reconstructed_data.append(
+                                                Chromatogram.from_json(item),
+                                            )
+                                        except (json.JSONDecodeError, ValueError):
+                                            reconstructed_data.append(None)
+
+                                data[col] = reconstructed_data
+                            case "ms2_scans":
+                                data_col = features_group[col][:]
+                                # Convert JSON strings back to lists of integers
+                                reconstructed_data = []
+                                for item in data_col:
+                                    if isinstance(item, bytes):
+                                        item = item.decode("utf-8")
+
+                                    if item == "None":
+                                        reconstructed_data.append(None)
+                                    else:
+                                        try:
+                                            # Parse JSON string to get list of integers
+                                            scan_list = json.loads(item)
+                                            reconstructed_data.append(scan_list)
+                                        except (json.JSONDecodeError, ValueError):
+                                            reconstructed_data.append(None)
+
+                                data[col] = reconstructed_data
+                            case "ms2_specs":
+                                data_col = features_group[col][:]
+                                # Convert JSON strings back to lists of Spectrum objects
+                                reconstructed_data = []
+                                for item in data_col:
+                                    if isinstance(item, bytes):
+                                        item = item.decode("utf-8")
+
+                                    # Parse the outer JSON (list of JSON strings)
+                                    json_list = json.loads(item)
+
+                                    if json_list == ["None"]:
+                                        # This was originally None
+                                        reconstructed_data.append(None)
+                                    else:
+                                        # This was originally a list of Spectrum objects
+                                        spectrum_list: list[Any] = []
+                                        for json_str in json_list:
+                                            if json_str == "None":
+                                                spectrum_list.append(None)
+                                            else:
+                                                spectrum_list.append(
+                                                    Spectrum.from_json(json_str),
+                                                )
+                                        reconstructed_data.append(spectrum_list)
+
+                                data[col] = reconstructed_data
+                            case _:
+                                self.logger.debug(f"Unexpected Object column '{col}'")
+                                data[col] = None
+                                missing_columns.append(col)
+
+                    case _:
+                        data[col] = features_group[col][:]
+
+            # create polars DataFrame from data
+            if data:
+                # Build schema for DataFrame creation to handle Object columns properly
+                df_schema = {}
+                for col, values in data.items():
+                    if col in schema.get("features_df", {}).get("columns", {}):
+                        dtype_str = schema["features_df"]["columns"][col]["dtype"]
+                        if dtype_str == "pl.Object":
+                            df_schema[col] = pl.Object
+                        else:
+                            # Let Polars infer the type initially, we'll cast later
+                            df_schema[col] = None
+                    else:
+                        df_schema[col] = None
+
+                # Create DataFrame with explicit Object types where needed
+                try:
+                    self.features_df = pl.DataFrame(data, schema=df_schema)
+                except Exception:
+                    # Fallback: create without schema and handle Object columns manually
+                    object_columns = {
+                        k: v
+                        for k, v in data.items()
+                        if k in schema.get("features_df", {}).get("columns", {})
+                        and schema["features_df"]["columns"][k]["dtype"] == "pl.Object"
+                    }
+                    regular_columns = {k: v for k, v in data.items() if k not in object_columns}
+
+                    # Create DataFrame with regular columns first
+                    if regular_columns:
+                        self.features_df = pl.DataFrame(regular_columns)
+                        # Add Object columns one by one
+                        for col, values in object_columns.items():
+                            self.features_df = self.features_df.with_columns([
+                                pl.Series(col, values, dtype=pl.Object),
+                            ])
+                    else:
+                        # Only Object columns
+                        self.features_df = pl.DataFrame()
+                        for col, values in object_columns.items():
+                            self.features_df = self.features_df.with_columns([
+                                pl.Series(col, values, dtype=pl.Object),
+                            ])
+
+                # update all columns with schema types (skip Object columns)
+                for col in self.features_df.columns:
+                    if col in schema.get("features_df", {}).get("columns", {}):
+                        try:
+                            dtype_str = schema["features_df"]["columns"][col]["dtype"]
+                            # Convert dtype string to actual polars dtype
+                            if dtype_str.startswith("pl."):
+                                # Skip Object columns - they're already properly reconstructed
+                                if "Object" in dtype_str:
+                                    continue
+                                # Handle different polars data types
+                                if "Int" in dtype_str:
+                                    # Convert to numeric first, handling different input types
+                                    if self.features_df[col].dtype == pl.Utf8:
+                                        # String data - convert to integer
+                                        self.features_df = self.features_df.with_columns(
+                                            pl.col(col).str.to_integer().cast(eval(dtype_str)),
+                                        )
+                                    elif self.features_df[col].dtype in [
+                                        pl.Float64,
+                                        pl.Float32,
+                                    ]:
+                                        # Float data - cast to integer with null handling for NaN values
+                                        self.features_df = self.features_df.with_columns(
+                                            pl.col(col).cast(eval(dtype_str), strict=False),
+                                        )
+                                    else:
+                                        # Handle special cases and try direct casting for other types
+                                        current_dtype = self.features_df[col].dtype
+                                        target_dtype = eval(dtype_str)
+
+                                        # Handle binary data that might need string conversion first
+                                        if "Binary" in str(current_dtype):
+                                            # Convert binary to string first, then to target type
+                                            if target_dtype == pl.Utf8:
+                                                self.features_df = self.features_df.with_columns(
+                                                    pl.col(col)
+                                                    .map_elements(
+                                                        lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                        return_dtype=pl.Utf8,
+                                                    )
+                                                    .cast(target_dtype),
+                                                )
+                                            elif "Int" in str(target_dtype):
+                                                self.features_df = self.features_df.with_columns(
+                                                    pl.col(col)
+                                                    .map_elements(
+                                                        lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                        return_dtype=pl.Utf8,
+                                                    )
+                                                    .str.to_integer()
+                                                    .cast(target_dtype),
+                                                )
+                                            elif "Float" in str(target_dtype):
+                                                self.features_df = self.features_df.with_columns(
+                                                    pl.col(col)
+                                                    .map_elements(
+                                                        lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                        return_dtype=pl.Utf8,
+                                                    )
+                                                    .str.to_decimal()
+                                                    .cast(target_dtype),
+                                                )
+                                            else:
+                                                # Try direct casting
+                                                self.features_df = self.features_df.with_columns(
+                                                    pl.col(col).cast(target_dtype),
+                                                )
+                                        else:
+                                            # Try direct casting for non-binary types
+                                            self.features_df = self.features_df.with_columns(
+                                                pl.col(col).cast(target_dtype),
+                                            )
+                                elif "Float" in dtype_str:
+                                    # Convert to float, handling different input types
+                                    if self.features_df[col].dtype == pl.Utf8:
+                                        # String data - convert to float
+                                        self.features_df = self.features_df.with_columns(
+                                            pl.col(col).str.to_decimal().cast(eval(dtype_str)),
+                                        )
+                                    else:
+                                        # Handle special cases and try direct casting for other types
+                                        current_dtype = self.features_df[col].dtype
+                                        target_dtype = eval(dtype_str)
+
+                                        # Handle binary data that might need string conversion first
+                                        if "Binary" in str(current_dtype):
+                                            # Convert binary to string first, then to target type
+                                            if target_dtype == pl.Utf8:
+                                                self.features_df = self.features_df.with_columns(
+                                                    pl.col(col)
+                                                    .map_elements(
+                                                        lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                        return_dtype=pl.Utf8,
+                                                    )
+                                                    .cast(target_dtype),
+                                                )
+                                            elif "Int" in str(target_dtype):
+                                                self.features_df = self.features_df.with_columns(
+                                                    pl.col(col)
+                                                    .map_elements(
+                                                        lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                        return_dtype=pl.Utf8,
+                                                    )
+                                                    .str.to_integer()
+                                                    .cast(target_dtype),
+                                                )
+                                            elif "Float" in str(target_dtype):
+                                                self.features_df = self.features_df.with_columns(
+                                                    pl.col(col)
+                                                    .map_elements(
+                                                        lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                        return_dtype=pl.Utf8,
+                                                    )
+                                                    .str.to_decimal()
+                                                    .cast(target_dtype),
+                                                )
+                                            else:
+                                                # Try direct casting
+                                                self.features_df = self.features_df.with_columns(
+                                                    pl.col(col).cast(target_dtype),
+                                                )
+                                        else:
+                                            # Try direct casting for non-binary types
+                                            self.features_df = self.features_df.with_columns(
+                                                pl.col(col).cast(target_dtype),
+                                            )
+                                elif "Utf8" in dtype_str:
+                                    # Ensure it's string type
+                                    self.features_df = self.features_df.with_columns(
+                                        pl.col(col).cast(pl.Utf8),
+                                    )
+                                else:
+                                    # Handle special cases and try direct casting for other types
+                                    current_dtype = self.features_df[col].dtype
+                                    target_dtype = eval(dtype_str)
+
+                                    # Handle binary data that might need string conversion first
+                                    if "Binary" in str(current_dtype):
+                                        # Convert binary to string first, then to target type
+                                        if target_dtype == pl.Utf8:
+                                            self.features_df = self.features_df.with_columns(
+                                                pl.col(col)
+                                                .map_elements(
+                                                    lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                    return_dtype=pl.Utf8,
+                                                )
+                                                .cast(target_dtype),
+                                            )
+                                        elif "Int" in str(target_dtype):
+                                            self.features_df = self.features_df.with_columns(
+                                                pl.col(col)
+                                                .map_elements(
+                                                    lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                    return_dtype=pl.Utf8,
+                                                )
+                                                .str.to_integer()
+                                                .cast(target_dtype),
+                                            )
+                                        elif "Float" in str(target_dtype):
+                                            self.features_df = self.features_df.with_columns(
+                                                pl.col(col)
+                                                .map_elements(
+                                                    lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x),
+                                                    return_dtype=pl.Utf8,
+                                                )
+                                                .str.to_decimal()
+                                                .cast(target_dtype),
+                                            )
+                                        else:
+                                            # Try direct casting
+                                            self.features_df = self.features_df.with_columns(
+                                                pl.col(col).cast(target_dtype),
+                                            )
+                                    else:
+                                        # Try direct casting for non-binary types
+                                        self.features_df = self.features_df.with_columns(
+                                            pl.col(col).cast(target_dtype),
+                                        )
+                        except Exception as e:
+                            self.logger.warning(
+                                f"Failed to cast column '{col}' in features_df: {e}",
+                            )
+                    else:
+                        self.logger.warning(
+                            f"Column '{col}' in features_df not found in schema, keeping original type.",
+                        )
+
+                # FINAL null conversion pass - after all type casting is done
+                # This ensures "None" strings introduced by failed conversions are properly handled
+                for col in self.features_df.columns:
+                    if self.features_df[col].dtype == pl.Utf8:  # String columns
+                        self.features_df = self.features_df.with_columns([
+                            pl.when(pl.col(col).is_in(["None", "", "null", "NULL"]))
+                            .then(None)
+                            .otherwise(pl.col(col))
+                            .alias(col),
+                        ])
+                    # Float columns
+                    elif self.features_df[col].dtype in [pl.Float64, pl.Float32]:
+                        self.features_df = self.features_df.with_columns([
+                            pl.col(col).fill_nan(None).alias(col),
+                        ])
+
+                # Ensure column order matches schema order
+                if "features_df" in schema and "columns" in schema["features_df"]:
+                    schema_column_order = list(schema["features_df"]["columns"].keys())
+                    # Only reorder columns that exist in both schema and DataFrame
+                    existing_columns = [col for col in schema_column_order if col in self.features_df.columns]
+                    if existing_columns:
+                        self.features_df = self.features_df.select(existing_columns)
+
+            else:
+                self.features_df = None
+        else:
+            self.features_df = None
+
+        # Load ms1_df
+        if "ms1" in f:
+            ms1_group = f["ms1"]
+            data = {}
+
+            # Get all datasets in the ms1 group
+            for col in ms1_group.keys():
+                data[col] = ms1_group[col][:]
+
+            if data:
+                # Create DataFrame directly with Polars
+                self.ms1_df = pl.DataFrame(data)
+
+                # Apply schema if available
+                if "ms1_df" in schema and "columns" in schema["ms1_df"]:
+                    schema_columns = schema["ms1_df"]["columns"]
+                    for col in self.ms1_df.columns:
+                        if col in schema_columns:
+                            dtype_str = schema_columns[col]["dtype"]
+                            try:
+                                if "Int" in dtype_str:
+                                    self.ms1_df = self.ms1_df.with_columns([
+                                        pl.col(col).cast(pl.Int64, strict=False),
+                                    ])
+                                elif "Float" in dtype_str:
+                                    self.ms1_df = self.ms1_df.with_columns([
+                                        pl.col(col).cast(pl.Float64, strict=False),
+                                    ])
+                            except Exception as e:
+                                self.logger.warning(
+                                    f"Failed to apply schema type {dtype_str} to column {col}: {e}",
+                                )
+
+                # Convert "None" strings and NaN values to proper null values
+                self.ms1_df = clean_null_values_polars(self.ms1_df)
+            else:
+                self.ms1_df = None
+        else:
+            self.ms1_df = None
+
+        # Parameters are now loaded from metadata JSON (see above)
+        # Lib and lib_match are no longer saved/loaded
+
+    if map:
+        featureXML = filename.replace(".sample5", ".featureXML")
+        if os.path.exists(featureXML):
+            self._load_featureXML(featureXML)
+            self._features_sync()
+        else:
+            self.logger.warning(
+                f"Feature XML file {featureXML} not found, skipping loading.",
+            )
+
+    # set self.file_path to *.sample5
+    self.file_path = filename
+    # set self.label to basename without extension
+    if self.label is None or self.label == "":
+        self.label = os.path.splitext(os.path.basename(filename))[0]
+    self.logger.info(f"Sample loaded successfully from {filename}")
+
+
+def load_schema(schema_path: str) -> Dict[str, Any]:
+    """
+    Load schema from JSON file with error handling.
+
+    Args:
+        schema_path: Path to the schema JSON file
+
+    Returns:
+        Dictionary containing the schema, empty dict if not found
+    """
+    try:
+        with open(schema_path) as f:
+            return json.load(f)  # type: ignore
+    except FileNotFoundError:
+        return {}
+
+
+def decode_metadata_attr(attr_value: Any) -> str:
+    """
+    Decode metadata attribute, handling both bytes and string types.
+
+    Args:
+        attr_value: The attribute value to decode
+
+    Returns:
+        String representation of the attribute
+    """
+    if isinstance(attr_value, bytes):
+        return attr_value.decode()
+    return str(attr_value) if attr_value is not None else ""
+
+
+def clean_null_values_polars(df: pl.DataFrame) -> pl.DataFrame:
+    """
+    Clean null values in a Polars DataFrame by converting string nulls to proper nulls.
+
+    Args:
+        df: The Polars DataFrame to clean
+
+    Returns:
+        Cleaned DataFrame
+    """
+    cleaned_df = df
+    for col in df.columns:
+        if df[col].dtype == pl.Utf8:  # String columns
+            cleaned_df = cleaned_df.with_columns([
+                pl.when(pl.col(col).is_in(["None", "", "null", "NULL"])).then(None).otherwise(pl.col(col)).alias(col),
+            ])
+        elif df[col].dtype in [pl.Float64, pl.Float32]:  # Float columns
+            cleaned_df = cleaned_df.with_columns([
+                pl.col(col).fill_nan(None).alias(col),
+            ])
+    return cleaned_df
+
+
+def cast_column_by_dtype(df: pl.DataFrame, col: str, dtype_str: str) -> pl.DataFrame:
+    """
+    Cast a Polars DataFrame column to the specified dtype with appropriate handling.
+
+    Args:
+        df: The Polars DataFrame
+        col: Column name to cast
+        dtype_str: Target dtype as string (e.g., 'pl.Int64')
+
+    Returns:
+        DataFrame with the column cast to the new type
+    """
+    if not dtype_str.startswith("pl.") or "Object" in dtype_str:
+        return df
+
+    try:
+        target_dtype = eval(dtype_str)
+        current_dtype = df[col].dtype
+
+        if "Int" in dtype_str:
+            return _cast_to_int(df, col, current_dtype, target_dtype)
+        elif "Float" in dtype_str:
+            return _cast_to_float(df, col, current_dtype, target_dtype)
+        elif "Utf8" in dtype_str:
+            return df.with_columns(pl.col(col).cast(pl.Utf8))
+        else:
+            return _cast_with_binary_handling(df, col, current_dtype, target_dtype)
+
+    except Exception:
+        return df
+
+
+def _cast_to_int(df: pl.DataFrame, col: str, current_dtype: pl.DataType, target_dtype: pl.DataType) -> pl.DataFrame:
+    """Helper function to cast column to integer type."""
+    if current_dtype == pl.Utf8:
+        return df.with_columns(
+            pl.col(col).str.to_integer().cast(target_dtype),
+        )
+    elif current_dtype in [pl.Float64, pl.Float32]:
+        return df.with_columns(pl.col(col).cast(target_dtype))
+    else:
+        return _cast_with_binary_handling(df, col, current_dtype, target_dtype)
+
+
+def _cast_to_float(df: pl.DataFrame, col: str, current_dtype: pl.DataType, target_dtype: pl.DataType) -> pl.DataFrame:
+    """Helper function to cast column to float type."""
+    if current_dtype == pl.Utf8:
+        return df.with_columns(
+            pl.col(col).str.to_decimal().cast(target_dtype),
+        )
+    else:
+        return _cast_with_binary_handling(df, col, current_dtype, target_dtype)
+
+
+def _cast_with_binary_handling(
+    df: pl.DataFrame,
+    col: str,
+    current_dtype: pl.DataType,
+    target_dtype: pl.DataType,
+) -> pl.DataFrame:
+    """Helper function to handle binary data conversion."""
+    if "Binary" in str(current_dtype):
+        if target_dtype == pl.Utf8:
+            return df.with_columns(
+                pl.col(col)
+                .map_elements(lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
+                .cast(target_dtype),
+            )
+        elif "Int" in str(target_dtype):
+            return df.with_columns(
+                pl.col(col)
+                .map_elements(lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
+                .str.to_integer()
+                .cast(target_dtype),
+            )
+        elif "Float" in str(target_dtype):
+            return df.with_columns(
+                pl.col(col)
+                .map_elements(lambda x: x.decode("utf-8") if isinstance(x, bytes) else str(x), return_dtype=pl.Utf8)
+                .str.to_decimal()
+                .cast(target_dtype),
+            )
+
+    # Fallback: try direct casting
+    return df.with_columns(pl.col(col).cast(target_dtype))
+
+
+def apply_schema_to_dataframe(df: pl.DataFrame, schema: Dict[str, Any], df_name: str) -> pl.DataFrame:
+    """
+    Apply schema type casting to a Polars DataFrame.
+
+    Args:
+        df: The DataFrame to modify
+        schema: The schema dictionary
+        df_name: Name of the DataFrame in the schema (e.g., 'scans_df', 'features_df')
+
+    Returns:
+        DataFrame with schema types applied
+    """
+    df_schema = schema.get(df_name, {}).get("columns", {})
+
+    for col in df.columns:
+        if col in df_schema:
+            dtype_str = df_schema[col]["dtype"]
+            df = cast_column_by_dtype(df, col, dtype_str)
+
+    return df
+
+
+def reconstruct_object_column(data_col: np.ndarray, col_name: str) -> List[Any]:
+    """
+    Reconstruct object columns from serialized data.
+
+    Args:
+        data_col: Array containing serialized data
+        col_name: Name of the column for type-specific reconstruction
+
+    Returns:
+        List of reconstructed objects
+    """
+    reconstructed_data: list[Any] = []
+
+    for item in data_col:
+        if isinstance(item, bytes):
+            item = item.decode("utf-8")
+
+        if item == "None" or item == "":
+            reconstructed_data.append(None)
+            continue
+
+        try:
+            if col_name == "chrom":
+                reconstructed_data.append(Chromatogram.from_json(item))
+            elif col_name == "ms2_scans":
+                scan_list = json.loads(item)
+                reconstructed_data.append(scan_list)
+            elif col_name == "ms2_specs":
+                json_list = json.loads(item)
+                if json_list == ["None"]:
+                    reconstructed_data.append(None)
+                else:
+                    spectrum_list: list[Any] = []
+                    for json_str in json_list:
+                        if json_str == "None":
+                            spectrum_list.append(None)
+                        else:
+                            spectrum_list.append(Spectrum.from_json(json_str))
+                    reconstructed_data.append(spectrum_list)
+            else:
+                # Unknown object column
+                reconstructed_data.append(None)
+        except (json.JSONDecodeError, ValueError):
+            reconstructed_data.append(None)
+
+    return reconstructed_data
+
+
+def load_dataframe_from_h5_group(
+    group: h5py.Group,
+    schema: Dict[str, Any],
+    df_name: str,
+    logger: Optional[Any] = None,
+) -> Tuple[Optional[pl.DataFrame], List[str]]:
+    """
+    Load a Polars DataFrame from an HDF5 group using schema.
+
+    Args:
+        group: The HDF5 group containing the DataFrame data
+        schema: The schema dictionary
+        df_name: Name of the DataFrame in the schema
+        logger: Optional logger for warnings
+
+    Returns:
+        Tuple of (DataFrame or None, list of missing columns)
+    """
+    data: dict[str, Any] = {}
+    missing_columns = []
+
+    # Load columns according to schema
+    schema_columns = schema.get(df_name, {}).get("columns", [])
+
+    for col in schema_columns:
+        if col not in group:
+            if logger:
+                logger.warning(f"Column '{col}' not found in {df_name}.")
+            data[col] = None
+            missing_columns.append(col)
+            continue
+
+        dtype = schema[df_name]["columns"][col].get("dtype", "native")
+
+        if dtype == "pl.Object":
+            # Handle object columns specially
+            data[col] = reconstruct_object_column(group[col][:], col)
+        else:
+            data[col] = group[col][:]
+
+    if not data:
+        return None, missing_columns
+
+    # Create DataFrame with proper schema for Object columns
+    df_schema = {}
+    for col, values in data.items():
+        if col in schema_columns:
+            dtype_str = schema[df_name]["columns"][col]["dtype"]
+            if dtype_str == "pl.Object":
+                df_schema[col] = pl.Object
+
+    try:
+        if df_schema:
+            df = pl.DataFrame(data, schema=df_schema)
+        else:
+            df = pl.DataFrame(data)
+    except Exception:
+        # Fallback: handle Object columns manually
+        df = _create_dataframe_with_object_columns(data, schema, df_name)
+
+    # Clean null values
+    df = clean_null_values_polars(df)
+
+    # Apply schema type casting
+    df = apply_schema_to_dataframe(df, schema, df_name)
+
+    return df, missing_columns
+
+
+def _create_dataframe_with_object_columns(
+    data: Dict[str, Any],
+    schema: Dict[str, Any],
+    df_name: str,
+) -> pl.DataFrame:
+    """
+    Create DataFrame handling Object columns manually when schema creation fails.
+
+    Args:
+        data: Dictionary of column data
+        schema: The schema dictionary
+        df_name: Name of the DataFrame in the schema
+
+    Returns:
+        Polars DataFrame with Object columns properly handled
+    """
+    schema_columns = schema.get(df_name, {}).get("columns", {})
+
+    object_columns = {
+        k: v for k, v in data.items() if k in schema_columns and schema_columns[k]["dtype"] == "pl.Object"
+    }
+    regular_columns = {k: v for k, v in data.items() if k not in object_columns}
+
+    # Create DataFrame with regular columns first
+    if regular_columns:
+        df = pl.DataFrame(regular_columns)
+        # Add Object columns one by one
+        for col, values in object_columns.items():
+            df = df.with_columns([pl.Series(col, values, dtype=pl.Object)])
+    else:
+        # Only Object columns
+        df = pl.DataFrame()
+        for col, values in object_columns.items():
+            df = df.with_columns([pl.Series(col, values, dtype=pl.Object)])
+
+    return df
+
+
+def load_ms1_dataframe_from_h5_group(
+    group: h5py.Group,
+    schema: Dict[str, Any],
+    logger: Optional[Any] = None,
+) -> Optional[pl.DataFrame]:
+    """
+    Load MS1 DataFrame from HDF5 group.
+
+    Args:
+        group: The HDF5 group containing MS1 data
+        schema: The schema dictionary
+        logger: Optional logger for warnings
+
+    Returns:
+        Polars DataFrame or None
+    """
+    data = {}
+
+    # Get all datasets in the ms1 group
+    for col in group.keys():
+        data[col] = group[col][:]
+
+    if not data:
+        return None
+
+    # Create DataFrame directly with Polars
+    ms1_df = pl.DataFrame(data)
+
+    # Apply schema if available
+    if "ms1_df" in schema and "columns" in schema["ms1_df"]:
+        schema_columns = schema["ms1_df"]["columns"]
+        for col in ms1_df.columns:
+            if col in schema_columns:
+                dtype_str = schema_columns[col]["dtype"]
+                try:
+                    if "Int" in dtype_str:
+                        ms1_df = ms1_df.with_columns([
+                            pl.col(col).cast(pl.Int64, strict=False),
+                        ])
+                    elif "Float" in dtype_str:
+                        ms1_df = ms1_df.with_columns([
+                            pl.col(col).cast(pl.Float64, strict=False),
+                        ])
+                except Exception as e:
+                    if logger:
+                        logger.warning(
+                            f"Failed to apply schema type {dtype_str} to column {col}: {e}",
+                        )
+
+    # Convert "None" strings and NaN values to proper null values
+    return clean_null_values_polars(ms1_df)
+
+
+def load_parameters_from_metadata(metadata_group: h5py.Group) -> Optional[Dict[str, Any]]:
+    """
+    Load parameters from HDF5 metadata group.
+
+    Args:
+        metadata_group: The HDF5 metadata group containing parameters
+
+    Returns:
+        Dictionary of parameters or None if not found
+    """
+    if "parameters" in metadata_group.attrs:
+        try:
+            params_json = decode_metadata_attr(metadata_group.attrs["parameters"])
+            # Ensure params_json is a string before attempting JSON decode
+            if isinstance(params_json, str) and params_json.strip():
+                result = json.loads(params_json)
+                # Ensure the result is a dictionary
+                if isinstance(result, dict):
+                    return result
+        except (json.JSONDecodeError, ValueError, TypeError) as e:
+            # Log the error for debugging
+            print(f"Warning: Failed to parse parameters JSON: {e}")
+            print(f"Raw parameter data type: {type(params_json)}")
+            print(f"Raw parameter data: {repr(params_json)}")
+    return None
+
+
+def create_h5_metadata_group(
+    f: h5py.File,
+    file_path: Optional[str],
+    file_source: Optional[str],
+    file_type: Optional[str],
+    label: Optional[str],
+) -> None:
+    """
+    Create and populate metadata group in HDF5 file.
+
+    Args:
+        f: The HDF5 file object
+        file_path: Source file path
+        file_source: Original source file path
+        file_type: Source file type
+        label: Sample label
+    """
+    metadata_group = f.create_group("metadata")
+    metadata_group.attrs["format"] = "master-sample5-1"
+    metadata_group.attrs["file_path"] = str(file_path) if file_path is not None else ""
+    metadata_group.attrs["file_source"] = str(file_source) if file_source is not None else ""
+    metadata_group.attrs["file_type"] = str(file_type) if file_type is not None else ""
+    metadata_group.attrs["label"] = str(label) if label is not None else ""