CoreMS 4.0.0__py3-none-any.whl

corems/molecular_id/search/database_interfaces.py

@@ -0,0 +1,1527 @@
+import os
+import re
+import warnings
+from abc import ABC
+from io import StringIO
+from pathlib import Path
+import sqlite3
+
+import numpy as np
+import requests
+import pandas as pd
+from ms_entropy import FlashEntropySearch
+
+from corems.molecular_id.factory.EI_SQL import (
+    EI_LowRes_SQLite,
+    Metadatar,
+    MetaboliteMetadata,
+)
+from corems.molecular_id.factory.lipid_molecular_metadata import LipidMetadata
+from corems.mass_spectra.calc.lc_calc import find_closest
+
+
+class SpectralDatabaseInterface(ABC):
+    """
+    Base class that facilitates connection to spectral reference databases,
+    such as EMSL's Metabolomics Reference Database (MetabRef).
+
+    """
+
+    def __init__(self, key=None):
+        """
+        Initialize instance.
+
+        Parameters
+        ----------
+        key : str
+            Token key.
+
+        """
+
+        self.key = key
+
+    def set_token(self, path):
+        """
+        Set environment variable for MetabRef database token.
+
+        Parameters
+        ----------
+        path : str
+            Path to token.
+
+        """
+
+        # Read token from file
+        with open(path, "r", encoding="utf-8") as f:
+            token = f.readline().strip()
+
+        # Set environment variable
+        os.environ[self.key] = token
+
+    def get_token(self):
+        """
+        Get environment variable for database token.
+
+        Returns
+        -------
+        str
+            Token string.
+
+        """
+
+        # Check for token
+        if self.key not in os.environ:
+            raise ValueError("Must set {} environment variable.".format(self.key))
+
+        # Get token from environment variables
+        return os.environ.get(self.key)
+
+    def get_header(self):
+        """
+        Access stored database token and prepare as header.
+
+        Returns
+        -------
+        str
+            Header string.
+
+        """
+
+        # Get token
+        token = self.get_token()
+
+        # Pad header information
+        header = {"Authorization": f"Bearer {token}", "Content-Type": "text/plain"}
+
+        return header
+
+    def get_query(self, url, use_header=True):
+        """
+        Request payload from URL according to `get` protocol.
+
+        Parameters
+        ----------
+        url : str
+            URL for request.
+        use_header: bool
+            Whether or not the query should include the header
+
+        Returns
+        -------
+        dict
+            Response as JSON.
+
+        """
+
+        # Query URL via `get`
+        if use_header:
+            response = requests.get(url, headers=self.get_header())
+        else:
+            response = requests.get(url)
+
+        # Check response
+        response.raise_for_status()
+
+        # Return as JSON
+        return response.json()
+
+    def post_query(self, url, variable, values, tolerance):
+        """
+        Request payload from URL according to `post` protocol.
+
+        Parameters
+        ----------
+        url : str
+            URL for request.
+        variable : str
+            Variable to query.
+        values : str
+            Specific values of `variable` to query.
+        tolerance : str
+            Query tolerance relative to `values`.
+
+        Returns
+        -------
+        dict
+            Response as JSON.
+
+        """
+
+        # Coerce to string
+        if not isinstance(variable, str):
+            variable = str(variable).replace(" ", "")
+
+        if not isinstance(values, str):
+            values = str(values).replace(" ", "")
+
+        if not isinstance(tolerance, str):
+            tolerance = str(tolerance).replace(" ", "")
+
+        # Query URL via `post`
+        response = requests.post(
+            os.path.join(url, variable, tolerance),
+            data=values,
+            headers=self.get_header(),
+        )
+
+        # Check response
+        response.raise_for_status()
+
+        # Return as JSON
+        return response.json()
+
+    def _check_flash_entropy_kwargs(self, fe_kwargs):
+        """
+        Check FlashEntropy keyword arguments.
+
+        Parameters
+        ----------
+        fe_kwargs : dict
+            Keyword arguments for FlashEntropy search.
+
+
+        Raises
+        ------
+        ValueError
+            If "min_ms2_difference_in_da" or "max_ms2_tolerance_in_da" are present in `fe_kwargs` and they
+            are not equal.
+
+        """
+        # If "min_ms2_difference_in_da" in fe_kwargs, check that "max_ms2_tolerance_in_da" is also present and that min_ms2_difference_in_da = 2xmax_ms2_tolerance_in_da
+        if (
+            "min_ms2_difference_in_da" in fe_kwargs
+            or "max_ms2_tolerance_in_da" in fe_kwargs
+        ):
+            if (
+                "min_ms2_difference_in_da" not in fe_kwargs
+                or "max_ms2_tolerance_in_da" not in fe_kwargs
+            ):
+                raise ValueError(
+                    "Both 'min_ms2_difference_in_da' and 'max_ms2_tolerance_in_da' must be specified."
+                )
+            if (
+                fe_kwargs["min_ms2_difference_in_da"]
+                != 2 * fe_kwargs["max_ms2_tolerance_in_da"]
+            ):
+                raise ValueError(
+                    "The values of 'min_ms2_difference_in_da' must be exactly 2x 'max_ms2_tolerance_in_da'."
+                )
+
+    def _get_format_func(self, format):
+        """
+        Obtain format function by key.
+
+        Returns
+        -------
+        func
+            Formatting function.
+        """
+
+        if format.lower() in self.format_map.keys():
+            return self.format_map[format.lower()]
+
+        raise ValueError(("{} not a supported format.").format(format))
+
+    def _dict_to_dataclass(self, source_dict, data_class):
+        """
+        Convert dictionary to dataclass.
+
+        Notes
+        -----
+        This function will pull the attributes a dataclass and its parent class
+        and convert the dictionary to a dataclass instance with the appropriate
+        attributes.
+
+        Parameters
+        ----------
+        data_class : :obj:`~dataclasses.dataclass`
+            Dataclass to convert to.
+        source_dict : dict
+            Dictionary object to convert to dataclass.
+
+        Returns
+        -------
+        :obj:`~dataclasses.dataclass`
+            Dataclass instance.
+
+        """
+
+        # Get list of expected attributes of data_class
+        data_class_keys = list(data_class.__annotations__.keys())
+
+        # Does the data_class inherit from another class, if so, get the attributes of the parent class as well
+        if len(data_class.__mro__) > 2:
+            parent_class_keys = list(data_class.__bases__[0].__annotations__.keys())
+            data_class_keys = list(set(data_class_keys + parent_class_keys))
+
+        # Remove keys that are not in the data_class from the input dictionary
+        input_dict = {k: v for k, v in source_dict.items() if k in data_class_keys}
+
+        # Add keys that are in the data class but not in the input dictionary as None
+        for key in data_class_keys:
+            if key not in input_dict.keys():
+                input_dict[key] = None
+        return data_class(**input_dict)
+
+    def _spectrum_to_array(self, spectrum, normalize=True):
+        """
+        Convert a parenthesis-delimited spectrum string to array.
+
+        Parameters
+        ----------
+        spectrum : str
+            Spectrum string, i.e. list of (m/z,abundance) pairs.
+        normalize : bool
+            Normalize the spectrum by its magnitude.
+
+        Returns
+        -------
+        :obj:`~numpy.array`
+            Array of shape (N, 2), with m/z in the first column and abundance in
+            the second.
+        """
+
+        arr = np.array(
+            re.findall(r"\(([^,]+),([^)]+)\)", spectrum), dtype=float
+        ).reshape(-1, 2)
+
+        if normalize:
+            arr = self.normalize_peaks(arr)
+
+        return arr
+
+    @staticmethod
+    def normalize_peaks(arr):
+        """
+        Normalize peaks in an array.
+
+        Parameters
+        ----------
+        arr : :obj:`~numpy.array`
+            Array of shape (N, 2), with m/z in the first column and abundance in
+            the second.
+
+        Returns
+        -------
+        :obj:`~numpy.array`
+            Normalized array of shape (N, 2), with m/z in the first column and
+            normalized abundance in the second.
+        """
+        # Normalize the array
+        arr[:, -1] = arr[:, -1] / arr[:, -1].sum()
+
+        return arr
+
+    @staticmethod
+    def _build_flash_entropy_index(fe_lib, fe_kwargs={}, clean_spectra=True):
+        """
+        Build FlashEntropy index.
+
+        Parameters
+        ----------
+        fe_lib : list
+            List of spectra to build index from. Can be a list of dictionaries or
+            a FlashEntropy search instance.
+        fe_kwargs : dict, optional
+            Keyword arguments for FlashEntropy search.
+        clean_spectra : bool, optional
+            Clean spectra before building index. Default is True.
+
+        Returns
+        -------
+        :obj:`~ms_entropy.FlashEntropySearch`
+            FlashEntropy search instance.
+
+        """
+        # Initialize FlashEntropy
+        fe_init_kws = [
+            "max_ms2_tolerance_in_da",
+            "mz_index_step",
+            "low_memory",
+            "path_data",
+        ]
+        fe_init_kws = {k: v for k, v in fe_kwargs.items() if k in fe_init_kws}
+        fes = FlashEntropySearch(**fe_init_kws)
+
+        # Build FlashEntropy index
+        fe_index_kws = [
+            "max_indexed_mz",
+            "precursor_ions_removal_da",
+            "noise_threshold",
+            "min_ms2_difference_in_da",
+            "max_peak_num",
+        ]
+        fe_index_kws = {k: v for k, v in fe_kwargs.items() if k in fe_index_kws}
+        fes.build_index(fe_lib, **fe_index_kws, clean_spectra=clean_spectra)
+
+        return fes
+
+
+class MetabRefInterface(SpectralDatabaseInterface):
+    """
+    DEPRECATED interface retained for backward compatibility only.
+    """
+
+    def __init__(self):
+        """
+        Initialize instance with deprecation warning.
+
+        """
+
+        super().__init__(key=None)
+
+        if self.__class__ is MetabRefInterface:
+            warnings.warn(
+                "MetabRefInterface is deprecated. Instantiate a concrete interface "
+                "such as GCMSLibraryInterface or LCLipidLibraryInterface instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+
+class GCMSLibraryInterface(SpectralDatabaseInterface):
+    """
+    Interface to bundled GCMS spectral libraries in MSP format.
+    
+    Loads GCMS compound library and FAMES calibration library from local MSP files.
+    Default files are bundled with CoreMS, but can be overridden via environment variables.
+    """
+
+    def __init__(self):
+        """
+        Initialize instance.
+        """
+        super().__init__(key=None)
+        
+        # Local data file paths
+        from pathlib import Path
+        
+        # Default to bundled data files
+        data_dir = Path(__file__).parent.parent / "data"
+        self.gcms_library_file = os.getenv(
+            "GCMS_LIBRARY_PATH", 
+            str(data_dir / "PNNLMetV20191015.msp")
+        )
+        self.fames_library_file = os.getenv(
+            "FAMES_LIBRARY_PATH",
+            str(data_dir / "FAMES_REF.msp")
+        )
+
+        self.__init_format_map__()
+
+    def __init_format_map__(self):
+        """
+        Initialize database format mapper, enabling multiple format requests.
+
+        """
+
+        # Define format workflows
+        self.format_map = {
+            "json": lambda x, normalize, fe_kwargs: x,
+            "dict": lambda x,
+            normalize,
+            fe_kwargs: self._to_LowResolutionEICompound_dict(x, normalize),
+            "sql": lambda x,
+            normalize,
+            fe_kwargs: self._LowResolutionEICompound_dict_to_sqlite(
+                self._to_LowResolutionEICompound_dict(x, normalize)
+            ),
+        }
+
+        # Add aliases
+        self.format_map["metabref"] = self.format_map["json"]
+        self.format_map["datadict"] = self.format_map["dict"]
+        self.format_map["data-dict"] = self.format_map["dict"]
+        self.format_map["lowreseicompound"] = self.format_map["dict"]
+        self.format_map["lowres"] = self.format_map["dict"]
+        self.format_map["lowresgc"] = self.format_map["dict"]
+        self.format_map["sqlite"] = self.format_map["sql"]
+
+    def available_formats(self):
+        """
+        View list of available formats.
+
+        Returns
+        -------
+        list
+            Format map keys.
+        """
+
+        return list(self.format_map.keys())
+
+    def get_library(self, format="json", normalize=False):
+        """
+        Load GC/MS library from local MSP file.
+
+        Parameters
+        ----------
+        format : str
+            Format of requested library, i.e. "json", "sql", "dict".
+            See `available_formats` method for aliases.
+        normalize : bool
+            Normalize the spectrum by its magnitude.
+
+        Returns
+        -------
+        Library in requested format.
+
+        """
+        # Load from local MSP file
+        library_data = self._load_msp_file(self.gcms_library_file, normalize)
+        
+        # Init format function
+        format_func = self._get_format_func(format)
+        
+        # Apply format conversion
+        return format_func(library_data, normalize, {})
+
+    def get_fames(self, format="json", normalize=False):
+        """
+        Load GC/MS FAMEs library from local MSP file.
+
+        Parameters
+        ----------
+        format : str
+            Format of requested library, i.e. "json", "sql", "dict".
+            See `available_formats` method for aliases.
+        normalize : bool
+            Normalize the spectrum by its magnitude.
+
+        Returns
+        -------
+        Library in requested format.
+
+        """
+        # Load from local MSP file
+        library_data = self._load_msp_file(self.fames_library_file, normalize)
+        
+        # Init format function
+        format_func = self._get_format_func(format)
+        
+        # Apply format conversion
+        return format_func(library_data, normalize, {})
+    
+    def _load_msp_file(self, file_path, normalize=False):
+        """
+        Load and parse MSP file into format compatible with existing pipeline.
+        
+        Parameters
+        ----------
+        file_path : str
+            Path to MSP file
+        normalize : bool
+            Normalize spectra
+            
+        Returns
+        -------
+        list of dict
+            Library data in format compatible with _to_LowResolutionEICompound_dict
+        """
+        from pathlib import Path
+        
+        file_path = Path(file_path)
+        if not file_path.exists():
+            raise FileNotFoundError(
+                f"Library file not found: {file_path}. "
+                f"Set GCMS_LIBRARY_PATH or FAMES_LIBRARY_PATH environment variable to specify location."
+            )
+        
+        # Parse MSP file
+        spectra = []
+        spectrum = {}
+        peaks = []
+        
+        with open(file_path, 'r') as f:
+            for line in f:
+                line = line.strip()
+                
+                # Empty line marks end of spectrum
+                if not line:
+                    if spectrum and peaks:
+                        # Convert peaks to the format expected by downstream code
+                        # Format: "(mz,abundance)(mz,abundance)..."
+                        peak_str = "".join([f"({int(mz)},{int(abun)})" for mz, abun in peaks])
+                        spectrum['mz'] = peak_str
+                        spectra.append(spectrum)
+                    spectrum = {}
+                    peaks = []
+                    continue
+                
+                # Check if line contains peak data (starts with digit)
+                if line and line[0].isdigit():
+                    parts = line.split()
+                    if len(parts) >= 2:
+                        peaks.append((float(parts[0]), float(parts[1])))
+                    continue
+                
+                # Handle metadata fields
+                if ":" in line:
+                    key, value = line.split(":", 1)
+                    key = key.strip().lower()
+                    value = value.strip()
+                    
+                    # Map MSP fields to expected format
+                    field_mapping = {
+                        "name": "molecule_name",
+                        "formula": "formula",
+                        "cas": "casno",
+                        "retentiontime": "retention_time",
+                        "ri": "ri",
+                        "comment": "comments",
+                        "num peaks": "peak_count",
+                        "derivative": "derivative"
+                    }
+                    
+                    # Metadata fields that go into the metadata dict
+                    metadata_fields = {
+                        "inchikey": "inchikey",
+                        "inchi": "inchi",
+                        "smiles": "smiles",
+                        "pubchem": "pubchem",
+                        "chebi": "chebi",
+                        "kegg": "kegg",
+                        "refmet": "refmet",
+                        "iupac_name": "iupac_name"
+                    }
+                    
+                    if key in field_mapping:
+                        mapped_key = field_mapping[key]
+                        # Convert numeric fields
+                        if key in ["retentiontime", "ri"]:
+                            try:
+                                value = float(value)
+                            except:
+                                pass
+                        elif key == "num peaks":
+                            try:
+                                value = int(value)
+                            except:
+                                pass
+                        spectrum[mapped_key] = value
+                    elif key in metadata_fields:
+                        # Store in nested metadata dict
+                        if "metadata" not in spectrum:
+                            spectrum["metadata"] = {}
+                        spectrum["metadata"][metadata_fields[key]] = value
+                    else:
+                        # Keep unmapped fields
+                        spectrum[key] = value
+        
+        # Add last spectrum if file doesn't end with blank line
+        if spectrum and peaks:
+            peak_str = "".join([f"({int(mz)},{int(abun)})" for mz, abun in peaks])
+            spectrum['mz'] = peak_str
+            spectra.append(spectrum)
+        
+        return spectra
+
+    def _to_LowResolutionEICompound_dict(self, metabref_lib, normalize=False):
+        """
+        Convert MetabRef-formatted library to CoreMS LowResolutionEICompound-formatted
+        dictionary for local ingestion.
+
+        Parameters
+        ----------
+        metabref_lib : dict
+            MetabRef GC-MS library in JSON format.
+        normalize : bool
+            Normalize each spectrum by its magnitude.
+
+        Returns
+        -------
+        list of dict
+            List of each spectrum contained in dictionary.
+
+        """
+
+        # All below key:value lookups are based on CoreMS class definitions
+        # NOT MetabRef content. For example, MetabRef has keys for PubChem,
+        # USI, etc. that are not considered below.
+
+        # Dictionary to map metabref keys to corems keys
+        metadatar_cols = {
+            "casno": "cas",
+            "inchikey": "inchikey",
+            "inchi": "inchi",
+            "chebi": "chebi",
+            "smiles": "smiles",
+            "kegg": "kegg",
+            "iupac_name": "iupac_name",
+            "traditional_name": "traditional_name",  # Not present in metabref
+            "common_name": "common_name",  # Not present in metabref
+        }
+
+        # Dictionary to map metabref keys to corems keys
+        lowres_ei_compound_cols = {
+            "id": "metabref_id",
+            "molecule_name": "name",  # Is this correct?
+            "classify": "classify",  # Not present in metabref
+            "formula": "formula",
+            "ri": "ri",
+            "rt": "retention_time",
+            "source": "source",  # Not present in metabref
+            "casno": "casno",
+            "comments": "comment",
+            "source_temp_c": "source_temp_c",  # Not present in metabref
+            "ev": "ev",  # Not present in metabref
+            "peak_count": "peaks_count",
+            "mz": "mz",
+            "abundance": "abundance",
+        }
+
+        # Local result container
+        corems_lib = []
+
+        # Enumerate spectra
+        for i, source_ in enumerate(metabref_lib):
+            # Copy source to prevent modification
+            source = source_.copy()
+
+            # Parse target data
+            target = {
+                lowres_ei_compound_cols[k]: v
+                for k, v in source.items()
+                if k in lowres_ei_compound_cols
+            }
+
+            # Explicitly add this to connect with LowResCompoundRef later
+            if "retention_time" in source:
+                target["rt"] = source["retention_time"]
+            elif "rt" in source:
+                target["rt"] = source["rt"]
+
+            # Parse (mz, abundance)
+            arr = self._spectrum_to_array(target["mz"], normalize=normalize)
+            target["mz"] = arr[:, 0]
+            target["abundance"] = arr[:, 1]
+
+            # Parse meta data
+            target["metadata"] = {
+                metadatar_cols[k]: v for k, v in source.items() if k in metadatar_cols
+            }
+
+            # Add anything else
+            for k in source:
+                if k not in lowres_ei_compound_cols:
+                    target[k] = source[k]
+
+            # Add to CoreMS list
+            corems_lib.append(target)
+
+        return corems_lib
+
+    def _LowResolutionEICompound_dict_to_sqlite(
+        self, lowres_ei_compound_dict, url="sqlite://"
+    ):
+        """
+        Convert CoreMS LowResolutionEICompound-formatted dictionary to SQLite
+        database for local ingestion.
+
+        Parameters
+        ----------
+        lowres_ei_compound_dict : dict
+            CoreMS GC-MS library formatted for LowResolutionEICompound.
+        url : str
+            URL to SQLite prefix.
+
+        Returns
+        -------
+        sqlite database
+            Spectra contained in SQLite database.
+
+        """
+
+        # Dictionary to map corems keys to all-caps keys
+        capped_cols = {
+            "name": "NAME",
+            "formula": "FORM",
+            "ri": "RI",
+            "retention_time": "RT",
+            "source": "SOURCE",
+            "casno": "CASNO",
+            "comment": "COMMENT",
+            "peaks_count": "NUM PEAKS",
+        }
+
+        # Initialize SQLite object
+        sqlite_obj = EI_LowRes_SQLite(url=url)
+
+        # Iterate spectra
+        for _data_dict in lowres_ei_compound_dict:
+            # Copy source to prevent modification
+            data_dict = _data_dict.copy()
+
+            # Add missing capped values
+            for k, v in capped_cols.items():
+                # Key exists
+                if k in data_dict:
+                    # # This will replace the key
+                    # data_dict[v] = data_dict.pop(k)
+
+                    # This will keep both keys
+                    data_dict[v] = data_dict[k]
+
+            # Parse number of peaks
+            if not data_dict.get("NUM PEAKS"):
+                data_dict["NUM PEAKS"] = len(data_dict.get("mz"))
+
+            # Parse CAS number
+            if not data_dict.get("CASNO"):
+                data_dict["CASNO"] = data_dict.get("CAS")
+
+            if not data_dict["CASNO"]:
+                data_dict["CASNO"] = 0
+
+            # Build linked metadata table
+            if "metadata" in data_dict:
+                metadata = data_dict.pop("metadata")
+                # Only create metadata entry if we have required fields and valid data
+                # Filter to only include fields that Metadatar model supports
+                supported_metadata_fields = [
+                    'cas', 'inchikey', 'inchi', 'chebi', 'smiles', 
+                    'kegg', 'iupac_name', 'traditional_name', 'common_name'
+                ]
+                filtered_metadata = {
+                    k: v for k, v in metadata.items() 
+                    if k in supported_metadata_fields and v
+                }
+                # Inchikey is required by the database model
+                if filtered_metadata and filtered_metadata.get("inchikey"):
+                    data_dict["metadatar"] = Metadatar(**filtered_metadata)
+
+            # Attempt addition to sqlite
+            try:
+                sqlite_obj.add_compound(data_dict)
+            except:
+                print(data_dict["NAME"])
+
+        return sqlite_obj
+
+
+class MetabRefGCInterface(GCMSLibraryInterface):
+    """
+    DEPRECATED: Use GCMSLibraryInterface instead.
+    
+    This interface is maintained for backward compatibility only.
+    MetabRef API has been discontinued as of 2026.
+    """
+
+    def __init__(self):
+        """
+        Initialize instance with deprecation warning.
+        """
+        warnings.warn(
+            "MetabRefGCInterface is deprecated. Use GCMSLibraryInterface instead. "
+            "MetabRef API has been discontinued; all data now loads from bundled local MSP files.",
+            DeprecationWarning,
+            stacklevel=2
+        )
+        super().__init__()
+
+
+class LCLipidLibraryInterface(SpectralDatabaseInterface):
+    """
+    Interface to a local sqlite lipid library for LC-MS spectral searches.
+    """
+
+    DEFAULT_DOWNLOAD_URL = (
+        "https://nmdcdemo.emsl.pnnl.gov/minio/lipidomics/parameter_files/"
+        "202412_lipid_ref.sqlite"
+    )
+
+    def __init__(self, db_location=None):
+        """
+        Initialize instance.
+
+        Parameters
+        ----------
+        db_location : str | Path, optional
+            Local path to the sqlite lipid library. If omitted, the
+            COREMS_LIPIDOMICS_SQLITE_PATH environment variable is used.
+        """
+
+        super().__init__(key=None)
+        self.db_location = db_location
+        self.__init_format_map__()
+
+    def _to_flashentropy(self, spectral_library, normalize=True, fe_kwargs={}):
+        """
+        Convert a spectral library to FlashEntropy format.
+
+        Parameters
+        ----------
+        spectral_library : dict
+            MS2 library in JSON format or FlashEntropy search instance
+            (for reformatting at different MS2 separation).
+        normalize : bool
+            Normalize each spectrum by its magnitude.
+        fe_kwargs : dict, optional
+            Keyword arguments for instantiation of FlashEntropy search and building index for FlashEntropy search;
+            any keys not recognized will be ignored. By default, all parameters set to defaults.
+
+        Returns
+        -------
+        :obj:`~ms_entropy.FlashEntropySearch`
+            MS2 library as FlashEntropy search instance.
+
+        Raises
+        ------
+        ValueError
+            If "min_ms2_difference_in_da" or "max_ms2_tolerance_in_da" are present in `fe_kwargs` and they are not equal.
+
+        """
+        self._check_flash_entropy_kwargs(fe_kwargs)
+
+        # Initialize empty library
+        fe_lib = []
+
+        # Enumerate spectra
+        for i, source in enumerate(spectral_library):
+            if "spectrum_data" in source.keys():
+                spectrum = source["spectrum_data"]
+            else:
+                spectrum = source
+
+            if "precursor_mz" not in spectrum.keys():
+                spectrum["precursor_mz"] = spectrum.pop("precursor_ion")
+
+            spectrum["peaks"] = self._spectrum_to_array(
+                spectrum["mz"], normalize=normalize
+            )
+            fe_lib.append(spectrum)
+
+        fe_search = self._build_flash_entropy_index(fe_lib, fe_kwargs=fe_kwargs)
+
+        return fe_search
+
+    def __init_format_map__(self):
+        """
+        Initialize database format mapper, enabling multiple format requests.
+        """
+
+        self.format_map = {
+            "json": lambda x, normalize, fe_kwargs: x,
+            "flashentropy": lambda x, normalize, fe_kwargs: self._to_flashentropy(
+                x, normalize, fe_kwargs
+            ),
+            "dataframe": lambda x, normalize, fe_kwargs: pd.DataFrame(x),
+        }
+
+        self.format_map["fe"] = self.format_map["flashentropy"]
+        self.format_map["flash-entropy"] = self.format_map["flashentropy"]
+        self.format_map["df"] = self.format_map["dataframe"]
+
+    def available_formats(self):
+        """
+        View list of available formats.
+
+        Returns
+        -------
+        list
+            Format map keys.
+        """
+
+        return list(self.format_map.keys())
+
+    def _resolve_db_location(self):
+        """
+        Resolve and validate sqlite database location.
+
+        Returns
+        -------
+        Path
+            Existing sqlite database file path.
+        """
+
+        db_location = self.db_location or os.getenv("COREMS_LIPIDOMICS_SQLITE_PATH")
+        if not db_location:
+            raise ValueError(
+                "A local lipid sqlite library path is required. "
+                "Set COREMS_LIPIDOMICS_SQLITE_PATH or pass db_location."
+            )
+
+        db_path = Path(db_location).expanduser()
+        if not db_path.exists():
+            raise FileNotFoundError(
+                f"Lipid sqlite library not found at {db_path}. "
+                f"Download it from {self.DEFAULT_DOWNLOAD_URL} "
+                "and set COREMS_LIPIDOMICS_SQLITE_PATH."
+            )
+
+        return db_path
+
+    def _get_candidate_spectra(self, connection, mz_list, polarity, mz_tol_ppm):
+        """
+        Fetch candidate spectra rows by precursor m/z and polarity.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Filtered rows from lipidMassSpectrumObject.
+        """
+
+        mz_observed = np.sort(np.asarray(mz_list, dtype=float))
+        if mz_observed.size == 0:
+            return pd.DataFrame()
+
+        mz_all = pd.read_sql_query(
+            "SELECT id, polarity, precursor_mz FROM lipidMassSpectrumObject", connection
+        )
+        mz_all = mz_all[mz_all["polarity"] == polarity].copy()
+        if mz_all.empty:
+            return pd.DataFrame()
+
+        mz_all = mz_all.sort_values(by="precursor_mz").reset_index(drop=True)
+
+        if mz_observed.size == 1:
+            mz_all["closest_mz_obs"] = mz_observed[0]
+        else:
+            mz_all["closest_mz_obs"] = mz_observed[
+                find_closest(mz_observed, mz_all.precursor_mz.values)
+            ]
+
+        mz_all["ppm_error"] = (
+            (mz_all["precursor_mz"] - mz_all["closest_mz_obs"])
+            / mz_all["precursor_mz"]
+            * 1e6
+        )
+
+        mz_all = mz_all[np.abs(mz_all["ppm_error"]) <= mz_tol_ppm]
+        if mz_all.empty:
+            return pd.DataFrame()
+
+        mz_ids = tuple(mz_all["id"].tolist())
+        return pd.read_sql_query(
+            f"SELECT * FROM lipidMassSpectrumObject WHERE id IN {mz_ids}",
+            connection,
+        )
+
+    def get_lipid_library(
+        self,
+        mz_list,
+        polarity,
+        mz_tol_ppm,
+        mz_tol_da_api=None,
+        format="json",
+        normalize=True,
+        fe_kwargs={},
+        api_delay=5,
+        api_attempts=10,
+    ):
+        """
+        Retrieve lipid spectra and metadata from a local sqlite library.
+
+        Parameters
+        ----------
+        mz_list : list
+            List of precursor m/z values.
+        polarity : str
+            Ionization polarity, either "positive" or "negative".
+        mz_tol_ppm : float
+            Tolerance in ppm for precursor matching.
+        mz_tol_da_api : float, optional
+            Unused, kept for backward compatibility.
+        format : str, optional
+            Format of requested library, e.g. "json" or "flashentropy".
+        normalize : bool, optional
+            Normalize spectrum intensities.
+        fe_kwargs : dict, optional
+            Keyword arguments for FlashEntropy search.
+        api_delay : int, optional
+            Unused, kept for backward compatibility.
+        api_attempts : int, optional
+            Unused, kept for backward compatibility.
+
+        Returns
+        -------
+        tuple
+            Library in requested format and lipid metadata dictionary.
+        """
+
+        if not isinstance(mz_list, (list, np.ndarray)):
+            raise ValueError("mz_list must be a list or numpy array")
+        if not all(isinstance(mz, (float, int)) for mz in mz_list):
+            raise ValueError("All elements in mz_list must be float or int")
+        if polarity not in {"positive", "negative"}:
+            raise ValueError("polarity must be either 'positive' or 'negative'")
+        if not isinstance(mz_tol_ppm, (float, int)):
+            raise ValueError("mz_tol_ppm must be a float or int")
+
+        db_path = self._resolve_db_location()
+        connection = sqlite3.connect(str(db_path))
+        try:
+            # Step 1: Get candidate spectra records based on m/z and polarity
+            spectra_df = self._get_candidate_spectra(
+                connection=connection,
+                mz_list=mz_list,
+                polarity=polarity,
+                mz_tol_ppm=float(mz_tol_ppm),
+            )
+
+            if spectra_df.empty:
+                format_func = self._get_format_func(format)
+                return format_func([], normalize=normalize, fe_kwargs=fe_kwargs), {}
+
+            # Step 2: Get corresponding lipid metadata for candidate spectra from lipidTree view
+            mol_ids = tuple(spectra_df["molecular_data_id"].tolist())
+            mol_df = pd.read_sql_query(
+                f"SELECT * FROM lipidTree WHERE id IN {mol_ids}",
+                connection,
+            )
+        finally:
+            connection.close()
+
+        mol_df["id_index"] = mol_df["id"]
+        mol_df = mol_df.set_index("id_index")
+        mol_records = mol_df.to_dict(orient="index")
+        lipid_metadata = {
+            int(k): self._dict_to_dataclass(v, LipidMetadata)
+            for k, v in mol_records.items()
+        }
+
+        spectra_records = spectra_df.to_dict(orient="records")
+        format_func = self._get_format_func(format)
+        library = format_func(spectra_records, normalize=normalize, fe_kwargs=fe_kwargs)
+        return library, lipid_metadata
+
+
+class MSPInterface(SpectralDatabaseInterface):
+    """
+    Interface to parse NIST MSP files
+    """
+
+    def __init__(self, file_path):
+        """
+        Initialize instance.
+
+        Parameters
+        ----------
+        file_path : str
+            Path to a local MSP file.
+
+        Attributes
+        ----------
+        file_path : str
+            Path to the MSP file.
+        _file_content : str
+            Content of the MSP file.
+        _data_frame : :obj:`~pandas.DataFrame`
+            DataFrame of spectra from the MSP file with unaltered content.
+        """
+        super().__init__(key=None)
+
+        self.file_path = file_path
+        if not os.path.exists(self.file_path):
+            raise FileNotFoundError(
+                f"File {self.file_path} does not exist. Please check the file path."
+            )
+        with open(self.file_path, "r") as f:
+            self._file_content = f.read()
+
+        self._data_frame = self._read_msp_file()
+        self.__init_format_map__()
+
+    def __init_format_map__(self):
+        """
+        Initialize database format mapper, enabling multiple format requests.
+
+        """
+
+        # x is a pandas dataframe similar to self._data_frame format
+        # Define format workflows
+        self.format_map = {
+            "msp": lambda x, normalize, fe_kwargs: self._to_msp(x, normalize),
+            "flashentropy": lambda x, normalize, fe_kwargs: self._to_flashentropy(
+                x, normalize, fe_kwargs
+            ),
+            "df": lambda x, normalize, fe_kwargs: self._to_df(x, normalize),
+        }
+
+        # Add aliases
+        self.format_map["fe"] = self.format_map["flashentropy"]
+        self.format_map["flash-entropy"] = self.format_map["flashentropy"]
+        self.format_map["dataframe"] = self.format_map["df"]
+        self.format_map["data-frame"] = self.format_map["df"]
+
+    def _read_msp_file(self):
+        """
+        Reads the MSP files into the pandas dataframe, and sort/remove zero intensity ions in MS/MS spectra.
+
+        Returns
+        -------
+        :obj:`~pandas.DataFrame`
+            DataFrame of spectra from the MSP file, exacly as it is in the file (no sorting, filtering etc)
+        """
+        # If input_dataframe is provided, return it it
+        spectra = []
+        spectrum = {}
+
+        f = StringIO(self._file_content)
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue  # Skip empty lines
+
+            # Handle metadata
+            if ":" in line:
+                key, value = line.split(":", 1)
+                key = key.strip().lower()
+                value = value.strip()
+
+                if key == "name":
+                    # Save current spectrum and start a new one
+                    if spectrum:
+                        spectra.append(spectrum)
+                    spectrum = {"name": value, "peaks": []}
+                else:
+                    spectrum[key] = value
+
+            # Handle peak data (assumed to start with a number)
+            elif line[0].isdigit():
+                peaks = line.split()
+                m_z = float(peaks[0])
+                intensity = float(peaks[1])
+                spectrum["peaks"].append(([m_z, intensity]))
+        # Save the last spectrum
+        if spectrum:
+            spectra.append(spectrum)
+
+        df = pd.DataFrame(spectra)
+        for column in df.columns:
+            if column != "peaks":  # Skip 'peaks' column
+                try:
+                    df[column] = pd.to_numeric(df[column], errors="raise")
+                except:
+                    pass
+        
+        # Standardize spectra ID column name
+        # Check for common variations and create a standard 'spectra_id' column
+        spectra_id_variants = ['spectrum_id', 'gnps_spectra_id']
+        for variant in spectra_id_variants:
+            if variant in df.columns and 'spectra_id' not in df.columns:
+                df['spectra_id'] = df[variant]
+                break
+        
+        # If no spectra_id column exists after checking variants, create one with sequential IDs
+        if 'spectra_id' not in df.columns:
+            df['spectra_id'] = [f"spectrum_{i:06d}" for i in range(len(df))]
+        
+        # Standardize compound name column
+        # Ensure 'compound_name' column exists, using 'name' field
+        if 'name' in df.columns and 'compound_name' not in df.columns:
+            df['compound_name'] = df['name']
+        elif 'compound_name' not in df.columns:
+            warnings.warn(
+                "MSP file does not contain 'name' or 'compound_name' field. "
+                "Compound names will be set to 'Unknown'. This may affect plot labels and annotations.",
+                UserWarning
+            )
+            df['compound_name'] = 'Unknown'
+        
+        return df
+
+    def _to_df(self, input_dataframe, normalize=True):
+        """
+        Convert MSP-derived library to FlashEntropy library. 
+
+        Parameters
+        ----------
+        input_dataframe : :obj:`~pandas.DataFrame`
+            Input DataFrame containing MSP-formatted spectra.
+        normalize : bool, optional
+            Normalize each spectrum by its magnitude.
+            Default is True.
+
+        Returns
+        -------
+        :obj:`~pandas.DataFrame`
+            DataFrame of with desired normalization
+        """
+        if not normalize:
+            return input_dataframe
+        else:
+            # Convert to dictionary
+            db_dict = input_dataframe.to_dict(orient="records")
+
+            # Initialize empty library
+            lib = []
+
+            # Enumerate spectra
+            for i, source in enumerate(db_dict):
+                spectrum = source
+                # Check that spectrum["peaks"] exists
+                if "peaks" not in spectrum.keys():
+                    raise KeyError(
+                        "MSP not interpretted correctly, 'peaks' key not found in spectrum, check _dataframe attribute."
+                    )
+
+                # Convert spectrum["peaks"] to numpy array
+                if not isinstance(spectrum["peaks"], np.ndarray):
+                    spectrum["peaks"] = np.array(spectrum["peaks"])
+
+                # Normalize peaks, if requested
+                if normalize:
+                    spectrum["peaks"] = self.normalize_peaks(spectrum["peaks"])
+                    spectrum["num peaks"] = len(spectrum["peaks"])
+
+                # Add spectrum to library
+                lib.append(spectrum)
+            
+            # Convert to DataFrame
+            df = pd.DataFrame(lib)
+            return df
+    
+    def _to_flashentropy(self, input_dataframe, normalize=True, fe_kwargs={}):
+        """
+        Convert MSP-derived library to FlashEntropy library.
+
+        Parameters
+        ----------
+        input_dataframe : :obj:`~pandas.DataFrame`
+            Input DataFrame containing MSP-formatted spectra.
+        normalize : bool
+            Normalize each spectrum by its magnitude.
+        fe_kwargs : dict, optional
+            Keyword arguments for instantiation of FlashEntropy search and building index for FlashEntropy search;
+            any keys not recognized will be ignored. By default, all parameters set to defaults.
+
+        Returns
+        -------
+        :obj:`~ms_entropy.FlashEntropySearch`
+            MS2 library as FlashEntropy search instance.
+
+        Raises
+        ------
+        ValueError
+            If "min_ms2_difference_in_da" or "max_ms2_tolerance_in_da" are present in `fe_kwargs` and they
+        """
+        self._check_flash_entropy_kwargs(fe_kwargs)
+
+        db_df = input_dataframe
+
+        # Convert to dictionary
+        db_dict = db_df.to_dict(orient="records")
+
+        # Initialize empty library
+        fe_lib = []
+
+        # Enumerate spectra
+        for i, source in enumerate(db_dict):
+            # Reorganize source dict, if necessary
+            if "spectrum_data" in source.keys():
+                spectrum = source["spectrum_data"]
+            else:
+                spectrum = source
+
+            # Rename precursor_mz key for FlashEntropy
+            if "precursor_mz" not in spectrum.keys():
+                if "precursormz" in spectrum:
+                    spectrum["precursor_mz"] = spectrum.pop("precursormz")
+                elif "precursor_ion" in spectrum:
+                    spectrum["precursor_mz"] = spectrum.pop("precursor_ion")
+                else:
+                    raise KeyError(
+                        "MSP must have either 'precursormz' or 'precursor_ion' key to be converted to FlashEntropy format."
+                    )
+
+            # Check that spectrum["peaks"] exists
+            if "peaks" not in spectrum.keys():
+                raise KeyError(
+                    "MSP not interpretted correctly, 'peaks' key not found in spectrum, check _dataframe attribute."
+                )
+
+            # Convert spectrum["peaks"] to numpy array
+            if not isinstance(spectrum["peaks"], np.ndarray):
+                spectrum["peaks"] = np.array(spectrum["peaks"])
+
+            # Normalize peaks, if requested
+            if normalize:
+                spectrum["peaks"] = self.normalize_peaks(spectrum["peaks"])
+
+            # Add spectrum to library
+            fe_lib.append(spectrum)
+
+        # Build FlashEntropy index
+        fe_search = self._build_flash_entropy_index(fe_lib, fe_kwargs=fe_kwargs)
+
+        return fe_search
+    
+    def _check_msp_compatibility(self):
+        """
+        Check if the MSP file is compatible with the get_metabolomics_spectra_library method and provide feedback if it is not.
+        """
+        # Check polarity
+        if (
+            "polarity" not in self._data_frame.columns
+            and "ionmode" not in self._data_frame.columns
+        ):
+            raise ValueError(
+                "Neither 'polarity' nor 'ionmode' columns found in the input MSP metadata. Please check the file."
+            )
+        polarity_column = (
+            "polarity" if "polarity" in self._data_frame.columns else "ionmode"
+        )
+
+        # Check if polarity_column contents is either "positive" or "negative"
+        if not all(self._data_frame[polarity_column].isin(["positive", "negative"])):
+            raise ValueError(
+                f"Input field on MSP '{polarity_column}' must contain only 'positive' or 'negative' values."
+            )
+
+        # Check if the MSP file contains the required columns for metabolite metadata
+        # either formula or molecular_formula, not null
+        
+        if (
+            "formula" not in self._data_frame.columns
+            and "molecular_formula" not in self._data_frame.columns
+        ):
+            raise ValueError(
+                "Input field on MSP must contain either 'formula' or 'molecular_formula' columns."
+            )
+        molecular_formula_column = (
+            "formula" if "formula" in self._data_frame.columns else "molecular_formula"
+        )
+        if not all(self._data_frame[molecular_formula_column].notnull()):
+            raise ValueError(
+                f"Input field on MSP '{molecular_formula_column}' must contain only non-null values."
+            )
+
+    def get_metabolomics_spectra_library(
+        self,
+        polarity,
+        metabolite_metadata_mapping={},
+        format="fe",
+        normalize=True,
+        fe_kwargs={},
+        molecular_id_field="inchikey",
+    ):
+        """
+        Prepare metabolomics spectra library and associated metabolite metadata
+
+        Parameters
+        ----------
+        polarity : str
+            Polarity of the spectra to extract. Must be 'positive' or 'negative'.
+        metabolite_metadata_mapping : dict, optional
+            Mapping of MSP field names to MetaboliteMetadata attribute names.
+            Default uses common mappings (e.g., 'molecular_formula' -> 'formula').
+        format : str, optional
+            Output format for the spectral library. Options: 'fe', 'flashentropy', 'msp', 'df'.
+            Default is 'fe' (FlashEntropy).
+        normalize : bool, optional
+            Whether to normalize spectra. Default is True.
+        fe_kwargs : dict, optional
+            Additional keyword arguments for FlashEntropy library creation.
+        molecular_id_field : str, optional
+            Field name to use as the unique molecular identifier for linking spectra to metadata.
+            Default is 'inchikey'. The specified field must exist in the MSP file and contain
+            non-null values for all entries.
+
+        Returns
+        -------
+        tuple
+            (spectral_library, metabolite_metadata_dict) where spectral_library is in the
+            requested format and metabolite_metadata_dict maps molecular IDs to MetaboliteMetadata objects.
+
+        Notes
+        -----
+        The molecular_id_field parameter allows flexibility for different MSP file formats:
+        - Use 'inchikey' for standard metabolite databases (default)
+        - Use 'name' or 'spectra_id' for custom or isotope-labeled standards
+        - The specified field must exist and contain non-null values for all entries
+
+        """
+        # Check if the MSP file is compatible with the get_metabolomics_spectra_library method
+        self._check_msp_compatibility()
+
+        # Check if the polarity parameter is valid and if a polarity column exists in the dataframe
+        if polarity not in ["positive", "negative"]:
+            raise ValueError("Polarity must be 'positive' or 'negative'")
+        polarity_column = (
+            "polarity" if "polarity" in self._data_frame.columns else "ionmode"
+        )
+
+        # Get a subset of the initial dataframea by polarity
+        db_df = self._data_frame[self._data_frame[polarity_column] == polarity].copy()
+
+        # Rename the columns of the db_df to match the MetaboliteMetadata dataclass using the metabolite_metadata_mapping
+        # If the mapping is not provided, use the default mapping
+        if not metabolite_metadata_mapping:
+            metabolite_metadata_mapping = {
+                "chebi_id": "chebi",
+                "kegg_id": "kegg",
+                "refmet_name": "common_name",
+                "molecular_formula": "formula",
+                "gnps_spectra_id":"id",
+                "precursormz": "precursor_mz",
+                "precursortype":"ion_type"
+            }
+        db_df.rename(columns=metabolite_metadata_mapping, inplace=True)
+        
+        # Create molecular_data_id from the specified field
+        if molecular_id_field not in db_df.columns:
+            raise ValueError(
+                f"Specified molecular_id_field '{molecular_id_field}' not found in MSP data. "
+                f"Available columns: {', '.join(db_df.columns)}"
+            )
+        
+        if not db_df[molecular_id_field].notnull().all():
+            raise ValueError(
+                f"Specified molecular_id_field '{molecular_id_field}' contains null values. "
+                f"All entries must have non-null values for the molecular ID field."
+            )
+        
+        # Use the specified field as the molecular ID
+        db_df["molecular_data_id"] = db_df[molecular_id_field].astype(str)
+        
+        # Ensure 'id' field exists for spectra identification
+        # If not present, create from spectra_id or use a sequential index
+        if "id" not in db_df.columns:
+            if "spectra_id" in db_df.columns:
+                db_df["id"] = db_df["spectra_id"].astype(str)
+            else:
+                # Generate sequential IDs
+                db_df["id"] = [f"spectrum_{i:06d}" for i in range(len(db_df))]
+
+        # Check if the resulting dataframe has the required columns for the flash entropy search
+        required_columns = ["molecular_data_id", "precursor_mz", "ion_type", "id"]
+        for col in required_columns:
+            if col not in db_df.columns:
+                raise ValueError(
+                    f"Input field on MSP must contain '{col}' column for FlashEntropy search."
+                )
+
+        # Pull out the metabolite metadata from the dataframe and put it into a different dataframe
+        # First get a list of the possible attributes of the MetaboliteMetadata dataclass
+        metabolite_metadata_keys = list(MetaboliteMetadata.__annotations__.keys())
+        # Replace id with molecular_data_id in metabolite_metadata_keys
+        metabolite_metadata_keys = [
+            "molecular_data_id" if x == "id" else x for x in metabolite_metadata_keys
+        ]
+        metabolite_metadata_df = db_df[
+            db_df.columns[db_df.columns.isin(metabolite_metadata_keys)]
+        ].copy()
+
+        # Make unique and recast the id column for metabolite metadata
+        metabolite_metadata_df.drop_duplicates(subset=["molecular_data_id"], inplace=True)
+        metabolite_metadata_df["id"] = metabolite_metadata_df["molecular_data_id"]
+
+        # Convert to a dictionary using the molecular_data_id as the key
+        metabolite_metadata_dict = metabolite_metadata_df.to_dict(
+            orient="records"
+        )
+        metabolite_metadata_dict = {
+            v["id"]: self._dict_to_dataclass(v, MetaboliteMetadata)
+            for v in metabolite_metadata_dict
+        }
+
+        # Remove the metabolite metadata columns from the original dataframe
+        for key in metabolite_metadata_keys:
+            if key != "molecular_data_id":
+                if key in db_df.columns:
+                    db_df.drop(columns=key, inplace=True)
+
+        # Format the spectral library
+        format_func = self._get_format_func(format)
+        lib = format_func(db_df, normalize=normalize, fe_kwargs=fe_kwargs)
+        return (lib, metabolite_metadata_dict)