nerdd-module 0.3.49__tar.gz → 0.3.51__tar.gz

{nerdd_module-0.3.49 → nerdd_module-0.3.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nerdd-module
-Version: 0.3.49
+Version: 0.3.51
 Summary: Base package to create NERDD modules
 Author-email: Steffen Hirte <steffen.hirte@univie.ac.at>
 Maintainer-email: Steffen Hirte <steffen.hirte@univie.ac.at>

{nerdd_module-0.3.49 → nerdd_module-0.3.51}/nerdd_module/input/depth_first_explorer.py

@@ -8,16 +8,17 @@ __all__ = ["DepthFirstExplorer"]
 
 
 class InvalidInputReader(Reader):
-    def __init__(self) -> None:
+    def __init__(self, message: str = "Invalid input") -> None:
         super().__init__()
+        self.message = message
 
     def read(self, input: Any, explore: ExploreCallable) -> Iterator[MoleculeEntry]:
         yield MoleculeEntry(
             raw_input=input,
             input_type="unknown",
-            source=("input",),
+            source=("raw_input",),
             mol=None,
-            errors=[Problem("invalid_input", "Invalid input")],
+            errors=[Problem("invalid_input", self.message)],
         )
 
     def __repr__(self) -> str:
@@ -120,6 +121,8 @@ class DepthFirstExplorer(Explorer):
             if best_mode == "builtin":
                 parent["first_guess"].append(best_reader)
 
+        # In order to get more fine-grained error messages, we do not handle exceptions here and
+        # rely on the readers to do so.
         yield from sample
         yield from generator
 

{nerdd_module-0.3.49 → nerdd_module-0.3.51}/nerdd_module/input/gzip_reader.py

@@ -1,6 +1,7 @@
 import gzip
 from typing import Any, Iterator
 
+from ..problem import Problem
 from .reader import ExploreCallable, MoleculeEntry, Reader
 
 __all__ = ["GzipReader"]
@@ -22,7 +23,16 @@ class GzipReader(Reader):
             f.read(1)
             f.seek(0)
 
-            yield from explore(f)
+            try:
+                yield from explore(f)
+            except Exception as e:
+                yield MoleculeEntry(
+                    raw_input="<gzip>",
+                    input_type="gzip",
+                    source=("raw_input",),
+                    mol=None,
+                    errors=[Problem("invalid_input", f"Invalid gzip file: {e}")],
+                )
 
     def __repr__(self) -> str:
         return "GzipReader()"

{nerdd_module-0.3.49 → nerdd_module-0.3.51}/nerdd_module/input/list_reader.py

@@ -1,6 +1,7 @@
 from io import IOBase
 from typing import Any, Iterable, Iterator
 
+from ..problem import Problem
 from .reader import ExploreCallable, MoleculeEntry, Reader
 
 __all__ = ["ListReader"]
@@ -16,7 +17,19 @@ class ListReader(Reader):
         ), f"input must be an iterable, but is {type(input_iterable)}"
 
         for entry in input_iterable:
-            yield from explore(entry)
+            try:
+                yield from explore(entry)
+            except Exception as e:
+                raw_input = str(entry)
+                if len(raw_input) > 100:
+                    raw_input = raw_input[:97] + "..."
+                yield MoleculeEntry(
+                    raw_input=raw_input,
+                    input_type="unknown",
+                    source=(),
+                    mol=None,
+                    errors=[Problem("invalid_list_entry", f"Could not read list entry: {e}")],
+                )
 
     def __repr__(self) -> str:
         return "ListReader()"

{nerdd_module-0.3.49 → nerdd_module-0.3.51}/nerdd_module/input/sdf_reader.py

@@ -11,7 +11,7 @@ __all__ = ["SdfReader"]
 
 
 class SdfReader(StreamReader):
-    def __init__(self, max_num_lines_mol_block: int = 10_000) -> None:
+    def __init__(self, max_num_lines_mol_block: int = 100_000) -> None:
         super().__init__()
         self.max_num_lines_mol_block = max_num_lines_mol_block
 

nerdd_module-0.3.51/nerdd_module/input/tar_reader.py

@@ -0,0 +1,50 @@
+import tarfile
+from typing import Any, Iterator, Tuple
+
+from ..problem import Problem
+from .reader import ExploreCallable, MoleculeEntry, Reader
+
+__all__ = ["TarReader"]
+
+
+class TarReader(Reader):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def read(self, input_stream: Any, explore: ExploreCallable) -> Iterator[MoleculeEntry]:
+        if not hasattr(input_stream, "read") or not hasattr(input_stream, "seek"):
+            raise TypeError("input must be a stream-like object")
+
+        input_stream.seek(0)
+
+        with tarfile.open(fileobj=input_stream, mode="r") as tar:
+            for member in tar.getmembers():
+                if not member.isfile():
+                    continue
+
+                try:
+                    for entry in explore(tar.extractfile(member)):
+                        # the underlying reader only sees the file content as a stream
+                        # -> it might believe that the source is "raw_input"
+                        # -> we need to correct that here
+                        if len(entry.source) == 1 and entry.source[0] == "raw_input":
+                            source: Tuple[str, ...] = tuple()
+                        else:
+                            source = entry.source
+                        yield entry._replace(source=(member.name, *source))
+                except Exception as e:
+                    yield MoleculeEntry(
+                        raw_input="<tar>",
+                        input_type="unknown",
+                        source=(member.name,),
+                        mol=None,
+                        errors=[
+                            Problem(
+                                "invalid_tar_member",
+                                f"Could not read tar member '{member.name}': {e}",
+                            )
+                        ],
+                    )
+
+    def __repr__(self) -> str:
+        return "TarReader()"

nerdd_module-0.3.51/nerdd_module/input/zip_reader.py

@@ -0,0 +1,52 @@
+import zipfile
+from typing import Any, Iterator, Tuple
+
+from ..problem import Problem
+from .reader import ExploreCallable, MoleculeEntry, Reader
+
+__all__ = ["ZipReader"]
+
+
+class ZipReader(Reader):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def read(self, input_stream: Any, explore: ExploreCallable) -> Iterator[MoleculeEntry]:
+        if not hasattr(input_stream, "read") or not hasattr(input_stream, "seek"):
+            raise TypeError("input must be a stream-like object")
+
+        input_stream.seek(0)
+
+        with zipfile.ZipFile(input_stream, "r") as zipf:
+            for member in zipf.namelist():
+                # check if the member is a file
+                if member.endswith("/"):
+                    continue
+
+                try:
+                    with zipf.open(member, "r") as f:
+                        for entry in explore(f):
+                            # the underlying reader only sees the file content as a stream
+                            # -> it might believe that the source is "raw_input"
+                            # -> we need to correct that here
+                            if len(entry.source) == 1 and entry.source[0] == "raw_input":
+                                source: Tuple[str, ...] = tuple()
+                            else:
+                                source = entry.source
+
+                            yield entry._replace(source=(member, *source))
+                except Exception as e:
+                    yield MoleculeEntry(
+                        raw_input="<zip>",
+                        input_type="unknown",
+                        source=(member,),
+                        mol=None,
+                        errors=[
+                            Problem(
+                                "invalid_zip_member", f"Could not read zip member '{member}': {e}"
+                            )
+                        ],
+                    )
+
+    def __repr__(self) -> str:
+        return "ZipReader()"

nerdd_module-0.3.51/nerdd_module/preprocessing/__init__.py

@@ -0,0 +1,81 @@
+"""
+Molecular preprocessing pipeline components.
+
+This package provides a comprehensive set of preprocessing steps for molecular data processing
+pipelines. These steps can be chained together to clean, standardize, and validate molecular
+datasets commonly used in cheminformatics and drug discovery.
+
+The preprocessing steps inherit from the base `PreprocessingStep` class and can be easily combined
+to create custom preprocessing pipelines. Each step operates on molecular records and can transform
+molecules, report problems, or filter out invalid structures.
+
+Available Preprocessing Steps
+-----------------------------
+
+- `CheckValidSmiles` : Validates molecules through SMILES round-trip conversion
+- `Sanitize` : Validates and corrects molecular structures using RDKit sanitization
+- `FilterByWeight` : Filters molecules based on molecular weight thresholds
+- `FilterByElement` : Filters molecules based on allowed elemental composition
+- `StandardizeWithCsp` : Standardizes molecules using ChEMBL Structure Pipeline
+- `GetParentMolWithCsp` : Extracts parent molecules using ChEMBL Structure Pipeline
+- `RemoveHydrogens` : Removes hydrogen atoms from molecular representations
+- `RemoveSmallFragments` : Removes small fragments, keeping only the largest component
+- `RemoveStereochemistry` : Removes stereochemical information from molecules
+
+Base Classes
+------------
+
+- `PreprocessingStep` : Abstract base class for all preprocessing steps
+
+Examples
+--------
+
+Basic usage of individual preprocessing steps:
+
+>>> from nerdd_module.preprocessing import FilterByWeight, RemoveHydrogens, Sanitize
+>>>
+>>> # Create preprocessing steps
+>>> weight_filter = FilterByWeight(min_weight=150, max_weight=500)
+>>> hydrogen_remover = RemoveHydrogens()
+>>> sanitizer = Sanitize()
+
+Creating a complete preprocessing pipeline:
+
+>>> from nerdd_module.preprocessing import (
+...     CheckValidSmiles, FilterByElement, RemoveSmallFragments,
+...     Sanitize, StandardizeWithCsp, ORGANIC_SUBSET
+... )
+>>>
+>>> # Define a comprehensive preprocessing pipeline
+>>> pipeline_steps = [
+...     Sanitize(),                       # Sanitize molecules
+...     CheckValidSmiles(),               # Validate SMILES representation
+...     RemoveSmallFragments(),           # Remove salts and solvents
+...     FilterByElement(ORGANIC_SUBSET),  # Keep only organic molecules
+...     StandardizeWithCsp(),             # Standardize using chembl_structure_pipeline
+... ]
+
+
+Notes
+-----
+* All preprocessing steps follow the same interface defined by `PreprocessingStep`
+* Steps can be chained together to create comprehensive preprocessing pipelines
+* Problems encountered during preprocessing are accumulated in the record's "problems" list
+* Some steps require optional dependencies (e.g., `chembl_structure_pipeline`)
+* The order of preprocessing steps can significantly impact the final results
+
+See Also
+--------
+nerdd_module.steps : Base classes for pipeline steps nerdd_module.problem : Problem reporting
+classes used by preprocessing steps
+"""
+
+from .check_valid_smiles import *
+from .chembl_structure_pipeline import *
+from .filter_by_element import *
+from .filter_by_weight import *
+from .preprocessing_step import *
+from .remove_hydrogens import *
+from .remove_small_fragments import *
+from .remove_stereochemistry import *
+from .sanitize import *

nerdd_module-0.3.51/nerdd_module/preprocessing/check_valid_smiles.py

@@ -0,0 +1,74 @@
+"""
+SMILES validation preprocessing step for molecular data.
+
+This module provides functionality to validate molecular representations by converting them to
+SMILES format and attempting to parse them back. This round-trip validation ensures that molecules
+can be properly serialized and deserialized as SMILES strings.
+"""
+
+from typing import List, Optional, Tuple
+
+from rdkit.Chem import Mol, MolFromSmiles, MolToSmiles
+
+from ..problem import InvalidSmiles, Problem
+from .preprocessing_step import PreprocessingStep
+
+__all__ = ["CheckValidSmiles"]
+
+
+class CheckValidSmiles(PreprocessingStep):
+    """
+    Preprocessing step that validates molecules through SMILES round-trip conversion.
+
+    This class validates molecular representations by converting them to SMILES format and then
+    attempting to parse the SMILES back to a molecule object. This round-trip validation ensures
+    that molecules can be properly represented as SMILES strings, which is an indicator for a valid
+    molecular structure. Molecules that fail the round-trip test are considered invalid and removed.
+
+    Parameters
+    ----------
+    None
+
+    Examples
+    --------
+    >>> # Create a SMILES validation step
+    >>> smiles_check = CheckValidSmiles()
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def _preprocess(self, mol: Mol) -> Tuple[Optional[Mol], List[Problem]]:
+        """
+        Validate a molecule through SMILES round-trip conversion.
+
+        Converts the input molecule to a canonical SMILES string and then attempts to parse it back
+        to a molecule object. If the round-trip conversion fails, the molecule is considered
+        invalid.
+
+        Parameters
+        ----------
+        mol : Mol
+            RDKit Mol object representing the molecule to be validated.
+
+        Returns
+        -------
+        Tuple[Optional[Mol], List[Problem]]
+            A tuple containing:
+            * The original molecule if SMILES validation succeeded, or None if validation failed
+            * An empty list if validation succeeded, or a list containing an InvalidSmiles problem
+              if validation failed
+
+        Notes
+        -----
+        The validation process converts the molecule to canonical SMILES.
+        """
+        problems = []
+
+        smi = MolToSmiles(mol, True)
+        check_mol = MolFromSmiles(smi)
+        if check_mol is None:
+            problems.append(InvalidSmiles())
+            mol = None
+
+        return mol, problems

nerdd_module-0.3.51/nerdd_module/preprocessing/chembl_structure_pipeline.py

@@ -0,0 +1,183 @@
+"""
+ChEMBL Structure Pipeline preprocessing steps for molecular data.
+
+This module provides preprocessing steps that utilize the ChEMBL Structure Pipeline library for
+molecule standardization and parent molecule extraction.
+"""
+
+import warnings
+from typing import List, Optional, Tuple
+
+from rdkit.Chem import Mol
+
+from ..polyfills import BlockLogs
+from ..problem import Problem
+from .preprocessing_step import PreprocessingStep
+
+# before importing chembl_structure_pipeline, we need to suppress RDKit warnings
+warnings.filterwarnings(
+    "ignore",
+    category=DeprecationWarning,
+    module="rdkit.Chem.MolStandardize",
+)
+
+# We check if chembl_structure_pipeline is installed. Since importing this library already logs
+# messages, we suppress them using RDKit's BlockLogs.
+with BlockLogs():
+    try:
+        from chembl_structure_pipeline import get_parent_mol, standardize_mol
+
+        import_error = None
+    except ImportError as e:
+        # raise ImportError later when using this class
+        # --> this allows to use the rest of the package without chembl_structure_pipeline
+        import_error = e
+
+__all__ = ["GetParentMolWithCsp", "StandardizeWithCsp"]
+
+
+class StandardizeWithCsp(PreprocessingStep):
+    """
+    Preprocessing step that standardizes molecules using ChEMBL Structure Pipeline.
+
+    This class applies the ChEMBL Structure Pipeline standardization procedures to normalize
+    molecular representations. The standardization includes tautomer normalization, charge
+    neutralization, and other structural standardizations commonly used in pharmaceutical databases.
+
+    Parameters
+    ----------
+    None
+
+    Raises
+    ------
+    ImportError
+        If the chembl_structure_pipeline library is not installed.
+
+    Examples
+    --------
+    >>> # Create a standardization step (requires chembl_structure_pipeline)
+    >>> standardize_step = StandardizeWithCsp()
+
+    Notes
+    -----
+    * Requires the chembl_structure_pipeline library to be installed
+    * Automatically removes 3D conformers as the pipeline cannot handle them
+    * Uses ChEMBL's standardize_mol function which applies comprehensive molecular standardization
+      procedures
+    * If standardization fails, the original molecule is returned with a problem
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+
+        if import_error is not None:
+            raise import_error
+
+    def _preprocess(self, mol: Mol) -> Tuple[Optional[Mol], List[Problem]]:
+        """
+        Standardize a molecule using ChEMBL Structure Pipeline.
+
+        Applies ChEMBL's standardization procedures to normalize the molecular representation. The
+        process removes 3D conformers before applying the standardize_mol function.
+
+        Parameters
+        ----------
+        mol : Mol
+            RDKit Mol object representing the molecule to be standardized.
+
+        Returns
+        -------
+        Tuple[Optional[Mol], List[Problem]]
+            A tuple containing:
+            * The standardized molecule if successful, or the original molecule if standardization
+              failed
+            * An empty list if standardization succeeded, or a list containing a Problem instance
+              with code "csp_error" if standardization failed
+        """
+        problems: List[Problem] = []
+
+        # chembl structure pipeline cannot handle molecules with 3D coordinates
+        # --> delete conformers
+        mol.RemoveAllConformers()
+
+        # standardization via chembl structure pipeline
+        preprocessed_mol = standardize_mol(mol)
+
+        if preprocessed_mol is None:
+            problems.append(Problem("csp_error", "Could not standardize the molecule."))
+            preprocessed_mol = mol
+
+        return preprocessed_mol, problems
+
+
+class GetParentMolWithCsp(PreprocessingStep):
+    """
+    Preprocessing step that extracts parent molecules using ChEMBL Structure Pipeline.
+
+    This class uses the ChEMBL Structure Pipeline to identify and extract the parent molecule from
+    complex molecular structures. This process removes salts, solvents, and other fragments while
+    applying ChEMBL's standardization rules.
+
+    Parameters
+    ----------
+    None
+
+    Raises
+    ------
+    ImportError
+        If the chembl_structure_pipeline library is not installed.
+
+    Examples
+    --------
+    >>> # Create a parent molecule extraction step
+    >>> get_parent_step = GetParentMolWithCsp()
+
+    Notes
+    -----
+    * Requires the chembl_structure_pipeline library to be installed
+    * Automatically removes 3D conformers as the pipeline cannot handle them
+    * Applies the get_parent_mol function from the chembl_structure_pipeline library
+    * If parent extraction fails or is flagged for exclusion, the original molecule is returned with
+      a Problem instance
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+
+        if import_error is not None:
+            raise import_error
+
+    def _preprocess(self, mol: Mol) -> Tuple[Optional[Mol], List[Problem]]:
+        """
+        Extract the parent molecule using ChEMBL Structure Pipeline.
+
+        Identifies and returns the main molecular component. The process removes 3D conformers,
+        because chembl_structure_pipeline cannot handle them.
+
+        Parameters
+        ----------
+        mol : Mol
+            RDKit Mol object representing the molecule from which to extract the parent structure.
+
+        Returns
+        -------
+        Tuple[Optional[Mol], List[Problem]]
+            A tuple containing:
+            * The parent molecule if successful, or the original molecule if extraction failed
+            * An empty list if extraction succeeded, or a list containing a Problem instance with
+              code "csp_error" if extraction failed or was flagged for exclusion
+        """
+        problems = []
+
+        # chembl structure pipeline cannot handle molecules with 3D coordinates
+        # --> delete conformers
+        mol.RemoveAllConformers()
+
+        # get parent molecule via chembl structure pipeline
+        preprocessed_mol, exclude_flag = get_parent_mol(mol)
+        if exclude_flag or preprocessed_mol is None:
+            problems.append(Problem("csp_error", "Could not remove small fragments."))
+        if preprocessed_mol is None:
+            preprocessed_mol = mol
+
+        return preprocessed_mol, problems