nerdd-module 0.1.12__tar.gz → 0.2.0__tar.gz

nerdd-module-0.2.0/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.1
+Name: nerdd-module
+Version: 0.2.0
+Summary: Base package to create NERDD modules
+Home-page: https://github.com/molinfo-vienna/nerdd-module.git
+Maintainer: Steffen Hirte
+Maintainer-email: steffen.hirte@univie.ac.at
+License: BSD 3-Clause License
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: C
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development
+Classifier: Topic :: Scientific/Engineering
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: Unix
+Classifier: Operating System :: MacOS
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: rdkit>=2022.3.3
+Requires-Dist: pandas>=1.2.1
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: filetype~=1.2.0
+Requires-Dist: rich-click>=1.7.1
+Requires-Dist: stringcase>=1.2.0
+Requires-Dist: decorator>=5.1.1
+Requires-Dist: importlib-resources>=5; python_version < "3.10"
+Requires-Dist: importlib-metadata>=4.6; python_version < "3.10"
+Requires-Dist: chembl_structure_pipeline>=1.0.0
+Provides-Extra: dev
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-sugar; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-asyncio; extra == "test"
+Requires-Dist: pytest-bdd; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-watch; extra == "test"
+Requires-Dist: hypothesis; extra == "test"
+Requires-Dist: hypothesis-rdkit; extra == "test"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: mkdocstrings; extra == "docs"
+
+# NERDD Module
+
+This package provides the basis to implement molecular prediction modules in the
+NERDD ecosystem.
+
+## Installation
+
+```bash
+pip install -U nerdd-module
+```
+
+
+## Contribute
+
+1. Fork and clone the code
+2. Install test dependencies with ```pip install -e .[test]```
+3. Run tests via ```pytest``` or ```pytest-watch``` (short: ```ptw```)
+4. Build docs via ```pip install -e .[docs]``` and ```mkdocs serve```

nerdd-module-0.2.0/README.md

@@ -0,0 +1,18 @@
+# NERDD Module
+
+This package provides the basis to implement molecular prediction modules in the
+NERDD ecosystem.
+
+## Installation
+
+```bash
+pip install -U nerdd-module
+```
+
+
+## Contribute
+
+1. Fork and clone the code
+2. Install test dependencies with ```pip install -e .[test]```
+3. Run tests via ```pytest``` or ```pytest-watch``` (short: ```ptw```)
+4. Build docs via ```pip install -e .[docs]``` and ```mkdocs serve```

{nerdd-module-0.1.12 → nerdd-module-0.2.0}/nerdd_module/__init__.py

@@ -1,6 +1,7 @@
 from .abstract_model import *
 from .cli import *
 from .config import *
+from .problem import *
 from .version import *
 
 # import entry_points from importlib.metadata or fall back to pkg_resources
@@ -9,11 +10,13 @@ try:
 
     def get_entry_points(group):
         return entry_points().get(group, [])
+
 except ImportError:
     import pkg_resources
-    
+
     def get_entry_points(group):
         return pkg_resources.iter_entry_points(group)
 
+
 for entry_point in get_entry_points("nerdd-module.plugins"):
     entry_point.load()

{nerdd-module-0.1.12 → nerdd-module-0.2.0}/nerdd_module/abstract_model.py

@@ -5,18 +5,19 @@ import pandas as pd
 from rdkit.Chem import Mol, MolToSmiles
 
 from .config import AutoConfiguration, Configuration
-from .io import MoleculeEntry, guess_and_read
+from .io import DepthFirstExplorer, MoleculeEntry
 from .preprocessing import Pipeline, Step, registry
+from .problem import Problem
 
 __all__ = ["AbstractModel"]
 
 
 class CustomPreprocessingStep(Step):
-    def __init__(self, fn: Callable[[Mol], Tuple[Mol, List[str]]]):
+    def __init__(self, fn: Callable[[Mol], Tuple[Mol, List[Problem]]]):
         super().__init__()
         self.fn = fn
 
-    def _run(self, mol: Mol) -> Tuple[Mol, List[str]]:
+    def _run(self, mol: Mol) -> Tuple[Mol, List[Problem]]:
         return self.fn(mol)
 
 
@@ -69,7 +70,7 @@ class AbstractModel(ABC):
         #
         self.num_processes = num_processes
 
-    def _preprocess_single_mol(self, mol: Mol) -> Tuple[Mol, List[str]]:
+    def _preprocess_single_mol(self, mol: Mol) -> Tuple[Mol, List[Problem]]:
         # if this method is called, the preprocessing_pipeline was set to "custom"
         # and this method has to be overwritten
         raise NotImplementedError()
@@ -117,13 +118,6 @@ class AbstractModel(ABC):
             for mol in df_preprocess.input_mol
         ]
 
-        # add smiles columns for web UI
-        def _to_smiles(mol):
-            try:
-                return MolToSmiles(mol)
-            except:
-                return None
-
         #
         # PREPARE PREDICTION OF MOLECULES
         #
@@ -223,10 +217,8 @@ class AbstractModel(ABC):
         df_result.drop(columns=["missing", "preprocessing_errors"], inplace=True)
 
         # convert errors to string
-        if "errors" in df_result.columns:
-            df_result["errors"] = df_result.errors.map(lambda x: ", ".join(set(x)))
-        else:
-            df_result["errors"] = ""
+        if "errors" not in df_result.columns:
+            df_result["errors"] = []
 
         # delete mol column (not needed anymore)
         df_load.drop(columns=["mol"], inplace=True)
@@ -236,7 +228,7 @@ class AbstractModel(ABC):
 
         # merge errors from loading and prediction
         df_result["errors"] = [
-            ", ".join(set(load_errors + [prediction_errors]))
+            load_errors + prediction_errors
             for load_errors, prediction_errors in zip(
                 df_result.load_errors, df_result.errors
             )
@@ -266,7 +258,7 @@ class AbstractModel(ABC):
         input_type=None,
         **kwargs,
     ):
-        entries = guess_and_read(inputs)
+        entries = DepthFirstExplorer().explore(inputs)
 
         return self._predict_entries(entries, **kwargs)
 

{nerdd-module-0.1.12 → nerdd-module-0.2.0}/nerdd_module/cli.py

@@ -62,7 +62,7 @@ def auto_cli(f, *args, **kwargs):
     if len(examples) > 0:
         footer = "Examples:\n"
         for example in examples:
-            footer += f"* {command_name} {example}\n"
+            footer += f'* {command_name} "{example}"\n'
     else:
         footer = ""
 

{nerdd-module-0.1.12 → nerdd-module-0.2.0}/nerdd_module/config/default_configuration.py

@@ -7,9 +7,11 @@ class DefaultConfiguration(Configuration):
     def __init__(self, nerdd_module):
         super().__init__()
 
-        # we do not use default values at the moment
-        # feel free to add values here if needed
-        self.config = {}
+        self.config = dict(
+            task="molecular_property_prediction",
+            job_parameters=[],
+            result_properties=[],
+        )
 
     def _get_dict(self):
         return self.config

{nerdd-module-0.1.12 → nerdd-module-0.2.0}/nerdd_module/io/__init__.py

@@ -1,17 +1,17 @@
 from .csv_writer import *
-from .elementary_reader import *
-from .guess_and_read import *
+from .depth_first_explorer import *
+from .file_reader import *
+from .gzip_reader import *
 from .inchi_reader import *
-from .elementary_inchi_reader import *
 from .list_reader import *
-from .elementary_mol_block_reader import *
-from .elementary_rdkit_mol_reader import *
+from .mol_reader import *
 from .reader import *
 from .reader_registry import *
 from .sdf_reader import *
 from .sdf_writer import *
 from .smiles_reader import *
-from .elementary_smiles_reader import *
-from .splitting_reader import *
+from .string_reader import *
+from .tar_reader import *
 from .writer import *
 from .writer_registry import *
+from .zip_reader import *

{nerdd-module-0.1.12 → nerdd-module-0.2.0}/nerdd_module/io/csv_writer.py

@@ -1,7 +1,6 @@
 import csv
-from io import TextIOWrapper
 from itertools import chain
-from typing import BinaryIO, Dict, Iterable, TextIO, Union
+from typing import Dict, Iterable
 
 from rdkit.Chem import Mol, MolToSmiles
 

nerdd-module-0.2.0/nerdd_module/io/depth_first_explorer.py

@@ -0,0 +1,111 @@
+from itertools import chain, islice, repeat
+from typing import Generator, Iterable, Optional
+
+from .explorer import Explorer
+from .reader import MoleculeEntry, Problem, Reader
+from .reader_registry import ReaderRegistry
+
+__all__ = ["DepthFirstExplorer"]
+
+
+class InvalidInputReader(Reader):
+    def __init__(self):
+        super().__init__()
+
+    def read(self, input, explore) -> Generator[MoleculeEntry, None, None]:
+        yield MoleculeEntry(
+            raw_input=input,
+            input_type="unknown",
+            source=tuple(["input"]),
+            mol=None,
+            errors=[Problem("invalid_input", "Invalid input")],
+        )
+
+    def __repr__(self) -> str:
+        return "InvalidInputReader()"
+
+
+class DepthFirstExplorer(Explorer):
+    def __init__(
+        self,
+        readers: Optional[Iterable[Reader]] = None,
+        num_test_entries: int = 10,
+        threshold: float = 0.5,
+        maximum_depth: int = 50,
+    ):
+        super().__init__()
+
+        if readers is None:
+            self.reader_registry = ReaderRegistry()
+        else:
+            self.reader_registry = None
+
+        self.num_test_entries = num_test_entries
+        self.threshold = threshold
+        self.state_stack = [self.empty_state()]
+        self.maximum_depth = maximum_depth
+
+    def empty_state(self):
+        return dict(first_guess=[])
+
+    def explore(self, input) -> Generator[MoleculeEntry, None, None]:
+        # create a new child node and set it as the current node
+        state = self.empty_state()
+        parent = self.state_stack[-1]
+        self.state_stack.append(state)
+
+        depth = len(self.state_stack)
+        if depth > self.maximum_depth:
+            raise ValueError(f"Maximum depth of {self.maximum_depth} reached")
+
+        readers_iter = chain(
+            zip(parent["first_guess"], repeat("guess")),
+            zip(self.reader_registry, repeat("builtin")),
+        )
+
+        # try all readers and take a sample of the first num_test_entries
+        # the reader with most valid molecule entries will be used
+        best_reader: Optional[Reader] = None
+        best_mode = None
+        best_score = 0
+        best_ratio = 0.0
+        generator = None
+        sample = []
+        for reader, mode in readers_iter:
+            try:
+                # read at most num_test_entries entries
+                generator = reader.read(input, self.explore)
+                sample = list(islice(generator, self.num_test_entries))
+                valid_entries = [entry for entry in sample if entry.mol is not None]
+
+                score = len(valid_entries)
+                ratio = len(valid_entries) / len(sample)
+
+                if score > best_score or (score == best_score and ratio > best_ratio):
+                    best_reader = reader
+                    best_mode = mode
+                    best_score = score
+                    best_ratio = ratio
+
+                    if score == self.num_test_entries:
+                        break
+            except Exception:
+                pass
+
+            # clean up tree
+            while len(self.state_stack) > depth:
+                self.state_stack.pop()
+            generator = None
+
+        if generator is None:
+            if best_reader is None:
+                generator = InvalidInputReader().read(input, self.explore)
+            else:
+                generator = best_reader.read(input, self.explore)
+                sample = list(islice(generator, self.num_test_entries))
+        else:
+            if best_mode is not None and best_mode != "guess":
+                parent["first_guess"].append(best_reader)
+
+        yield from sample
+        yield from generator

nerdd-module-0.2.0/nerdd_module/io/explorer.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from typing import Generator
+
+from .reader import MoleculeEntry
+
+
+class Explorer(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def explore(self, input) -> Generator[MoleculeEntry, None, None]:
+        pass

nerdd-module-0.2.0/nerdd_module/io/file_reader.py

@@ -0,0 +1,28 @@
+import os
+from typing import Generator
+
+from .reader import MoleculeEntry, Reader
+from .reader_registry import register_reader
+
+__all__ = ["FileReader"]
+
+
+@register_reader
+class FileReader(Reader):
+    def __init__(self):
+        super().__init__()
+
+    def read(self, filename, explore) -> Generator[MoleculeEntry, None, None]:
+        if not isinstance(filename, str) or not os.path.exists(filename):
+            raise TypeError("input must be a valid filename")
+
+        with open(filename, "rb") as f:
+            for entry in explore(f):
+                if len(entry.source) == 1 and entry.source[0] == "raw_input":
+                    source = tuple()
+                else:
+                    source = entry.source
+                yield entry._replace(source=tuple([filename, *source]))
+
+    def __repr__(self):
+        return f"FileReader()"

nerdd-module-0.2.0/nerdd_module/io/gzip_reader.py

@@ -0,0 +1,30 @@
+import gzip
+from typing import Generator
+
+from .reader import MoleculeEntry, Reader
+from .reader_registry import register_reader
+
+__all__ = ["GzipReader"]
+
+
+@register_reader
+class GzipReader(Reader):
+    def __init__(self):
+        super().__init__()
+
+    def read(self, input_stream, explore) -> Generator[MoleculeEntry, None, None]:
+        if not hasattr(input_stream, "read") or not hasattr(input_stream, "seek"):
+            raise TypeError("input must be a stream-like object")
+
+        input_stream.seek(0)
+
+        with gzip.open(input_stream, "rb") as f:
+            # gzip.open will not raise an exception if the file is not a valid gzip file
+            # --> check by attempting to read the first byte
+            f.read(1)
+            f.seek(0)
+
+            yield from explore(f)
+
+    def __repr__(self) -> str:
+        return "GzipReader()"

nerdd-module-0.2.0/nerdd_module/io/inchi_reader.py

@@ -0,0 +1,59 @@
+from codecs import getreader
+from typing import Generator
+
+from rdkit.Chem import MolFromInchi
+from rdkit.rdBase import BlockLogs
+
+from ..problem import Problem
+from .reader import MoleculeEntry, Reader
+from .reader_registry import register_reader
+
+__all__ = ["InchiReader"]
+
+StreamReader = getreader("utf-8")
+
+
+@register_reader
+class InchiReader(Reader):
+    def __init__(self):
+        super().__init__()
+
+    def read(self, input_stream, explore) -> Generator[MoleculeEntry, None, None]:
+        if not hasattr(input_stream, "read") or not hasattr(input_stream, "seek"):
+            raise TypeError("input must be a stream-like object")
+
+        input_stream.seek(0)
+
+        reader = StreamReader(input_stream)
+
+        # suppress RDKit warnings
+        with BlockLogs():
+            for line in reader:
+                # skip empty lines
+                if line.strip() == "":
+                    continue
+
+                # skip comments
+                if line.strip().startswith("#"):
+                    continue
+
+                try:
+                    mol = MolFromInchi(line, sanitize=False)
+                except:
+                    mol = None
+
+                if mol is None:
+                    errors = [Problem("invalid_inchi", "Invalid InChI")]
+                else:
+                    errors = []
+
+                yield MoleculeEntry(
+                    raw_input=line,
+                    input_type="inchi",
+                    source=tuple(["raw_input"]),
+                    mol=mol,
+                    errors=errors,
+                )
+
+    def __repr__(self) -> str:
+        return "InchiReader()"

nerdd-module-0.2.0/nerdd_module/io/list_reader.py

@@ -0,0 +1,24 @@
+from io import BytesIO, StringIO
+from typing import BinaryIO, Generator, Iterable
+
+from .reader import MoleculeEntry, Reader
+from .reader_registry import register_reader
+
+__all__ = ["ListReader"]
+
+
+@register_reader
+class ListReader(Reader):
+    def __init__(self):
+        super().__init__()
+
+    def read(self, input_iterable, explore) -> Generator[MoleculeEntry, None, None]:
+        assert isinstance(input_iterable, Iterable) and not isinstance(
+            input_iterable, (str, bytes, BytesIO, StringIO, BinaryIO)
+        ), f"input must be an iterable, but is {type(input_iterable)}"
+
+        for entry in input_iterable:
+            yield from explore(entry)
+
+    def __repr__(self) -> str:
+        return "ListReader()"

nerdd-module-0.2.0/nerdd_module/io/mol_reader.py

@@ -0,0 +1,25 @@
+from typing import Generator
+
+from rdkit.Chem import Mol
+
+from .reader import MoleculeEntry, Reader
+from .reader_registry import register_reader
+
+
+@register_reader
+class MolReader(Reader):
+    def __init__(self):
+        super().__init__()
+
+    def read(self, mol, explore) -> Generator[MoleculeEntry, None, None]:
+        assert isinstance(mol, Mol)
+        yield MoleculeEntry(
+            raw_input=mol,
+            input_type="rdkit_mol",
+            source=tuple(["raw_input"]),
+            mol=mol,
+            errors=[],
+        )
+
+    def __repr__(self) -> str:
+        return "MolReader()"

nerdd-module-0.2.0/nerdd_module/io/reader.py

@@ -0,0 +1,25 @@
+from abc import ABC, abstractmethod
+from typing import Generator, List, NamedTuple, Optional, Tuple
+
+from rdkit.Chem import Mol
+
+from ..problem import Problem
+
+__all__ = ["MoleculeEntry", "Reader"]
+
+
+class MoleculeEntry(NamedTuple):
+    raw_input: str
+    input_type: str
+    source: Tuple[str, ...]
+    mol: Optional[Mol]
+    errors: List[Problem]
+
+
+class Reader(ABC):
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def read(self, input, explore) -> Generator[MoleculeEntry, None, None]:
+        pass

nerdd-module-0.2.0/nerdd_module/io/reader_registry.py

@@ -0,0 +1,30 @@
+from functools import lru_cache
+from typing import Generator, Type
+
+from .reader import Reader
+
+__all__ = ["ReaderRegistry", "register_reader"]
+
+
+# lru_cache makes the registry a singleton
+@lru_cache(maxsize=1)
+class ReaderRegistry:
+    def __init__(self):
+        self._factories = []
+
+    def register(self, ReaderClass: Type[Reader], *args, **kwargs):
+        assert issubclass(ReaderClass, Reader)
+        self._factories.append(lambda: ReaderClass(*args, **kwargs))
+
+    def readers(self) -> Generator[Reader, None, None]:
+        for reader in self._factories:
+            yield reader()
+
+    def __iter__(self):
+        return iter(map(lambda f: f(), self._factories))
+
+
+def register_reader(clazz, *args, **kwargs):
+    # TODO: implement both decorator modes
+    ReaderRegistry().register(clazz, *args, **kwargs)
+    return clazz

nerdd-module-0.2.0/nerdd_module/io/sdf_reader.py

@@ -0,0 +1,81 @@
+from codecs import getreader
+from typing import Generator
+
+from rdkit.Chem import MolFromMolBlock
+from rdkit.rdBase import BlockLogs
+
+from ..problem import Problem
+from .reader import MoleculeEntry, Reader
+from .reader_registry import register_reader
+
+__all__ = ["SdfReader"]
+
+StreamReader = getreader("utf-8")
+
+
+@register_reader
+class SdfReader(Reader):
+    def __init__(self, max_num_lines_mol_block: int = 10000):
+        super().__init__()
+        self.max_num_lines_mol_block = max_num_lines_mol_block
+
+    def read(self, input_stream, explore) -> Generator[MoleculeEntry, None, None]:
+        if not hasattr(input_stream, "read") or not hasattr(input_stream, "seek"):
+            raise TypeError("input must be a stream-like object")
+
+        input_stream.seek(0)
+
+        reader = StreamReader(input_stream)
+
+        # suppress RDKit warnings
+        with BlockLogs():
+
+            # We do not use SDMolSupplier, because it does not accept a stream-like
+            # object as input. The ForwadSDMolSupplier is not suitable either, because
+            # it does not allow to return the raw text.
+            while True:
+                # collect lines to parse as a mol block
+                mol_block = ""
+                num_lines = 0
+                line = reader.readline()
+                while line:
+                    mol_block += line
+                    if line.strip() == "$$$$":
+                        break
+
+                    num_lines += 1
+                    if num_lines > self.max_num_lines_mol_block:
+                        break
+
+                    # read next line
+                    line = reader.readline()
+
+                if mol_block.strip() != "":
+                    try:
+                        mol = MolFromMolBlock(mol_block, sanitize=False, removeHs=False)
+                    except:
+                        mol = None
+
+                    if mol is None:
+                        errors = [Problem("invalid_mol_block", "Invalid mol block")]
+                    else:
+                        errors = []
+
+                    yield MoleculeEntry(
+                        raw_input=mol_block,
+                        input_type="mol_block",
+                        source=tuple(["raw_input"]),
+                        mol=mol,
+                        errors=errors,
+                    )
+
+                # We stop reading if
+                # (1) we have reached the end of the file OR
+                # (2) the last entry had more than MAX_NUM_LINES_MOL_BLOCK lines
+                #     (this entry is probably not a valid mol block and everything after
+                #      it is probably not a valid mol block either)
+                if (not line) or (num_lines > self.max_num_lines_mol_block):
+                    break
+
+    def __repr__(self) -> str:
+        return f"SdfReader(max_num_lines_mol_block={self.max_num_lines_mol_block})"