nerdd-module 0.2.6__tar.gz → 0.3.3__tar.gz

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nerdd-module
-Version: 0.2.6
+Version: 0.3.3
 Summary: Base package to create NERDD modules
 Home-page: https://github.com/molinfo-vienna/nerdd-module
 Maintainer: Steffen Hirte
@@ -35,7 +35,11 @@ Requires-Dist: importlib-resources>=5; python_version < "3.10"
 Requires-Dist: importlib-metadata>=4.6; python_version < "3.10"
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
-Requires-Dist: isort; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: pandas-stubs; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: types-decorator; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
 Provides-Extra: csp
 Requires-Dist: chembl_structure_pipeline>=1.0.0; extra == "csp"
 Provides-Extra: test

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/__init__.py

@@ -1,10 +1,10 @@
-from .abstract_model import *
 from .cli import *
-from .config import *
+from .input import ReaderRegistry
+from .model import *
+from .output import WriterRegistry
+from .polyfills import get_entry_points
 from .problem import *
 from .version import *
-from .polyfills import get_entry_points
-
 
 for entry_point in get_entry_points("nerdd-module.plugins"):
     entry_point.load()

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/cli.py

@@ -4,8 +4,7 @@ import sys
 
 import rich_click as click
 from decorator import decorator
-from nerdd_module.output import WriterRegistry
-from stringcase import spinalcase
+from stringcase import spinalcase  # type: ignore
 
 __all__ = ["auto_cli"]
 
@@ -14,7 +13,7 @@ input_description = """{description}
 INPUT molecules are provided as file paths or strings. The following formats are 
 supported:
 
-{format_list}
+{input_format_list}
 
 Note that input formats shouldn't be mixed.
 """
@@ -43,17 +42,19 @@ def auto_cli(f, *args, **kwargs):
     # get the model
     model = f()
 
-    config = model.get_config().get_dict()
+    config = model.get_config()
 
     # compose cli description
     description = config.get("description", "")
 
-    format_list = "\n".join([f"* {fmt}" for fmt in ["smiles", "sdf", "inchi"]])
+    input_format_list = "\n".join([f"* {fmt}" for fmt in ["smiles", "sdf", "inchi"]])
 
     help_text = input_description.format(
-        description=description, format_list=format_list
+        description=description, input_format_list=input_format_list
     )
 
+    output_format_list = ["sdf", "csv"]
+
     # compose footer with examples
     examples = []
     if "example_smiles" in config:
@@ -88,21 +89,15 @@ def auto_cli(f, *args, **kwargs):
     ):
         logging.basicConfig(level=log_level.upper())
 
-        df_result = model.predict(input, **kwargs)
-
         # write results
-        assert format in WriterRegistry().supported_formats
-        writer = WriterRegistry().get_writer(format)
+        assert format in output_format_list, f"Unknown output format: {format}"
 
-        if output.lower() == "stdout":
-            assert not writer.writes_bytes, "stdout does not support binary output"
+        if str(output).lower() == "stdout":
             output_handle = sys.stdout
         else:
-            mode = "wb" if writer.writes_bytes else "w"
-            output_handle = click.open_file(output, mode)
+            output_handle = click.open_file(str(output), "wb")
 
-        entries = (tup._asdict() for tup in df_result.itertuples(index=False))
-        writer.write(output_handle, entries)
+        model.predict(input, output_format=format, output_file=output_handle, **kwargs)
 
     #
     # Add job parameters
@@ -130,7 +125,7 @@ def auto_cli(f, *args, **kwargs):
     main = click.option(
         "--format",
         default="csv",
-        type=click.Choice(WriterRegistry().supported_formats, case_sensitive=False),
+        type=click.Choice(output_format_list, case_sensitive=False),
         help="The output format.",
     )(main)
 

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/config/__init__.py

@@ -1,7 +1,7 @@
-from .auto_configuration import *
 from .configuration import *
 from .default_configuration import *
 from .dict_configuration import *
 from .merged_configuration import *
 from .package_configuration import *
+from .search_yaml_configuration import *
 from .yaml_configuration import *

nerdd_module-0.3.3/nerdd_module/config/configuration.py

@@ -0,0 +1,71 @@
+from abc import ABC, abstractmethod
+from functools import lru_cache
+from typing import List
+
+__all__ = ["Configuration"]
+
+
+def get_property_columns_of_type(config, t) -> List[dict]:
+    return [c for c in config["result_properties"] if c.get("level", "molecule") == t]
+
+
+class Configuration(ABC):
+    def __init__(self):
+        pass
+
+    @lru_cache
+    def get_dict(self) -> dict:
+        config = self._get_dict()
+
+        if "result_properties" not in config:
+            config["result_properties"] = []
+
+        # check that a module can only predict atom or derivative properties, not both
+        num_atom_properties = len(get_property_columns_of_type(config, "atom"))
+        num_derivative_properties = len(
+            get_property_columns_of_type(config, "derivative")
+        )
+        assert (
+            num_atom_properties == 0 or num_derivative_properties == 0
+        ), "A module can only predict atom or derivative properties, not both."
+
+        return config
+
+    @abstractmethod
+    def _get_dict(self) -> dict:
+        pass
+
+    def is_empty(self) -> bool:
+        return self.get_dict() == {}
+
+    def molecular_property_columns(self) -> List[dict]:
+        return get_property_columns_of_type(self, "molecule")
+
+    def atom_property_columns(self) -> List[dict]:
+        return get_property_columns_of_type(self, "atom")
+
+    def derivative_property_columns(self) -> List[dict]:
+        return get_property_columns_of_type(self, "derivative")
+
+    def get_task(self) -> str:
+        # if task is specified in the config, use that
+        config = self.get_dict()
+        if "task" in config:
+            return config["task"]
+
+        # try to derive the task from the result_properties
+        num_atom_properties = len(self.atom_property_columns())
+        num_derivative_properties = len(self.derivative_property_columns())
+
+        if num_atom_properties > 0:
+            return "atom_property_prediction"
+        elif num_derivative_properties > 0:
+            return "derivative_property_prediction"
+        else:
+            return "molecular_property_prediction"
+
+    def __getitem__(self, key):
+        return self.get_dict()[key]
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self._get_dict()})"

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/config/default_configuration.py

@@ -1,15 +1,13 @@
-from stringcase import snakecase
+from stringcase import snakecase  # type: ignore
 
 from ..polyfills import version
-from .configuration import Configuration
+from .dict_configuration import DictConfiguration
 
 __all__ = ["DefaultConfiguration"]
 
 
-class DefaultConfiguration(Configuration):
+class DefaultConfiguration(DictConfiguration):
     def __init__(self, nerdd_module):
-        super().__init__()
-
         # generate a name from the module name
         class_name = nerdd_module.__class__.__name__
         if class_name.endswith("Model"):
@@ -25,17 +23,15 @@ class DefaultConfiguration(Configuration):
         try:
             module = nerdd_module.__module__
             root_module = module.split(".", 1)[0]
-            version_ = version(root_module)
+            package_version = version(root_module)
         except ModuleNotFoundError:
-            pass
+            package_version = "0.0.1"
 
-        self.config = dict(
+        config = dict(
             name=name,
-            version=version_,
-            task="molecular_property_prediction",
+            version=package_version,
             job_parameters=[],
             result_properties=[],
         )
 
-    def _get_dict(self):
-        return self.config
+        super().__init__(config)

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/config/dict_configuration.py

@@ -4,10 +4,9 @@ __all__ = ["DictConfiguration"]
 
 
 class DictConfiguration(Configuration):
-    def __init__(self, config):
+    def __init__(self, config: dict) -> None:
         super().__init__()
+        self._config = config
 
-        self.config = config
-
-    def _get_dict(self):
-        return self.config
+    def _get_dict(self) -> dict:
+        return self._config

nerdd_module-0.3.3/nerdd_module/config/merged_configuration.py

@@ -0,0 +1,44 @@
+from collections import Counter
+
+from .configuration import Configuration
+from .dict_configuration import DictConfiguration
+
+__all__ = ["MergedConfiguration"]
+
+
+def merge(*args):
+    assert len(args) > 0
+
+    first_entry = args[0]
+    assert all(isinstance(d, type(first_entry)) for d in args)
+
+    if isinstance(first_entry, list):
+        return [e for d in args for e in d]
+    if isinstance(first_entry, dict):
+        count_fields = Counter([k for d in args for k in d.keys()])
+
+        # merge fields that occur in multiple dicts
+        overlapping_fields = [k for k, v in count_fields.items() if v > 1]
+        merged_overlapping_fields = {
+            k: merge(*[d[k] for d in args if k in d]) for k in overlapping_fields
+        }
+
+        # collect fields that occur in only one dict
+        non_overlapping_fields = [k for k, v in count_fields.items() if v == 1]
+        merged_non_overlapping_fields = {
+            k: v for d in args for k, v in d.items() if k in non_overlapping_fields
+        }
+
+        return {
+            **merged_non_overlapping_fields,
+            **merged_overlapping_fields,
+        }
+    else:
+        # merge all configurations starting from the first one
+        # --> last configuration has the highest priority
+        return args[-1]
+
+
+class MergedConfiguration(DictConfiguration):
+    def __init__(self, *configs: Configuration):
+        super().__init__(merge(*[c.get_dict() for c in configs]))

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/config/package_configuration.py

@@ -1,3 +1,5 @@
+import logging
+
 from ..polyfills import files
 from .configuration import Configuration
 from .dict_configuration import DictConfiguration
@@ -5,6 +7,8 @@ from .yaml_configuration import YamlConfiguration
 
 __all__ = ["PackageConfiguration"]
 
+logger = logging.getLogger(__name__)
+
 
 class PackageConfiguration(Configuration):
     def __init__(self, package):
@@ -23,6 +27,7 @@ class PackageConfiguration(Configuration):
             config_file = root_dir / "nerdd.yml"
 
             if config_file is not None and config_file.exists():
+                logger.info(f"Found configuration file in package: {config_file}")
                 self.config = YamlConfiguration(config_file, base_path=root_dir)
             else:
                 self.config = DictConfiguration({})

nerdd_module-0.3.3/nerdd_module/config/search_yaml_configuration.py

@@ -0,0 +1,40 @@
+import logging
+import os
+import sys
+from typing import Any, Optional
+
+from .configuration import Configuration
+from .dict_configuration import DictConfiguration
+from .yaml_configuration import YamlConfiguration
+
+__all__ = ["SearchYamlConfiguration"]
+
+logger = logging.getLogger(__name__)
+
+
+class SearchYamlConfiguration(DictConfiguration):
+    def __init__(self, start: str, base_path: Optional[str] = None) -> None:
+        # provide a default configuration if no configuration file is found
+        config: Configuration = DictConfiguration({})
+
+        if start is not None:
+            # start at the directory containing the file where nerdd_module_class is
+            # defined and go up the directory tree until nerdd.yml is found (or root is
+            # reached)
+            leaf = start
+            while True:
+                if os.path.isfile(os.path.join(leaf, "nerdd.yml")):
+                    default_config_file = os.path.join(leaf, "nerdd.yml")
+                    break
+                elif leaf == os.path.dirname(leaf):  # reached root
+                    default_config_file = None
+                    break
+                leaf = os.path.dirname(leaf)
+
+            if default_config_file is not None:
+                logger.info(
+                    f"Found configuration file in project directory: {default_config_file}"
+                )
+                config = YamlConfiguration(default_config_file, base_path)
+
+        super().__init__(config.get_dict())

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/config/yaml_configuration.py

@@ -1,8 +1,10 @@
 import base64
 import os
 import pathlib
+from typing import Optional, Union
+from pathlib import Path
 
-import filetype
+import filetype  # type: ignore
 import yaml
 
 from .configuration import Configuration
@@ -26,7 +28,9 @@ def image_constructor(loader, node):
 
 
 class YamlConfiguration(Configuration):
-    def __init__(self, handle, base_path=None):
+    def __init__(
+        self, handle: Union[str, Path], base_path: Optional[Union[str, Path]] = None
+    ) -> None:
         super().__init__()
 
         if base_path is None:

nerdd_module-0.3.3/nerdd_module/converters/__init__.py

@@ -0,0 +1,2 @@
+from .converter import *
+from .converter_registry import *

nerdd_module-0.3.3/nerdd_module/converters/converter.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+__all__ = ["Converter"]
+
+
+class Converter(ABC):
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def _convert(self, input: Any, context: dict, **kwargs) -> Any:
+        pass
+
+    def convert(self, input: Any, context: dict, **kwargs) -> Any:
+        return self._convert(input, context, **kwargs)

nerdd_module-0.3.3/nerdd_module/converters/converter_registry.py

@@ -0,0 +1,61 @@
+from functools import lru_cache, partial
+from typing import Callable, Dict, Tuple, Type
+
+from ..util import call_with_mappings, class_decorator
+from .converter import Converter
+from .identity_converter import IdentityConverter
+
+__all__ = [
+    "ConverterRegistry",
+    "register_representation",
+]
+
+
+ConverterFactory = Callable[[dict], Converter]
+
+
+# lru_cache makes the registry a singleton
+@lru_cache(maxsize=1)
+class ConverterRegistry:
+    def __init__(self) -> None:
+        self._factories: Dict[Tuple[str, str], ConverterFactory] = {}
+
+    def register(
+        self,
+        data_type: str,
+        output_format: str,
+        ConverterClass: Type[Converter],
+        *args: str,
+        **kwargs: str,
+    ):
+        assert issubclass(ConverterClass, Converter)
+        assert all([isinstance(arg, str) for arg in args])
+        assert all(
+            [isinstance(k, str) and isinstance(v, str) for k, v in kwargs.items()]
+        )
+
+        self._factories[(data_type, output_format)] = partial(
+            call_with_mappings, ConverterClass, args_mapping=args, kwargs_mapping=kwargs
+        )
+
+    def get_converter(
+        self, data_type: str, output_format: str, return_default=True, **kwargs
+    ) -> Converter:
+        if (data_type, output_format) not in self._factories:
+            if return_default:
+                return IdentityConverter()
+            else:
+                raise ValueError(
+                    f"Unknown data type '{data_type}' or output format '{output_format}'"
+                )
+        return self._factories[(data_type, output_format)](kwargs)
+
+    def get_output_formats(self) -> frozenset:
+        return frozenset(self._factories.keys())
+
+
+@class_decorator
+def register_representation(
+    cls: Type[Converter], data_type: str, output_format: str, *args, **kwargs
+):
+    ConverterRegistry().register(data_type, output_format, cls, *args, **kwargs)

nerdd_module-0.3.3/nerdd_module/converters/identity_converter.py

@@ -0,0 +1,5 @@
+from .converter import Converter
+
+
+class IdentityConverter(Converter):
+    pass

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/input/__init__.py

@@ -1,4 +1,5 @@
 from .depth_first_explorer import *
+from .explorer import *
 from .file_reader import *
 from .gzip_reader import *
 from .inchi_reader import *

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/input/depth_first_explorer.py

@@ -1,5 +1,5 @@
 from itertools import chain, islice, repeat
-from typing import Generator, Iterable, Optional
+from typing import Iterable, Iterator, Optional
 
 from .explorer import Explorer
 from .reader import MoleculeEntry, Problem, Reader
@@ -12,7 +12,7 @@ class InvalidInputReader(Reader):
     def __init__(self):
         super().__init__()
 
-    def read(self, input, explore) -> Generator[MoleculeEntry, None, None]:
+    def read(self, input, explore) -> Iterator[MoleculeEntry]:
         yield MoleculeEntry(
             raw_input=input,
             input_type="unknown",
@@ -36,31 +36,31 @@ class DepthFirstExplorer(Explorer):
         super().__init__()
 
         if readers is None:
-            self.reader_registry = ReaderRegistry()
+            self._reader_registry = list(ReaderRegistry().get_readers())
         else:
-            self.reader_registry = readers
+            self._reader_registry = list(readers)
 
-        self.num_test_entries = num_test_entries
-        self.threshold = threshold
-        self.state_stack = [self.empty_state()]
-        self.maximum_depth = maximum_depth
+        self._num_test_entries = num_test_entries
+        self._threshold = threshold
+        self._state_stack = [self._empty_state()]
+        self._maximum_depth = maximum_depth
 
-    def empty_state(self):
+    def _empty_state(self):
         return dict(first_guess=[])
 
-    def explore(self, input) -> Generator[MoleculeEntry, None, None]:
+    def explore(self, input) -> Iterator[MoleculeEntry]:
         # create a new child node and set it as the current node
-        state = self.empty_state()
-        parent = self.state_stack[-1]
-        self.state_stack.append(state)
+        state = self._empty_state()
+        parent = self._state_stack[-1]
+        self._state_stack.append(state)
 
-        depth = len(self.state_stack)
-        if depth > self.maximum_depth:
-            raise ValueError(f"Maximum depth of {self.maximum_depth} reached")
+        depth = len(self._state_stack)
+        if depth > self._maximum_depth:
+            raise ValueError(f"Maximum depth of {self._maximum_depth} reached")
 
         readers_iter = chain(
             zip(parent["first_guess"], repeat("guess")),
-            zip(self.reader_registry, repeat("builtin")),
+            zip(self._reader_registry, repeat("builtin")),
         )
 
         # try all readers and take a sample of the first num_test_entries
@@ -69,40 +69,56 @@ class DepthFirstExplorer(Explorer):
         best_mode = None
         best_score = 0
         best_ratio = 0.0
+        best_num_invalid_results = 0
         generator = None
         sample = []
         for reader, mode in readers_iter:
             try:
                 # read at most num_test_entries entries
-                generator = reader.read(input, self.explore)
-                sample = list(islice(generator, self.num_test_entries))
+                generator = self._read(reader, input)
+                sample = list(islice(generator, self._num_test_entries))
                 valid_entries = [entry for entry in sample if entry.mol is not None]
 
                 score = len(valid_entries)
                 ratio = len(valid_entries) / len(sample)
-
-                if score > best_score or (score == best_score and ratio > best_ratio):
+                num_invalid_results = len(sample) - len(valid_entries)
+
+                if (
+                    score > best_score
+                    # if the score is the same, prefer the reader with higher ratio
+                    # of valid entries
+                    or (score == best_score and ratio > best_ratio)
+                    # if the ratio is the same, prefer the reader with less invalid
+                    # results
+                    or (
+                        score == best_score
+                        and ratio == best_ratio
+                        and num_invalid_results < best_num_invalid_results
+                    )
+                ):
                     best_reader = reader
                     best_mode = mode
                     best_score = score
                     best_ratio = ratio
+                    best_num_invalid_results = num_invalid_results
 
-                    if score == self.num_test_entries:
+                    if score == self._num_test_entries:
                         break
             except Exception:
                 pass
 
             # clean up tree
-            while len(self.state_stack) > depth:
-                self.state_stack.pop()
+            while len(self._state_stack) > depth:
+                self._state_stack.pop()
             generator = None
 
         if generator is None:
             if best_reader is None:
-                generator = InvalidInputReader().read(input, self.explore)
+                generator = self._read(InvalidInputReader(), input)
+                sample = []
             else:
-                generator = best_reader.read(input, self.explore)
-                sample = list(islice(generator, self.num_test_entries))
+                generator = self._read(best_reader, input)
+                sample = list(islice(generator, self._num_test_entries))
         else:
             if best_mode is not None and best_mode != "guess":
                 parent["first_guess"].append(best_reader)

nerdd_module-0.3.3/nerdd_module/input/explorer.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from typing import Iterator
+
+from .reader import MoleculeEntry, Reader
+
+
+class Explorer(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def explore(self, input) -> Iterator[MoleculeEntry]:
+        pass
+
+    def _read(self, reader: Reader, input) -> Iterator[MoleculeEntry]:
+        return reader.read(input, self.explore)

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/input/file_reader.py

@@ -1,5 +1,6 @@
+from os import PathLike
 from pathlib import Path
-from typing import Generator, Tuple
+from typing import Iterator, Optional, Tuple, Union
 
 from .reader import MoleculeEntry, Reader
 from .reader_registry import register_reader
@@ -7,15 +8,15 @@ from .reader_registry import register_reader
 __all__ = ["FileReader"]
 
 
-@register_reader("data_dir")
+@register_reader
 class FileReader(Reader):
-    def __init__(self, data_dir=None):
+    def __init__(self, data_dir: Union[str, PathLike, None] = None):
         super().__init__()
         self.data_dir = data_dir
         if self.data_dir is not None:
             self.data_dir = Path(self.data_dir)
 
-    def read(self, filename, explore) -> Generator[MoleculeEntry, None, None]:
+    def read(self, filename, explore) -> Iterator[MoleculeEntry]:
         assert isinstance(filename, str), "input must be a string"
 
         # convert filename to path

{nerdd_module-0.2.6 → nerdd_module-0.3.3}/nerdd_module/input/gzip_reader.py

@@ -1,5 +1,5 @@
 import gzip
-from typing import Generator
+from typing import Iterator
 
 from .reader import MoleculeEntry, Reader
 from .reader_registry import register_reader
@@ -12,7 +12,7 @@ class GzipReader(Reader):
     def __init__(self):
         super().__init__()
 
-    def read(self, input_stream, explore) -> Generator[MoleculeEntry, None, None]:
+    def read(self, input_stream, explore) -> Iterator[MoleculeEntry]:
         if not hasattr(input_stream, "read") or not hasattr(input_stream, "seek"):
             raise TypeError("input must be a stream-like object")