pinexq-procon 2.1.0.dev3__py3-none-any.whl

pinexq/procon/dataslots/abstractionlayer.py

@@ -0,0 +1,215 @@
+import enum
+import typing
+from typing import TYPE_CHECKING, Any, Callable, Final, NewType, TypeVar
+
+from ..core.exceptions import ProConDataslotError
+from .annotation import RETURN_SLOT_NAME, dataslot
+from .dataslots import DataSlot, DataSlotDescription, SlotType
+from .metadata import MetadataHandler
+
+
+if TYPE_CHECKING:
+    from ..step.introspection import FunctionSchema
+
+R = TypeVar('R')
+P = NewType('P', dict[str, Any])
+F = NewType('F', Callable[[P], R])
+
+
+# fmt: off
+class NotSetYetType(enum.IntEnum):
+    token = 0
+NOTSET: Final = NotSetYetType.token  # noqa: E305
+# fmt: on
+
+
+class DataslotLayer:
+    """Abstraction layer that handles reading and writing from/to storage locations
+    and presents them as regular parameters to the wrapped function."""
+
+    _function: F
+    _regular_parameters: P | NotSetYetType = NOTSET
+    _dataslot_parameters: P | NotSetYetType = NOTSET
+    _result: R | NotSetYetType = NOTSET
+    _signature: "FunctionSchema"
+    _dataslots_in: dict[str, DataSlot]
+    _dataslots_out: dict[str, DataSlot]
+    _sig_by_alias: dict[str, dataslot]
+    _result_slot: DataSlot | None
+    _metadata_handler: MetadataHandler | None
+
+    def __init__(
+            self,
+            function: F,
+            parameters: P,
+            signature: "FunctionSchema",
+            dataslots_in_descr: dict[str, DataSlotDescription],
+            dataslots_out_descr: dict[str, DataSlotDescription],
+            metadata_handler: MetadataHandler | None = None
+    ):
+        """
+
+        Args:
+            function: A function (potentially) with dataslots for parameters
+                or return value.
+            parameters: Dictionary with the parameters the function will be
+                called with.
+            dataslots_in_descr: Dictionary with `DataSlotDescription` objects
+                for each input dataslot parameter.
+            dataslots_out_descr: Dictionary with `DataSlotDescription` objects
+                for all output dataslots. The return value of the function is
+                internally defined by the name '__returns__'.
+            signature: A `FunctionSchema` object created during the function's
+                introspection (optional). If not provided, the introspection will
+                be called on the function internally.
+            metadata_handler: An optional `MetadataHandler` object uses to access
+                metadata from the currently used backend.
+        """
+        self._function = function
+        self._regular_parameters = parameters or {}  # avoid parameters being None
+        self._metadata_handler = metadata_handler
+
+        self._signature = signature
+        # Dataslots are presented to the outside by their alias name
+        self._sig_by_alias = {(s.alias or s.name): s for n, s in self._signature.dataslots.items()}
+
+        self._dataslots_in = self._init_dataslots(dataslots_in_descr)
+
+        # filter out special output slot `__returns__` (if present) since it isn't part of the function's parameters
+        output_slots = self._init_dataslots(dataslots_out_descr)
+        self._result_slot = None
+        for alias_name, ds in output_slots.items():
+            if ds.annotation.name == RETURN_SLOT_NAME:
+                self._result_slot = ds
+                output_slots.pop(alias_name)
+                break
+        self._dataslots_out = output_slots
+
+    def _init_dataslots(self, dataslot_description: dict[str, DataSlotDescription]) -> dict[str, DataSlot]:
+        """Creates the actual `DataSlot` object and match parameter alias names with
+        dataslots from the signature."""
+        slots = {}
+        for name, descr in dataslot_description.items():
+            try:
+                slots[name] = DataSlot(
+                    description=descr,
+                    annotation=self._sig_by_alias[name],
+                    metadata_handler=self._metadata_handler,
+                )
+            except KeyError as ex:
+                raise ProConDataslotError(
+                    f"Dataslot '{name}' not found in function '{self._signature.name}'. "
+                    f"Possibly a typo or the Dataslot is renamed by an alias? "
+                    f"(available dataslots: {list(self._sig_by_alias.keys())})") from ex
+        return slots
+
+    def __enter__(self) -> "DataslotLayer":
+        """Sync all files in the dataslots; i.e. if not local, get them"""
+        self._resolve_parameters()
+        return self
+
+    def _resolve_parameters(self):
+        """Resolve all implicit dataslots.
+        If a dataslot is NOT defined as type `Dataslot`, open it and deserialize the data if necessary"""
+        self._dataslot_parameters = typing.cast(P, {})
+        for n, ds in (self._dataslots_in | self._dataslots_out).items():
+            param_name = self._sig_by_alias[n].name
+            self._dataslot_parameters[param_name] = self._extract_parameter(ds)
+
+        self._check_for_disjoint_names()
+
+    def _extract_parameter(self, ds: DataSlot) -> Any:
+        # Function parameter is explicitly annotated with ":DataSlot"
+        if ds.annotation.dtype is DataSlot:
+            return ds
+        elif ds.annotation.slot_type == SlotType.INPUT:
+            return ds.read_data_from_slots()
+        elif ds.annotation.slot_type == SlotType.OUTPUT:
+            return self._init_output_dataslot_parameter(ds)
+        else:
+            raise Exception("Tried to init a return dataslot as parameter.")
+
+    def _check_for_disjoint_names(self):
+        dataslot_names = set(self._dataslot_parameters.keys())
+        parameter_names = set(self._regular_parameters.keys())
+        if not dataslot_names.isdisjoint(parameter_names):
+            raise ProConDataslotError("Name collision between dataslot- and parameter-names!"
+                                      "The following names appear in both: "
+                                      f"{', '.join(dataslot_names & parameter_names)}")
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Sync all files in the dataslots; i.e. if not local, push them to their destination"""
+        # Skip syncing return-Dataslots when there was an Exception, as the function did not return anything.
+        # Sync all other Dataslots, even in case of an error.
+        if exc_type is None:
+            self._resolve_result()
+        # self._resolve_result(include_result_slot=exc_type is None)
+
+    def _resolve_result(self, include_result_slot: bool = True):
+        """Resolve the result value to a dataslot; i.e. serializing and saving
+
+        Args:
+            include_result_slot: Write data for return-Dataslots (default: True)
+        """
+        if self._result_slot and include_result_slot:
+            self._result_slot.write_data_to_slots(self._result)
+
+        for dataslot_name, ds in self._dataslots_out.items():
+            if ds.annotation.dtype is not DataSlot:
+                ds.write_data_to_slots(self._dataslot_parameters[dataslot_name])
+
+    @property
+    def parameters(self) -> P | NotSetYetType:
+        """The given parameters and the resolved data from all input dataslots combined."""
+        if self._dataslot_parameters is NOTSET:
+            return NOTSET
+        return typing.cast(
+            P, self._regular_parameters | self._dataslot_parameters
+        )
+
+    # @property
+    # def function(self) -> F:
+    #     return self._function
+
+    def call_function(self) -> R:
+        """Call the function and return the *raw parameters*."""
+        self._result = self._function(
+            **self.parameters
+        )
+        return self._result
+
+    @property
+    def result(self) -> R | NotSetYetType:
+        """The return value of the function call or None if it was written to an output dataslot"""
+        return self._result if not self._result_slot else None
+
+    def has_results_data_slot(self) -> bool:
+        """Return True if the function has a `return.dataslot` annotation."""
+        return self._result_slot is not None
+
+    @staticmethod
+    def _init_output_dataslot_parameter(ds) -> Any:
+        """Initialize a dataslot with an empty instance of the annotated type.
+        This requires, that the type's constructor can be called without parameter.
+        E.g. for `list` calling `list()` will return and empty list."""
+        try:
+            return ds.annotation.dtype()
+        except Exception as ex:
+            raise ProConDataslotError(f"Can not create default instance of {ds.annotation.dtype.__name__}"
+                                      f" for output dataslot {ds.annotation.name}."
+                                      f" Type must be creatable without parameters.") from ex
+
+    def update_parameters(self, new: dict[str, Any]) -> None:
+        """Update the internal state of the parameters. Needed to update parameters that use
+        "return by reference", i.e. Output-Dataslots, after calling the function."""
+        for name, value in new.items():
+            if name in self._dataslot_parameters:
+                self._dataslot_parameters[name] = value
+
+    def update_result(self, new: Any) -> None:
+        """Set the return value of the function. Needed to funnel the return value to a Return-DataSlot."""
+        self._result = new
+
+    @property
+    def function(self) -> F:
+        return self._function

pinexq/procon/dataslots/annotation.py

@@ -0,0 +1,389 @@
+import dataclasses
+import logging
+from dataclasses import dataclass, field
+from typing import IO, Any, Callable, Literal, TypeAlias
+
+from ..core.types import UNSET, UNSETTYPE
+from ..dataslots.datatypes import MediaTypes, SlotType
+
+
+log = logging.getLogger(__name__)
+
+DATASLOT_ANNOTATION_NAME = "__dataslots__"
+RETURN_SLOT_NAME = "__returns__"
+
+FileMode: TypeAlias = Literal["r", "rb", "w", "wb"]
+ReaderType: TypeAlias = Callable[[IO], Any]
+WriterType: TypeAlias = Callable[[IO, Any], None]
+CollectionReaderType: TypeAlias = Callable[[list[IO]], Any]
+CollectionWriterType: TypeAlias = Callable[[[IO], Any], None]
+
+
+def media_type_2_file_mode(media_type: MediaTypes, is_write=False) -> FileMode:
+    access_mode = "w" if is_write else "r"
+    mapping = {
+        MediaTypes.OCTETSTREAM: "b",
+        MediaTypes.JSON: "",
+        MediaTypes.FORMDATA: "",
+        MediaTypes.SIREN: "",
+        MediaTypes.XML: "",
+        MediaTypes.ZIP: "b",
+        MediaTypes.PDF: "b",
+        MediaTypes.TEXT: "",
+        MediaTypes.HTML: "",
+        MediaTypes.CSV: "",
+        MediaTypes.SVG: "",
+        MediaTypes.PNG: "b",
+        MediaTypes.JPEG: "b",
+        MediaTypes.BMP: "b",
+        MediaTypes.WORKFLOW_DEFINITION: "",
+        MediaTypes.WORKFLOW_REPORT: "",
+    }
+    # if we do not know the mediatype we assume binary
+    binary_suffix = mapping.get(media_type, None)
+    if binary_suffix is None:
+        binary_suffix = "b"
+        log.warning(
+            f"Unknown media type {media_type} will assume binary format. "
+            f"If this does not fit, please set the mode on the dataslot explicitly."
+        )
+    return access_mode + binary_suffix
+
+
+@dataclass
+class dataslot:
+    """Decorator to attach metadata for a dataslot to a function.
+    You can use the general Dataslot decorator `@dataslot` to parametrize all possible options
+    manually or use the specialized constructors, which offer default settings and only the
+    necessary options for every dataslot type.
+
+    The available constructors are:
+        - `dataslot.input`
+        - `dataslot.output`
+        - `dataslot.input_collection`
+        - `dataslot.output_collection`
+        - `dataslot.returns`
+        - `dataslot.returns_collection`
+
+    Attributes:
+        name: The name has to match the functions parameter that will be used as dataslot.
+        alias: The name for this dataslot presented in the manifest and expected in a job offer.
+        title: Brief description of the dataslot.
+        description: More detailed description of the dataslot.
+        media_type: The media type as `MediaTypes` Enum.
+        reader: A callable accepting a file object as parameter and returning the datatype
+            expected by the functions' parameter.
+        writer: A callable accepting a file object and data to be written as a parameter.
+        mode: File mode of the IO object representing the dataslot. Usually set by the constructor
+            to a reasonable value ('r' for inputs, 'w' for outputs, 'b' for binary).
+        dtype: The annotated type of implicitly defined dataslots. Set by the introspection internally.
+        collection: Whether the dataslot accepts more than one file as slots. (default: False)
+    """
+
+    name: str
+    alias: str | None = field(default=None, kw_only=True)
+    slot_type: SlotType = field(default=SlotType.INPUT, kw_only=True)
+    title: str = field(default_factory=str, kw_only=True)
+    description: str = field(default_factory=str, kw_only=True)
+    media_type: str = field(default=MediaTypes.OCTETSTREAM, kw_only=True)
+    reader: ReaderType | None = field(default=None, kw_only=True)
+    writer: WriterType | None = field(default=None, kw_only=True)
+    collection_reader: CollectionReaderType | None = field(default=None, kw_only=True)
+    collection_writer: CollectionWriterType | None = field(default=None, kw_only=True)
+    mode: FileMode = field(default="r", kw_only=True)
+    dtype: type | None = field(default=None, kw_only=True)
+    collection: bool = field(default=False, kw_only=True)
+    min_slots: int | UNSETTYPE = field(default=UNSET, kw_only=True)
+    max_slots: int | None | UNSETTYPE = field(default=UNSET, kw_only=True)
+
+    def __post_init__(self):
+        if self.min_slots is UNSET:
+            self.min_slots = 1
+        if self.max_slots is UNSET:
+            self.max_slots = self.min_slots
+        if (self.max_slots is not None) and (self.min_slots > self.max_slots):
+            raise ValueError("Dataslot attribute 'min_slots' has to smaller than 'max_slots'!")
+
+        if (self.collection_reader or self.collection_writer) and not self.collection:
+            raise ValueError(
+                "'collection_reader' and '*_writer' can only be used for collection-DataSlots"
+                " (when the parameter 'collection=True')!"
+            )
+
+    @classmethod
+    def input(
+        cls,
+        name: str,
+        title: str = "",
+        description: str = "",
+        media_type: MediaTypes = MediaTypes.OCTETSTREAM,
+        mode: FileMode | None = None,
+        alias: str | None = None,
+        reader: ReaderType | None = None,
+        dtype: type | None = None,
+    ):
+        """Define the parameter `name` as input dataslot.
+
+        Attributes:
+            name: The name has to match the functions parameter that will be used as dataslot.
+            alias: The name for this dataslot presented in the manifest and expected in a job offer.
+            title: Brief description of the dataslot.
+            description: More detailed description of the dataslot.
+            media_type: The media type as `MediaTypes` Enum.
+            reader: A callable accepting a file object as parameter and returning the datatype
+                expected by the functions' parameter.
+            mode: File mode of the IO object representing the dataslot. Usually set by the constructor
+                to a reasonable value ('r' for inputs, 'w' for outputs, 'b' for binary).
+            dtype: The annotated type of implicitly defined dataslots. Set by the introspection internally.
+        """
+        # The typechecker will complain that `_mode` could be None in the return statement below, which can't be.
+        _mode = media_type_2_file_mode(media_type, is_write=False) if mode is None else mode
+        # Using the following if/else clause it's recognized correctly, even though it's functionally the same.
+        # if mode is None:
+        #     _mode = media_type_2_file_mode(media_type, is_write=False)
+        # else:
+        #     _mode = mode
+        return cls(
+            name,
+            slot_type=SlotType.INPUT,
+            title=title,
+            description=description,
+            media_type=media_type,
+            mode=_mode,
+            alias=alias,
+            reader=reader,
+            dtype=dtype,
+        )
+
+    @classmethod
+    def input_collection(
+        cls,
+        name: str,
+        alias: str | None = None,
+        title: str = "",
+        description: str = "",
+        media_type: MediaTypes = MediaTypes.OCTETSTREAM,
+        mode: FileMode | None = None,
+        collection_reader: CollectionReaderType | None = None,
+        dtype: type | None = None,
+        min_slots: int | UNSETTYPE = UNSET,
+        max_slots: int | None | UNSETTYPE = UNSET,
+    ):
+        """Define the input dataslot as collection of files
+
+        Attributes:
+            name: The name has to match the functions parameter that will be used as dataslot.
+            alias: The name for this dataslot presented in the manifest and expected in a job offer.
+            title: Brief description of the dataslot.
+            description: More detailed description of the dataslot.
+            media_type: The media type as `MediaTypes` Enum.
+            mode: File mode of the IO object representing the dataslot. Usually set by the constructor
+                to a reasonable value ('r' for inputs, 'w' for outputs, 'b' for binary).
+            dtype: The annotated type of implicitly defined dataslots. Set by the introspection internally.
+        """
+        _mode = media_type_2_file_mode(media_type, is_write=False) if mode is None else mode
+        return cls(
+            name,
+            slot_type=SlotType.INPUT,
+            collection=True,
+            title=title,
+            description=description,
+            media_type=media_type,
+            mode=_mode,  # see comment in input()
+            alias=alias,
+            collection_reader=collection_reader,
+            dtype=dtype,
+            min_slots=min_slots,
+            max_slots=max_slots,
+        )
+
+    @classmethod
+    def output(
+        cls,
+        name: str,
+        title: str = "",
+        description: str = "",
+        media_type: MediaTypes = MediaTypes.OCTETSTREAM,
+        mode: FileMode | None = None,
+        alias: str | None = None,
+        writer: WriterType | None = None,
+        dtype: type | None = None,
+    ):
+        """Define the parameter `name` as output dataslot.
+
+        Attributes:
+            name: The name has to match the functions parameter that will be used as dataslot.
+            alias: The name for this dataslot presented in the manifest and expected in a job offer.
+            title: Brief description of the dataslot.
+            description: More detailed description of the dataslot.
+            media_type: The media type as `MediaTypes` Enum.
+            writer: A callable accepting a file object and data to be written as a parameter.
+            mode: File mode of the IO object representing the dataslot. Usually set by the constructor
+                to a reasonable value ('r' for inputs, 'w' for outputs, 'b' for binary).
+            dtype: The annotated type of implicitly defined dataslots. Set by the introspection internally.
+        """
+        _mode = media_type_2_file_mode(media_type, is_write=True) if mode is None else mode
+        return cls(
+            name,
+            slot_type=SlotType.OUTPUT,
+            title=title,
+            description=description,
+            media_type=media_type,
+            mode=_mode,  # see comment in input()
+            alias=alias,
+            writer=writer,
+            dtype=dtype,
+        )
+
+    @classmethod
+    def output_collection(
+        cls,
+        name: str,
+        title: str = "",
+        description: str = "",
+        media_type: MediaTypes = MediaTypes.OCTETSTREAM,
+        mode: FileMode | None = None,
+        alias: str | None = None,
+        collection_writer: CollectionWriterType | None = None,
+        dtype: type | None = None,
+        min_slots: int | UNSETTYPE = UNSET,
+        max_slots: int | None | UNSETTYPE = UNSET,
+    ):
+        """Define the output dataslot as collection of files
+
+        Attributes:
+            name: The name has to match the functions parameter that will be used as dataslot.
+            alias: The name for this dataslot presented in the manifest and expected in a job offer.
+            title: Brief description of the dataslot.
+            description: More detailed description of the dataslot.
+            media_type: The media type as `MediaTypes` Enum.
+            mode: File mode of the IO object representing the dataslot. Usually set by the constructor
+                to a reasonable value ('r' for inputs, 'w' for outputs, 'b' for binary).
+            dtype: The annotated type of implicitly defined dataslots. Set by the introspection internally.
+        """
+        _mode = media_type_2_file_mode(media_type, is_write=True) if mode is None else mode
+        return cls(
+            name,
+            slot_type=SlotType.OUTPUT,
+            collection=True,
+            title=title,
+            description=description,
+            media_type=media_type,
+            mode=_mode,  # see comment in input()
+            alias=alias,
+            collection_writer=collection_writer,
+            dtype=dtype,
+            min_slots=min_slots,
+            max_slots=max_slots,
+        )
+
+    @classmethod
+    def returns(
+        cls,
+        title: str = "",
+        description: str = "",
+        media_type: MediaTypes = MediaTypes.OCTETSTREAM,
+        mode: FileMode | None = None,
+        writer: WriterType | None = None,
+        dtype: type | None = None,
+    ):
+        """Define the return value of the function to be written to a dataslot.
+
+        Attributes:
+            title: Brief description of the dataslot.
+            description: More detailed description of the dataslot.
+            media_type: The media type as `MediaTypes` Enum.
+            writer: A callable accepting a file object and data to be written as a parameter.
+            mode: File mode of the IO object representing the dataslot. Usually set by the constructor
+                to a reasonable value ('r' for inputs, 'w' for outputs, 'b' for binary).
+            dtype: The annotated type of implicitly defined dataslots. Set by the introspection internally.
+        """
+        _mode = media_type_2_file_mode(media_type, is_write=True) if mode is None else mode
+        return cls(
+            RETURN_SLOT_NAME,
+            slot_type=SlotType.RETURN,
+            collection=False,
+            title=title,
+            description=description,
+            media_type=media_type,
+            mode=_mode,  # see comment in input()
+            writer=writer,
+            dtype=dtype,
+        )
+
+    @classmethod
+    def returns_collection(
+        cls,
+        title: str = "",
+        description: str = "",
+        media_type: MediaTypes = MediaTypes.OCTETSTREAM,
+        mode: FileMode | None = None,
+        collection_writer: CollectionWriterType | None = None,
+        dtype: type | None = None,
+        min_slots: int | UNSETTYPE = UNSET,
+        max_slots: int | None | UNSETTYPE = UNSET,
+    ):
+        """Define the return value of the function will be written to a return dataslot.
+
+        Attributes:
+            title: Brief description of the dataslot.
+            description: More detailed description of the dataslot.
+            media_type: The media type as `MediaTypes` Enum.
+            mode: File mode of the IO object representing the dataslot. Usually set by the constructor
+                to a reasonable value ('r' for inputs, 'w' for outputs, 'b' for binary).
+            dtype: The annotated type of implicitly defined dataslots. Set by the introspection internally.
+        """
+        _mode = media_type_2_file_mode(media_type, is_write=True) if mode is None else mode
+        return cls(
+            RETURN_SLOT_NAME,
+            slot_type=SlotType.RETURN,
+            collection=True,
+            title=title,
+            description=description,
+            mode=_mode,  # see comment in input()
+            media_type=media_type,
+            collection_writer=collection_writer,
+            dtype=dtype,
+            min_slots=min_slots,
+            max_slots=max_slots,
+        )
+
+    # @classmethod
+    # def watch(cls, name: str, **kwargs):
+    #     """Watch the local filesystem and upload written files to this dataslot."""
+    #     raise NotImplementedError()
+
+    def __call__(self, function: Callable[[Any], Any]):
+        """Called when used as a decorator to attach metadata to 'func'."""
+        metadata = function.__dict__.setdefault(DATASLOT_ANNOTATION_NAME, {})
+        if self.name in metadata:
+            raise NameError(f"A dataslot with the name '{self.name}' is already defined on '{function.__name__}()'!")
+        if (self.name == RETURN_SLOT_NAME) and (self.slot_type is not SlotType.RETURN):
+            raise NameError(f"Invalid dataslot name.The name '{RETURN_SLOT_NAME}' is reserved for internal use.")
+        metadata[self.name] = self
+
+        return function
+
+    def update(self, d: "dataslot"):
+        """Update this dataslot with information from another dataslot"""
+        if not isinstance(d, self.__class__):
+            raise TypeError("Can only update with dataslots of the same type!")
+
+        fields_with_default_value = {
+            f.name  #
+            for f in dataclasses.fields(d)
+            if getattr(d, f.name) == f.default
+        }
+        set_fields = {
+            name: value  #
+            for name, value in dataclasses.asdict(d).items()
+            if name not in fields_with_default_value
+        }
+        return dataclasses.replace(self, **set_fields)
+
+    def __or__(self, other):
+        return self.update(other)
+
+
+def get_dataslot_metadata(function: Callable[[Any], Any]) -> dict[str, dataslot]:
+    return getattr(function, DATASLOT_ANNOTATION_NAME, {})