pinexq-procon 2.1.0.dev3__py3-none-any.whl

pinexq/procon/step/schema.py

@@ -0,0 +1,99 @@
+"""
+Building blocks for all schemas
+
+All objects for the schema description of function signatures are here
+represented as Pydantic BaseModel classes.
+"""
+import warnings
+from enum import Enum
+from typing import Any, Annotated, Type, Mapping
+
+from pydantic import ConfigDict, BaseModel, constr, Field, model_validator
+
+from pydantic.json_schema import (
+    GenerateJsonSchema, DEFAULT_REF_TEMPLATE, JsonSchemaMode, JsonSchemaValue, CoreSchema
+)
+
+
+# Custom JSON schema generators to create schemas with '$schema' identifier
+# -> copied from: https://github.com/pydantic/pydantic/blob/73373c3e08fe5fe23e4b05f549ea34e0da6a16b7/tests/test_json_schema.py#L3090-L3116
+
+class GenerateJsonSchemaWithDialect(GenerateJsonSchema):
+    def generate(self, schema: CoreSchema, mode: JsonSchemaMode = 'validation') -> JsonSchemaValue:
+        json_schema = super().generate(schema)
+        json_schema['$schema'] = self.schema_dialect
+        return json_schema
+
+
+class BaseModelWithSchema(BaseModel):
+    @classmethod
+    def model_json_schema(
+            cls,
+            by_alias: bool = True,
+            ref_template: str = DEFAULT_REF_TEMPLATE,
+            schema_generator: Type[GenerateJsonSchema] = GenerateJsonSchemaWithDialect,
+            mode: JsonSchemaMode = 'validation'
+    ) -> dict[str, Any]:
+        return super().model_json_schema(by_alias, ref_template, schema_generator, mode)
+
+
+# Models defining the structure of the "manifest" generated by the "signature" command
+class DataslotModel(BaseModel):
+    """Description of a single DataSlot"""
+    name: str
+    title: str
+    description: str
+    mediatype: str
+    metadata: dict
+    min_slots: Annotated[int, Field(ge=0, strict=True, serialization_alias='minSlots')]
+    max_slots: Annotated[int | None, Field(ge=1, strict=True, serialization_alias='maxSlots')]
+
+
+class FunctionModel(BaseModel):
+    """Structure of a Step-function manifest"""
+    version: Annotated[str, Field(min_length=0, max_length=40)]
+    function_name: Annotated[str, Field(min_length=1, max_length=100)]
+    short_description: str
+    long_description: str
+    parameters: Any = None  # 'DynamicParameters' actually, but embedded at runtime as schema
+    returns: Any = None  # 'DynamicReturns' actually, but embedded at runtime as schema
+    input_dataslots: list[DataslotModel]
+    output_dataslots: list[DataslotModel]
+    procon_version: str
+
+
+class DynamicParameters(BaseModelWithSchema):
+    """Basis for a dynamically created model of the functions parameters."""
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        extra="forbid",
+        strict=True,
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def _convert_enums(cls, values: Mapping[str, Any]) -> Mapping[str, Any]:
+        values = dict(values)  # Ensure mutable dict
+        for field_name, model_field in cls.model_fields.items():
+            field_type = model_field.annotation
+            if isinstance(field_type, type) and issubclass(field_type, Enum):
+                value = values.get(field_name)
+                if isinstance(value, str):
+                    try:
+                        values[field_name] = field_type(value)
+                    except ValueError:
+                        warnings.warn(
+                            f"Failed to convert '{value}' to '{field_type.__name__}' for field '{field_name}'."
+                        )
+        return values
+
+
+
+
+class DynamicReturns(BaseModelWithSchema):
+    """Basis for a dynamically created model of the functions return value."""
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        extra="ignore",
+        strict=True,
+    )

pinexq/procon/step/step.py

@@ -0,0 +1,119 @@
+from dataclasses import dataclass, field
+from typing import Any, TypeVar
+
+import pydantic
+
+from ..core.exceptions import ProConException, ProConSchemaValidationError
+from ..dataslots import DataSlotDescription, DataslotLayer
+from ..dataslots.metadata import MetadataHandler
+from .introspection import FunctionSchema, StepClassInfo
+
+
+@dataclass
+class ExecutionContext:
+    """Wraps all information to call a function in a Step container"""
+
+    function_name: str
+    parameters: dict[str, Any] = field(default_factory=dict)
+    input_dataslots: dict[str, "DataSlotDescription"] = field(default_factory=dict)
+    output_dataslots: dict[str, "DataSlotDescription"] = field(default_factory=dict)
+    metadata_handler: MetadataHandler | None = None
+
+
+ExecutionContextType = TypeVar("ExecutionContextType", bound="ExecutionContext")
+
+
+class Step:
+    _signatures: dict[str, FunctionSchema]
+    _context: ExecutionContextType | None = None
+
+    def __init__(self, use_cli=True):
+        self._signatures = StepClassInfo(self).get_func_schemas()
+
+        if use_cli:
+            # Do the import here to avoid circular imports
+            from pinexq.procon.core.cli import cli
+
+            cli.main(
+                obj=self,  # This Step object *self* is available as *obj* attribute in the context of each cli command
+                auto_envvar_prefix="PROCON",
+            )
+
+    def _call(self, context: ExecutionContextType) -> Any:
+        """Calls a function from this container"""
+        try:
+            # Check if this class contains the function?
+            function = getattr(self, context.function_name)
+            # is it one of the exposed functions?
+            signature = self._signatures[context.function_name]
+        except (KeyError, AttributeError):
+            raise ProConException(
+                f"No function with such name: '{context.function_name}'"
+            )
+
+        # Connect to the dataslot sources and load the data into parameters
+        with DataslotLayer(
+            function=function,
+            parameters=context.parameters,
+            dataslots_in_descr=context.input_dataslots,
+            dataslots_out_descr=context.output_dataslots,
+            signature=signature,
+            metadata_handler=context.metadata_handler,
+        ) as ds_handle:
+            # Match parameters with the function's signature
+            try:
+                # The validation may fail when e.g. Enums are used within the
+                # parameters. TODO: find a way how to verify nested enums.
+                function_parameter_model = signature.get_parameters_model().model_validate(ds_handle.parameters)
+            except pydantic.ValidationError as ex:
+                raise ProConSchemaValidationError(
+                    "Parameters don't match the functions signature!"
+                ) from ex
+
+            # Call the function
+            self._context = context
+            try:
+                # Get the parameter model as a dictionary. Unlike .model_dump(), casting
+                # it with `dict()` will only convert the root level Basemodel and not
+                # any embedded Basemodels.
+                function_parameters = dict(function_parameter_model)
+
+                # The actual function call
+                result = ds_handle.function(**function_parameters)
+
+                # Update parameters to handle "return-by-reference" via Output-Dataslots
+                ds_handle.update_parameters(function_parameters)
+                # Update return value to handle Return-Dataslots
+                ds_handle.update_result(result)
+            except ProConException:
+                raise
+            except Exception as ex:
+                raise ProConException(
+                    f"Exception during execution of '{context.function_name}'!"
+                ) from ex
+            finally:
+                self._context = None
+
+            # Match the results signature
+            #  But only if there is no return-Dataslot, in which case there is no return value
+            # FixMe: Data in the return.dataslot is actually not validated against the signature!
+            if not ds_handle.has_results_data_slot():
+                try:
+                    signature.get_returns_model().model_validate({"value": result})
+                except pydantic.ValidationError as ex:
+                    raise ProConSchemaValidationError(
+                        "The functions return values doesn't match its signature!"
+                    ) from ex
+
+            # The result value or None, if a `return.dataslot` is defined
+            func_result = ds_handle.result
+
+        return func_result
+
+    @property
+    def step_context(self) -> ExecutionContextType | None:
+        return self._context
+
+    @property
+    def step_signatures(self) -> dict[str, FunctionSchema]:
+        return self._signatures

pinexq/procon/step/versioning.py

@@ -0,0 +1,84 @@
+import re
+from typing import Any, Callable
+
+from pydantic import Field
+from pydantic.dataclasses import dataclass
+
+from ..core.types import UNSET, UNSETTYPE
+
+
+METADATA_ANNOTATION_NAME = "__pxq_metadata__"
+
+# Version string regex
+VERSION_PATTERN = r"""
+    ^(?:                                                # start of the string
+        (?P<version>[0-9]{1,8}(?:\.[0-9]{1,8}){,2})     # version number
+        (?:-(?P<postfix>[0-9a-zA-Z_-]+))?               # postfix string
+    )$                                                  # end of the string
+"""
+version_regex = re.compile(VERSION_PATTERN, flags=re.VERBOSE)
+
+# Regex pattern for just the postfix substring for pydantic
+POSTFIX_PATTERN = r"^[0-9a-zA-Z_-]*$"
+postfix_regex = re.compile(POSTFIX_PATTERN)
+
+# Restrictions
+# <major:int>.<minor:int>.<patch:int>-<string> mit max 40 chars
+# maj/min/patch each <= 8 chars , defaults == 0
+
+MAX_VERSION_LENGTH = 40
+
+
+@dataclass
+class version:
+    """Decorator to attach version information to a Step function.
+
+    Attributes:
+        version: ...
+
+    """
+
+    version: str | UNSETTYPE = Field(default=UNSET, pattern=version_regex)
+    major: int | UNSETTYPE = Field(default=UNSET, ge=0, lt=100_000_000, kw_only=True)
+    minor: int | UNSETTYPE = Field(default=UNSET, ge=0, lt=100_000_000, kw_only=True)
+    patch: int | UNSETTYPE = Field(default=UNSET, ge=0, lt=100_000_000, kw_only=True)
+    postfix: str | UNSETTYPE = Field(default=UNSET, pattern=postfix_regex, kw_only=True)
+
+
+    def __post_init__(self):
+        version_was_set = self.version is not UNSET
+        any_specifier_was_set = (self.major is not UNSET or self.minor is not UNSET
+                                 or self.patch is not UNSET or self.postfix is not UNSET)
+        if not (version_was_set or any_specifier_was_set):
+            self.major = 0  # default value when no parameter was given
+        elif not (version_was_set ^ any_specifier_was_set):
+            raise ValueError("You have to define the version either as a string or explicitly via the "
+                             "'major', 'minor', 'patch', 'postfix' keyword, but not both!")
+
+        if not version_was_set:
+            self._set_version_from_kwargs()
+
+        if len(self.version) > MAX_VERSION_LENGTH:
+            raise ValueError(f"The version string is too long! "
+                             f"Max {MAX_VERSION_LENGTH} chars are allowed, but it has {len(self.version)} chars.")
+
+    def _set_version_from_kwargs(self):
+        """Generate the version string from single kwarg variables"""
+        # The default string will always be "0.0.0" and all values and postfix are optional
+        self.version = (f"{self.major or '0'}.{self.minor or '0'}.{self.patch or '0'}"
+                        f"{f'-{self.postfix}' if self.postfix else ''}")
+
+
+    def __call__(self, function: Callable[[Any], Any]):
+        """Called when used as a decorator to attach metadata to 'func'."""
+        metadata = function.__dict__.setdefault(METADATA_ANNOTATION_NAME, {})
+        metadata["version"] = self
+
+        return function
+
+    def __str__(self) -> str:
+        return self.version
+
+def get_version_metadata(function: Callable[[Any], Any]) -> version:
+    metadata = getattr(function, METADATA_ANNOTATION_NAME, {})
+    return metadata.get("version", version())

pinexq_procon-2.1.0.dev3.dist-info/METADATA

@@ -0,0 +1,83 @@
+Metadata-Version: 2.3
+Name: pinexq-procon
+Version: 2.1.0.dev3
+Summary: Framework to create containers for DataCybernetic's PineXQ computing platform
+Author: Sebastian Höfer, Carsten Blank, Mathias Reichardt
+Author-email: Sebastian Höfer <hoefer@data-cybernetics.com>, Carsten Blank <blank@data-cybernetics.com>, Mathias Reichardt <reichardt@data-cybernetics.com>
+License: Copyright (C) data cybernetics ssc GmbH - All Rights Reserved
+         <contactus@data-cybernetics.com>, December 2023
+Requires-Dist: aio-pika==9.5.5
+Requires-Dist: click>=8.1.3
+Requires-Dist: docstring-parser==0.*
+Requires-Dist: httpx-caching>=0.1a4
+Requires-Dist: httpx==0.*
+Requires-Dist: pinexq-client>=0.10
+Requires-Dist: pydantic>=2.10.0
+Requires-Dist: pyjwt>=2.10.0
+Requires-Dist: rich>=13.3.2
+Requires-Dist: stamina>=24.2.0
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# PineXQ ProCon Framework
+
+Computations in DC-Cloud are done by **Workers** running inside a **ProcessingContainer**, or short "ProCon", which is also the name of the framework.
+ProCon provides an unobtrusive wrapper around function definitions without introducing new semantics, allowing for a clean definition of the computational task, while handling all cloud-related communication and data-management transparently in the background.
+This removes the code and configuration required from the function implementation.
+
+### Installation
+
+To install the package use the `pip` command, to either install from a package feed:
+```
+pip install pinexq-procon
+```
+
+
+### Creating a container
+
+To publish a function in a container it has to be a method of a class inheriting from the `Step` class.
+
+```python
+from pinexq.procon.step import Step # import package
+
+class MyStepCollection(Step):  # define the container class
+    def calculate_square(self, x: float) -> float:  # define a step function
+        """Calculate the square of x
+
+        :param x: a float number
+        :returns: the square of x
+        """
+        return x ** 2
+
+    # More step functions can go in the same class    
+
+if __name__ == '__main__':  # add script guard
+    MyStepCollection()  # run the container - this will spawn the cli
+```
+
+It is mandatory to annotate the types of parameters and return value.
+Docstrings are optional, but highly recommended.
+
+The [documentation](doc/ProCon.md) has a detailed section about [implementing processing-steps](doc/Implementing-a-Step.md).
+
+
+### Running a Step-function locally
+
+The Python file with the container is itself a cli-tool. 
+You get a list of all available commands with the `--help` parameter.
+
+```
+python ./my_step_file.py --help
+```
+
+With the `run` option you can call a function in the container directly and the result is written to the console.
+
+```
+python ./my_step_file.py run --function calculate_square --parameters "{'x': 5}"
+25
+```
+
+You can find full list of available commands in the [cli documentation](doc/Using-Procon-From-Cli.md).
+All possible parameters and environment variables are listed [here](doc/Parameters.md).
+
+

pinexq_procon-2.1.0.dev3.dist-info/RECORD

@@ -0,0 +1,35 @@
+pinexq/procon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pinexq/procon/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pinexq/procon/core/cli.py,sha256=8MPdTi9sCluFn7Y9TfaBKJavTsUmR5SG5Eup1SAMaZM,16602
+pinexq/procon/core/exceptions.py,sha256=0SXvC54-H6E65Fi06mQc6FlTnWB91GHH27ci7ypBYRE,1677
+pinexq/procon/core/helpers.py,sha256=2nnd99o0_oPwAptOF_6LySeeDv8ZPf6EO2R88yUBbuY,1996
+pinexq/procon/core/logconfig.py,sha256=UlIqgzShK3UETmRz2iSDniypQxnIGOuwz-gZetfaths,1503
+pinexq/procon/core/naming.py,sha256=K2g3dFZdz46WevbRpeK-75v6427ph56F43evDARWxHk,916
+pinexq/procon/core/types.py,sha256=w6XmboyUSly4JI-zM-A3Otc-f-F2NrfzGvpC7HLOcwA,226
+pinexq/procon/dataslots/__init__.py,sha256=uiRtBPBbK7thyte7Ch7x-S00LYaX_qJ_ELtPfqaydi4,365
+pinexq/procon/dataslots/abstractionlayer.py,sha256=ll86ONfQNsmbn55IbF4aE9D9jpe32s5L7UmX23JIx-4,9257
+pinexq/procon/dataslots/annotation.py,sha256=O-oAi2vrrwZaAD0tOB4viAqYsGTe_ICftfAa2YH_5nk,16487
+pinexq/procon/dataslots/dataslots.py,sha256=6RQwncq8E_9ILPBhxthnWfy9K6_W8IY6jouTNAZLhXI,14274
+pinexq/procon/dataslots/datatypes.py,sha256=TpbHCAjuFokHUUG8Dweu7RlULmu4QIZYB4lvnk09ROg,1292
+pinexq/procon/dataslots/default_reader_writer.py,sha256=f9edmeaTDO11D5im_cqIqBgFGfqh9Sb5cw6DtxO2KvM,1003
+pinexq/procon/dataslots/filebackend.py,sha256=xGAXwo5XzDtznzoQMLwReKrabNCeRQuYYjHGjWugcyI,3935
+pinexq/procon/dataslots/metadata.py,sha256=Qi5vetHF6_3epubYeoSZ3KFopboxrbjDUSId7SLbrP8,4382
+pinexq/procon/jobmanagement/__init__.py,sha256=W4UC5s70RzL_EwnEuvXsXS7Hph2awsblCQvZ_ROM-DQ,190
+pinexq/procon/jobmanagement/api_helpers.py,sha256=xEtNEOEgQUtIIz6wtK-fiJThfgCIdLBQcKMb9L394VU,10224
+pinexq/procon/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pinexq/procon/remote/messages.py,sha256=Mr6rz3VRxRxUM3l2Y0vU7CDmpc9Mi1NuhvEndoZbKEo,6904
+pinexq/procon/remote/rabbitmq.py,sha256=2Kwsct35sN0pameRVCDER3WlPgnCI8YzCq9dlbGQgw8,16070
+pinexq/procon/runtime/__init__.py,sha256=K-pSghTgjHAJJSeJpG8F_sYHDp8j3CL2X6izHS76DT0,119
+pinexq/procon/runtime/foreman.py,sha256=YrKA8u1UieOU1aM22sbTv7PuppwuP8KII7WuDy7aQJc,5022
+pinexq/procon/runtime/job.py,sha256=HDQRGIGj12McUQQ6F8bB2E_Wkuu-M-sf1in1MCxy1OQ,13883
+pinexq/procon/runtime/settings.py,sha256=owSCPBEEpaf7zSDMdgy-jX_ZQ5--A7705nZcsPFx2sU,444
+pinexq/procon/runtime/tool.py,sha256=PJ9Gtz3OuLvtOopPz955qJJvrrRj1ZyXKNYKIPPgQAc,653
+pinexq/procon/runtime/worker.py,sha256=JOYyoZUoIOmVV-sK5fkmSouAiYU4M1pB5F6_XNektlA,17845
+pinexq/procon/step/__init__.py,sha256=QZCtYx2uumBdzfZxeLoVlYqZS6l8nnVewqOtzYucblw,92
+pinexq/procon/step/introspection.py,sha256=OF1hSherQGg9i4-WnyudUvslu0CDaxRzUrV5iKrD6K4,9814
+pinexq/procon/step/schema.py,sha256=TlPjn1ETx3bTTy0Gb_6vU0RZ1QW4OIC6hYmBjWNhxiE,3664
+pinexq/procon/step/step.py,sha256=cmErYmKzEilcMPVmMtSJMY7zGd1rb23x5VTVzmZU_hM,4862
+pinexq/procon/step/versioning.py,sha256=2RzEtz0Y5_yrMFw8hUXVdagLUDgTtD5LqmrZnLogpJc,3304
+pinexq_procon-2.1.0.dev3.dist-info/WHEEL,sha256=ZyFSCYkV2BrxH6-HRVRg3R9Fo7MALzer9KiPYqNxSbo,79
+pinexq_procon-2.1.0.dev3.dist-info/METADATA,sha256=gWOhHZbbIU97BepESzg4mTA2_GvLHZ__ZbTzYBNl_dk,3038
+pinexq_procon-2.1.0.dev3.dist-info/RECORD,,

pinexq_procon-2.1.0.dev3.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.18
+Root-Is-Purelib: true
+Tag: py3-none-any