procodile 0.0.7__tar.gz → 0.0.9__tar.gz

{procodile-0.0.7 → procodile-0.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: procodile
-Version: 0.0.7
+Version: 0.0.9
 Summary: A light-weight processor development framework
 Project-URL: Documentation, https://eo-tools.github.io/eozilla
 Project-URL: Repository, https://github.com/eo-tools/eozilla

{procodile-0.0.7 → procodile-0.0.9}/procodile/__init__.py

@@ -4,22 +4,37 @@
 
 from importlib.metadata import version
 
-
 __version__ = version("procodile")
 
 from gavicore.util.request import ExecutionRequest
+
+from .artifacts import ArtifactRef, ArtifactStore, ExecutionContext
 from .job import Job, JobCancelledException, JobContext
-from .process import Process
+from .process import Process, additional_parameters
 from .registry import ProcessRegistry
+from .workflow import (
+    FromMain,
+    FromStep,
+    Workflow,
+    WorkflowStepRegistry,
+)
 
 """Processes development API."""
 
 __all__ = [
+    "additional_parameters",
+    "ArtifactRef",
+    "ArtifactStore",
     "__version__",
     "ExecutionRequest",
+    "ExecutionContext",
+    "FromStep",
+    "FromMain",
     "Job",
     "JobContext",
     "JobCancelledException",
-    "ProcessRegistry",
     "Process",
+    "ProcessRegistry",
+    "WorkflowStepRegistry",
+    "Workflow",
 ]

procodile-0.0.9/procodile/artifacts.py

@@ -0,0 +1,192 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Mapping, TypeAlias
+
+NormalizedOutputs: TypeAlias = dict[str, Any]
+"""
+Normalized mapping of output names to resolved values produced by a step 
+execution in a workflow.
+"""
+
+
+@dataclass(frozen=True)
+class ArtifactRef:
+    """
+    Lightweight reference to an externally stored artifact.
+
+    An ``ArtifactRef`` is returned when a value is materialized by an
+    ArtifactStore and can later be resolved back into the original object.
+    """
+
+    path: str
+    loader: str
+
+
+class ArtifactStore(ABC):
+    """
+    Abstract base class for persisting and loading large objects
+    (e.g. xarray datasets, large arrays, model artifacts).
+
+    Concrete implementations define how objects are stored
+    (filesystem, object store, etc.).
+    """
+
+    @abstractmethod
+    def is_artifact_big(self, obj: Any) -> bool:
+        """
+        Determine whether an object should be treated as a 'big object'
+        and stored externally instead of being passed as is.
+
+        Args:
+            obj: Object to inspect.
+
+        Returns:
+            True if the object is considered large and should be stored
+            via this store, False otherwise.
+        """
+        raise NotImplementedError  # pragma: no cover
+
+    @abstractmethod
+    def save_artifact(self, obj: Any) -> ArtifactRef:
+        """
+        Persist a large object and return a reference to it.
+
+        Args:
+            obj: The object to be stored.
+
+        Returns:
+            ArtifactRef: A reference that can later be used to load the object.
+
+        Raises:
+            TypeError: If the object type is not supported by the implementation.
+        """
+        raise NotImplementedError  # pragma: no cover
+
+    @abstractmethod
+    def load_artifact(self, ref: ArtifactRef) -> Any:
+        """
+        Load a previously stored object using an ArtifactRef.
+
+        Args:
+            ref: Reference returned by `save`.
+
+        Returns:
+            The reconstructed object.
+
+        Raises:
+            ValueError: If the loader specified in the reference is unknown.
+        """
+        raise NotImplementedError  # pragma: no cover
+
+
+class NullArtifactStore(ArtifactStore):
+    """
+    No-op artifact store.
+
+    This store never treats objects as 'big' and does not
+    support saving or loading artifacts.
+    """
+
+    def is_artifact_big(self, obj: Any) -> bool:
+        """
+        No object is considered big in the null store.
+        """
+        return False
+
+    def save_artifact(self, obj: Any):
+        """
+        Saving artifacts is not supported in the null store.
+        """
+        raise RuntimeError("NullArtifactStore does not support saving artifacts")
+
+    def load_artifact(self, ref):
+        """
+        Loading artifacts is not supported in the null store.
+        """
+        raise RuntimeError("NullArtifactStore does not support loading artifacts")
+
+
+class ExecutionContext:
+    """
+    Holds execution-time state for a workflow run.
+
+    The execution context stores resolved outputs from the main step and
+    individual steps, manages artifact materialization, and normalizes outputs.
+    """
+
+    def __init__(self, store: ArtifactStore) -> None:
+        self.store = store
+        self.main: dict[str, Any] = {}
+        self.steps: dict[str, dict[str, Any]] = {}
+
+    def resolve(self, value: Any) -> Any:
+        """Recursively resolve the inputs."""
+        if isinstance(value, ArtifactRef):
+            return self.store.load_artifact(value)
+
+        if isinstance(value, dict):
+            return {k: self.resolve(v) for k, v in value.items()}
+
+        if isinstance(value, tuple):
+            return tuple(self.resolve(v) for v in value)
+
+        if isinstance(value, list):
+            return [self.resolve(v) for v in value]
+
+        return value
+
+    def normalize_outputs(
+        self, result: Any, output_spec: Mapping[str, Any] | None, store: ArtifactStore
+    ) -> NormalizedOutputs:
+        """
+        Normalize function outputs into a dict[str, Any].
+
+        Rules:
+        - If output_spec is None → {"return_value": result}
+        - Tuple → positional mapping
+        - Dict → key mapping
+        - Single value → single output
+        """
+        if output_spec is None:
+            return {"return_value": self.materialize_artifact(result, store)}
+
+        output_keys = list(output_spec.keys())
+
+        if len(output_keys) == 1:
+            return {output_keys[0]: self.materialize_artifact(result, store)}
+
+        if isinstance(result, dict):
+            missing = set(output_keys) - result.keys()
+            if missing:
+                raise ValueError(f"Missing outputs in return dict: {missing}")
+            return {k: self.materialize_artifact(result[k], store) for k in output_keys}
+
+        if isinstance(result, tuple):
+            if len(result) != len(output_keys):
+                raise ValueError("Tuple output length does not match declared outputs")
+            return {
+                k: self.materialize_artifact(v, store)
+                for k, v in zip(output_keys, result, strict=True)
+            }
+
+        raise TypeError(
+            f"Invalid return type for declared outputs. result: {result}, output_spec: {output_spec}",
+        )
+
+    def materialize_artifact(self, value: Any, store: ArtifactStore) -> Any:
+        """
+        Recursively materialize big objects.
+        """
+        if store.is_artifact_big(value):
+            return store.save_artifact(value)
+
+        if isinstance(value, dict):
+            return {k: self.materialize_artifact(v, store) for k, v in value.items()}
+
+        if isinstance(value, tuple):
+            return tuple(self.materialize_artifact(v, store) for v in value)
+
+        if isinstance(value, list):
+            return [self.materialize_artifact(v, store) for v in value]
+
+        return value

{procodile-0.0.7 → procodile-0.0.9}/procodile/cli/__init__.py

@@ -5,6 +5,6 @@
 # No exports here. To avoid unnecessary imports, CLI apps should pick
 # only what they need from dedicated submodules.
 
-from .cli import get_cli
+from .cli import new_cli
 
-__all__ = ["get_cli"]
+__all__ = ["new_cli"]

procodile-0.0.9/procodile/cli/cli.py

@@ -0,0 +1,259 @@
+#  Copyright (c) 2025 by the Eozilla team and contributors
+#  Permissions are hereby granted under the terms of the Apache 2.0 License:
+#  https://opensource.org/license/apache-2-0.
+
+import json
+from typing import TYPE_CHECKING, Annotated, Any, Callable, Optional, Union
+
+import click
+import typer
+
+from gavicore.util.cli.group import AliasedGroup
+from gavicore.util.cli.parameters import (
+    DOT_PATH_OPTION,
+    PROCESS_ID_ARGUMENT,
+    REQUEST_FILE_OPTION,
+    REQUEST_INPUT_OPTION,
+    REQUEST_SUBSCRIBER_OPTION,
+)
+
+if TYPE_CHECKING:  # pragma: no cover
+    from procodile import ProcessRegistry
+
+PROCESS_REGISTRY_GETTER_KEY = "get_process_registry"
+
+DEFAULT_NAME = "procodile"
+
+DEFAULT_SUMMARY = """`{name}` is a command-line tool to describe and execute
+one or more registered processes."""
+
+DEFAULT_HELP = """{summary}
+
+You can use shorter command name aliases, e.g., use command name `ep`
+for `execute-process`, or `lp` for `list-processes`.
+"""
+
+
+# noinspection PyShadowingBuiltins
+def new_cli(
+    registry: Union[
+        str,
+        "ProcessRegistry",
+        Callable[[], "ProcessRegistry"],
+    ],
+    name: str,
+    version: str,
+    help: str | None = None,
+    summary: str | None = None,
+    context: dict[str, Any] | None = None,
+) -> typer.Typer:
+    """
+    Get the CLI instance configured to use the process registry
+    that is given either by
+
+    - a reference of the form "path.to.module:attribute",
+    - or process registry instance,
+    - or as a no-arg process registry getter function.
+
+    The process registry is usually a singleton in your application.
+
+    The context object `obj` of the returned CLI object
+    will be of type `dict` and will contain
+    a process registry getter function using the key
+    `get_process_registry`.
+
+    The function must be called before any CLI command or
+    callback has been invoked. Otherwise, the provided
+    `get_process_registry` getter will not be recognized and
+    all commands that require the process registry will
+    fail with an `AssertionError`.
+
+    Args:
+        name: The name of the CLI application.
+        registry: A registry reference string,
+            or a registry instance, or a no-arg
+            function that returns a registry instance.
+        help: Optional CLI application help text. If not provided, the default
+            `cuiman` help text will be used.
+        summary: A one-sentence human-readable description of the tool that
+            will be used by the default help text. Hence, used only,
+            if `help` is not provided. Should end with a dot '.'.
+        version: Optional version string. If not provided, the
+            `cuiman` version will be used.
+        context: Additional context values that will be registered with the CLI
+            and can be accessed by commands that you add to the
+            returned `typer.Typer` instance.
+
+    Return:
+        a `typer.Typer` instance
+    """
+    assert bool(registry), "registry argument must be provided"
+    assert bool(name), "name argument must be provided"
+    assert bool(version), "version argument must be provided"
+
+    t = typer.Typer(
+        cls=AliasedGroup,
+        name=name,
+        help=(
+            help
+            or DEFAULT_HELP.format(summary=summary or DEFAULT_SUMMARY.format(name=name))
+        ),
+        add_completion=False,
+        invoke_without_command=True,
+        context_settings={
+            "obj": {
+                PROCESS_REGISTRY_GETTER_KEY: _parse_process_registry_getter(registry),
+                **(context or {}),
+            },
+        },
+    )
+
+    @t.callback()
+    def main(
+        version_: Annotated[
+            bool, typer.Option("--version", help="Show version and exit.")
+        ] = False,
+    ):
+        if version_:
+            from procodile import __version__ as procodile_version
+
+            typer.echo(f"{version} (procodile {procodile_version})")
+            return
+
+    @t.command("execute-process")
+    def execute_process(
+        ctx: typer.Context,
+        process_id: Annotated[Optional[str], PROCESS_ID_ARGUMENT] = None,
+        dotpath: Annotated[bool, DOT_PATH_OPTION] = False,
+        request_inputs: Annotated[Optional[list[str]], REQUEST_INPUT_OPTION] = None,
+        request_subscribers: Annotated[
+            Optional[list[str]], REQUEST_SUBSCRIBER_OPTION
+        ] = None,
+        request_file: Annotated[Optional[str], REQUEST_FILE_OPTION] = None,
+    ):
+        """
+        Execute a process.
+
+        The process request to be submitted may be read from a file given
+        by `--request`, or from `stdin`, or from the `process_id` argument
+        with zero, one, or more `--input` (or `-i`) options.
+
+        The `process_id` argument and any given `--input` options will override
+        settings with the same name found in the given request file or `stdin`,
+        if any.
+        """
+        from procodile import ExecutionRequest, Job
+
+        registry = _get_process_registry(ctx)
+        execution_request = ExecutionRequest.create(
+            process_id=process_id,
+            dotpath=dotpath,
+            inputs=request_inputs,
+            subscribers=request_subscribers,
+            request_path=request_file,
+        )
+        process_id_ = execution_request.process_id
+        process = registry.get(process_id_)
+        if process is None:
+            raise click.ClickException(f"Process {process_id_!r} not found.")
+
+        job = Job.create(process, request=execution_request.to_process_request())
+        job_results = job.run()
+        if job_results is not None:
+            typer.echo(job_results.model_dump_json(indent=2))
+        else:
+            typer.echo(job.job_info.model_dump_json(indent=2))
+
+    @t.command("list-processes", help="List all processes.")
+    def list_processes(ctx: typer.Context):
+        registry = _get_process_registry(ctx)
+        typer.echo(
+            json.dumps(
+                {
+                    k: v.description.model_dump(
+                        mode="json",
+                        by_alias=True,
+                        exclude_none=True,
+                        exclude_defaults=True,
+                        exclude_unset=True,
+                        exclude={"inputs", "outputs"},
+                    )
+                    for k, v in registry.items()
+                },
+                indent=2,
+            )
+        )
+
+    @t.command("get-process", help="Get details of a process.")
+    def get_process(
+        ctx: typer.Context,
+        process_id: Annotated[str, PROCESS_ID_ARGUMENT],
+    ):
+        import json
+
+        registry = _get_process_registry(ctx)
+        process = registry.get(process_id)
+        if process is None:
+            raise click.ClickException(f"Process {process_id!r} not found.")
+
+        typer.echo(
+            json.dumps(
+                process.description.model_dump(
+                    mode="json",
+                    by_alias=True,
+                    exclude_defaults=True,
+                    exclude_none=True,
+                    exclude_unset=True,
+                ),
+                indent=2,
+            )
+        )
+
+    return t
+
+
+__all__ = [
+    "new_cli",
+]
+
+
+def _parse_process_registry_getter(
+    process_registry: Union[
+        str,
+        "ProcessRegistry",
+        Callable[[], "ProcessRegistry"],
+    ],
+) -> Callable[[], "ProcessRegistry"]:
+    process_registry_getter: Callable
+    if isinstance(process_registry, str):
+
+        def process_registry_getter():
+            from gavicore.util.dynimp import import_value
+            from procodile import ProcessRegistry
+
+            return import_value(
+                process_registry, name="process registry", type=ProcessRegistry
+            )
+
+        return process_registry_getter
+
+    elif callable(process_registry):
+        return process_registry
+    else:
+
+        def process_registry_getter():
+            return process_registry
+
+        return process_registry_getter
+
+
+def _get_process_registry(
+    ctx: typer.Context,
+) -> "ProcessRegistry":
+    from procodile import ProcessRegistry
+
+    process_registry_getter = ctx.obj.get(PROCESS_REGISTRY_GETTER_KEY)
+    assert process_registry_getter is not None and callable(process_registry_getter)
+    process_registry = process_registry_getter()
+    assert isinstance(process_registry, ProcessRegistry)
+    return process_registry

{procodile-0.0.7 → procodile-0.0.9}/procodile/job.py

@@ -71,7 +71,8 @@ class JobContext(ABC):
             del frame
         # noinspection PyUnreachableCode
         warnings.warn(
-            "cannot determine current job context; using non-functional dummy"
+            "cannot determine current job context; using non-functional dummy",
+            stacklevel=2,
         )
         return NullJobContext()
 
@@ -141,6 +142,7 @@ class Job(JobContext):
             job_id: Optional job identifier.
                 If omitted, a unique identifier will be generated (UUID4).
 
+
         Returns:
             A new job instance.
 
@@ -148,6 +150,7 @@ class Job(JobContext):
             pydantic.ValidationError: if an input value is not valid
                 with respect to its process input description.
         """
+
         process_desc = process.description
         input_params = request.inputs or {}
         input_default_params = {
@@ -276,6 +279,12 @@ class Job(JobContext):
     def _get_job_results(self, function_result: Any) -> JobResults:
         assert self.job_info.status == JobStatus.successful
         assert self.job_info.processID is not None
+
+        # Outputs are already validated and normalized by
+        # `ExecutionContext.normalize_outputs()`.
+        if isinstance(function_result, dict):
+            return JobResults(**function_result)
+
         outputs = self.process.description.outputs or {}
         output_count = len(outputs)
         return JobResults(