dagflows 0.1.0__py3-none-any.whl

dagflows/__init__.py

@@ -0,0 +1,72 @@
+"""
+dagflows - author and run workflow nodes.
+"""
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+from ._runtime.content import ContentType
+from ._runtime.envelope import Ctx, Input, Inputs
+from ._runtime.errors import (
+    EXECUTION,
+    INFRASTRUCTURE,
+    PERMANENT,
+    TIMEOUT,
+    Fail,
+    InputTooLarge,
+    InputUnavailable,
+    OutputTooLarge,
+)
+from ._runtime.result import Result
+from ._runtime.runner import run
+
+if TYPE_CHECKING:
+    from ._authoring.config import ExecutionConfig, RetryConfig
+    from ._authoring.node import NodeRef
+    from ._authoring.workflow import Workflow
+
+# Authoring names resolve on first use rather than at import. A node running in
+# a microVM imports this module and touches none of them, so the guest never
+# loads code that only ever runs on a laptop or in the builder.
+_AUTHORING = {
+    "ExecutionConfig": "._authoring.config",
+    "RetryConfig": "._authoring.config",
+    "NodeRef": "._authoring.node",
+    "Workflow": "._authoring.workflow",
+}
+
+
+def __getattr__(name: str) -> Any:
+    module = _AUTHORING.get(name)
+    if module is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+    return getattr(import_module(module, __name__), name)
+
+
+def __dir__() -> list[str]:
+    return sorted(__all__)
+
+
+__all__ = [
+    "ContentType",
+    "Ctx",
+    "Input",
+    "Inputs",
+    "Result",
+    "Fail",
+    "InputTooLarge",
+    "InputUnavailable",
+    "OutputTooLarge",
+    "PERMANENT",
+    "INFRASTRUCTURE",
+    "TIMEOUT",
+    "EXECUTION",
+    "run",
+    "Workflow",
+    "ExecutionConfig",
+    "RetryConfig",
+    "NodeRef",
+]
+
+__version__ = "0.1.0"

dagflows/__main__.py

@@ -0,0 +1,67 @@
+"""The python SDK's command line interface.
+
+    python -m dagflows build manifest <module>    emit dagflows-manifest.json
+    python -m dagflows build validate <module>    check a project without writing
+    python -m dagflows dev run <entrypoint>       run a node locally
+    python -m dagflows dev fixture <entrypoint>   write a starting fixture
+
+Invoked via `python -m dagflows` to avoid command collisions with the standalone
+CLI and to ensure execution uses the current Python environment.
+
+The internal `invoke` command is reserved for platform runtime execution and
+reports failures via output envelopes. All other CLI commands report failures
+to stderr with standard process exit codes.
+"""
+
+from __future__ import annotations
+
+import sys
+
+from ._cli import USAGE as USAGE_CODE
+from ._cli import CommandError, report, take_flag
+
+HELP = """python -m dagflows - author and run workflow nodes
+
+  build manifest <module> [-o dagflows-manifest.json]
+  build validate <module>
+  dev   run <entrypoint> [--input <parent>=<file|json>] [options]
+  dev   fixture <entrypoint> [-o fixture.json]
+
+  --json    machine readable output, on any command above
+
+Exit codes: 0 success, 1 the operation failed, 2 the command was wrong."""
+
+
+def main(argv: list[str]) -> int:
+    # The internal invoke command handles runtime execution directly.
+    if argv and argv[0] == "invoke":
+        from ._runtime.dispatch import main as invoke
+
+        return invoke(argv)
+
+    argv, as_json = take_flag(argv, "--json")
+
+    if not argv or argv[0] in ("-h", "--help", "help"):
+        print(HELP)
+        return 0 if argv else USAGE_CODE
+
+    command, rest = argv[0], argv[1:]
+
+    try:
+        match command:
+            case "build":
+                from ._cli.build import main as build
+
+                return build(rest, as_json)
+            case "dev":
+                from ._cli.dev import main as dev
+
+                return dev(rest, as_json)
+            case _:
+                raise CommandError(f"unknown command {command!r}\n\n{HELP}", USAGE_CODE)
+    except CommandError as error:
+        return report(command, error, as_json)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

dagflows/_authoring/__init__.py

@@ -0,0 +1 @@
+"""The authoring half: runs on a laptop and in the builder, never in a microVM."""

dagflows/_authoring/config.py

@@ -0,0 +1,58 @@
+"""Per-node settings that reach the manifest."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+def _non_negative(name: str, value: int | None) -> None:
+    if value is not None and value < 0:
+        raise ValueError(f"{name} cannot be negative, got {value}")
+
+
+@dataclass(frozen=True)
+class ExecutionConfig:
+    """Resource and timeout configuration for a node."""
+
+    timeout: int | None = None
+    memory_limit_mb: int | None = None
+    milli_cores: int | None = None
+    # Declaring this is what gets the node a multipart upload, letting it emit
+    # more than it can hold. The platform may issue less than asked for, so a
+    # node reads what it actually got from ctx rather than assuming.
+    max_output_mb: int | None = None
+
+    def __post_init__(self) -> None:
+        _non_negative("timeout", self.timeout)
+        _non_negative("memory_limit_mb", self.memory_limit_mb)
+        _non_negative("milli_cores", self.milli_cores)
+        _non_negative("max_output_mb", self.max_output_mb)
+
+    def as_config(self) -> dict[str, int]:
+        """The settings that belong in the node's config map."""
+        out: dict[str, int] = {}
+        if self.memory_limit_mb is not None:
+            out["memory_limit_mb"] = self.memory_limit_mb
+        if self.milli_cores is not None:
+            out["milli_cores"] = self.milli_cores
+        if self.max_output_mb is not None:
+            out["max_output_mb"] = self.max_output_mb
+        return out
+
+
+@dataclass(frozen=True)
+class RetryConfig:
+    """Retry configuration for a node."""
+
+    max_attempts: int = 0
+    initial_backoff_ms: int = 1000
+
+    def __post_init__(self) -> None:
+        _non_negative("max_attempts", self.max_attempts)
+        _non_negative("initial_backoff_ms", self.initial_backoff_ms)
+
+    def as_manifest(self) -> dict[str, int]:
+        return {
+            "max_attempts": self.max_attempts,
+            "initial_backoff_ms": self.initial_backoff_ms,
+        }

dagflows/_authoring/node.py

@@ -0,0 +1,44 @@
+"""Node handles.
+
+``depends`` takes these, never strings: an undefined name is then a NameError
+at import, before anything is built, and an editor can list exactly the nodes
+that exist. Renaming a node in a visual builder is routine, which is why the
+one place a string survives is ``external_node``.
+"""
+
+from __future__ import annotations
+
+import functools
+from typing import Any, Callable
+
+
+class NodeRef:
+    """A registered node reference that remains directly callable."""
+
+    def __init__(self, key: str, func: Callable[..., Any] | None, entrypoint: str = "") -> None:
+        self.key = key
+        self.entrypoint = entrypoint
+        self._func = func
+        
+        if func is not None:
+            functools.update_wrapper(self, func)
+
+    @property
+    def external(self) -> bool:
+        """Whether this node lives in another project."""
+        return self._func is None
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        if self._func is None:
+            raise TypeError(
+                f"{self.key!r} is an external node declared in this project, "
+                "so it has no body here to call"
+            )
+        
+        return self._func(*args, **kwargs)
+
+    def __repr__(self) -> str:
+        if self.external:
+            return f"<NodeRef {self.key!r} external>"
+        
+        return f"<NodeRef {self.key!r} {self.entrypoint}>"

dagflows/_authoring/workflow.py

@@ -0,0 +1,211 @@
+"""Declaring a workflow and emitting its manifest.
+
+Node bodies never run here. Decoration records metadata and ``manifest()``
+serialises the registry, so importing a workflow module is safe on a laptop
+and inside the builder alike.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+from typing import Any, Callable, Iterable, Sequence
+
+from .config import ExecutionConfig, RetryConfig
+from .node import NodeRef
+
+# Builder validation constants and patterns.
+MANIFEST_VERSION = 1
+LANGUAGE = "python"
+NODE_KEY = re.compile(r"^[a-zA-Z][a-zA-Z0-9_-]{0,63}$")
+RUNTIME_VERSION = re.compile(r"^[0-9][0-9A-Za-z.\-]{0,31}$")
+
+# Registry of declared workflows in the current process.
+_declared: list[Workflow] = []
+
+
+def declared() -> list[Workflow]:
+    """Workflows created so far in this interpreter."""
+    return list(_declared)
+
+
+def _entrypoint(func: Callable[..., Any]) -> str:
+    """The module:function form the platform dispatches on."""
+    module = getattr(func, "__module__", "")
+
+    if module == "__main__":
+        module = _module_of_main(func)
+
+    if not module:
+        raise ValueError(
+            f"cannot tell which module defines {func.__name__!r}; emit the manifest with "
+            "`python -m dagflows build manifest <module>` so the node is imported under its real name"
+        )
+
+    return f"{module}:{func.__name__}"
+
+
+def _module_of_main(func: Callable[..., Any]) -> str:
+    """Recover a dotted name for a module being run as a script."""
+    main = sys.modules.get("__main__")
+    path = getattr(main, "__file__", "")
+    if not path:
+        return ""
+
+    try:
+        relative = Path(path).resolve().relative_to(Path.cwd())
+    except ValueError:
+        # Outside the project root.
+        return ""
+
+    return ".".join(relative.with_suffix("").parts)
+
+
+class Workflow:
+    """Workflow definition and node registry for a project."""
+
+    def __init__(
+        self,
+        name: str = "",
+        *,
+        version: str = "",
+        max_concurrent_nodes: int = 0,
+        max_cycle_count: int = 0,
+    ) -> None:
+        if version and not RUNTIME_VERSION.match(version):
+            raise ValueError(
+                f"version {version!r} invalid, want a version like '3.12', not an image reference"
+            )
+        
+        if max_concurrent_nodes < 0 or max_cycle_count < 0:
+            raise ValueError("workflow limits cannot be negative")
+
+        self.name = name
+        self.version = version
+        self.max_concurrent_nodes = max_concurrent_nodes
+        self.max_cycle_count = max_cycle_count
+        self._nodes: list[dict[str, Any]] = []
+        self._keys: set[str] = set()
+        _declared.append(self)
+
+    def node(
+        self,
+        key: str = "",
+        *,
+        depends: Sequence[NodeRef] = (),
+        execution: ExecutionConfig | None = None,
+        retry: RetryConfig | None = None,
+        config: dict[str, Any] | None = None,
+        type: str = "",
+    ) -> Callable[[Callable[..., Any]], NodeRef]:
+        """Register the decorated function as a workflow node."""
+
+        def register(func: Callable[..., Any]) -> NodeRef:
+            node_key = key or func.__name__
+            self._claim(node_key)
+
+            local, external = _split_depends(node_key, depends)
+            entry: dict[str, Any] = {"key": node_key, "entrypoint": _entrypoint(func)}
+
+            if type:
+                entry["type"] = type
+            
+            if local:
+                entry["depends"] = local
+            
+            if external:
+                entry["external_depends"] = external
+
+            settings = dict(config or {})
+
+            if execution is not None:
+                settings.update(execution.as_config())
+                if execution.timeout is not None:
+                    entry["timeout_seconds"] = execution.timeout
+            
+            if settings:
+                entry["config"] = settings
+            
+            if retry is not None:
+                entry["retry"] = retry.as_manifest()
+
+            self._nodes.append(entry)
+            return NodeRef(node_key, func, entry["entrypoint"])
+
+        return register
+
+    def external_node(self, key: str) -> NodeRef:
+        """Declare an external dependency on a node defined in another project."""
+        _check_key(key)
+        return NodeRef(key, None)
+
+    def manifest(self) -> dict[str, Any]:
+        """The dagflows-manifest.json body for this project."""
+        if not self._nodes:
+            raise ValueError("this workflow declares no nodes, so there is nothing to build")
+
+        for node in self._nodes:
+            for parent in node.get("depends", ()):
+                if parent not in self._keys:
+                    # Reject node references from other workflow instances.
+                    raise ValueError(
+                        f"node {node['key']!r} depends on {parent!r}, which this workflow "
+                        f"does not define; its nodes are: {', '.join(sorted(self._keys))}"
+                    )
+
+        runtime: dict[str, Any] = {"language": LANGUAGE}
+        if self.version:
+            runtime["version"] = self.version
+
+        out: dict[str, Any] = {"v": MANIFEST_VERSION, "runtime": runtime}
+
+        if self.name:
+            settings: dict[str, Any] = {"name": self.name}
+
+            if self.max_concurrent_nodes:
+                settings["max_concurrent_nodes"] = self.max_concurrent_nodes
+            
+            if self.max_cycle_count:
+                settings["max_cycle_count"] = self.max_cycle_count
+            
+            out["workflow"] = settings
+
+        out["nodes"] = [dict(node) for node in self._nodes]
+        return out
+
+    def _claim(self, key: str) -> None:
+        _check_key(key)
+
+        if key in self._keys:
+            raise ValueError(f"duplicate node key {key!r} in this project")
+        
+        self._keys.add(key)
+
+
+def _check_key(key: str) -> None:
+    if not NODE_KEY.match(key):
+        raise ValueError(
+            f"node key {key!r} invalid, want a letter then letters, digits, _ or -, max 64 chars"
+        )
+
+
+def _split_depends(node_key: str, depends: Iterable[Any]) -> tuple[list[str], list[str]]:
+    """Separate this project's parents from the ones another project defines."""
+    local: list[str] = []
+    external: list[str] = []
+
+    for parent in depends:
+        if isinstance(parent, str):
+            raise TypeError(
+                f"node {node_key!r} depends on the string {parent!r}; pass the handle the "
+                "decorator returned, or wf.external_node(...) for a node in another project"
+            )
+        
+        if not isinstance(parent, NodeRef):
+            raise TypeError(
+                f"node {node_key!r} depends on {parent!r}, which is not a node handle"
+            )
+        (external if parent.external else local).append(parent.key)
+
+    return local, external

dagflows/_cli/__init__.py

@@ -0,0 +1,57 @@
+"""
+Developer CLI commands, exit codes, and output formatting helpers.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+
+# Standard CLI process exit codes.
+OK = 0
+FAILED = 1  # the operation ran and did not succeed
+USAGE = 2  # the command was wrong
+
+
+class CommandError(Exception):
+    """Base exception for user-facing CLI command errors."""
+
+    def __init__(self, message: str, code: int = FAILED) -> None:
+        super().__init__(message)
+        self.message = message
+        self.code = code
+
+
+def fail(message: str) -> CommandError:
+    return CommandError(message, FAILED)
+
+
+def misuse(message: str) -> CommandError:
+    return CommandError(message, USAGE)
+
+
+def report(command: str, error: CommandError, as_json: bool) -> int:
+    """Emit an error message to stderr or structured JSON."""
+    if as_json:
+        emit({"ok": False, "command": command, "error": error.message}, as_json=True)
+    else:
+        print(f"dagflows {command}: {error.message}", file=sys.stderr)
+    return error.code
+
+
+def emit(payload: dict[str, Any], as_json: bool, human: str = "") -> None:
+    """Emit command output as structured JSON or human-readable text."""
+    if as_json:
+        json.dump(payload, sys.stdout, indent=2)
+        sys.stdout.write("\n")
+        return
+
+    if human:
+        print(human)
+
+
+def take_flag(argv: list[str], *names: str) -> tuple[list[str], bool]:
+    """Extract a boolean flag from argv and return remaining arguments."""
+    rest = [item for item in argv if item not in names]
+    return rest, len(rest) != len(argv)