palmengine 0.7.4__py3-none-any.whl

palm/__init__.py

@@ -0,0 +1,23 @@
+"""
+Palm Engine — lightweight orchestration for multi-step transactional workflows.
+
+The ``palm`` package is organized in layers:
+
+- ``palm.app`` — application orchestrator (:class:`~palm.app.PalmApp`, settings, multi-runtime)
+- ``palm.core`` — pure foundational engines (no imports from outside core)
+- ``palm.common`` — shared coordination (plans, submission, hooks, persistence)
+- ``palm.instances`` — durable process instance snapshots
+- ``palm.patterns`` / ``palm.providers`` / ``palm.storages`` — extensible plugin apps
+- ``palm.definitions`` — flow and process definition models
+- ``palm.runtimes`` — CLI, embedded, server, and daemon surfaces
+
+Public API version: ``palm.__version__`` (currently 0.7.4).
+
+PyPI distribution name: ``palmengine`` (``pip install palmengine``).
+"""
+
+from __future__ import annotations
+
+__version__ = "0.7.4"
+
+__all__ = ["__version__"]

palm/app/__init__.py

@@ -0,0 +1,22 @@
+"""
+Palm application layer — configuration, bootstrap, and multi-runtime orchestration.
+
+Use :class:`~palm.app.app.PalmApp` as the top-level entrypoint when embedding
+Palm in services, tests, or multi-process deployments.
+"""
+
+from palm.app.app import CLI_RUNTIME_NAME, PalmApp
+from palm.app.registry import RuntimeHandle, RuntimeKind, RuntimeRegistry
+from palm.app.session import create_cli_app, create_console
+from palm.app.settings import PalmSettings
+
+__all__ = [
+    "CLI_RUNTIME_NAME",
+    "PalmApp",
+    "PalmSettings",
+    "RuntimeHandle",
+    "RuntimeKind",
+    "RuntimeRegistry",
+    "create_cli_app",
+    "create_console",
+]

palm/app/app.py

@@ -0,0 +1,356 @@
+"""
+PalmApp — central application orchestrator for Palm Engine.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Self
+
+from palm.app.bootstrap import (
+    ensure_plugins,
+    load_definitions_for_repository,
+    runtime_start_options,
+)
+from palm.app.registry import RuntimeHandle, RuntimeKind, RuntimeRegistry
+from palm.app.settings import PalmSettings
+from palm.common.managers import InstanceManager
+from palm.common.persistence.instance_repository import InstanceRepository
+from palm.core.storage import StorageEngine
+
+if TYPE_CHECKING:
+    from palm.common.persistence.definition_repository import DefinitionRepository
+    from palm.core.orchestration import Job
+    from palm.definitions.flow import FlowDefinition
+    from palm.definitions.process import ProcessDefinition
+    from palm.instances import ProcessInstance, StateSnapshot
+    from palm.runtimes.base import BaseRuntime
+
+CLI_RUNTIME_NAME = "cli"
+
+
+class PalmApp:
+    """
+    Top-level Palm application — configuration, shared storage, and runtimes.
+
+    A single ``PalmApp`` can host multiple runtimes (embedded, daemon, server)
+    that share one :class:`~palm.core.storage.StorageEngine` for durable
+    definitions and instances.
+
+    Typical usage::
+
+        app = PalmApp().bootstrap()
+        embedded = app.create_runtime("embedded", autostart=True)
+        daemon = app.create_runtime("daemon", name="worker", autostart=True)
+        app.load_definitions()
+    """
+
+    def __init__(
+        self,
+        settings: PalmSettings | None = None,
+        *,
+        storage: StorageEngine | None = None,
+    ) -> None:
+        self.settings = settings or PalmSettings()
+        self._owns_storage = storage is None
+        self.storage = storage if storage is not None else StorageEngine()
+        self._instance_repository = InstanceRepository(self.storage)
+        self._instance_manager = InstanceManager(
+            self._instance_repository,
+            settings=self.settings,
+        )
+        self._runtimes = RuntimeRegistry()
+        self._primary: str | None = None
+        self._bootstrapped = False
+
+    @property
+    def is_bootstrapped(self) -> bool:
+        return self._bootstrapped
+
+    @property
+    def primary_name(self) -> str | None:
+        return self._primary
+
+    @property
+    def instance_manager(self) -> InstanceManager:
+        """Shared instance lifecycle coordinator across runtimes."""
+        return self._instance_manager
+
+    def bootstrap(self) -> Self:
+        """Load plugin apps and mark the application ready for runtime creation."""
+        ensure_plugins()
+        self._bootstrapped = True
+        return self
+
+    def bootstrap_cli(self, **start_options: Any) -> BaseRuntime:
+        """Register the CLI embedded runtime, start it, and load definitions."""
+        self._require_bootstrapped()
+        runtime = self.create_runtime(
+            "embedded",
+            name=CLI_RUNTIME_NAME,
+            autostart=True,
+            set_primary=True,
+            **start_options,
+        )
+        self.load_definitions(name=CLI_RUNTIME_NAME)
+        return runtime
+
+    def create_runtime(
+        self,
+        kind: RuntimeKind,
+        *,
+        name: str | None = None,
+        autostart: bool = False,
+        set_primary: bool | None = None,
+        **start_options: Any,
+    ) -> BaseRuntime:
+        """
+        Construct and optionally start a named runtime sharing app storage.
+
+        Parameters
+        ----------
+        kind:
+            ``embedded``, ``daemon``, or ``server``.
+        name:
+            Registry key. Defaults to ``kind`` or ``{kind}-{n}`` when taken.
+        autostart:
+            When ``True``, call :meth:`start` before returning.
+        set_primary:
+            When ``True``, make this runtime the default for :attr:`runtime`.
+            Defaults to ``True`` when this is the first registered runtime.
+        """
+        self._require_bootstrapped()
+        runtime_name = name or self._default_runtime_name(kind)
+        runtime = self._build_runtime(kind, **start_options)
+        handle = RuntimeHandle(name=runtime_name, kind=kind, runtime=runtime)
+        self._runtimes.register(handle)
+
+        if set_primary is True or (set_primary is None and self._primary is None):
+            self._primary = runtime_name
+
+        if autostart:
+            self.start(runtime_name, **start_options)
+        return runtime
+
+    def runtime(self, name: str | None = None) -> BaseRuntime:
+        """Return a registered runtime (primary by default)."""
+        handle = self._runtimes.get(name or self._require_primary_name())
+        return handle.runtime
+
+    def repository(self, *, runtime_name: str | None = None) -> DefinitionRepository:
+        """Return the definition repository for a registered runtime."""
+        return self.runtime(runtime_name).repository
+
+    def resolve_flow(self, ref: str, *, runtime_name: str | None = None) -> FlowDefinition:
+        """Resolve a flow by display name, falling back to definition id."""
+        from palm.app.resolvers import resolve_flow_for_app
+
+        return resolve_flow_for_app(self, ref, runtime_name=runtime_name)
+
+    def resolve_process(
+        self, ref: str, *, runtime_name: str | None = None
+    ) -> ProcessDefinition:
+        """Resolve a process by display name, falling back to definition id."""
+        from palm.app.resolvers import resolve_process_for_app
+
+        return resolve_process_for_app(self, ref, runtime_name=runtime_name)
+
+    def submit_flow(
+        self,
+        ref: FlowDefinition | str,
+        *,
+        runtime_name: str | None = None,
+        by_id: bool = False,
+        job_id: str | None = None,
+        state: Any = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> Job:
+        """Submit a flow on a registered runtime (primary by default)."""
+        return self.runtime(runtime_name).submit_flow(
+            ref,
+            by_id=by_id,
+            job_id=job_id,
+            state=state,
+            metadata=metadata,
+        )
+
+    def submit_process(
+        self,
+        ref: ProcessDefinition | str,
+        *,
+        runtime_name: str | None = None,
+        by_id: bool = False,
+        job_id: str | None = None,
+        state: Any = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> Job | list[Job]:
+        """Submit a process on a registered runtime (primary by default)."""
+        return self.runtime(runtime_name).submit_process(
+            ref,
+            by_id=by_id,
+            job_id=job_id,
+            state=state,
+            metadata=metadata,
+        )
+
+    def resume_process(self, instance_id: str, *, runtime_name: str | None = None) -> Job:
+        """Resume a persisted process instance on a registered runtime."""
+        return self.runtime(runtime_name).resume_process(instance_id)
+
+    def provide_input(
+        self, job_id: str, value: Any, *, runtime_name: str | None = None
+    ) -> str | None:
+        """Deliver interactive input and resume the job on a registered runtime."""
+        return self.runtime(runtime_name).provide_input(job_id, value)
+
+    def get_job(self, job_id: str, *, runtime_name: str | None = None) -> Job:
+        """Return an orchestration job from a registered runtime."""
+        return self.runtime(runtime_name).get_job(job_id)
+
+    def get_instance(self, instance_id: str, *, runtime_name: str | None = None) -> ProcessInstance:
+        """Load a persisted process instance from the shared manager."""
+        _ = runtime_name
+        return self._instance_manager.get(instance_id)
+
+    def list_instances(self, *, runtime_name: str | None = None) -> list[ProcessInstance]:
+        """List durable process instances (full load via manager)."""
+        _ = runtime_name
+        return self._instance_manager.list_instances()
+
+    def list_instance_summaries(self, *, runtime_name: str | None = None) -> list:
+        """List lightweight instance summaries without loading full payloads."""
+        _ = runtime_name
+        return self._instance_manager.list_summaries()
+
+    def list_instance_snapshots(
+        self, instance_id: str, *, runtime_name: str | None = None
+    ) -> list[StateSnapshot]:
+        """Return point-in-time state snapshots for a persisted instance."""
+        _ = runtime_name
+        return self._instance_manager.list_state_snapshots(instance_id)
+
+    def list_flows(self, *, runtime_name: str | None = None) -> list[FlowDefinition]:
+        """List flow definitions from a registered runtime repository."""
+        return self.repository(runtime_name=runtime_name).list_flows()
+
+    def list_processes(self, *, runtime_name: str | None = None) -> list[ProcessDefinition]:
+        """List process definitions from a registered runtime repository."""
+        return self.repository(runtime_name=runtime_name).list_processes()
+
+    def current_wizard_step(self, job_id: str, *, runtime_name: str | None = None) -> str | None:
+        """Return the active wizard step slug when applicable."""
+        return self.runtime(runtime_name).current_wizard_step(job_id)
+
+    def resume_job(self, job_id: str, *, runtime_name: str | None = None) -> None:
+        """Resume orchestration for a registered job."""
+        self.runtime(runtime_name).orchestration.resume_job(job_id)
+
+    def persist_job(self, job: Job, *, runtime_name: str | None = None) -> None:
+        """Persist job state through a registered runtime executor."""
+        self.runtime(runtime_name).executor.persist_job(job)
+
+    def is_runtime_started(self, name: str | None = None) -> bool:
+        """Return whether a registered runtime has been started."""
+        return self.runtime(name).is_started
+
+    def get_handle(self, name: str) -> RuntimeHandle:
+        """Return the registry record for a named runtime."""
+        return self._runtimes.get(name)
+
+    def set_primary(self, name: str) -> None:
+        """Choose the default runtime returned by :attr:`runtime`."""
+        self._runtimes.get(name)  # validate
+        self._primary = name
+
+    def start(self, name: str, **options: Any) -> BaseRuntime:
+        """Start a registered runtime using app settings merged with ``options``."""
+        handle = self._runtimes.get(name)
+        if handle.runtime.is_started:
+            return handle.runtime
+        merged = runtime_start_options(self.settings, **options)
+        handle.runtime.start(**merged)
+        return handle.runtime
+
+    def stop(self, name: str) -> None:
+        """Stop a single runtime without shutting down shared storage."""
+        self._runtimes.get(name).runtime.stop()
+
+    def load_definitions(self, *, name: str | None = None) -> int:
+        """
+        Hydrate definition catalogs for one or all registered runtimes.
+
+        Returns the total number of definition records touched.
+        """
+        if name is not None:
+            handle = self._runtimes.get(name)
+            return load_definitions_for_repository(handle.runtime.repository, self.settings)
+
+        total = 0
+        for handle in self._runtimes.items():
+            total += load_definitions_for_repository(handle.runtime.repository, self.settings)
+        return total
+
+    def running(self) -> list[str]:
+        """Return names of runtimes that are currently started."""
+        return [handle.name for handle in self._runtimes.items() if handle.is_started]
+
+    def shutdown(self) -> None:
+        """Stop all runtimes and release shared storage when owned by the app."""
+        for handle in self._runtimes.items():
+            if handle.is_started:
+                handle.runtime.stop()
+        self._instance_manager.shutdown()
+        if self._owns_storage and self.storage.is_initialized:
+            self.storage.shutdown()
+        self._runtimes.clear()
+        self._primary = None
+
+    def __enter__(self) -> Self:
+        if not self._bootstrapped:
+            self.bootstrap()
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.shutdown()
+
+    def _build_runtime(self, kind: RuntimeKind, **options: Any) -> BaseRuntime:
+        if kind == "embedded":
+            from palm.runtimes.embedded import EmbeddedRuntime
+
+            return EmbeddedRuntime(
+                storage=self.storage,
+                instance_manager=self._instance_manager,
+            )
+        if kind == "daemon":
+            from palm.runtimes.daemon import DaemonRuntime
+
+            return DaemonRuntime(
+                storage=self.storage,
+                instance_manager=self._instance_manager,
+            )
+        if kind == "server":
+            from palm.runtimes.server import ServerRuntime
+
+            return ServerRuntime(
+                storage=self.storage,
+                instance_manager=self._instance_manager,
+                host=str(options.pop("host", "127.0.0.1")),
+                port=int(options.pop("port", 8080)),
+            )
+        raise ValueError(f"Unknown runtime kind {kind!r}")
+
+    def _default_runtime_name(self, kind: RuntimeKind) -> str:
+        if kind not in self._runtimes.names():
+            return kind
+        index = 1
+        while f"{kind}-{index}" in self._runtimes:
+            index += 1
+        return f"{kind}-{index}"
+
+    def _require_primary_name(self) -> str:
+        if self._primary is None:
+            raise RuntimeError("No primary runtime; call create_runtime() first")
+        return self._primary
+
+    def _require_bootstrapped(self) -> None:
+        if not self._bootstrapped:
+            raise RuntimeError("PalmApp is not bootstrapped; call bootstrap() first")

palm/app/bootstrap.py

@@ -0,0 +1,124 @@
+"""
+Application bootstrap — plugin loading and definition catalog hydration.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+from typing import Any
+
+import palm.patterns  # — autoload pattern apps
+import palm.providers  # — autoload provider apps
+import palm.storages  # noqa: F401 — autoload core storage apps
+from palm.app.settings import PalmSettings
+from palm.common.persistence.definition_repository import DefinitionRepository
+from palm.common.storage import StorageFactory
+
+
+def ensure_plugins() -> None:
+    """Import extensible plugin packages so registries are populated."""
+    # Side-effect imports above register patterns, providers, and storages.
+    return None
+
+
+def hydrate_definitions_from_storage(repository: DefinitionRepository) -> int:
+    """Load flow/process definitions from storage into the in-memory cache."""
+    count = 0
+    for flow in repository.list_flows():
+        repository.register_flow(flow)
+        count += 1
+    for process in repository.list_processes():
+        repository.register_process(process)
+        count += 1
+    return count
+
+
+def load_definition_modules(
+    repository: DefinitionRepository,
+    *,
+    roots: list[Path],
+) -> int:
+    """Import ``register_definitions`` modules from the given directories."""
+    loaded = 0
+    seen: set[Path] = set()
+    for root in roots:
+        if not root.is_dir():
+            continue
+        for path in sorted(root.glob("*.py")):
+            if path.name.startswith("_") or path in seen:
+                continue
+            seen.add(path)
+            if _import_register(path, repository):
+                loaded += 1
+    return loaded
+
+
+def package_definition_roots(settings: PalmSettings) -> list[Path]:
+    """Built-in example definition paths bundled with Palm."""
+    if not settings.load_example_definitions:
+        return []
+    package_root = Path(__file__).resolve().parents[3]
+    return [package_root / "examples" / "definitions"]
+
+
+def all_definition_roots(settings: PalmSettings) -> list[Path]:
+    """Merge configured, cwd, and packaged definition directories."""
+    roots: list[Path] = []
+    if settings.data_dir is not None:
+        roots.append(settings.data_dir / "definitions")
+    roots.append(Path.cwd() / "examples" / "definitions")
+    roots.extend(package_definition_roots(settings))
+    # Preserve order while deduplicating
+    unique: list[Path] = []
+    seen: set[Path] = set()
+    for root in roots:
+        resolved = root.resolve()
+        if resolved not in seen:
+            seen.add(resolved)
+            unique.append(root)
+    return unique
+
+
+def load_definitions_for_repository(
+    repository: DefinitionRepository,
+    settings: PalmSettings,
+) -> int:
+    """Hydrate storage-backed definitions and import code-defined catalogs."""
+    count = hydrate_definitions_from_storage(repository)
+    count += load_definition_modules(repository, roots=all_definition_roots(settings))
+    return count
+
+
+def runtime_start_options(settings: PalmSettings, **overrides: Any) -> dict[str, Any]:
+    """Build keyword arguments for :meth:`~palm.runtimes.base.BaseRuntime.start`."""
+    options: dict[str, Any] = {
+        "storage_backend": settings.storage_backend,
+        "backend_options": StorageFactory.backend_options(settings=settings),
+        "observability": settings.observability,
+        "auth_enforce": settings.auth_enforce,
+        "auth_roles": list(settings.auth_roles),
+    }
+    if settings.max_concurrent_jobs is not None:
+        options["max_concurrent_jobs"] = settings.max_concurrent_jobs
+    options["enable_state_snapshot"] = settings.enable_state_snapshot
+    options["snapshot_on_status"] = list(settings.snapshot_on_status)
+    options["max_snapshots_per_instance"] = settings.max_snapshots_per_instance
+    options["max_loaded_instances"] = settings.max_loaded_instances
+    options["max_concurrent_active"] = settings.max_concurrent_active
+    options["reconcile_on_startup"] = settings.reconcile_instances_on_startup
+    options.update(overrides)
+    return options
+
+
+def _import_register(path: Path, repository: DefinitionRepository) -> bool:
+    spec = importlib.util.spec_from_file_location(f"palm_app_definitions_{path.stem}", path)
+    if spec is None or spec.loader is None:
+        return False
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    register = getattr(module, "register_definitions", None)
+    if not callable(register):
+        return False
+    register(repository)
+    return True

palm/app/cli_settings.py

@@ -0,0 +1,63 @@
+"""
+CLI settings resolution — env-first, flag overrides only when explicit.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from palm.app.settings import PalmSettings, SchedulerPolicy
+from palm.common.storage import StorageFactory
+
+DURABLE_STORAGE_BACKENDS: frozenset[str] = frozenset({"filesystem", "postgres", "mongodb"})
+
+
+def is_durable_storage(backend: str | None) -> bool:
+    """Return whether ``backend`` persists data across process restarts."""
+    if not backend:
+        return False
+    return backend.strip().lower() in DURABLE_STORAGE_BACKENDS
+
+
+def resolve_cli_settings(
+    *,
+    storage_backend: str | None = None,
+    data_dir: Path | None = None,
+    settings: PalmSettings | None = None,
+    align_shared_storage: str | None = None,
+    enable_state_snapshot: bool | None = None,
+    max_loaded_instances: int | None = None,
+    max_concurrent_active: int | None = None,
+    default_scheduler: SchedulerPolicy | None = None,
+) -> PalmSettings:
+    """
+    Build CLI settings with environment variables as the base.
+
+    Precedence (highest last):
+
+    1. ``PALM_*`` environment variables via :class:`~palm.app.settings.PalmSettings`
+    2. ``--config`` file (loaded into ``PalmSettings`` before this merge)
+    3. Explicit ``settings`` argument (when passed to bootstrap)
+    4. CLI flags (only when not ``None``)
+    5. ``align_shared_storage`` — backend name from a pre-opened shared engine
+    """
+    cfg = settings.model_copy(deep=True) if settings is not None else PalmSettings()
+
+    if storage_backend is not None:
+        cfg.storage_backend = storage_backend
+    if data_dir is not None:
+        cfg.data_dir = data_dir
+    if align_shared_storage is not None:
+        cfg.storage_backend = align_shared_storage
+    if enable_state_snapshot is not None:
+        cfg.enable_state_snapshot = enable_state_snapshot
+    if max_loaded_instances is not None:
+        cfg.max_loaded_instances = max_loaded_instances
+    if max_concurrent_active is not None:
+        cfg.max_concurrent_active = max_concurrent_active
+    if default_scheduler is not None:
+        cfg.default_scheduler = default_scheduler
+
+    if is_durable_storage(cfg.storage_backend) and cfg.data_dir is None:
+        cfg.data_dir = StorageFactory.resolve_data_dir(None)
+    return cfg

palm/app/registry.py

@@ -0,0 +1,72 @@
+"""
+Runtime registry — named runtime handles managed by :class:`~palm.app.app.PalmApp`.
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Literal
+
+if TYPE_CHECKING:
+    from palm.runtimes.base import BaseRuntime
+
+RuntimeKind = Literal["embedded", "daemon", "server"]
+
+
+@dataclass
+class RuntimeHandle:
+    """A named runtime instance registered on the application."""
+
+    name: str
+    kind: RuntimeKind
+    runtime: BaseRuntime
+
+    @property
+    def is_started(self) -> bool:
+        return self.runtime.is_started
+
+
+class RuntimeRegistry:
+    """Thread-safe in-memory map of named runtimes."""
+
+    def __init__(self) -> None:
+        self._entries: dict[str, RuntimeHandle] = {}
+        self._lock = threading.RLock()
+
+    def __len__(self) -> int:
+        with self._lock:
+            return len(self._entries)
+
+    def __contains__(self, name: str) -> bool:
+        with self._lock:
+            return name in self._entries
+
+    def names(self) -> list[str]:
+        with self._lock:
+            return sorted(self._entries)
+
+    def register(self, handle: RuntimeHandle) -> RuntimeHandle:
+        with self._lock:
+            if handle.name in self._entries:
+                raise ValueError(f"Runtime {handle.name!r} is already registered")
+            self._entries[handle.name] = handle
+            return handle
+
+    def get(self, name: str) -> RuntimeHandle:
+        with self._lock:
+            try:
+                return self._entries[name]
+            except KeyError as exc:
+                available = ", ".join(sorted(self._entries)) or "(none)"
+                raise KeyError(
+                    f"Unknown runtime {name!r}. Registered: {available}"
+                ) from exc
+
+    def items(self) -> list[RuntimeHandle]:
+        with self._lock:
+            return [self._entries[name] for name in sorted(self._entries)]
+
+    def clear(self) -> None:
+        with self._lock:
+            self._entries.clear()