runfox 0.0.2__py3-none-any.whl

runfox/__init__.py

@@ -0,0 +1,5 @@
+from .backend.base import Backend
+from .results import (Complete, Dispatch, DispatchJob, Halt, Pending,
+                      StateChangeEvent)
+from .status import StepStatus, WorkflowStatus
+from .workflow import Workflow

runfox/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.0.2'
+__version_tuple__ = version_tuple = (0, 0, 2)
+
+__commit_id__ = commit_id = 'gf1470cbc4'

runfox/backend/__init__.py

@@ -0,0 +1,9 @@
+from .models import StepRecord, WorkflowRecord
+from .base import Backend
+from .inprocess_runner import InProcessRunner
+from .inprocess_worker import InProcessWorker
+from .runner import Runner
+from .sqlite_runner import SqliteRunner
+from .store import Store
+from .inmemory_store import InMemoryStore
+from .sqlite_store import SqliteStore

runfox/backend/base.py

@@ -0,0 +1,268 @@
+"""
+base.py -- Backend
+
+Composes a Store and a Runner. All workflow lifecycle operations are
+implemented here in terms of self._store.load() and self._store.write().
+dispatch() and gather() delegate to self._runner.
+
+Construction
+------------
+  Backend(store=s, runner=r)  -- explicit store and runner
+
+Composite key accessors
+-----------------------
+workflow_execution_id(record)
+step_key(wf_exec_id, step_id)
+step_run_key(wf_exec_id, step_id, run_id)
+"""
+
+import dataclasses
+import datetime
+import hashlib
+import json
+import random
+import socket
+import string
+from typing import Any
+
+from runfox.status import StepStatus, WorkflowStatus
+
+from .models import StepRecord, WorkflowRecord
+from .inprocess_runner import InProcessRunner
+from .inprocess_worker import InProcessWorker
+from .inmemory_store import InMemoryStore
+
+class Backend:
+
+    def __init__(
+        self,
+        executor=None,
+        store=None,
+        runner=None,
+        poll_interval: float = 0.1,
+        on_state_change=None,
+    ):
+        """
+        on_state_change: optional callback fired after every state merge.
+            Signature: (workflow_execution_id, previous_state, new_state) -> None.
+            Must be pure: no side effects, no exceptions, no backend calls.
+            The callback fires inside a write cycle; any mutation of backend
+            state from within it will produce inconsistent records.
+        """
+        if store is None:
+            store = InMemoryStore()
+        if runner is None:
+            runner = InProcessRunner()
+        self._store = store
+        self._runner = runner
+        self._worker = InProcessWorker(runner, executor) if executor else None
+        self.poll_interval = poll_interval
+        self._on_state_change = on_state_change
+
+    # ------------------------------------------------------------------
+    # Private ID generation
+    # ------------------------------------------------------------------
+
+    def _make_workflow_id(self, spec: dict) -> str:
+        canonical = json.dumps(spec, sort_keys=True, separators=(",", ":"))
+        return hashlib.md5(canonical.encode()).hexdigest()
+
+    def _make_execution_id(self) -> str:
+        ts = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S")
+        suffix = "".join(random.choices(string.hexdigits[:16], k=4)).lower()
+        return f"{ts}-{suffix}"
+
+    # ------------------------------------------------------------------
+    # Public composite key accessors
+    # ------------------------------------------------------------------
+
+    def workflow_execution_id(self, record: WorkflowRecord) -> str:
+        return f"{record.workflow_id}#{record.execution_id}"
+
+    def step_key(self, wf_exec_id: str, step_id: str) -> str:
+        return f"{wf_exec_id}#{step_id}"
+
+    def step_run_key(self, wf_exec_id: str, step_id: str, run_id: int) -> str:
+        return f"{wf_exec_id}#{step_id}#{run_id}"
+
+    # ------------------------------------------------------------------
+    # Utilities
+    # ------------------------------------------------------------------
+
+    def _now_iso(self) -> str:
+        return datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+    def _make_step_record(self, step_id: str) -> StepRecord:
+        return StepRecord(id=step_id)
+
+    # ------------------------------------------------------------------
+    # Store pass-throughs (used by Workflow and tests)
+    # ------------------------------------------------------------------
+
+    def load(self, workflow_execution_id: str) -> WorkflowRecord:
+        return self._store.load(workflow_execution_id)
+
+    def write(self, record: WorkflowRecord) -> None:
+        self._store.write(record)
+
+    # ------------------------------------------------------------------
+    # create
+    # ------------------------------------------------------------------
+
+    def create(self, spec: dict, inputs: dict = None) -> str:
+        record = WorkflowRecord(
+            workflow_id=self._make_workflow_id(spec),
+            execution_id=self._make_execution_id(),
+            spec=spec,
+            inputs=inputs or {},
+            state={},
+            steps={
+                step["id"]: self._make_step_record(step["id"]) for step in spec["steps"]
+            },
+            status=WorkflowStatus.PENDING,
+        )
+        self._store.write(record)
+        return self.workflow_execution_id(record)
+
+    # ------------------------------------------------------------------
+    # Named step operations
+    # ------------------------------------------------------------------
+
+    def mark_in_progress(self, workflow_execution_id: str, step_id: str) -> None:
+        record = self._store.load(workflow_execution_id)
+        new_step = dataclasses.replace(
+            record.steps[step_id],
+            status=StepStatus.IN_PROGRESS,
+            start_time=self._now_iso(),
+            host=socket.gethostname(),
+        )
+        self._store.write(
+            dataclasses.replace(
+                record,
+                status=WorkflowStatus.IN_PROGRESS,
+                steps={**record.steps, step_id: new_step},
+            )
+        )
+
+    def mark_complete(self, workflow_execution_id: str, step_id: str) -> None:
+        record = self._store.load(workflow_execution_id)
+        new_step = dataclasses.replace(
+            record.steps[step_id],
+            status=StepStatus.COMPLETE,
+            end_time=self._now_iso(),
+        )
+        self._store.write(
+            dataclasses.replace(
+                record,
+                steps={**record.steps, step_id: new_step},
+            )
+        )
+
+    def mark_halted(self, workflow_execution_id: str, step_id: str) -> None:
+        record = self._store.load(workflow_execution_id)
+        new_step = dataclasses.replace(
+            record.steps[step_id],
+            status=StepStatus.HALTED,
+            end_time=self._now_iso(),
+        )
+        self._store.write(
+            dataclasses.replace(
+                record,
+                status=WorkflowStatus.HALTED,
+                steps={**record.steps, step_id: new_step},
+            )
+        )
+
+    def write_step_output(
+        self, workflow_execution_id: str, step_id: str, output: dict
+    ) -> None:
+        record = self._store.load(workflow_execution_id)
+        new_step = dataclasses.replace(record.steps[step_id], output=output)
+        self._store.write(
+            dataclasses.replace(
+                record,
+                steps={**record.steps, step_id: new_step},
+            )
+        )
+
+    def merge_workflow_state(self, workflow_execution_id, output, event=None):
+        """
+        Update the internal workflow state with a new output.
+        Latest-wins update of keys.
+
+        on_state_change is called with
+        (workflow_execution_id, prev_state, new_state, event) if set.
+        event is a StateChangeEvent identifying the step that triggered
+        the merge, or None if called outside a step result context.
+        """
+        if not output:
+            return
+        record = self._store.load(workflow_execution_id)
+        new_state = {**record.state, **output}
+        if self._on_state_change:
+            # Callback must be pure. Side effects here are undefined behaviour --
+            # the store write has completed but the workflow has not advanced.
+            self._on_state_change(workflow_execution_id, record.state, new_state, event)
+        self._store.write(dataclasses.replace(record, state=new_state))
+
+    def write_workflow_outcome(self, workflow_execution_id: str, outcome: Any) -> None:
+        """send the result of the workflow to the store for writing"""
+        record = self._store.load(workflow_execution_id)
+        status = (
+            WorkflowStatus.HALTED
+            if record.status == WorkflowStatus.HALTED
+            else WorkflowStatus.COMPLETE
+        )
+        self._store.write(dataclasses.replace(record, outcome=outcome, status=status))
+
+    def reset_step(self, workflow_execution_id: str, step_id: str) -> None:
+        record = self._store.load(workflow_execution_id)
+        new_step = dataclasses.replace(
+            record.steps[step_id],
+            status=StepStatus.READY,
+            output=None,
+            start_time=None,
+            end_time=None,
+            run_id=record.steps[step_id].run_id + 1,
+        )
+        self._store.write(
+            dataclasses.replace(
+                record,
+                steps={**record.steps, step_id: new_step},
+            )
+        )
+
+    def reset_for_retry(self, workflow_execution_id: str, step_id: str) -> None:
+        record = self._store.load(workflow_execution_id)
+        new_step = dataclasses.replace(
+            record.steps[step_id],
+            status=StepStatus.RETRY,
+            run_id=record.steps[step_id].run_id + 1,
+            start_time=None,
+            end_time=None,
+        )
+        self._store.write(
+            dataclasses.replace(
+                record,
+                steps={**record.steps, step_id: new_step},
+            )
+        )
+
+    # ------------------------------------------------------------------
+    # Runner pass-throughs
+    # ------------------------------------------------------------------
+
+    def dispatch(self, workflow_execution_id: str, jobs: list) -> None:
+        self._runner.dispatch(workflow_execution_id, jobs)
+
+    def gather(self, workflow_execution_id: str) -> list:
+        return self._runner.gather(workflow_execution_id)
+
+    def pending_tasks(self) -> list:
+        return self._runner.list_pending_jobs()
+
+    def take_tasks(self) -> list:
+        return self._runner.take_pending_jobs()
+
+    def submit_result(self, workflow_execution_id, step_id, output):
+        self._runner.submit_work_result(workflow_execution_id, step_id, output)

runfox/backend/inmemory_store.py

@@ -0,0 +1,36 @@
+"""
+store.py -- Store
+
+Two primitives:
+
+  load(workflow_execution_id) -> WorkflowRecord
+  write(record) -> None
+
+Implementations: InMemoryStore, SqliteStore.
+SqliteStore manages the workflows table only. The tasks table belongs to SqliteRunner.
+"""
+
+import copy
+import dataclasses
+import json
+import sqlite3
+
+from runfox.status import StepStatus, WorkflowStatus
+
+from .models import StepRecord, WorkflowRecord
+from .store import Store
+
+
+class InMemoryStore(Store):
+
+    def __init__(self):
+        self._store: dict[str, WorkflowRecord] = {}
+
+    def load(self, workflow_execution_id: str) -> WorkflowRecord:
+        if workflow_execution_id not in self._store:
+            raise KeyError(workflow_execution_id)
+        return copy.deepcopy(self._store[workflow_execution_id])
+
+    def write(self, record: WorkflowRecord) -> None:
+        key = f"{record.workflow_id}#{record.execution_id}"
+        self._store[key] = copy.deepcopy(record)

runfox/backend/inprocess_runner.py

@@ -0,0 +1,74 @@
+"""
+runner.py -- Runner
+
+Two primitives:
+
+  dispatch(workflow_execution_id, jobs) -> None
+  gather(workflow_execution_id) -> list[tuple[str, dict]]
+
+gather() always returns immediately. Returns an empty list if no results
+are ready. The runner never calls on_step_result; Workflow.run() does that.
+
+The runner is a job queue. dispatch() enqueues; gather() dequeues results.
+The caller drives execution between those two calls -- a local function, a
+thread, a Lambda, an SQS consumer. The executor (fn, inputs -> dict) has
+no runfox dependency regardless of which runner is used.
+
+InProcessRunner -- dict-backed queue. Semantically identical to SqliteRunner;
+                   the dict is the tasks table. Use InProcessWorker to drive
+                   local execution against it.
+
+SqliteRunner    -- SQLite tasks-table queue. An external worker owns execution.
+                   See worker protocol in class docstring.
+
+InProcessWorker -- local worker harness for InProcessRunner. Mirrors the
+                   SqliteRunner worker protocol. The executor remains a plain
+                   callable with no runfox dependency.
+"""
+
+import datetime
+import json
+import sqlite3
+from typing import Callable
+
+from ..results import DispatchJob
+from .runner import Runner
+
+
+class InProcessRunner(Runner):
+    """
+    Dict-backed job queue.
+
+    Semantically identical to SqliteRunner: dispatch() writes to _pending
+    (equivalent to INSERT PENDING), gather() reads from _results (equivalent
+    to SELECT COMPLETE and mark PROCESSED). InProcessWorker drives execution
+    between those two calls, mirroring what an external worker does against
+    the SQLite tasks table.
+    """
+
+    def __init__(self):
+        self._pending: dict[str, list] = {}
+        self._results: dict[str, list] = {}
+
+    def dispatch(self, workflow_execution_id: str, jobs: list) -> None:
+        existing = self._pending.get(workflow_execution_id, [])
+        self._pending[workflow_execution_id] = existing + list(jobs)
+
+    def gather(self, workflow_execution_id: str) -> list:
+        return self._results.pop(workflow_execution_id, [])
+
+    def list_pending_jobs(self) -> list:
+        """Non-destructive snapshot. Does not alter queue state."""
+        return [job for jobs in self._pending.values() for job in jobs]
+
+    def take_pending_jobs(self) -> list:
+        """Consume all pending jobs. Clears the queue."""
+        jobs = [job for jobs in self._pending.values() for job in jobs]
+        self._pending.clear()
+        return jobs
+
+    def submit_work_result(
+        self, workflow_execution_id: str, step_id: str, output: dict
+    ) -> None:
+        existing = self._results.get(workflow_execution_id, [])
+        self._results[workflow_execution_id] = existing + [(step_id, output)]

runfox/backend/inprocess_worker.py

@@ -0,0 +1,68 @@
+"""
+runner.py -- Runner
+
+Two primitives:
+
+  dispatch(workflow_execution_id, jobs) -> None
+  gather(workflow_execution_id) -> list[tuple[str, dict]]
+
+gather() always returns immediately. Returns an empty list if no results
+are ready. The runner never calls on_step_result; Workflow.run() does that.
+
+The runner is a job queue. dispatch() enqueues; gather() dequeues results.
+The caller drives execution between those two calls -- a local function, a
+thread, a Lambda, an SQS consumer. The executor (fn, inputs -> dict) has
+no runfox dependency regardless of which runner is used.
+
+InProcessRunner -- dict-backed queue. Semantically identical to SqliteRunner;
+                   the dict is the tasks table. Use InProcessWorker to drive
+                   local execution against it.
+
+SqliteRunner    -- SQLite tasks-table queue. An external worker owns execution.
+                   See worker protocol in class docstring.
+
+InProcessWorker -- local worker harness for InProcessRunner. Mirrors the
+                   SqliteRunner worker protocol. The executor remains a plain
+                   callable with no runfox dependency.
+"""
+
+import datetime
+import json
+import sqlite3
+from typing import Callable
+
+from .inprocess_runner import InProcessRunner
+
+
+class InProcessWorker:
+    """
+    Local worker harness for InProcessRunner.
+
+    Mirrors the SqliteRunner worker protocol using the runner's internal
+    dicts in place of the tasks table. The executor remains a plain callable
+    with no runfox dependency.
+
+    Equivalent remote pattern
+    -------------------------
+    This harness:
+        for job in runner.pending(wf_exec_id):
+            output = executor(job.fn, job.inputs)
+            runner.submit_work_result(wf_exec_id, job.step_id, output)
+
+    SQS/Lambda equivalent:
+        message = sqs.receive()
+        output  = executor(message.fn, message.inputs)
+        dynamodb.put(task_key, output, status="COMPLETE")
+    """
+
+    def __init__(self, runner: InProcessRunner, executor: Callable[[str, dict], dict]):
+        self._runner = runner
+        self._executor = executor
+
+    def run(self, workflow_execution_id: str) -> None:
+        for job in self._runner.take_pending_jobs():
+            try:
+                output = self._executor(job.fn, job.inputs)
+            except Exception as exc:
+                output = {"error": str(exc), "ok": False}
+            self._runner.submit_work_result(workflow_execution_id, job.step_id, output)

runfox/backend/models.py

@@ -0,0 +1,76 @@
+"""
+base.py -- Backend
+
+Composes a Store and a Runner. All workflow lifecycle operations are
+implemented here in terms of self._store.load() and self._store.write().
+dispatch() and gather() delegate to self._runner.
+
+Construction
+------------
+  Backend(store=s, runner=r)  -- explicit store and runner
+
+Composite key accessors
+-----------------------
+workflow_execution_id(record)
+step_key(wf_exec_id, step_id)
+step_run_key(wf_exec_id, step_id, run_id)
+"""
+
+import dataclasses
+import datetime
+import hashlib
+import json
+import random
+import socket
+import string
+from typing import Any
+
+from runfox.status import StepStatus, WorkflowStatus
+
+
+@dataclasses.dataclass
+class StepRecord:
+    """
+    Runtime state of a single step.
+
+    id         -- step identifier
+    status     -- current lifecycle status
+    output     -- written by executor on completion; None until then
+    start_time -- ISO-8601 UTC; set when claimed
+    end_time   -- ISO-8601 UTC; set on terminal status
+    host       -- hostname of the claiming process
+    run_id     -- incremented on every dispatch (retry or set-branch reset)
+    """
+
+    id: str
+    status: StepStatus = StepStatus.READY
+    output: dict | None = None
+    start_time: str | None = None
+    end_time: str | None = None
+    host: str | None = None
+    run_id: int = 0
+
+
+@dataclasses.dataclass
+class WorkflowRecord:
+    """
+    Runtime state of a workflow execution as returned by store.load().
+
+    workflow_id  -- MD5 of canonical spec JSON
+    execution_id -- timestamp + short suffix; identifies one run
+    spec         -- parsed workflow definition (immutable after create)
+    inputs       -- workflow-level inputs (immutable after create)
+    state        -- mutable shared accumulator
+    steps        -- dict[step_id -> StepRecord]
+    status       -- current workflow lifecycle status
+    outcome      -- resolved outputs on completion, branch payload on halt
+    """
+
+    workflow_id: str
+    execution_id: str
+    spec: dict
+    inputs: dict
+    state: dict
+    steps: dict
+    status: WorkflowStatus
+    outcome: Any = None

runfox/backend/runner.py

@@ -0,0 +1,69 @@
+"""
+runner.py -- Runner
+
+Two primitives:
+
+  dispatch(workflow_execution_id, jobs) -> None
+  gather(workflow_execution_id) -> list[tuple[str, dict]]
+
+gather() always returns immediately. Returns an empty list if no results
+are ready. The runner never calls on_step_result; Workflow.run() does that.
+
+The runner is a job queue. dispatch() enqueues; gather() dequeues results.
+The caller drives execution between those two calls -- a local function, a
+thread, a Lambda, an SQS consumer. The executor (fn, inputs -> dict) has
+no runfox dependency regardless of which runner is used.
+
+InProcessRunner -- dict-backed queue. Semantically identical to SqliteRunner;
+                   the dict is the tasks table. Use InProcessWorker to drive
+                   local execution against it.
+
+SqliteRunner    -- SQLite tasks-table queue. An external worker owns execution.
+                   See worker protocol in class docstring.
+
+InProcessWorker -- local worker harness for InProcessRunner. Mirrors the
+                   SqliteRunner worker protocol. The executor remains a plain
+                   callable with no runfox dependency.
+"""
+
+import datetime
+import json
+import sqlite3
+from typing import Callable
+
+from ..results import DispatchJob
+
+
+class Runner:
+
+    def dispatch(self, workflow_execution_id: str, jobs: list) -> None:
+        """Enqueue jobs. jobs is a list of DispatchJob."""
+        raise NotImplementedError
+
+    def gather(self, workflow_execution_id: str) -> list:
+        """
+        Return completed (step_id, output) pairs. Always returns immediately.
+        Returns an empty list if no results are ready.
+        """
+        raise NotImplementedError
+
+    def list_pending_jobs(self) -> list:
+        """
+        Non-destructive snapshot of all pending jobs across all workflows.
+        Safe to call for diagnostics; does not affect queue state.
+        """
+        raise NotImplementedError
+
+    def take_pending_jobs(self) -> list:
+        """
+        Consume and return all pending jobs across all workflows.
+        Called by worker harnesses. Each returned job will not be
+        returned again by a subsequent call.
+        """
+        raise NotImplementedError
+
+    def submit_work_result(
+        self, workflow_execution_id: str, step_id: str, output: dict
+    ) -> None:
+        """Write a result back from a worker."""
+        raise NotImplementedError