substantial 0.0.1__py3-none-any.whl

pyproject.toml

@@ -0,0 +1,51 @@
+[tool.poetry]
+name = "substantial"
+version = "0.0.1"
+description = "Brokerless durable execution for Python."
+authors = []
+license = "MPL-2.0"
+readme = "README.md"
+include = [
+    "dataconf", "pyproject.toml"
+]
+classifiers = [
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12"
+]
+keywords = [
+    "durable",
+    "execution",
+    "redis",
+    "s3",
+    "filesystem",
+]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+asyncio = "^3.4.3"
+uvloop = "^0.21.0"
+aioprocessing = { version = "2.0.1", extras = ["dill"] }
+pydantic = "^2.9.2"
+redis = "^5.2.0"
+betterproto = { version = "2.0.0b7", extras = ["compiler"]}
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.3"
+pytest-asyncio = "^0.23.7"
+commitizen = ">=3.29.0"
+
+[tool.commitizen]
+name = "cz_conventional_commits"
+version = "3.3.0"
+tag_format = "v$major.$minor.$patch$prerelease"
+version_files = [
+    "pyproject.toml:version",
+]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

substantial/__init__.py

@@ -0,0 +1,14 @@
+from substantial.workflows.workflow import workflow, Workflow
+from substantial.workflows.context import Context
+from substantial.workflows.run import Run
+from substantial.conductor import Conductor
+from substantial.backends.fs import FSBackend
+
+__all__ = [
+    "workflow",
+    "Workflow",
+    "Context",
+    "Conductor",
+    "Run",
+    "FSBackend",
+]

substantial/agent.py

@@ -0,0 +1,70 @@
+import asyncio
+from contextlib import suppress
+from substantial.backends.backend import Backend
+import uvloop
+
+from substantial.workflows.run import Run
+
+
+# move to backend?
+lease_seconds = 10
+renew_seconds = 8
+pool_interval = 1
+
+
+class Agent:
+    def __init__(self, backend: Backend, queue: str):
+        self.backend = backend
+        self.queue = queue
+
+    def run_sync(self):
+        with asyncio.Runner(loop_factory=uvloop.new_event_loop) as runner:
+            runner.run(self.run())
+
+    async def run(self):
+        while True:
+            await self.poll_with_lease()
+            await asyncio.sleep(pool_interval)
+
+    async def poll_with_lease(self):
+        active_leases = await self.backend.active_leases(lease_seconds)
+        next_run = await self.backend.next_run(self.queue, active_leases)
+        print("run_id", next_run)
+
+        if next_run is None:
+            print("no runs")
+            await asyncio.sleep(1)
+            return
+
+        run_id, schedule = next_run
+
+        async def heartbeat():
+            while True:
+                await asyncio.sleep(renew_seconds)
+                renewed = await self.backend.renew_lease(run_id, lease_seconds)
+                if not renewed:
+                    return
+
+        leased = await self.backend.acquire_lease(run_id, lease_seconds)
+        if not leased:
+            return
+
+        renew_task = asyncio.create_task(heartbeat())
+        run = Run(run_id, self.queue, self.backend)
+        process_task = asyncio.create_task(run.replay(schedule))
+
+        done, pending = await asyncio.wait(
+            [
+                renew_task,
+                process_task,
+            ],
+            return_when=asyncio.FIRST_COMPLETED,
+        )
+        done = done.pop()
+        pending = pending.pop()
+
+        pending.cancel()
+        with suppress(asyncio.CancelledError):
+            await pending
+
+        await self.backend.remove_lease(run_id, lease_seconds)

substantial/backends/backend.py

@@ -0,0 +1,54 @@
+from datetime import datetime
+from typing import List, Tuple, Union
+from substantial.protos.events import Event, Records
+from substantial.protos.metadata import Metadata
+
+
+class Backend:
+    # run related
+
+    async def read_events(self, run_id: str) -> Union[Records, None]:
+        raise NotImplementedError()
+
+    async def write_events(self, run_id: str, content: Records) -> None:
+        raise NotImplementedError()
+
+    async def read_all_metadata(self, run_id: str) -> List[Metadata]:
+        raise NotImplementedError()
+
+    async def append_metadata(
+        self, run_id: str, schedule: datetime, content: str
+    ) -> None:
+        raise NotImplementedError()
+
+    async def add_schedule(
+        self, queue: str, run_id: str, schedule: datetime, content: Union[Event, None]
+    ) -> None:
+        raise NotImplementedError()
+
+    async def read_schedule(
+        self, queue: str, run_id: str, schedule: datetime
+    ) -> Union[Event, None]:
+        raise NotImplementedError()
+
+    async def close_schedule(self, queue: str, run_id: str, schedule: datetime) -> None:
+        raise NotImplementedError()
+
+    # agent related
+
+    async def next_run(
+        self, queue: str, excludes: list[str]
+    ) -> Union[Tuple[str, datetime], None]:
+        raise NotImplementedError()
+
+    async def active_leases(self, lease_seconds: int) -> List[str]:
+        raise NotImplementedError()
+
+    async def acquire_lease(self, run_id: str, lease_seconds: int) -> bool:
+        raise NotImplementedError()
+
+    async def renew_lease(self, run_id: str, lease_seconds: int) -> bool:
+        raise NotImplementedError()
+
+    async def remove_lease(self, run_id: str, lease_seconds: int) -> None:
+        raise NotImplementedError()

substantial/backends/fs.py

@@ -0,0 +1,160 @@
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import List, Tuple, Union
+from uuid import uuid4
+from substantial.backends.backend import Backend
+from substantial.protos.events import Event, Records
+
+
+class FSBackend(Backend):
+    """
+    This backend is for testing purposes only as it is not scalable:
+      - there is no file order in POSIX
+      - all the runs need to be loaded in memory
+      - not all required operations are atomic or using compare-and-swap
+    """
+
+    def __init__(self, root: str):
+        self.root = Path(root)
+        for d in ["runs", "schedules", "leases"]:
+            (self.root / d).mkdir(parents=True, exist_ok=True)
+
+    async def read_events(self, run_id: str) -> Union[Records, None]:
+        f = self.root / "runs" / run_id / "events"
+        if not f.exists():
+            return None
+        return Records().from_json(f.read_text())
+
+    async def write_events(self, run_id: str, content: Records) -> None:
+        f = self.root / "runs" / run_id / "events"
+        f.parent.mkdir(parents=True, exist_ok=True)
+        f.write_text(content.to_json(indent=4))
+
+    async def read_all_metadata(self, run_id: str) -> List[str]:
+        f = self.root / "runs" / run_id / "logs"
+        ret = []
+        for log in f.iterdir():
+            # could be polymorphic
+            ret.append(log.read_text())
+        return ret
+
+    async def append_metadata(self, run_id: str, schedule: datetime, content: str):
+        f = self.root / "runs" / run_id / "logs" / schedule.isoformat()
+        f.parent.mkdir(parents=True, exist_ok=True)
+        f.write_text(content)
+
+    async def next_run(
+        self, queue: str, excludes: list[str]
+    ) -> Union[Tuple[str, datetime], None]:
+        f = self.root / "schedules" / queue
+        excludes_set = set(excludes)  # Note: lease related
+
+        for schedule in sorted(f.iterdir()):
+            for run_id in schedule.iterdir():
+                if run_id.name not in excludes_set:
+                    return run_id.name, datetime.fromisoformat(schedule.name)
+
+        return None
+
+    async def add_schedule(
+        self, queue: str, run_id: str, schedule: datetime, content: Union[Event, None]
+    ) -> None:
+        q = self.root / "schedules" / queue
+
+        if q.exists():
+            # Note: new schedule should always overwrite replays (but not scheduled events)
+            # This is for keeping the run consistent,
+            # Case 1: ======= r1 == ev1 == r2 ====== r3 ========>
+            #                              x r4
+            # Result: ============== ev1 = r4 ====== r3 ========>
+            # Rationale being that the scheduled replays are often triggered by Interrupts and such
+            # for more sane runs, fuse planned replays (induced by add_schedule calls)?
+
+            for sched in sorted(q.iterdir()):
+                for planned in sched.iterdir():
+                    planned_date = datetime.fromisoformat(sched.name)
+                    if planned.name == run_id and planned_date <= schedule:
+                        event = await self.read_schedule(queue, run_id, planned_date)
+                        if event is None:  # event => None == scheduled replays..
+                            await self.close_schedule(queue, run_id, planned_date)
+
+        f1 = q / schedule.isoformat() / run_id
+        f1.parent.mkdir(parents=True, exist_ok=False)
+        f1.write_text("" if content is None else content.to_json())
+
+    async def read_schedule(
+        self, queue: str, run_id: str, schedule: datetime
+    ) -> Union[Event, None]:
+        f = self.root / "schedules" / queue / schedule.isoformat() / run_id
+        if not f.exists():
+            raise Exception(f"run not found: {f}")
+        ret = f.read_text()
+        return None if ret == "" else Event().from_json(ret)
+
+    async def close_schedule(self, queue: str, run_id: str, schedule: datetime) -> None:
+        f = self.root / "schedules" / queue / schedule.isoformat() / run_id
+        if not f.exists():
+            # raise Exception(f"run not found: {f}")
+            return
+        print(f"closed {f}")
+        f.unlink()
+
+    async def active_leases(self, lease_seconds: int) -> List[str]:
+        f = self.root / "leases"
+        ret = []
+        for lease in f.iterdir():
+            if not lease_held(lease, lease_seconds):
+                continue
+            ret.append(lease.name)
+        return ret
+
+    async def acquire_lease(self, run_id: str, lease_seconds: int) -> bool:
+        f = self.root / "leases" / run_id
+        return leasing_cas(f, "acquire", lease_seconds)
+
+    async def renew_lease(self, run_id: str, lease_seconds: int) -> bool:
+        f = self.root / "leases" / run_id
+
+        if not f.exists():
+            raise Exception(f"lease not found: {f}")
+
+        return leasing_cas(f, "renew", lease_seconds)
+
+    async def remove_lease(self, run_id: str, lease_seconds: int):
+        f = self.root / "leases" / run_id
+
+        if not f.exists():
+            raise Exception(f"lease not found: {f}")
+
+        still_holding = leasing_cas(f, "remove", lease_seconds)
+        if still_holding:
+            f.unlink()
+
+
+def leasing_cas(f: Path, suffix: str, lease_seconds: int) -> bool:
+    if lease_held(f, lease_seconds):
+        return False
+
+    return cas(f, suffix)
+
+
+def lease_held(f: Path, lease_seconds: int) -> bool:
+    return (
+        f.exists()
+        and datetime.fromtimestamp(f.stat().st_mtime) + timedelta(seconds=lease_seconds)
+        > datetime.now()
+    )
+
+
+def cas(f: Path, suffix: str) -> bool:
+    nonce = str(uuid4())
+    witness = f.parent / f"{f.name}.{suffix}"
+    witness.write_text(nonce)
+
+    try:
+        witness.rename(f)
+        return f.read_text() == nonce
+    except FileExistsError:
+        # another rename has already happened
+        witness.unlink()
+        return False

substantial/backends/redis.py

@@ -0,0 +1,181 @@
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import List, Tuple, Union
+from uuid import uuid4
+
+import redis
+from substantial.backends.backend import Backend
+from substantial.protos.events import Event, Records
+
+
+class RedisBackend(Backend):
+    def __init__(self, host: str, port: int, **kwargs):
+        self.redis = redis.Redis(host=host, port=port, decode_responses=True, **kwargs)
+        # self.redis.flushall()
+
+    async def read_events(self, run_id: str) -> Union[Records, None]:
+        key = "_".join(["runs", run_id, "events"])
+        val = self.redis.get(key)
+        return None if val is None else Records().from_json(val)
+
+    async def write_events(self, run_id: str, content: Records) -> None:
+        key = "_".join(["runs", run_id, "events"])
+        self.redis.set(key, content.to_json(indent=4))
+
+    async def read_all_metadata(self, run_id: str) -> List[str]:
+        base_key = "_".join(["runs", run_id, "logs"])  # queue
+        ret = []
+        for schedule in self.redis.lrange(base_key, 0, -1):
+            log = self.redis.get("_".join[run_id, schedule])
+            # could be polymorphic
+            ret.append(log)
+        return ret
+
+    async def append_metadata(self, run_id: str, schedule: datetime, content: str):
+        base_key = "_".join(["runs", run_id, "logs"])  # queue
+        sched_key = "_".join([run_id, schedule.isoformat()])
+
+        self.redis.register_script("""
+            local base_key = KEYS[1]
+            local sched_key = KEYS[2]
+            local content = ARGV[1]
+
+            redis.call("LPUSH", base_key, sched_key)
+            redis.call("ZREM", sched_key, content)
+        """)(keys=[base_key, sched_key], args=[content])
+
+    async def next_run(
+        self, queue: str, excludes: list[str]
+    ) -> Union[Tuple[str, datetime], None]:
+        q_key = "_".join(["schedules", queue])  # priority queue
+        excludes_set = set(excludes)
+
+        for schedule in self.redis.zrange(q_key, 0, -1):
+            for run_id in self.redis.zrange(schedule, 0, -1):
+                if run_id not in excludes_set:
+                    return run_id, datetime.fromisoformat(schedule)
+
+        return None
+
+    async def add_schedule(
+        self, queue: str, run_id: str, schedule: datetime, content: Union[Event, None]
+    ) -> None:
+        q_key = "_".join(["schedules", queue])  # priority queue
+
+        # avoid having a bunch of replays happening
+        for sched_ref in self.redis.zrange(q_key, 0, -1):
+            for planned_key in self.redis.zrange(sched_ref, 0, -1):
+                _dt, planned_id = planned_key.split("_")
+                planned_date = datetime.fromisoformat(sched_ref)
+                if planned_id == run_id and planned_date <= schedule:
+                    event = await self.read_schedule(queue, run_id, planned_date)
+                    if event is None:  # event => None == scheduled replays..
+                        await self.close_schedule(queue, run_id, planned_date)
+
+        sched_ref = schedule.isoformat()
+        sched_score = 1 / schedule.timestamp()
+        sched_key = "_".join([sched_ref, run_id])
+
+        self.redis.register_script("""
+            local q_key = KEYS[1]
+            local sched_ref = KEYS[2]
+            local sched_key = KEYS[3]
+            local sched_score = tonumber(ARGV[1])
+            local run_id = ARGV[2]
+            local content = ARGV[3]
+
+            redis.call("ZADD", q_key, 0, sched_ref)
+            redis.call("ZADD", sched_ref, sched_score, run_id)
+            redis.call("SET", sched_key, content)
+        """)(
+            keys=[q_key, sched_ref, sched_key],
+            args=[sched_score, run_id, "" if content is None else content.to_json()],
+        )
+
+    async def read_schedule(
+        self, queue: str, run_id: str, schedule: datetime
+    ) -> Union[Event, None]:
+        sched_key = "_".join([schedule.isoformat(), run_id])
+        ret = self.redis.get(sched_key)
+        if ret is None:
+            raise Exception(f"run not found: {sched_key}")
+        return None if ret == "" else Event().from_json(ret)
+
+    async def close_schedule(self, queue: str, run_id: str, schedule: datetime) -> None:
+        q_key = "_".join(["schedules", queue])
+        sched_ref = schedule.isoformat()
+        sched_key = "_".join([sched_ref, run_id])
+
+        self.redis.register_script("""
+            local q_key = KEYS[1]
+            local sched_ref = KEYS[2]
+            local sched_key = KEYS[3]
+            local run_id = ARGV[1]
+
+            redis.call("ZREM", q_key, sched_ref)
+            redis.call("ZREM", sched_ref, run_id)
+            redis.call("DEL", sched_key)
+        """)(keys=[q_key, sched_ref, sched_key], args=[run_id])
+
+        print(f"closed {run_id}")
+
+    # TODO
+    async def active_leases(self, lease_seconds: int) -> List[str]:
+        f = Path("logs") / "leases"
+        ret = []
+        for lease in f.iterdir():
+            if not lease_held(lease, lease_seconds):
+                continue
+            ret.append(lease.name)
+        return ret
+
+    async def acquire_lease(self, run_id: str, lease_seconds: int) -> bool:
+        f = Path("logs") / "leases" / run_id
+        return leasing_cas(f, "acquire", lease_seconds)
+
+    async def renew_lease(self, run_id: str, lease_seconds: int) -> bool:
+        f = Path("logs") / "leases" / run_id
+
+        if not f.exists():
+            raise Exception(f"lease not found: {f}")
+
+        return leasing_cas(f, "renew", lease_seconds)
+
+    async def remove_lease(self, run_id: str, lease_seconds: int):
+        f = Path("logs") / "leases" / run_id
+
+        if not f.exists():
+            raise Exception(f"lease not found: {f}")
+
+        still_holding = leasing_cas(f, "remove", lease_seconds)
+        if still_holding:
+            f.unlink()
+
+
+def leasing_cas(f: Path, suffix: str, lease_seconds: int) -> bool:
+    if lease_held(f, lease_seconds):
+        return False
+
+    return cas(f, suffix)
+
+
+def lease_held(f: Path, lease_seconds: int) -> bool:
+    return (
+        f.exists()
+        and datetime.fromtimestamp(f.stat().st_mtime) + timedelta(seconds=lease_seconds)
+        > datetime.now()
+    )
+
+
+def cas(f: Path, suffix: str) -> bool:
+    nonce = str(uuid4())
+    witness = f.parent / f"{f.name}.{suffix}"
+    witness.write_text(nonce)
+
+    try:
+        witness.rename(f)
+        return f.read_text() == nonce
+    except FileExistsError:
+        # another rename has already happened
+        witness.unlink()
+        return False

substantial/conductor.py

@@ -0,0 +1,50 @@
+import asyncio
+from uuid import uuid4
+
+from substantial.agent import Agent
+from substantial.backends.backend import Backend
+from substantial.workflows import Store
+from substantial.workflows.run import Run
+from substantial.workflows.workflow import Workflow
+import aioprocessing
+
+
+class Conductor:
+    def __init__(self, backend: Backend):
+        self.backend = backend
+
+    def register(self, workflow: Workflow):
+        Store.register(workflow)
+
+    async def start(
+        self,
+        workflow: Workflow,
+        kwargs=None,
+        queue="default",
+    ):
+        if kwargs is None:
+            kwargs = {}
+
+        run_id = f"{workflow.id}-{uuid4()}"
+        run = Run(run_id, queue, self.backend)
+        await run.start(kwargs)
+        return run
+
+    def run_for(self, run_id, queue="default"):
+        return Run(run_id, queue, self.backend)
+
+    def run(self, queue="default"):
+        return asyncio.create_task(self.run_as_process(queue))
+
+    async def run_as_process(self, queue):
+        agent = Agent(self.backend, queue)
+        p = aioprocessing.AioProcess(target=agent.run_sync)
+        try:
+            p.start()
+            await p.coro_join()
+            return p
+        except asyncio.CancelledError:
+            p.terminate()
+            print("Cancelled")
+        except Exception:
+            raise

substantial/protos/events/__init__.py

@@ -0,0 +1,58 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# sources: protocol/events.proto
+# plugin: python-betterproto
+# This file has been @generated
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import List
+
+import betterproto
+import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf
+
+
+@dataclass(eq=False, repr=False)
+class Start(betterproto.Message):
+    kwargs: "betterproto_lib_google_protobuf.Struct" = betterproto.message_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class Save(betterproto.Message):
+    id: int = betterproto.uint32_field(1)
+    value: str = betterproto.string_field(2)
+    counter: int = betterproto.int32_field(3)
+
+
+@dataclass(eq=False, repr=False)
+class Sleep(betterproto.Message):
+    id: int = betterproto.uint32_field(1)
+    start: datetime = betterproto.message_field(2)
+    end: datetime = betterproto.message_field(3)
+
+
+@dataclass(eq=False, repr=False)
+class Send(betterproto.Message):
+    name: str = betterproto.string_field(1)
+    value: str = betterproto.string_field(2)
+
+
+@dataclass(eq=False, repr=False)
+class Stop(betterproto.Message):
+    ok: str = betterproto.string_field(1, group="result")
+    err: str = betterproto.string_field(2, group="result")
+
+
+@dataclass(eq=False, repr=False)
+class Event(betterproto.Message):
+    at: datetime = betterproto.message_field(1)
+    start: "Start" = betterproto.message_field(10, group="of")
+    save: "Save" = betterproto.message_field(11, group="of")
+    sleep: "Sleep" = betterproto.message_field(12, group="of")
+    send: "Send" = betterproto.message_field(13, group="of")
+    stop: "Stop" = betterproto.message_field(14, group="of")
+
+
+@dataclass(eq=False, repr=False)
+class Records(betterproto.Message):
+    run_id: str = betterproto.string_field(1)
+    events: List["Event"] = betterproto.message_field(2)

substantial/protos/metadata/__init__.py

@@ -0,0 +1,35 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# sources: protocol/metadata.proto
+# plugin: python-betterproto
+# This file has been @generated
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import List
+
+import betterproto
+
+
+@dataclass(eq=False, repr=False)
+class Info(betterproto.Message):
+    message: str = betterproto.string_field(1)
+
+
+@dataclass(eq=False, repr=False)
+class Error(betterproto.Message):
+    message: str = betterproto.string_field(1)
+    stack: str = betterproto.string_field(2)
+    type: str = betterproto.string_field(3)
+
+
+@dataclass(eq=False, repr=False)
+class Metadata(betterproto.Message):
+    at: datetime = betterproto.message_field(1)
+    info: "Info" = betterproto.message_field(10, group="of")
+    error: "Error" = betterproto.message_field(11, group="of")
+
+
+@dataclass(eq=False, repr=False)
+class Records(betterproto.Message):
+    run_id: str = betterproto.string_field(1)
+    metadata: List["Metadata"] = betterproto.message_field(2)