aethergraph 0.1.0a1__py3-none-any.whl

aethergraph/services/container/default_container.py

@@ -0,0 +1,320 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import os
+from pathlib import Path
+from typing import Any
+
+# ---- core services ----
+from aethergraph.config.config import AppSettings
+
+# ---- optional services (not used by default) ----
+from aethergraph.contracts.services.llm import LLMClientProtocol
+
+# ---- scheduler ---- TODO: move to a separate server to handle scheduling across threads/processes
+from aethergraph.contracts.services.state_stores import GraphStateStore
+from aethergraph.core.execution.global_scheduler import GlobalForwardScheduler
+
+# ---- artifact services ----
+from aethergraph.services.artifacts.fs_store import FSArtifactStore  # AsyncArtifactStore
+from aethergraph.services.artifacts.jsonl_index import JsonlArtifactIndex  # AsyncArtifactIndex
+from aethergraph.services.auth.dev import AllowAllAuthz, DevTokenAuthn
+from aethergraph.services.channel.channel_bus import ChannelBus
+
+# ---- channel services ----
+from aethergraph.services.channel.factory import build_bus, make_channel_adapters_from_env
+from aethergraph.services.clock.clock import SystemClock
+from aethergraph.services.continuations.stores.fs_store import (
+    FSContinuationStore,  # AsyncContinuationStore
+)
+from aethergraph.services.eventbus.inmem import InMemoryEventBus
+
+# ---- kv services ----
+from aethergraph.services.kv.ephemeral import EphemeralKV
+from aethergraph.services.kv.sqlite_kv import SQLiteKV
+from aethergraph.services.llm.factory import build_llm_clients
+from aethergraph.services.llm.service import LLMService
+from aethergraph.services.logger.std import LoggingConfig, StdLoggerService
+from aethergraph.services.mcp.service import MCPService
+
+# ---- memory services ----
+from aethergraph.services.memory.factory import MemoryFactory
+from aethergraph.services.memory.hotlog_kv import KVHotLog
+from aethergraph.services.memory.indices import KVIndices
+from aethergraph.services.memory.persist_fs import FSPersistence
+from aethergraph.services.metering.noop import NoopMetering
+from aethergraph.services.prompts.file_store import FilePromptStore
+from aethergraph.services.rag.chunker import TextSplitter
+from aethergraph.services.rag.facade import RAGFacade
+
+# ---- RAG components ----
+from aethergraph.services.rag.index_factory import create_vector_index
+from aethergraph.services.redactor.simple import RegexRedactor  # Simple PII redactor
+from aethergraph.services.registry.unified_registry import UnifiedRegistry
+from aethergraph.services.resume.multi_scheduler_resume_bus import MultiSchedulerResumeBus
+from aethergraph.services.resume.router import ResumeRouter
+from aethergraph.services.schedulers.registry import SchedulerRegistry
+from aethergraph.services.secrets.env import EnvSecrets
+from aethergraph.services.state_stores.json_store import JsonGraphStateStore
+from aethergraph.services.tracing.noop import NoopTracer
+from aethergraph.services.waits.wait_registry import WaitRegistry
+from aethergraph.services.wakeup.memory_queue import ThreadSafeWakeupQueue
+
+SERVICE_KEYS = [
+    # core
+    "registry",
+    "logger",
+    "clock",
+    "channels",
+    # continuations and resume
+    "cont_store",
+    "sched_registry",
+    "wait_registry",
+    "resume_bus",
+    "resume_router",
+    "wakeup_queue",
+    # storage and artifacts
+    "kv_hot",
+    "kv_durable",
+    "artifacts",
+    "artifact_index",
+    # memory
+    "memory_factory",
+    # optional
+    "llm",
+    "event_bus",
+    "prompts",
+    "authn",
+    "authz",
+    "redactor",
+    "metering",
+    "tracer",
+    "secrets",
+]
+
+
+@dataclass
+class DefaultContainer:
+    # root
+    root: str
+
+    # schedulers
+    schedulers: dict[str, Any]
+
+    # core
+    registry: UnifiedRegistry
+    logger: StdLoggerService
+    clock: SystemClock
+
+    # channels and interactions
+    channels: ChannelBus
+
+    # continuations and resume
+    cont_store: FSContinuationStore
+    sched_registry: SchedulerRegistry
+    wait_registry: WaitRegistry
+    resume_bus: MultiSchedulerResumeBus
+    resume_router: ResumeRouter
+    wakeup_queue: ThreadSafeWakeupQueue
+    state_store: GraphStateStore
+
+    # storage and artifacts
+    kv_hot: EphemeralKV
+    kv_durable: SQLiteKV
+    artifacts: FSArtifactStore
+    artifact_index: JsonlArtifactIndex
+
+    # memory
+    memory_factory: MemoryFactory
+
+    # optional llm service
+    llm: LLMClientProtocol | None = None
+    rag: RAGFacade | None = None
+    mcp: MCPService | None = None
+
+    # optional services (not used by default)
+    event_bus: InMemoryEventBus | None = None
+    prompts: FilePromptStore | None = None
+    authn: DevTokenAuthn | None = None
+    authz: AllowAllAuthz | None = None
+    redactor: RegexRedactor | None = None
+    metering: NoopMetering | None = None
+    tracer: NoopTracer | None = None
+    secrets: EnvSecrets | None = None
+
+    # extensible services
+    ext_services: dict[str, Any] = field(default_factory=dict)
+
+    # settings -- not a service, but useful to have around
+    settings: AppSettings | None = None
+
+
+def build_default_container(
+    *,
+    root: str | None = None,
+    cfg: AppSettings | None = None,
+) -> DefaultContainer:
+    """Build the default service container with standard services.
+    if "root" is provided, use it as the base directory for storage; else use from cfg/root.
+    if cfg is not provided, load from default AppSettings.
+    """
+    if cfg is None:
+        from aethergraph.config.context import set_current_settings
+        from aethergraph.config.loader import load_settings
+
+        cfg = load_settings()
+        set_current_settings(cfg)
+
+    root = root or cfg.root
+    # override root in cfg to match
+    cfg.root = root
+
+    # we use user specified root if provided, else from config/env
+    root_p = Path(root).resolve() if root else Path(cfg.root).resolve()
+    (root_p / "kv").mkdir(parents=True, exist_ok=True)
+    (root_p / "continuations").mkdir(parents=True, exist_ok=True)
+    (root_p / "index").mkdir(parents=True, exist_ok=True)
+    (root_p / "memory").mkdir(parents=True, exist_ok=True)
+    (root_p / "graph_states").mkdir(parents=True, exist_ok=True)
+
+    # core services
+    logger_factory = StdLoggerService.build(
+        LoggingConfig.from_cfg(cfg, log_dir=str(root_p / "logs"))
+    )
+    clock = SystemClock()
+    registry = UnifiedRegistry()
+
+    # continuations and resume
+    cont_store = FSContinuationStore(root=str(root_p / "continuations"), secret=os.urandom(32))
+    sched_registry = SchedulerRegistry()
+    wait_registry = WaitRegistry()
+    resume_bus = MultiSchedulerResumeBus(
+        registry=sched_registry, store=cont_store, logger=logger_factory.for_run()
+    )
+    resume_router = ResumeRouter(
+        store=cont_store,
+        runner=resume_bus,
+        logger=logger_factory.for_run(),
+        wait_registry=wait_registry,
+    )
+    wakeup_queue = ThreadSafeWakeupQueue()  # TODO: this is a placeholder, not fully implemented
+    state_store = JsonGraphStateStore(root=str(root_p / "graph_states"))
+
+    # global scheduler
+    global_sched = GlobalForwardScheduler(
+        registry=sched_registry,
+        global_max_concurrency=None,  # TODO: make configurable
+        logger=logger_factory.for_scheduler(),
+    )
+    schedulers = {
+        "global": global_sched,
+        "registry": sched_registry,
+    }
+
+    # channels
+    channel_adapters = make_channel_adapters_from_env(cfg)
+    channels = build_bus(
+        channel_adapters,
+        default="console:stdin",
+        logger=logger_factory.for_run(),
+        resume_router=resume_router,
+        cont_store=cont_store,
+    )
+
+    # storage and artifacts
+    kv_hot = EphemeralKV()
+    kv_durable = SQLiteKV(str(root_p / "kv" / "kv.sqlite"))
+    artifacts = FSArtifactStore(
+        str(root_p / "artifacts")
+    )  # async wrapper over FileArtifactStoreSync
+    artifact_index = JsonlArtifactIndex(str(root_p / "index" / "artifacts.jsonl"))
+
+    # memory
+    hotlog = KVHotLog(kv=kv_hot)
+    persistence = FSPersistence(base_dir=str(root_p / "memory"))
+    indices = KVIndices(kv=kv_durable, hot_ttl_s=7 * 24 * 3600)
+
+    # optional services
+    secrets = (
+        EnvSecrets()
+    )  # get secrets from env vars -- for local development; in prod, use a proper secrets manager
+    llm_clients = build_llm_clients(cfg.llm, secrets)  # return {profile: GenericLLMClient}
+    llm_service = LLMService(clients=llm_clients) if llm_clients else None
+
+    rag_cfg = cfg.rag
+    vec_index = create_vector_index(
+        backend=rag_cfg.backend, index_path=str(root_p / "rag" / "rag_index"), dim=rag_cfg.dim
+    )
+
+    rag_facade = RAGFacade(
+        corpus_root=str(root_p / "rag" / "rag_corpora"),
+        artifacts=artifacts,
+        embed_client=llm_service.get("default"),
+        llm_client=llm_service.get("default"),
+        index_backend=vec_index,
+        chunker=TextSplitter(),
+        logger=logger_factory.for_run(),
+    )
+    mcp = MCPService()  # empty MCP service; users can register clients as needed
+
+    memory_factory = MemoryFactory(
+        hotlog=hotlog,
+        persistence=persistence,
+        indices=indices,
+        artifacts=artifacts,
+        hot_limit=int(cfg.memory.hot_limit),
+        hot_ttl_s=int(cfg.memory.hot_ttl_s),
+        default_signal_threshold=float(cfg.memory.signal_threshold),
+        logger=logger_factory.for_run(),
+        llm_service=llm_service.get("default") if llm_service else None,
+        rag_facade=rag_facade,
+    )
+
+    return DefaultContainer(
+        root=str(root_p),
+        schedulers=schedulers,
+        registry=registry,
+        logger=logger_factory,
+        clock=clock,
+        channels=channels,
+        cont_store=cont_store,
+        sched_registry=sched_registry,
+        wait_registry=wait_registry,
+        resume_bus=resume_bus,
+        resume_router=resume_router,
+        wakeup_queue=wakeup_queue,
+        kv_hot=kv_hot,
+        kv_durable=kv_durable,
+        state_store=state_store,
+        artifacts=artifacts,
+        artifact_index=artifact_index,
+        memory_factory=memory_factory,
+        llm=llm_service,
+        rag=rag_facade,
+        mcp=mcp,
+        secrets=secrets,
+        event_bus=None,
+        prompts=None,
+        authn=None,
+        authz=None,
+        redactor=None,
+        metering=None,
+        tracer=None,
+        settings=cfg,
+    )
+
+
+# Singleton (used unless the host sets their own)
+DEFAULT_CONTAINER: DefaultContainer | None = None
+
+
+def get_container() -> DefaultContainer:
+    global DEFAULT_CONTAINER
+    if DEFAULT_CONTAINER is None:
+        DEFAULT_CONTAINER = build_default_container()
+    return DEFAULT_CONTAINER
+
+
+def set_container(c: DefaultContainer) -> None:
+    global DEFAULT_CONTAINER
+    DEFAULT_CONTAINER = c

aethergraph/services/continuations/continuation.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Correlator:
+    """Platform-agnostic correlation key for continuations."""
+
+    scheme: str  # e.g., "slack", "web", "email"
+    channel: str  # e.g., channel ID, email address
+    thread: str  # e.g., thread ID, conversation ID
+    message: str  # e.g., message ID, timestamp
+
+    def key(self) -> tuple[str, str, str, str]:
+        return (self.scheme, self.channel, self.thread or "", self.message or "")
+
+
+@dataclass
+class Continuation:
+    """Represents a continuation of a process or workflow."""
+
+    run_id: str
+    node_id: str
+    kind: str
+    token: str
+    prompt: str | None = None
+    resume_schema: dict | None = None
+    deadline: datetime | None = None
+    poll: dict | None = None
+    next_wakeup_at: datetime | None = None
+    attempts: int = 0
+    channel: str | None = None
+    created_at: datetime = datetime.utcnow()
+    closed: bool = False  # ← NEW
+    payload: dict[str, Any] | None = None  # set at creation time
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "node_id": self.node_id,
+            "kind": self.kind,
+            "token": self.token,
+            "prompt": self.prompt,
+            "resume_schema": self.resume_schema,
+            "deadline": self.deadline.isoformat() if self.deadline else None,
+            "poll": self.poll,
+            "next_wakeup_at": self.next_wakeup_at.isoformat() if self.next_wakeup_at else None,
+            "attempts": self.attempts,
+            "channel": self.channel,
+            "created_at": self.created_at.isoformat(),
+            "closed": self.closed,
+            "payload": self.payload,
+        }

aethergraph/services/continuations/factory.py

@@ -0,0 +1,34 @@
+import os
+
+from stores.fs_store import FSContinuationStore
+from stores.inmem_store import InMemoryContinuationStore
+
+from aethergraph.contracts.services.continuations import AsyncContinuationStore
+
+
+def make_continuation_store() -> AsyncContinuationStore:
+    """Factory to create a continuation store based on environment configuration.
+     Returns:
+        An instance of AsyncContinuationStore.
+
+    Currently supports:
+    - InMemoryContinuationStore (CONT_STORE="inmem")
+    - FSContinuationStore (CONT_STORE="fs")
+
+    We need env vars:
+    - CONT_STORE: "inmem" or "fs" (default: "fs")
+    - CONT_SECRET: optional secret for HMAC token generation
+    - CONT_ROOT: for FS store, root directory to store continuations (default: "./artifacts/continuations")
+    """
+    kind = (os.getenv("CONT_STORE", "fs")).lower()
+    secret = os.getenv("CONT_SECRET")
+    secret_bytes = secret.encode("utf-8") if secret else os.urandom(32)
+
+    if kind == "inmem":
+        return InMemoryContinuationStore(secret=secret_bytes)
+    elif kind == "fs":
+        return FSContinuationStore(
+            root=os.getenv("CONT_ROOT", "./artifacts/continuations"), secret=secret_bytes
+        )
+    else:
+        raise ValueError(f"Unknown continuation store kind: {kind}")

aethergraph/services/continuations/stores/fs_store.py

@@ -0,0 +1,264 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import asdict
+from datetime import datetime, timezone
+import hashlib
+import hmac
+import json
+import logging
+import os
+from pathlib import Path
+import re
+import threading
+from typing import Any
+
+from ..continuation import Continuation, Correlator
+
+
+class FSContinuationStore:  # implements AsyncContinuationStore
+    def __init__(self, root: str | Path, secret: bytes):
+        self.root = Path(root)
+        self.secret = secret
+        self._lock = threading.RLock()
+
+    # ---------- helpers ----------
+    def _cont_path(self, run_id: str, node_id: str) -> Path:
+        return self.root / "runs" / run_id / "nodes" / node_id / "continuation.json"
+
+    def _token_idx_path(self, token: str) -> Path:
+        return self.root / "index" / "tokens" / f"{token}.json"
+
+    def _rev_idx_path(self, token: str) -> Path:
+        return self.root / "index" / "rev" / f"{token}.json"
+
+    def _safe(self, s: str) -> str:
+        s = (s or "").replace("\\", "_").replace("/", "_").replace(":", "_")
+        s = re.sub(r"[^A-Za-z0-9._@-]", "_", s)
+        s = re.sub(r"_+", "_", s).strip("._ ") or "x"
+        if len(s) > 100:
+            h = hashlib.sha1(s.encode()).hexdigest()[:8]
+            s = f"{s[:92]}_{h}"
+        return s
+
+    def _corr_dir(self, scheme: str, channel: str, thread: str, message: str) -> Path:
+        return (
+            self.root
+            / "index"
+            / "corr"
+            / self._safe(scheme)
+            / self._safe(channel)
+            / self._safe(thread)
+            / self._safe(message)
+        )
+
+    def _corr_tokens_path(self, scheme: str, channel: str, thread: str, message: str) -> Path:
+        return self._corr_dir(scheme, channel, thread, message) / "tokens.json"
+
+    def _hmac(self, *parts: str) -> str:
+        h = hmac.new(self.secret, digestmod=hashlib.sha256)
+        for p in parts:
+            h.update(p.encode("utf-8"))
+        return h.hexdigest()
+
+    async def mint_token(self, run_id: str, node_id: str, attempts: int) -> str:
+        return self._hmac(run_id, node_id, str(attempts), os.urandom(8).hex())
+
+    # ---------- core ----------
+    async def save(self, cont: Continuation) -> None:
+        def _write():
+            path = self._cont_path(cont.run_id, cont.node_id)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            payload = cont.to_dict() if hasattr(cont, "to_dict") else asdict(cont)
+            for k in ("deadline", "next_wakeup_at", "created_at"):
+                v = payload.get(k)
+                if isinstance(v, datetime):
+                    payload[k] = v.astimezone(timezone.utc).isoformat()
+            tmp = path.with_suffix(".tmp")
+            tmp.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+            tmp.replace(path)
+            self._write_token_index(cont.run_id, cont.node_id, cont.token)
+
+        await asyncio.to_thread(_write)
+
+    async def get(self, run_id: str, node_id: str) -> Continuation | None:
+        def _read():
+            path = self._cont_path(run_id, node_id)
+            if not path.exists():
+                return None
+            raw = json.loads(path.read_text(encoding="utf-8"))
+            for k in ("deadline", "next_wakeup_at", "created_at"):
+                if raw.get(k):
+                    raw[k] = datetime.fromisoformat(raw[k])
+            raw["closed"] = bool(raw.get("closed", False))
+            return Continuation(**raw)
+
+        return await asyncio.to_thread(_read)
+
+    async def delete(self, run_id: str, node_id: str) -> None:
+        def _del():
+            p = self._cont_path(run_id, node_id)
+            if p.exists():
+                p.unlink()
+
+        await asyncio.to_thread(_del)
+
+    # ---------- token helpers ----------
+    def _write_token_index(self, run_id: str, node_id: str, token: str) -> None:
+        with self._lock:
+            p = self._token_idx_path(token)
+            p.parent.mkdir(parents=True, exist_ok=True)
+            p.write_text(
+                json.dumps({"run_id": run_id, "node_id": node_id}, indent=2), encoding="utf-8"
+            )
+
+    async def get_by_token(self, token: str) -> Continuation | None:
+        def _lookup():
+            p = self._token_idx_path(token)
+            if not p.exists():
+                return None
+            ref = json.loads(p.read_text(encoding="utf-8"))
+            return ref["run_id"], ref["node_id"]
+
+        ref = await asyncio.to_thread(_lookup)
+        if not ref:
+            return None
+        run_id, node_id = ref
+        return await self.get(run_id, node_id)
+
+    async def mark_closed(self, token: str) -> None:
+        c = await self.get_by_token(token)
+        if not c:
+            return
+        if not c.closed:
+            c.closed = True
+            await self.save(c)
+
+    async def verify_token(self, run_id: str, node_id: str, token: str) -> bool:
+        c = await self.get(run_id, node_id)
+        return bool(c and hmac.compare_digest(token, c.token))
+
+    # ---------- correlators ----------
+    async def bind_correlator(self, *, token: str, corr: Correlator) -> None:
+        def _bind():
+            scheme, channel, thread, message = corr.key()
+            tokens_path = self._corr_tokens_path(scheme, channel, thread, message)
+            tokens_path.parent.mkdir(parents=True, exist_ok=True)
+            toks: list[str] = []
+            if tokens_path.exists():
+                try:
+                    toks = json.loads(tokens_path.read_text(encoding="utf-8"))
+                except Exception:
+                    toks = []
+            if token not in toks:
+                toks.append(token)
+                tokens_path.write_text(json.dumps(toks, indent=2), encoding="utf-8")
+                # reverse index
+                r = self._rev_idx_path(token)
+                r.parent.mkdir(parents=True, exist_ok=True)
+                paths = []
+                if r.exists():
+                    try:
+                        paths = json.loads(r.read_text(encoding="utf-8"))
+                    except Exception:
+                        paths = []
+                key_path = str(tokens_path.relative_to(self.root))
+                if key_path not in paths:
+                    paths.append(key_path)
+                    r.write_text(json.dumps(paths, indent=2), encoding="utf-8")
+
+        await asyncio.to_thread(_bind)
+
+    async def find_by_correlator(self, *, corr: Correlator) -> Continuation | None:
+        def _read_toks():
+            scheme, channel, thread, message = corr.key()
+            p = self._corr_tokens_path(scheme, channel, thread, message)
+            if not p.exists():
+                return []
+            try:
+                return json.loads(p.read_text(encoding="utf-8")) or []
+            except Exception:
+                return []
+
+        toks = await asyncio.to_thread(_read_toks)
+        for tok in reversed(toks):
+            c = await self.get_by_token(tok)
+            if c and not c.closed:
+                if c.deadline and datetime.now(timezone.utc) > c.deadline.astimezone(timezone.utc):
+                    continue
+                return c
+        return None
+
+    async def last_open(self, *, channel: str, kind: str) -> Continuation | None:
+        # Optional slow scan (dev only)
+        def _scan():
+            waits = []
+            runs_path = self.root / "runs"
+            if not runs_path.exists():
+                return waits
+            for run_dir in runs_path.iterdir():
+                nodes_dir = run_dir / "nodes"
+                if not nodes_dir.exists():
+                    continue
+                for node_dir in nodes_dir.iterdir():
+                    cont_path = node_dir / "continuation.json"
+                    if cont_path.exists():
+                        waits.append(json.loads(cont_path.read_text(encoding="utf-8")))
+            return waits
+
+        waits = await asyncio.to_thread(_scan)
+        for raw in reversed(waits):
+            if raw.get("closed"):
+                continue
+            if raw.get("channel") == channel and raw.get("kind") == kind:
+                for k in ("deadline", "next_wakeup_at", "created_at"):
+                    if raw.get(k):
+                        raw[k] = datetime.fromisoformat(raw[k])
+                return Continuation(**raw)
+        return None
+
+    async def list_waits(self) -> list[dict[str, Any]]:
+        def _scan():
+            out = []
+            runs_path = self.root / "runs"
+            if not runs_path.exists():
+                return out
+            for run_dir in runs_path.iterdir():
+                nodes_dir = run_dir / "nodes"
+                if not nodes_dir.exists():
+                    continue
+                for node_dir in nodes_dir.iterdir():
+                    cont_path = node_dir / "continuation.json"
+                    if cont_path.exists():
+                        out.append(json.loads(cont_path.read_text(encoding="utf-8")))
+            return out
+
+        return await asyncio.to_thread(_scan)
+
+    async def clear(self) -> None:
+        def _clear():
+            for sub in ("runs", "index"):
+                p = self.root / sub
+                if p.exists():
+                    for root, dirs, files in os.walk(p, topdown=False):
+                        for f in files:
+                            try:
+                                os.remove(Path(root) / f)
+                            except Exception:
+                                logger = logging.getLogger(
+                                    "aethergraph.services.continuations.stores.fs_store"
+                                )
+                                logger.warning("Failed to remove file: %s", Path(root) / f)
+                        for d in dirs:
+                            try:
+                                os.rmdir(Path(root) / d)
+                            except Exception:
+                                logger = logging.getLogger(
+                                    "aethergraph.services.continuations.stores.fs_store"
+                                )
+                                logger.warning("Failed to remove dir: %s", Path(root) / d)
+
+        await asyncio.to_thread(_clear)
+
+    async def alias_for(self, token: str) -> str | None:
+        return token[:24]