aethergraph 0.1.0a1__py3-none-any.whl → 0.1.0a2__py3-none-any.whl

aethergraph/services/container/default_container.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-import os
 from pathlib import Path
 from typing import Any
 
@@ -12,17 +11,25 @@ from aethergraph.config.config import AppSettings
 from aethergraph.contracts.services.llm import LLMClientProtocol
 
 # ---- scheduler ---- TODO: move to a separate server to handle scheduling across threads/processes
+from aethergraph.contracts.services.metering import MeteringService
+from aethergraph.contracts.services.runs import RunStore
+from aethergraph.contracts.services.sessions import SessionStore
 from aethergraph.contracts.services.state_stores import GraphStateStore
+from aethergraph.contracts.storage.artifact_index import AsyncArtifactIndex
+from aethergraph.contracts.storage.artifact_store import AsyncArtifactStore
+from aethergraph.contracts.storage.event_log import EventLog
 from aethergraph.core.execution.global_scheduler import GlobalForwardScheduler
 
 # ---- artifact services ----
-from aethergraph.services.artifacts.fs_store import FSArtifactStore  # AsyncArtifactStore
-from aethergraph.services.artifacts.jsonl_index import JsonlArtifactIndex  # AsyncArtifactIndex
-from aethergraph.services.auth.dev import AllowAllAuthz, DevTokenAuthn
+from aethergraph.core.runtime.run_manager import RunManager
+from aethergraph.core.runtime.runtime_registry import current_registry, set_current_registry
+from aethergraph.services.auth.authn import DevTokenAuthn
+from aethergraph.services.auth.authz import AllowAllAuthz
 from aethergraph.services.channel.channel_bus import ChannelBus
 
 # ---- channel services ----
 from aethergraph.services.channel.factory import build_bus, make_channel_adapters_from_env
+from aethergraph.services.channel.ingress import ChannelIngress
 from aethergraph.services.clock.clock import SystemClock
 from aethergraph.services.continuations.stores.fs_store import (
     FSContinuationStore,  # AsyncContinuationStore
@@ -30,8 +37,6 @@ from aethergraph.services.continuations.stores.fs_store import (
 from aethergraph.services.eventbus.inmem import InMemoryEventBus
 
 # ---- kv services ----
-from aethergraph.services.kv.ephemeral import EphemeralKV
-from aethergraph.services.kv.sqlite_kv import SQLiteKV
 from aethergraph.services.llm.factory import build_llm_clients
 from aethergraph.services.llm.service import LLMService
 from aethergraph.services.logger.std import LoggingConfig, StdLoggerService
@@ -39,26 +44,40 @@ from aethergraph.services.mcp.service import MCPService
 
 # ---- memory services ----
 from aethergraph.services.memory.factory import MemoryFactory
-from aethergraph.services.memory.hotlog_kv import KVHotLog
-from aethergraph.services.memory.indices import KVIndices
-from aethergraph.services.memory.persist_fs import FSPersistence
-from aethergraph.services.metering.noop import NoopMetering
+from aethergraph.services.metering.eventlog_metering import EventLogMeteringService
 from aethergraph.services.prompts.file_store import FilePromptStore
 from aethergraph.services.rag.chunker import TextSplitter
 from aethergraph.services.rag.facade import RAGFacade
 
 # ---- RAG components ----
-from aethergraph.services.rag.index_factory import create_vector_index
+from aethergraph.services.rate_limit.inmem_rate_limit import SimpleRateLimiter
 from aethergraph.services.redactor.simple import RegexRedactor  # Simple PII redactor
 from aethergraph.services.registry.unified_registry import UnifiedRegistry
 from aethergraph.services.resume.multi_scheduler_resume_bus import MultiSchedulerResumeBus
 from aethergraph.services.resume.router import ResumeRouter
 from aethergraph.services.schedulers.registry import SchedulerRegistry
+from aethergraph.services.scope.scope_factory import ScopeFactory
 from aethergraph.services.secrets.env import EnvSecrets
-from aethergraph.services.state_stores.json_store import JsonGraphStateStore
 from aethergraph.services.tracing.noop import NoopTracer
+from aethergraph.services.viz.viz_service import VizService
 from aethergraph.services.waits.wait_registry import WaitRegistry
 from aethergraph.services.wakeup.memory_queue import ThreadSafeWakeupQueue
+from aethergraph.storage.factory import (
+    build_artifact_index,
+    build_artifact_store,
+    build_continuation_store,
+    build_doc_store,
+    build_event_log,
+    build_graph_state_store,
+    build_memory_hotlog,
+    build_memory_indices,
+    build_memory_persistence,
+    build_run_store,
+    build_session_store,
+    build_vector_index,
+)
+from aethergraph.storage.kv.inmem_kv import InMemoryKV as EphemeralKV
+from aethergraph.storage.metering.meter_event import EventLogMeteringStore
 
 SERVICE_KEYS = [
     # core
@@ -75,7 +94,6 @@ SERVICE_KEYS = [
     "wakeup_queue",
     # storage and artifacts
     "kv_hot",
-    "kv_durable",
     "artifacts",
     "artifact_index",
     # memory
@@ -98,6 +116,9 @@ class DefaultContainer:
     # root
     root: str
 
+    # scope
+    scope_factory: ScopeFactory
+
     # schedulers
     schedulers: dict[str, Any]
 
@@ -120,25 +141,35 @@ class DefaultContainer:
 
     # storage and artifacts
     kv_hot: EphemeralKV
-    kv_durable: SQLiteKV
-    artifacts: FSArtifactStore
-    artifact_index: JsonlArtifactIndex
+    artifacts: AsyncArtifactStore
+    artifact_index: AsyncArtifactIndex
+    eventlog: EventLog
 
     # memory
     memory_factory: MemoryFactory
 
+    # viz - only useful with frontend; otherwise this is a pure storage service for metrics and images
+    viz_service: VizService | None = None
+
     # optional llm service
     llm: LLMClientProtocol | None = None
     rag: RAGFacade | None = None
     mcp: MCPService | None = None
 
+    # run controls -- for http endpoints and run manager
+    run_store: RunStore | None = None
+    run_manager: RunManager | None = None  # RunManager
+    session_store: SessionStore | None = None  # SessionStore
+
     # optional services (not used by default)
     event_bus: InMemoryEventBus | None = None
     prompts: FilePromptStore | None = None
     authn: DevTokenAuthn | None = None
     authz: AllowAllAuthz | None = None
     redactor: RegexRedactor | None = None
-    metering: NoopMetering | None = None
+
+    metering: MeteringService | None = None
+    rate_limiter: SimpleRateLimiter | None = None
     tracer: NoopTracer | None = None
     secrets: EnvSecrets | None = None
 
@@ -148,6 +179,9 @@ class DefaultContainer:
     # settings -- not a service, but useful to have around
     settings: AppSettings | None = None
 
+    # channel ingress (set after init to avoid circular dependency)
+    channel_ingress: ChannelIngress | None = None  # set after init to avoid circular dependency
+
 
 def build_default_container(
     *,
@@ -172,20 +206,28 @@ def build_default_container(
     # we use user specified root if provided, else from config/env
     root_p = Path(root).resolve() if root else Path(cfg.root).resolve()
     (root_p / "kv").mkdir(parents=True, exist_ok=True)
-    (root_p / "continuations").mkdir(parents=True, exist_ok=True)
     (root_p / "index").mkdir(parents=True, exist_ok=True)
     (root_p / "memory").mkdir(parents=True, exist_ok=True)
-    (root_p / "graph_states").mkdir(parents=True, exist_ok=True)
+
+    # Scope factory
+    scope_factory = ScopeFactory()
+
+    # event log for metering and channel events --
+    # TODO: make configurable from cfg
+    eventlog = build_event_log(cfg)
 
     # core services
     logger_factory = StdLoggerService.build(
         LoggingConfig.from_cfg(cfg, log_dir=str(root_p / "logs"))
     )
     clock = SystemClock()
-    registry = UnifiedRegistry()
+    # registry = UnifiedRegistry()
+    registry: UnifiedRegistry = current_registry()
+    set_current_registry(registry)  # set global registry, ensure singleton (optional)
 
     # continuations and resume
-    cont_store = FSContinuationStore(root=str(root_p / "continuations"), secret=os.urandom(32))
+    cont_store = build_continuation_store(cfg)
+
     sched_registry = SchedulerRegistry()
     wait_registry = WaitRegistry()
     resume_bus = MultiSchedulerResumeBus(
@@ -198,7 +240,8 @@ def build_default_container(
         wait_registry=wait_registry,
     )
     wakeup_queue = ThreadSafeWakeupQueue()  # TODO: this is a placeholder, not fully implemented
-    state_store = JsonGraphStateStore(root=str(root_p / "graph_states"))
+    # state_store = JsonGraphStateStore(root=str(root_p / "graph_states"))
+    state_store = build_graph_state_store(cfg)
 
     # global scheduler
     global_sched = GlobalForwardScheduler(
@@ -212,7 +255,7 @@ def build_default_container(
     }
 
     # channels
-    channel_adapters = make_channel_adapters_from_env(cfg)
+    channel_adapters = make_channel_adapters_from_env(cfg, event_log=eventlog)
     channels = build_bus(
         channel_adapters,
         default="console:stdin",
@@ -221,18 +264,13 @@ def build_default_container(
         cont_store=cont_store,
     )
 
-    # storage and artifacts
+    # storage and artifacts -- kv_hot has special methods for hot data, do not use other persistent kv here
     kv_hot = EphemeralKV()
-    kv_durable = SQLiteKV(str(root_p / "kv" / "kv.sqlite"))
-    artifacts = FSArtifactStore(
-        str(root_p / "artifacts")
-    )  # async wrapper over FileArtifactStoreSync
-    artifact_index = JsonlArtifactIndex(str(root_p / "index" / "artifacts.jsonl"))
 
-    # memory
-    hotlog = KVHotLog(kv=kv_hot)
-    persistence = FSPersistence(base_dir=str(root_p / "memory"))
-    indices = KVIndices(kv=kv_durable, hot_ttl_s=7 * 24 * 3600)
+    artifacts = build_artifact_store(cfg)
+    artifact_index = build_artifact_index(cfg)
+
+    viz_service = VizService(event_log=eventlog)
 
     # optional services
     secrets = (
@@ -241,11 +279,8 @@ def build_default_container(
     llm_clients = build_llm_clients(cfg.llm, secrets)  # return {profile: GenericLLMClient}
     llm_service = LLMService(clients=llm_clients) if llm_clients else None
 
-    rag_cfg = cfg.rag
-    vec_index = create_vector_index(
-        backend=rag_cfg.backend, index_path=str(root_p / "rag" / "rag_index"), dim=rag_cfg.dim
-    )
-
+    # RAG facade
+    vec_index = build_vector_index(cfg)
     rag_facade = RAGFacade(
         corpus_root=str(root_p / "rag" / "rag_corpora"),
         artifacts=artifacts,
@@ -257,11 +292,17 @@ def build_default_container(
     )
     mcp = MCPService()  # empty MCP service; users can register clients as needed
 
+    # memory factory
+    persistence = build_memory_persistence(cfg)
+    hotlog = build_memory_hotlog(cfg)
+    indices = build_memory_indices(cfg)
+    docs = build_doc_store(cfg)
     memory_factory = MemoryFactory(
         hotlog=hotlog,
         persistence=persistence,
         indices=indices,
         artifacts=artifacts,
+        docs=docs,
         hot_limit=int(cfg.memory.hot_limit),
         hot_ttl_s=int(cfg.memory.hot_ttl_s),
         default_signal_threshold=float(cfg.memory.signal_threshold),
@@ -270,8 +311,35 @@ def build_default_container(
         rag_facade=rag_facade,
     )
 
-    return DefaultContainer(
+    # run store and manager
+    run_store = build_run_store(cfg)
+    run_manager = RunManager(
+        run_store=run_store,
+        registry=registry,
+        sched_registry=sched_registry,
+        max_concurrent_runs=cfg.rate_limit.max_concurrent_runs,
+    )
+    session_store = build_session_store(cfg)
+
+    # Metering service
+    # TODO: make metering service configurable
+    metering_store = EventLogMeteringStore(event_log=eventlog)
+    metering = EventLogMeteringService(store=metering_store)
+
+    # rate limiter
+    rl_settings = cfg.rate_limit
+    rate_limiter = SimpleRateLimiter(
+        max_events=rl_settings.burst_max_runs,
+        window_seconds=rl_settings.burst_window_seconds,
+    )
+
+    # auth services
+    authn = DevTokenAuthn()
+    authz = AllowAllAuthz()
+
+    container = DefaultContainer(
         root=str(root_p),
+        scope_factory=scope_factory,
         schedulers=schedulers,
         registry=registry,
         logger=logger_factory,
@@ -284,25 +352,36 @@ def build_default_container(
         resume_router=resume_router,
         wakeup_queue=wakeup_queue,
         kv_hot=kv_hot,
-        kv_durable=kv_durable,
         state_store=state_store,
         artifacts=artifacts,
         artifact_index=artifact_index,
+        viz_service=viz_service,
+        eventlog=eventlog,
         memory_factory=memory_factory,
         llm=llm_service,
         rag=rag_facade,
         mcp=mcp,
+        run_store=run_store,
+        run_manager=run_manager,
+        session_store=session_store,
         secrets=secrets,
         event_bus=None,
         prompts=None,
-        authn=None,
-        authz=None,
+        authn=authn,
+        authz=authz,
         redactor=None,
-        metering=None,
+        metering=metering,
+        rate_limiter=rate_limiter,
         tracer=None,
         settings=cfg,
     )
 
+    # channel ingress (after container is built to avoid circular dependency)
+    container.channel_ingress = ChannelIngress(
+        container=container, logger=logger_factory.for_channel()
+    )
+    return container
+
 
 # Singleton (used unless the host sets their own)
 DEFAULT_CONTAINER: DefaultContainer | None = None

aethergraph/services/continuations/continuation.py

@@ -37,6 +37,12 @@ class Continuation:
     closed: bool = False  # ← NEW
     payload: dict[str, Any] | None = None  # set at creation time
 
+    # new session, etc.
+    session_id: str | None = None
+    agent_id: str | None = None
+    app_id: str | None = None
+    graph_id: str | None = None
+
     def to_dict(self) -> dict[str, Any]:
         return {
             "run_id": self.run_id,

aethergraph/services/continuations/stores/fs_store.py

@@ -95,6 +95,25 @@ class FSContinuationStore:  # implements AsyncContinuationStore
 
         return await asyncio.to_thread(_read)
 
+    async def list_cont_by_run(self, run_id: str) -> list[Continuation]:
+        def _list():
+            out = []
+            run_path = self.root / "runs" / run_id / "nodes"
+            if not run_path.exists():
+                return out
+            for node_dir in run_path.iterdir():
+                cont_path = node_dir / "continuation.json"
+                if cont_path.exists():
+                    raw = json.loads(cont_path.read_text(encoding="utf-8"))
+                    for k in ("deadline", "next_wakeup_at", "created_at"):
+                        if raw.get(k):
+                            raw[k] = datetime.fromisoformat(raw[k])
+                    raw["closed"] = bool(raw.get("closed", False))
+                    out.append(Continuation(**raw))
+            return out
+
+        return await asyncio.to_thread(_list)
+
     async def delete(self, run_id: str, node_id: str) -> None:
         def _del():
             p = self._cont_path(run_id, node_id)

aethergraph/services/eventhub/event_hub.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Awaitable, Callable
+
+Subscriber = Callable[[dict], Awaitable[None]]
+
+"""
+Lightweight in-memory pub/sub hub for UI events.
+
+This sits *alongside* EventLog:
+
+- EventLog = durable storage (what HTTP polling uses)
+- EventHub = transient fan-out for live WebSocket subscribers
+
+Later:
+- You can swap this for Redis/Kafka/etc without changing the adapters.
+"""
+
+
+class EventHub:
+    """
+    Pub/sub keyed by (scope_id, kind).
+
+    For example:
+      scope_id="session:abc123", kind="session_chat"
+      scope_id="run:xyz456", kind="run_channel"
+    """
+
+    def __init__(self) -> None:
+        # (scope_id, kind) -> set of async callbacks
+        self._subs: dict[tuple[str, str], set[Subscriber]] = defaultdict(set)
+
+    def subscribe(self, scope_id: str, kind: str, cb: Subscriber) -> None:
+        """Register a callback that should receive new rows for this (scope, kind)."""
+        self._subs[(scope_id, kind)].add(cb)
+
+    def unsubscribe(self, scope_id: str, kind: str, cb: Subscriber) -> None:
+        """Remove a previously registered callback."""
+        key = (scope_id, kind)
+        subs = self._subs.get(key)
+        if not subs:
+            return
+        subs.discard(cb)
+        if not subs:
+            # Optional: cleanup empty sets
+            self._subs.pop(key, None)
+
+    async def broadcast(self, row: dict) -> None:
+        """
+        Push a new EventLog row to all subscribers.
+
+        Expected row schema:
+          {
+            "id": str,
+            "ts": float,
+            "scope_id": str,
+            "kind": str,
+            "payload": {...},
+          }
+        """
+        scope_id = row.get("scope_id")
+        kind = row.get("kind")
+        if not scope_id or not kind:
+            return
+
+        # Snapshot to avoid mutation during iteration
+        subs = list(self._subs.get((scope_id, kind), []))
+
+        for cb in subs:
+            try:
+                await cb(row)
+            except Exception:
+                # TODO: log error; maybe drop the subscriber if repeatedly failing
+                # For now, we ignore to avoid breaking others.
+                continue