aethergraph 0.1.0a1__py3-none-any.whl

aethergraph/services/memory/factory.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from aethergraph.contracts.services.artifacts import AsyncArtifactStore  # generic protocol
+from aethergraph.contracts.services.memory import HotLog, Indices, Persistence
+from aethergraph.services.memory.facade import MemoryFacade
+
+"""
+    # --- Artifacts (async FS store)
+    artifacts = FSArtifactStore(artifacts_dir)
+
+    # --- KV for hotlog/indices (choose EphemeralKV or SQLiteKV)
+    kv = SQLiteKV(f"{artifacts_dir}/kv.sqlite") if durable else EphemeralKV()
+
+    # --- HotLog + Indices
+    hotlog   = KVHotLog(kv, default_ttl_s=7*24*3600, default_limit=1000)
+    indices  = KVIndices(kv, ttl_s=7*24*3600)
+
+    # --- Persistence (JSONL under artifacts_dir/mem/<session>/events/...)
+    persistence = FSPersistence(base_dir=artifacts_dir)
+
+    # --- Factory
+    factory = MemoryFactory(
+        hotlog=hotlog,
+        persistence=persistence,
+        indices=indices,
+        artifacts=artifacts,
+        hot_limit=1000,
+        hot_ttl_s=7*24*3600,
+        default_signal_threshold=0.25,
+    )
+
+    # --- Global session handle (optional convenience)
+    global_mem = factory.for_session("global", run_id="global")
+"""
+
+
+@dataclass(frozen=True)
+class MemoryFactory:
+    """Factory for creating MemoryFacade instances with shared components."""
+
+    hotlog: HotLog
+    persistence: Persistence
+    indices: Indices  # key-value backed indices for fast lookups, not artifact storage index
+    artifacts: AsyncArtifactStore
+    hot_limit: int = 1000
+    hot_ttl_s: int = 7 * 24 * 3600
+    default_signal_threshold: float = 0.25
+    logger: Any | None = None
+    llm_service: Any | None = None  # LLMService
+    rag_facade: Any | None = None  # RAGFacade
+
+    def for_session(
+        self,
+        run_id: str,
+        *,
+        graph_id: str | None = None,
+        node_id: str | None = None,
+        agent_id: str | None = None,
+    ) -> MemoryFacade:
+        return MemoryFacade(
+            run_id=run_id,
+            graph_id=graph_id,
+            node_id=node_id,
+            agent_id=agent_id,
+            hotlog=self.hotlog,
+            persistence=self.persistence,
+            indices=self.indices,
+            artifact_store=self.artifacts,
+            hot_limit=self.hot_limit,
+            hot_ttl_s=self.hot_ttl_s,
+            default_signal_threshold=self.default_signal_threshold,
+            logger=self.logger,
+            rag=self.rag_facade,
+            llm=self.llm_service,
+        )

aethergraph/services/memory/hotlog_kv.py

@@ -0,0 +1,27 @@
+from aethergraph.contracts.services.kv import AsyncKV
+from aethergraph.contracts.services.memory import Event, HotLog
+
+
+def kv_hot_key(run_id: str) -> str:
+    return f"mem:{run_id}:hot"
+
+
+class KVHotLog(HotLog):
+    def __init__(self, kv: AsyncKV):
+        self.kv = kv
+
+    async def append(self, run_id: str, evt: Event, *, ttl_s: int, limit: int) -> None:
+        key = kv_hot_key(run_id)
+        buf = list((await self.kv.get(key, default=[])) or [])
+        buf.append(evt.__dict__)  # store as dict for JSON serializability
+        if len(buf) > limit:
+            buf = buf[-limit:]
+        await self.kv.set(key, buf, ttl_s=ttl_s)
+
+    async def recent(
+        self, run_id: str, *, kinds: list[str] | None = None, limit: int = 50
+    ) -> list[Event]:
+        buf = (await self.kv.get(kv_hot_key(run_id), default=[])) or []
+        if kinds:
+            buf = [e for e in buf if e.get("kind") in kinds]
+        return [Event(**e) for e in buf[-limit:]]

aethergraph/services/memory/indices.py

@@ -0,0 +1,74 @@
+from typing import Any
+
+from aethergraph.contracts.services.kv import AsyncKV
+from aethergraph.contracts.services.memory import Event, Indices
+
+
+def idx_by_ref_kind(run_id: str) -> str:
+    return f"mem:{run_id}:idx2:ref_kind"
+
+
+def idx_by_name(run_id: str) -> str:
+    return f"mem:{run_id}:idx2:name"
+
+
+def idx_by_topic(run_id: str) -> str:
+    return f"mem:{run_id}:idx2:topic"
+
+
+class KVIndices(Indices):
+    def __init__(self, kv: AsyncKV, hot_ttl_s: int):
+        self.kv = kv
+        self.ttl = hot_ttl_s
+
+    async def update(self, run_id: str, evt: Event) -> None:
+        ts, eid, tool = evt.ts, evt.event_id, (evt.tool or "")
+        outs = evt.outputs or []
+
+        by_kind = (await self.kv.get(idx_by_ref_kind(run_id), {})) or {}
+        by_name = (await self.kv.get(idx_by_name(run_id), {})) or {}
+        by_topic = (await self.kv.get(idx_by_topic(run_id), {})) or {}
+
+        for v in outs:
+            nm = v.get("name")
+            if not nm:
+                continue
+            by_name[nm] = {
+                "ts": ts,
+                "event_id": eid,
+                "vtype": v.get("vtype"),
+                "value": v.get("value"),
+            }
+            if v.get("vtype") == "ref" and isinstance(v.get("value"), dict):
+                kind = v["value"].get("kind")
+                uri = v["value"].get("uri")
+                if kind and uri:
+                    lst = by_kind.setdefault(kind, [])
+                    lst.append({"ts": ts, "event_id": eid, "name": nm, "uri": uri, "topic": tool})
+                    if len(lst) > 200:
+                        del lst[:-200]
+
+        if tool:
+            last = by_topic.get(tool, {}) or {}
+            last["ts"] = ts
+            last["event_id"] = eid
+            last["last_outputs"] = {v["name"]: v.get("value") for v in outs if v.get("name")}
+            by_topic[tool] = last
+
+        await self.kv.set(idx_by_ref_kind(run_id), by_kind, ttl_s=self.ttl)
+        await self.kv.set(idx_by_name(run_id), by_name, ttl_s=self.ttl)
+        await self.kv.set(idx_by_topic(run_id), by_topic, ttl_s=self.ttl)
+
+    async def last_by_name(self, run_id: str, name: str) -> dict[str, Any] | None:
+        by_name = await self.kv.get(idx_by_name(run_id), {}) or {}
+        return by_name.get(name)
+
+    async def latest_refs_by_kind(
+        self, run_id: str, kind: str, *, limit: int = 50
+    ) -> list[dict[str, Any]]:
+        by_kind = await self.kv.get(idx_by_ref_kind(run_id), {}) or {}
+        return list(reversed((by_kind.get(kind) or [])[-limit:]))
+
+    async def last_outputs_by_topic(self, run_id: str, topic: str) -> dict[str, Any] | None:
+        by_topic = await self.kv.get(idx_by_topic(run_id), {}) or {}
+        return by_topic.get(topic)

aethergraph/services/memory/io_helpers.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from aethergraph.contracts.services.memory import Value
+
+"""Create a Value of vtype 'ref' pointing to the given kind and uri.
+Args:
+    name: name of the Value slot
+    kind: kind of the referenced artifact, e.g. "spec", "design", "output", "tool_result"
+    uri: URI of the referenced artifact, e.g. "file://...", "mem://...", "db://..."
+    meta: optional additional metadata for the Ref
+Returns:
+    Value dict with vtype 'ref'
+
+Example:
+    v = ref(
+        name="my_ref",
+        kind="spec",
+        uri="file://path/to/spec",
+        title="My Spec",
+        mime="application/json"
+    )
+    print(v)
+    # Output: {
+    #   "name": "my_ref",
+    #   "vtype": "ref",
+    #   "value": {
+    #       "kind": "spec",
+    #       "uri": "file://path/to/spec",
+    #       "title": "My Spec",
+    #       "mime": "application/json"
+    #   }
+    # }
+"""
+
+
+def ref(name: str, kind: str, uri: str, **meta) -> Value:
+    v: Value = {
+        "name": name,
+        "vtype": "ref",
+        "value": {"kind": kind, "uri": uri, **meta},
+    }
+    return v
+
+
+def num(name: str, x: float) -> Value:
+    """Create a Value of vtype 'number'."""
+    return {"name": name, "vtype": "number", "value": float(x)}
+
+
+def text(name: str, s: str) -> Value:
+    """Create a Value of vtype 'string'."""
+    return {"name": name, "vtype": "string", "value": str(s)}
+
+
+def flag(name: str, b: bool) -> Value:
+    """Create a Value of vtype 'boolean'."""
+    return {"name": name, "vtype": "boolean", "value": bool(b)}
+
+
+def obj(name: str, d: dict) -> Value:
+    """Create a Value of vtype 'object'."""
+    return {"name": name, "vtype": "object", "value": dict(d)}
+
+
+def arr(name: str, lst: list) -> Value:
+    """Create a Value of vtype 'array'."""
+    return {"name": name, "vtype": "array", "value": list(lst)}
+
+
+def null(name: str) -> Value:
+    """Create a Value of vtype 'null'."""
+    return {"name": name, "vtype": "null", "value": None}

aethergraph/services/memory/persist_fs.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import asdict
+import json
+import os
+import time
+
+from aethergraph.contracts.services.memory import Event, Persistence
+
+
+class FSPersistence(Persistence):
+    def __init__(self, *, base_dir: str):
+        self.base_dir = os.path.abspath(base_dir)
+
+    async def append_event(self, run_id: str, evt: Event) -> None:
+        day = time.strftime("%Y-%m-%d", time.gmtime())
+        rel = os.path.join("mem", run_id, "events", f"{day}.jsonl")
+        path = os.path.join(self.base_dir, rel)
+
+        def _write():
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            with open(path, "a", encoding="utf-8") as f:
+                f.write(json.dumps(asdict(evt), ensure_ascii=False) + "\n")
+
+        await asyncio.to_thread(_write)
+
+    async def save_json(self, uri: str, obj: dict[str, any]) -> None:
+        assert uri.startswith("file://"), f"FSPersistence only supports file://, got {uri!r}"
+        rel = uri[len("file://") :].lstrip("/\\")
+        path = os.path.join(self.base_dir, rel)
+
+        def _write():
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            tmp = path + ".tmp"
+            with open(tmp, "w", encoding="utf-8") as f:
+                json.dump(obj, f, ensure_ascii=False, indent=2)
+            os.replace(tmp, path)
+
+        await asyncio.to_thread(_write)

aethergraph/services/memory/resolver.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from aethergraph.services.memory.facade import MemoryFacade
+
+# -------- Regexes (unchanged) --------
+STEP_RE = re.compile(r"^\s*\$step\[(?P<idx>-?\d+)\]\.refs\.(?P<key>\w+)\s*$")
+FROM_RE = re.compile(r"^\s*\$from:(\w+)\s*$")
+VAR_RE = re.compile(r"^\s*\$var:(\w+)\s*$")
+
+REF_KIND_RE = re.compile(r"^\s*\$resolve\s*:\s*ref\.kind\s*=\s*(\w+)\s*\|\s*last\s*$", re.I)
+NAME_RE = re.compile(r"^\s*\$resolve\s*:\s*name\s*=\s*(\w+)\s*\|\s*last\s*$", re.I)
+TOPIC_NAME_RE = re.compile(
+    r"^\s*\$resolve\s*:\s*topic\s*=\s*([\w\.\-\/]+)\s*\|\s*name\s*=\s*(\w+)\s*$", re.I
+)
+LEGACY_KIND_RE = re.compile(r"^\s*\$resolve\s*:\s*kind\s*=\s*(\w+)\s*\|\s*last\s*$", re.I)
+
+
+class ResolverContext:
+    def __init__(
+        self, mem: MemoryFacade, seq_ctx: dict | None = None, vars: dict[str, Any] | None = None
+    ):
+        self.mem = mem
+        self.seq_ctx = seq_ctx or {}
+        self.vars = vars or {}
+
+
+def _get_step_outputs(seq_ctx: dict, j: int) -> dict[str, Any] | None:
+    steps = seq_ctx.get("steps") or []
+    if 0 <= j < len(steps):
+        return steps[j].get("outputs") or {}
+    return None
+
+
+async def _latest_ref_by_kind(mem: MemoryFacade, kind: str) -> str | None:
+    arr = await mem.latest_refs_by_kind(kind, limit=1)
+    if arr:
+        return arr[0].get("uri")
+    # Fallback scan
+    events = await mem.recent(kinds=["tool_result", "checkpoint"], limit=400)
+    for e in reversed(events):
+        outs = e.outputs or []
+        for v in outs:
+            if (
+                v.get("vtype") == "ref"
+                and isinstance(v.get("value"), dict)
+                and v["value"].get("kind") == kind
+            ):
+                return v["value"].get("uri")
+        # legacy
+        if e.outputs_ref and f"{kind}_ref" in e.outputs_ref:
+            return e.outputs_ref.get(f"{kind}_ref")
+    return None
+
+
+async def _latest_value_by_name(mem: MemoryFacade, name: str) -> Any | None:
+    ent = await mem.last_by_name(name)
+    if ent:
+        return ent.get("value")
+    # Fallback scan
+    events = await mem.recent(kinds=["tool_result", "checkpoint"], limit=400)
+    for e in reversed(events):
+        outs = e.outputs or []
+        for v in outs:
+            if v.get("name") == name:
+                return v.get("value")
+        if e.outputs_ref and name in e.outputs_ref:
+            return e.outputs_ref.get(name)
+    return None
+
+
+async def _latest_value_by_topic_name(mem: MemoryFacade, topic: str, name: str) -> Any | None:
+    ent = await mem.last_outputs_by_topic(topic)
+    if ent:
+        last = ent.get("last_outputs") or {}
+        if name in last:
+            return last[name]
+    # Fallback scan
+    events = await mem.recent(kinds=["tool_result"], limit=400)
+    for e in reversed(events):
+        if (e.tool or "") != topic:
+            continue
+        outs = e.outputs or []
+        for v in outs:
+            if v.get("name") == name:
+                return v.get("value")
+        if e.outputs_ref and name in e.outputs_ref:
+            return e.outputs_ref.get(name)
+    return None
+
+
+async def resolve_params(raw: dict[str, Any], ctx: ResolverContext) -> dict[str, Any]:
+    out = dict(raw)
+
+    # 1) $step[i].refs.key
+    for k, v in list(out.items()):
+        if not isinstance(v, str):
+            continue
+        m = STEP_RE.match(v)
+        if not m:
+            continue
+        idx = int(m.group("idx"))
+        key = m.group("key")
+        steps = ctx.seq_ctx.get("steps") or []
+        j = idx if idx >= 0 else len(steps) + idx
+        refs = _get_step_outputs(ctx.seq_ctx, j)
+        if refs and key in refs:
+            out[k] = refs[key]
+        else:
+            if key.endswith("_ref"):
+                kind = key[:-4]
+                out[k] = await _latest_ref_by_kind(ctx.mem, kind)
+            else:
+                out[k] = None
+
+    # 2) $from:TAG (example strategy slot)
+    for k, v in list(out.items()):
+        if not isinstance(v, str):
+            continue
+        m = FROM_RE.match(v)
+        if m:
+            tag = m.group(1)
+            resolved = None
+            if tag == "last_opt_top1":
+                resolved = await _latest_value_by_topic_name(ctx.mem, "optimize.flow", "top1_ref")
+            out[k] = resolved
+
+    # 3) New selectors + legacy
+    for k, v in list(out.items()):
+        if not isinstance(v, str):
+            continue
+        if m := REF_KIND_RE.match(v):
+            out[k] = await _latest_ref_by_kind(ctx.mem, m.group(1).lower())
+            continue
+        if m := NAME_RE.match(v):
+            out[k] = await _latest_value_by_name(ctx.mem, m.group(1))
+            continue
+        if m := TOPIC_NAME_RE.match(v):
+            out[k] = await _latest_value_by_topic_name(ctx.mem, m.group(1), m.group(2))
+            continue
+        if m := LEGACY_KIND_RE.match(v):
+            out[k] = await _latest_ref_by_kind(ctx.mem, m.group(1).lower())
+            continue
+
+    # 4) $var:NAME
+    for k, v in list(out.items()):
+        if isinstance(v, str) and (m := VAR_RE.match(v)):
+            out[k] = ctx.vars.get(m.group(1))
+
+    return out

aethergraph/services/metering/noop.py

@@ -0,0 +1,4 @@
+# services/metering/noop.py
+class NoopMetering:
+    async def incr(self, metric: str, value: float = 1.0, **tags):
+        return None

aethergraph/services/prompts/file_store.py

@@ -0,0 +1,41 @@
+# services/prompts/file_store.py
+# Simple file-based prompt store
+from pathlib import Path
+
+
+class FilePromptStore:
+    def __init__(self, root: str = "./prompts"):
+        self.root = Path(root)
+
+    async def get(self, name: str, version: str | None = None) -> str:
+        """Get prompt by name and optional version.
+        If version is None, get the latest (unversioned) prompt.
+
+        Args:
+            name: Prompt name (filename without extension)
+            version: Optional version string
+            Returns:
+            Prompt content as string
+
+        Example:
+            prompt = await store.get("welcome_message", version="v1")
+            print(prompt)
+            # ./prompts/welcome_message@v1.md
+        """
+        p = self.root / (f"{name}@{version}.md" if version else f"{name}.md")
+        return p.read_text(encoding="utf-8")
+
+    async def render(self, name: str, **vars) -> str:
+        """Get and render prompt with variable substitution.
+
+        Args:
+            name: Prompt name (filename without extension)
+            **vars: Variables to substitute in the prompt
+        Returns:
+            Rendered prompt content as string
+        """
+        # Tiny {{var}} replacement; swap later for jinja/mustache
+        txt = await self.get(name)
+        for k, v in vars.items():
+            txt = txt.replace(f"{{{{{k}}}}}", str(v))
+        return txt

aethergraph/services/rag/chunker.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+
+class TextSplitter:
+    """A simple text splitter that splits text into chunks of approximately target_tokens,
+    with a specified overlap in tokens.
+
+    Example:
+        splitter = TextSplitter(target_tokens=400, overlap_tokens=60)
+        chunks = splitter.split(long_text)
+        for chunk in chunks:
+            print(chunk)
+    """
+
+    def __init__(self, target_tokens: int = 400, overlap_tokens: int = 60):
+        self.n = max(50, target_tokens)
+        self.o = max(0, min(self.n - 1, overlap_tokens))
+
+    def split(self, text: str) -> list[str]:
+        words = text.split()
+        if not words:
+            return []
+        step = self.n - self.o
+        chunks = []
+        for i in range(0, len(words), step):
+            chunk = " ".join(words[i : i + self.n])
+            if chunk.strip():
+                chunks.append(chunk)
+        return chunks