aethergraph 0.1.0a1__py3-none-any.whl → 0.1.0a3__py3-none-any.whl

aethergraph/storage/kv/inmem_kv.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import threading
+import time
+from typing import Any
+
+from aethergraph.contracts.storage.async_kv import AsyncKV
+
+
+@dataclass
+class KVEntry:
+    value: Any
+    expire_at: float | None = None
+
+
+class InMemoryKV(AsyncKV):
+    """
+    Simple in-memory KV.
+
+    - Process-local, not shared across processes.
+    - Thread-safe via RLock (sidecar + main thread can share safely).
+    - TTL managed best-effort on access / purge.
+    """
+
+    def __init__(self, *, prefix: str = ""):
+        self._data: dict[str, Any] = {}
+        self._expires_at: dict[str, float | None] = {}
+        self._lock = threading.RLock()
+        self._prefix = prefix
+
+    async def get(self, key: str, default: Any = None) -> Any:
+        now = time.time()
+        with self._lock:
+            if key not in self._data:
+                return default
+            exp = self._expires_at.get(key)
+            if exp is not None and exp < now:
+                # expired
+                self._data.pop(key, None)
+                self._expires_at.pop(key, None)
+                return default
+            return self._data[key]
+
+    async def set(self, key: str, value: Any, *, ttl_s: int | None = None) -> None:
+        with self._lock:
+            self._data[key] = value
+            self._expires_at[key] = time.time() + ttl_s if ttl_s is not None else None
+
+    async def delete(self, key: str) -> None:
+        with self._lock:
+            self._data.pop(key, None)
+            self._expires_at.pop(key, None)
+
+    async def mget(self, keys: list[str]) -> list[Any]:
+        # reuse get() so TTL is respected
+        return [await self.get(k) for k in keys]
+
+    async def mset(self, kv: dict[str, Any], *, ttl_s: int | None = None) -> None:
+        for k, v in kv.items():
+            await self.set(k, v, ttl_s=ttl_s)
+
+    async def expire(self, key: str, ttl_s: int) -> None:
+        with self._lock:
+            if key in self._data:
+                self._expires_at[key] = time.time() + ttl_s
+
+    async def purge_expired(self, limit: int = 1000) -> int:
+        now = time.time()
+        removed = 0
+        with self._lock:
+            for k in list(self._data.keys()):
+                if removed >= limit:
+                    break
+                exp = self._expires_at.get(k)
+                if exp is not None and exp < now:
+                    self._data.pop(k, None)
+                    self._expires_at.pop(k, None)
+                    removed += 1
+        return removed
+
+    # Helper to prefix keys
+    def _k(self, k: str) -> str:
+        return f"{self._prefix}{k}" if self._prefix else k
+
+    async def list_append_unique(
+        self, key: str, items: list[dict], *, id_key: str = "id", ttl_s: int | None = None
+    ) -> list[dict]:
+        """Append items to a list at `key`, ensuring uniqueness based on `id_key`."""
+        k = self._k(key)
+        with self._lock:
+            cur = list(self._data.get(k, KVEntry([])).value or [])
+            seen = {x.get(id_key) for x in cur if isinstance(x, dict)}
+            cur.extend([x for x in items if isinstance(x, dict) and x.get(id_key) not in seen])
+            self._data[k] = KVEntry(value=cur, expire_at=(time.time() + ttl_s) if ttl_s else None)
+            return cur
+
+    async def list_pop_all(self, key: str) -> list:
+        """Pop and return all items from the list at `key`."""
+        k = self._k(key)
+        with self._lock:
+            e = self._data.pop(k, None)
+            return list(e.value) if e and isinstance(e.value, list) else []

aethergraph/storage/kv/layered_kv.py

@@ -0,0 +1,52 @@
+# storage/kv/layered_kv.py
+from __future__ import annotations
+
+from typing import Any
+
+from aethergraph.contracts.storage.async_kv import AsyncKV
+
+
+class LayeredKV(AsyncKV):
+    """
+    Read-through / write-through KV:
+
+    - hot: typically InMemoryKV
+    - cold: persistent KV (SqliteKV, RedisKV, etc.)
+    """
+
+    def __init__(self, hot: AsyncKV, cold: AsyncKV):
+        self.hot = hot
+        self.cold = cold
+
+    async def get(self, key: str, default: Any = None) -> Any:
+        v = await self.hot.get(key, default=None)
+        if v is not None:
+            return v
+        v = await self.cold.get(key, default=default)
+        if v is not None:
+            await self.hot.set(key, v)
+        return v
+
+    async def set(self, key: str, value: Any, *, ttl_s: int | None = None) -> None:
+        await self.cold.set(key, value, ttl_s=ttl_s)
+        await self.hot.set(key, value, ttl_s=ttl_s)
+
+    async def delete(self, key: str) -> None:
+        await self.cold.delete(key)
+        await self.hot.delete(key)
+
+    async def mget(self, keys: list[str]) -> list[Any]:
+        return [await self.get(k) for k in keys]
+
+    async def mset(self, kv: dict[str, Any], *, ttl_s: int | None = None) -> None:
+        for k, v in kv.items():
+            await self.set(k, v, ttl_s=ttl_s)
+
+    async def expire(self, key: str, ttl_s: int) -> None:
+        await self.cold.expire(key, ttl_s)
+        await self.hot.expire(key, ttl_s)
+
+    async def purge_expired(self, limit: int = 1000) -> int:
+        n_cold = await self.cold.purge_expired(limit)
+        n_hot = await self.hot.purge_expired(limit)
+        return n_cold + n_hot

aethergraph/storage/kv/sqlite_kv.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from aethergraph.contracts.storage.async_kv import AsyncKV
+
+from .sqlite_kv_sync import SQLiteKVSync
+
+
+class SqliteKV(AsyncKV):
+    """
+    Async KV on top of SQLiteKVSync via asyncio.to_thread.
+    Safe across threads (RLock in sync core).
+    """
+
+    def __init__(self, path: str, *, prefix: str = ""):
+        self._sync = SQLiteKVSync(path, prefix=prefix)
+
+    async def get(self, key: str, default: Any = None) -> Any:
+        return await asyncio.to_thread(self._sync.get, key, default)
+
+    async def set(self, key: str, value: Any, *, ttl_s: int | None = None) -> None:
+        await asyncio.to_thread(self._sync.set, key, value, ttl_s)
+
+    async def delete(self, key: str) -> None:
+        await asyncio.to_thread(self._sync.delete, key)
+
+    async def mget(self, keys: list[str]) -> list[Any]:
+        return await asyncio.to_thread(self._sync.mget, keys)
+
+    async def mset(self, kv: dict[str, Any], *, ttl_s: int | None = None) -> None:
+        await asyncio.to_thread(self._sync.mset, kv, ttl_s)
+
+    async def expire(self, key: str, ttl_s: int) -> None:
+        await asyncio.to_thread(self._sync.expire, key, ttl_s)
+
+    async def purge_expired(self, limit: int = 1000) -> int:
+        return await asyncio.to_thread(self._sync.purge_expired, limit)

aethergraph/storage/kv/sqlite_kv_sync.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import threading
+import time
+from typing import Any
+
+"""
+SQLite Key-Value Store with TTL (synchronous). Only used by async wrapper SQLiteKV.
+"""
+
+
+class SQLiteKVSync:
+    """
+    Durable KV with TTL (JSON values), thread-safe via RLock.
+    """
+
+    def __init__(self, path: str, *, prefix: str = ""):
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        self._db = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
+        self._db.execute("PRAGMA journal_mode=WAL;")
+        self._db.execute("PRAGMA synchronous=NORMAL;")
+        self._db.execute(
+            """
+            CREATE TABLE IF NOT EXISTS kv (
+              k TEXT PRIMARY KEY,
+              v TEXT,
+              expire_at REAL
+            )
+            """
+        )
+        self._db.execute("CREATE INDEX IF NOT EXISTS kv_exp_idx ON kv(expire_at);")
+        self._lock = threading.RLock()
+        self._prefix = prefix
+
+    def _k(self, k: str) -> str:
+        return f"{self._prefix}{k}" if self._prefix else k
+
+    def get(self, key: str, default: Any = None) -> Any:
+        k = self._k(key)
+        with self._lock:
+            row = self._db.execute("SELECT v, expire_at FROM kv WHERE k=?", (k,)).fetchone()
+            if not row:
+                return default
+            v_txt, exp = row
+            if exp and exp < time.time():
+                self._db.execute("DELETE FROM kv WHERE k=?", (k,))
+                return default
+            try:
+                return json.loads(v_txt)
+            except Exception:
+                return default
+
+    def set(self, key: str, value: Any, ttl_s: int | None = None) -> None:
+        k = self._k(key)
+        exp = time.time() + ttl_s if ttl_s is not None else None
+        v_txt = json.dumps(value, ensure_ascii=False)
+        with self._lock:
+            self._db.execute(
+                """
+                INSERT INTO kv (k, v, expire_at) VALUES (?, ?, ?)
+                ON CONFLICT(k) DO UPDATE SET v=excluded.v, expire_at=excluded.expire_at
+                """,
+                (k, v_txt, exp),
+            )
+
+    def delete(self, key: str) -> None:
+        k = self._k(key)
+        with self._lock:
+            self._db.execute("DELETE FROM kv WHERE k=?", (k,))
+
+    def mget(self, keys: list[str]) -> list[Any]:
+        return [self.get(k) for k in keys]
+
+    def mset(self, kv: dict[str, Any], ttl_s: int | None = None) -> None:
+        for k, v in kv.items():
+            self.set(k, v, ttl_s=ttl_s)
+
+    def expire(self, key: str, ttl_s: int) -> None:
+        k = self._k(key)
+        exp = time.time() + ttl_s
+        with self._lock:
+            self._db.execute("UPDATE kv SET expire_at=? WHERE k=?", (exp, k))
+
+    def purge_expired(self, limit: int = 1000) -> int:
+        now = time.time()
+        with self._lock:
+            rows = self._db.execute(
+                "SELECT k FROM kv WHERE expire_at IS NOT NULL AND expire_at < ? LIMIT ?",
+                (now, limit),
+            ).fetchall()
+            keys = [r[0] for r in rows]
+            if not keys:
+                return 0
+            self._db.executemany("DELETE FROM kv WHERE k=?", [(k,) for k in keys])
+        return len(keys)

aethergraph/storage/memory/event_persist.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+import hashlib
+from typing import Any
+
+from aethergraph.contracts.services.memory import Event, Persistence
+from aethergraph.contracts.storage.doc_store import DocStore
+from aethergraph.contracts.storage.event_log import EventLog
+
+
+class EventLogPersistence(Persistence):
+    """
+    Persistence built on top of generic EventLog + DocStore.
+
+    - append_event: logs Event rows into EventLog with scope_id=run_id, kind="memory".
+    - save_json / load_json: store arbitrary JSON in DocStore using memdoc:// URIs.
+    """
+
+    def __init__(
+        self,
+        *,
+        log: EventLog,
+        docs: DocStore,
+        uri_prefix: str = "memdoc://",
+    ):
+        self._log = log
+        self._docs = docs
+        self._prefix = uri_prefix
+
+    # --------- helpers ---------
+    def _doc_id_from_uri(self, uri: str) -> str:
+        """
+        Accepts:
+          - memdoc://<id>  -> <id>
+          - anything-else  -> hashed to a stable doc_id.
+        """
+        if uri.startswith(self._prefix):
+            return uri[len(self._prefix) :]
+        # fallback: hash to avoid weird chars
+        h = hashlib.sha1(uri.encode("utf-8")).hexdigest()
+        return f"memdoc/{h}"
+
+    def _uri_from_doc_id(self, doc_id: str) -> str:
+        if doc_id.startswith("memdoc://"):
+            return doc_id
+        return f"{self._prefix}{doc_id}"
+
+    # --------- API ---------
+    async def append_event(self, run_id: str, evt: Event) -> None:
+        payload = asdict(evt)
+        payload.setdefault("scope_id", run_id)
+        payload.setdefault("kind", "memory")
+        # you can add tags like ["mem"] if useful
+        await self._log.append(payload)
+
+    async def save_json(self, uri: str, obj: dict[str, Any]) -> str:
+        doc_id = self._doc_id_from_uri(uri)
+        # Let DocStore own where/how it writes
+        await self._docs.put(doc_id, obj)
+        return self._uri_from_doc_id(doc_id)
+
+    async def load_json(self, uri: str) -> dict[str, Any]:
+        doc_id = self._doc_id_from_uri(uri)
+        doc = await self._docs.get(doc_id)
+        if doc is None:
+            raise FileNotFoundError(f"Memory JSON not found for URI: {uri}")
+        return doc

aethergraph/storage/memory/fs_persist.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import asdict
+import json
+import os
+from pathlib import Path
+import threading
+import time
+from typing import Any
+
+from aethergraph.contracts.services.memory import Event, Persistence
+
+
+class FSPersistence(Persistence):
+    """
+    File-system based persistence for memory events + JSON blobs.
+
+    - Events are written to:
+        <base_dir>/mem/<run_id>/events/YYYY-MM-DD.jsonl
+
+    - JSON docs are read/written via file:// URIs:
+        file://relative/path.json  -> <base_dir>/relative/path.json
+        file:///abs/path.json      -> /abs/path.json (not under base_dir)
+    """
+
+    def __init__(self, *, base_dir: str):
+        self.base_dir = Path(base_dir).resolve()
+        self._lock = threading.RLock()
+
+    # ---------- Event log (append-only JSONL) ----------
+
+    async def append_event(self, run_id: str, evt: Event) -> None:
+        day = time.strftime("%Y-%m-%d", time.gmtime())
+        path = self.base_dir / "mem" / run_id / "events" / f"{day}.jsonl"
+
+        def _write() -> None:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            raw = asdict(evt)
+            # Drop None values but keep [] / {} / 0.
+            data = {k: v for k, v in raw.items() if v is not None}
+            line = json.dumps(data, ensure_ascii=False) + "\n"
+            with self._lock, path.open("a", encoding="utf-8") as f:
+                f.write(line)
+
+        await asyncio.to_thread(_write)
+
+    # ---------- JSON blob helpers (file:// URIs) ----------
+
+    def _uri_to_path(self, uri: str) -> Path:
+        """
+        Convert a file:// URI into a local Path, resolving *relative* paths
+        against self.base_dir. Works cross-platform.
+        """
+        if not uri.startswith("file://"):
+            raise ValueError(f"FSPersistence only supports file:// URIs, got {uri!r}")
+
+        raw = uri[len("file://") :]
+
+        # Windows: normalize file:///C:/... -> C:/...
+        if (
+            os.name == "nt"
+            and raw.startswith("/")
+            and len(raw) > 2
+            and raw[1].isalpha()
+            and raw[2] == ":"
+        ):
+            raw = raw[1:]
+
+        p = Path(raw)
+
+        # Relative paths are resolved under base_dir
+        if not p.is_absolute():
+            p = self.base_dir / p
+
+        return p
+
+    def _path_to_uri(self, path: Path) -> str:
+        """
+        Convert a local Path to canonical file:// URI with forward slashes.
+        """
+        p = path.resolve()
+        s = p.as_posix()
+
+        # Ensure absolute paths appear as file:///... (add leading slash on Windows)
+        if p.is_absolute() and not s.startswith("/"):
+            s = "/" + s
+
+        return f"file://{s}"
+
+    async def save_json(self, uri: str, obj: dict[str, Any]) -> str:
+        """
+        Save JSON to the location specified by a file:// URI.
+        Returns the canonical file:// URI of the saved file.
+        """
+        path = self._uri_to_path(uri)
+
+        def _write() -> None:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            tmp = path.with_suffix(path.suffix + ".tmp")
+            with self._lock, tmp.open("w", encoding="utf-8") as f:
+                json.dump(obj, f, ensure_ascii=False, indent=2)
+            os.replace(tmp, path)
+
+        await asyncio.to_thread(_write)
+        return self._path_to_uri(path)
+
+    async def load_json(self, uri: str) -> dict[str, Any]:
+        """
+        Inverse of save_json: load JSON from a file:// URI.
+        """
+        path = self._uri_to_path(uri)
+
+        def _read() -> dict[str, Any]:
+            with self._lock, path.open("r", encoding="utf-8") as f:
+                return json.load(f)
+
+        return await asyncio.to_thread(_read)

aethergraph/{services/memory/hotlog_kv.py → storage/memory/hotlog.py}

@@ -1,6 +1,8 @@
 from aethergraph.contracts.services.kv import AsyncKV
 from aethergraph.contracts.services.memory import Event, HotLog
 
+# No specific backend is required; we use AsyncKV for storage.
+
 
 def kv_hot_key(run_id: str) -> str:
     return f"mem:{run_id}:hot"
@@ -13,13 +15,17 @@ class KVHotLog(HotLog):
     async def append(self, run_id: str, evt: Event, *, ttl_s: int, limit: int) -> None:
         key = kv_hot_key(run_id)
         buf = list((await self.kv.get(key, default=[])) or [])
-        buf.append(evt.__dict__)  # store as dict for JSON serializability
+        buf.append(evt.__dict__)  # store as dict for JSON-ability
         if len(buf) > limit:
             buf = buf[-limit:]
         await self.kv.set(key, buf, ttl_s=ttl_s)
 
     async def recent(
-        self, run_id: str, *, kinds: list[str] | None = None, limit: int = 50
+        self,
+        run_id: str,
+        *,
+        kinds: list[str] | None = None,
+        limit: int = 50,
     ) -> list[Event]:
         buf = (await self.kv.get(kv_hot_key(run_id), default=[])) or []
         if kinds:

aethergraph/{services → storage}/memory/indices.py

@@ -1,19 +1,22 @@
+from __future__ import annotations
+
 from typing import Any
 
-from aethergraph.contracts.services.kv import AsyncKV
 from aethergraph.contracts.services.memory import Event, Indices
+from aethergraph.contracts.storage.async_kv import AsyncKV
 
 
 def idx_by_ref_kind(run_id: str) -> str:
-    return f"mem:{run_id}:idx2:ref_kind"
+    return f"mem:{run_id}:idx:ref_kind"
 
 
 def idx_by_name(run_id: str) -> str:
-    return f"mem:{run_id}:idx2:name"
+    return f"mem:{run_id}:idx:name"
 
 
 def idx_by_topic(run_id: str) -> str:
-    return f"mem:{run_id}:idx2:topic"
+    # topic = tool / agent / flow name
+    return f"mem:{run_id}:idx:topic"
 
 
 class KVIndices(Indices):
@@ -29,25 +32,39 @@ class KVIndices(Indices):
         by_name = (await self.kv.get(idx_by_name(run_id), {})) or {}
         by_topic = (await self.kv.get(idx_by_topic(run_id), {})) or {}
 
+        # 1) Index by output name & ref.kind
         for v in outs:
             nm = v.get("name")
             if not nm:
                 continue
+
+            # name index
             by_name[nm] = {
                 "ts": ts,
                 "event_id": eid,
                 "vtype": v.get("vtype"),
                 "value": v.get("value"),
             }
+
+            # ref.kind index
             if v.get("vtype") == "ref" and isinstance(v.get("value"), dict):
                 kind = v["value"].get("kind")
                 uri = v["value"].get("uri")
                 if kind and uri:
                     lst = by_kind.setdefault(kind, [])
-                    lst.append({"ts": ts, "event_id": eid, "name": nm, "uri": uri, "topic": tool})
+                    lst.append(
+                        {
+                            "ts": ts,
+                            "event_id": eid,
+                            "name": nm,
+                            "uri": uri,
+                            "topic": tool,
+                        }
+                    )
                     if len(lst) > 200:
                         del lst[:-200]
 
+        # 2) Index by topic (tool / flow / agent)
         if tool:
             last = by_topic.get(tool, {}) or {}
             last["ts"] = ts
@@ -59,15 +76,22 @@ class KVIndices(Indices):
         await self.kv.set(idx_by_name(run_id), by_name, ttl_s=self.ttl)
         await self.kv.set(idx_by_topic(run_id), by_topic, ttl_s=self.ttl)
 
+    # ---------- queries ----------
     async def last_by_name(self, run_id: str, name: str) -> dict[str, Any] | None:
         by_name = await self.kv.get(idx_by_name(run_id), {}) or {}
         return by_name.get(name)
 
     async def latest_refs_by_kind(
-        self, run_id: str, kind: str, *, limit: int = 50
+        self,
+        run_id: str,
+        kind: str,
+        *,
+        limit: int = 50,
     ) -> list[dict[str, Any]]:
         by_kind = await self.kv.get(idx_by_ref_kind(run_id), {}) or {}
-        return list(reversed((by_kind.get(kind) or [])[-limit:]))
+        arr = (by_kind.get(kind) or [])[-limit:]
+        # latest first
+        return list(reversed(arr))
 
     async def last_outputs_by_topic(self, run_id: str, topic: str) -> dict[str, Any] | None:
         by_topic = await self.kv.get(idx_by_topic(run_id), {}) or {}

aethergraph/storage/metering/meter_event.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+from aethergraph.contracts.services.metering import MeteringStore
+from aethergraph.contracts.storage.event_log import EventLog
+
+METER_TAG = "meter"  # shared tag for all metering events
+
+
+@dataclass
+class EventLogMeteringStore(MeteringStore):
+    """
+    MeteringStore backed by a generic EventLog.
+
+    Convention:
+      - kind: e.g. "meter.llm", "meter.run", "meter.artifact", "meter.event"
+      - tags: always includes "meter" so queries don't mix with other app events
+    """
+
+    event_log: EventLog
+
+    async def append(self, event: dict[str, Any]) -> None:
+        # Enforce metering conventions
+        kind = event.get("kind")
+        if not kind or not kind.startswith("meter."):
+            raise ValueError(f"Metering event kind must start with 'meter.': {kind!r}")
+
+        tags = set(event.get("tags") or [])
+        tags.add(METER_TAG)
+        event["tags"] = list(tags)
+
+        await self.event_log.append(event)
+
+    async def query(
+        self,
+        *,
+        since: datetime | None = None,
+        until: datetime | None = None,
+        kinds: list[str] | None = None,
+        limit: int | None = None,
+        user_id: str | None = None,
+        org_id: str | None = None,
+    ) -> list[dict[str, Any]]:
+        # Always filter by meter tag
+        return await self.event_log.query(
+            scope_id=None,
+            since=since,
+            until=until,
+            kinds=kinds,
+            tags=[METER_TAG],
+            limit=limit,
+            user_id=user_id,
+            org_id=org_id,
+        )