aethergraph 0.1.0a1__py3-none-any.whl → 0.1.0a2__py3-none-any.whl

aethergraph/services/artifacts/jsonl_index.py

@@ -1,123 +0,0 @@
-# aethergraph/artifacts/index_jsonl.py
-from __future__ import annotations
-
-import asyncio
-import json
-import os
-import threading
-from typing import Literal
-
-from aethergraph.contracts.services.artifacts import Artifact
-
-
-class JsonlArtifactIndexSync:
-    """Simple JSONL-based artifact index for small to medium scale use cases.
-    Not suitable for very large scale (millions of artifacts) due to linear scans.
-    """
-
-    def __init__(self, path: str, occurrences_path: str | None = None):
-        self.path = path
-        self.occ_path = occurrences_path or (os.path.splitext(path)[0] + "_occurrences.jsonl")
-        os.makedirs(os.path.dirname(path), exist_ok=True)
-        # small in-memory map for quick lookup / dedup of last write
-        self._by_id = {}
-        self._lock = threading.Lock()
-        if os.path.exists(self.path):
-            with open(self.path, encoding="utf-8") as f:
-                for line in f:
-                    if not line.strip():
-                        continue
-                    rec = json.loads(line)
-                    self._by_id[rec["artifact_id"]] = rec
-
-    def upsert(self, a: Artifact) -> None:
-        """Upsert an artifact record."""
-        with self._lock:
-            rec = a.to_dict()
-            self._by_id[a.artifact_id] = rec
-            with open(self.path, "a", encoding="utf-8") as f:
-                f.write(json.dumps(rec) + "\n")
-
-    def list_for_run(self, run_id: str) -> list[Artifact]:
-        """List all artifacts for a given run_id."""
-        return [Artifact(**r) for r in self._by_id.values() if r.get("run_id") == run_id]
-
-    def search(
-        self,
-        *,
-        kind: str | None = None,
-        labels: dict[str, str] | None = None,
-        metric: str | None = None,
-        mode: Literal["max", "min"] | None = None,
-    ) -> list[Artifact]:
-        """Search artifacts by kind, labels (exact match), and metric (min/max)."""
-        rows = list(self._by_id.values())
-        if kind:
-            rows = [r for r in rows if r.get("kind") == kind]
-        if labels:
-            for k, v in labels.items():
-                rows = [r for r in rows if r.get("labels", {}).get(k) == v]
-        if metric and mode:
-            rows = [r for r in rows if metric in r.get("metrics", {})]
-            rows.sort(key=lambda r: r["metrics"][metric], reverse=(mode == "max"))
-        return [Artifact(**r) for r in rows]
-
-    def best(
-        self,
-        *,
-        kind: str,
-        metric: str,
-        mode: Literal["max", "min"],
-        filters: dict[str, str] | None = None,
-    ) -> Artifact | None:
-        """Get the best artifact by metric with optional filters."""
-        rows = self.search(kind=kind, labels=filters, metric=metric, mode=mode)
-        return rows[0] if rows else None
-
-    def pin(self, artifact_id: str, pinned: bool = True) -> None:
-        """Pin or unpin an artifact by artifact_id."""
-        if artifact_id in self._by_id:
-            self._by_id[artifact_id]["pinned"] = bool(pinned)
-            with open(self.path, "a", encoding="utf-8") as f:
-                f.write(json.dumps(self._by_id[artifact_id]) + "\n")
-
-    def record_occurrence(self, a: Artifact, extra_labels: dict | None = None):
-        """
-        Append-only log that this artifact appeared in this run/node at this time.
-        Keeps lineage even if bytes are identical across runs.
-        """
-        row = {
-            "artifact_id": a.artifact_id,
-            "run_id": a.run_id,
-            "graph_id": a.graph_id,
-            "node_id": a.node_id,
-            "tool_name": a.tool_name,
-            "tool_version": a.tool_version,
-            "created_at": a.created_at,
-            "labels": a.labels | (extra_labels or {}),
-        }
-        with open(self.occ_path, "a", encoding="utf-8") as f:
-            f.write(json.dumps(row) + "\n")
-
-
-class JsonlArtifactIndex:  # implements AsyncArtifactIndex
-    def __init__(self, path: str, occurrences_path: str | None = None):
-        self._sync = JsonlArtifactIndexSync(path, occurrences_path)
-
-    async def upsert(self, a: Artifact) -> None:
-        await asyncio.to_thread(self._sync.upsert, a)
-
-    async def list_for_run(self, run_id: str) -> list[Artifact]:
-        return await asyncio.to_thread(self._sync.list_for_run, run_id)
-
-    async def search(self, **kw) -> list[Artifact]:
-        return await asyncio.to_thread(self._sync.search, **kw)
-
-    async def best(self, **kw) -> Artifact | None:
-        return await asyncio.to_thread(self._sync.best, **kw)
-
-    async def pin(self, artifact_id: str, pinned: bool = True) -> None:
-        await asyncio.to_thread(self._sync.pin, artifact_id, pinned)
-
-    async def record_occurrence(self, a: Artifact, extra_labels: dict | None = None):
-        await asyncio.to_thread(self._sync.record_occurrence, a, extra_labels)

aethergraph/services/artifacts/sqlite_index.py

@@ -1,209 +0,0 @@
-# aethergraph/artifacts/index_sqlite.py
-from __future__ import annotations
-
-import asyncio
-import json
-import sqlite3
-from typing import Literal
-
-from aethergraph.contracts.services.artifacts import Artifact
-from aethergraph.services.artifacts.jsonl_index import JsonlArtifactIndexSync
-
-
-class SqliteArtifactIndexSync:
-    """SQLite-based artifact index for medium to large scale use cases.
-    Suitable for larger scale (millions of artifacts) with indexing.
-    """
-
-    def __init__(self, db_path: str):
-        self.db_path = db_path
-        self._init()
-
-    def _init(self):
-        con = sqlite3.connect(self.db_path)
-        cur = con.cursor()
-        cur.execute("""
-        CREATE TABLE IF NOT EXISTS artifacts (
-            artifact_id TEXT PRIMARY KEY,
-            uri TEXT NOT NULL,
-            kind TEXT NOT NULL,
-            bytes INTEGER,
-            sha256 TEXT,
-            mime TEXT,
-            run_id TEXT,
-            graph_id TEXT,
-            node_id TEXT,
-            tool_name TEXT,
-            tool_version TEXT,
-            created_at TEXT,
-            labels TEXT,
-            metrics TEXT,
-            params TEXT,
-            preview_uri TEXT,
-            pinned INTEGER DEFAULT 0
-        )""")
-        cur.execute("CREATE INDEX IF NOT EXISTS idx_kind ON artifacts(kind)")
-        cur.execute("CREATE INDEX IF NOT EXISTS idx_run ON artifacts(run_id)")
-        con.commit()
-        con.close()
-
-    def upsert(self, a: Artifact) -> None:
-        con = sqlite3.connect(self.db_path)
-        cur = con.cursor()
-        cur.execute(
-            """
-        INSERT INTO artifacts
-        (artifact_id, uri, kind, bytes, sha256, mime, run_id, graph_id, node_id,
-         tool_name, tool_version, created_at, labels, metrics, params, preview_uri, pinned)
-        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
-        ON CONFLICT(artifact_id) DO UPDATE SET
-            uri=excluded.uri, kind=excluded.kind, bytes=excluded.bytes,
-            sha256=excluded.sha256, mime=excluded.mime, run_id=excluded.run_id,
-            graph_id=excluded.graph_id, node_id=excluded.node_id,
-            tool_name=excluded.tool_name, tool_version=excluded.tool_version,
-            created_at=excluded.created_at, labels=excluded.labels,
-            metrics=excluded.metrics, params=excluded.params,
-            preview_uri=excluded.preview_uri, pinned=excluded.pinned
-        """,
-            (
-                a.artifact_id,
-                a.uri,
-                a.kind,
-                a.bytes,
-                a.sha256,
-                a.mime,
-                a.run_id,
-                a.graph_id,
-                a.node_id,
-                a.tool_name,
-                a.tool_version,
-                a.created_at,
-                json.dumps(a.labels),
-                json.dumps(a.metrics),
-                json.dumps(a.params),
-                a.preview_uri,
-                1 if a.pinned else 0,
-            ),
-        )
-        con.commit()
-        con.close()
-
-    def list_for_run(self, run_id: str) -> list[Artifact]:
-        con = sqlite3.connect(self.db_path)
-        cur = con.cursor()
-        cur.execute("SELECT * FROM artifacts WHERE run_id=? ORDER BY created_at", (run_id,))
-        rows = cur.fetchall()
-        con.close()
-        return [self._row_to_artifact(r) for r in rows]
-
-    def search(
-        self,
-        *,
-        kind: str | None = None,
-        labels: dict[str, str] | None = None,
-        metric: str | None = None,
-        mode: Literal["max", "min"] | None = None,
-    ) -> list[Artifact]:
-        con = sqlite3.connect(self.db_path)
-        cur = con.cursor()
-        q = "SELECT * FROM artifacts WHERE 1=1"
-        args = []
-        if kind:
-            q += " AND kind=?"
-        args.append(kind)
-        # naive label filter: all requested label kv must be contained in labels json
-        if labels:
-            for k, v in labels.items():
-                q += " AND json_extract(labels, ?) = ?"
-                args += (f"$.{k}", v)
-        cur.execute(q, args)
-        rows = [self._row_to_artifact(r) for r in cur.fetchall()]
-        con.close()
-        if metric and mode and rows:
-            rows = [r for r in rows if r.metrics and metric in r.metrics]
-            reverse = mode == "max"
-            rows.sort(key=lambda a: a.metrics.get(metric, float("-inf")), reverse=reverse)
-        return rows
-
-    def best(
-        self,
-        *,
-        kind: str,
-        metric: str,
-        mode: Literal["max", "min"],
-        filters: dict[str, str] | None = None,
-    ) -> Artifact | None:
-        rows = self.search(kind=kind, labels=filters, metric=metric, mode=mode)
-        return rows[0] if rows else None
-
-    def pin(self, artifact_id: str, pinned: bool = True) -> None:
-        con = sqlite3.connect(self.db_path)
-        cur = con.cursor()
-        cur.execute(
-            "UPDATE artifacts SET pinned=? WHERE artifact_id=?", (1 if pinned else 0, artifact_id)
-        )
-        con.commit()
-        con.close()
-
-    def _row_to_artifact(self, r) -> Artifact:
-        (
-            artifact_id,
-            uri,
-            kind,
-            bytes_,
-            sha256,
-            mime,
-            run_id,
-            graph_id,
-            node_id,
-            tool_name,
-            tool_version,
-            created_at,
-            labels,
-            metrics,
-            params,
-            preview_uri,
-            pinned,
-        ) = r
-        return Artifact(
-            artifact_id=artifact_id,
-            uri=uri,
-            kind=kind,
-            bytes=bytes_,
-            sha256=sha256,
-            mime=mime,
-            run_id=run_id,
-            graph_id=graph_id,
-            node_id=node_id,
-            tool_name=tool_name,
-            tool_version=tool_version,
-            created_at=created_at,
-            labels=json.loads(labels or "{}"),
-            metrics=json.loads(metrics or "{}"),
-            params=json.loads(params or "{}"),
-            preview_uri=preview_uri,
-            pinned=bool(pinned),
-        )
-
-
-class SqliteArtifactIndex:  # implements AsyncArtifactIndex
-    def __init__(self, path: str, occurrences_path: str | None = None):
-        self._sync = JsonlArtifactIndexSync(path, occurrences_path)
-
-    async def upsert(self, a: Artifact) -> None:
-        await asyncio.to_thread(self._sync.upsert, a)
-
-    async def list_for_run(self, run_id: str) -> list[Artifact]:
-        return await asyncio.to_thread(self._sync.list_for_run, run_id)
-
-    async def search(self, **kw) -> list[Artifact]:
-        return await asyncio.to_thread(self._sync.search, **kw)
-
-    async def best(self, **kw) -> Artifact | None:
-        return await asyncio.to_thread(self._sync.best, **kw)
-
-    async def pin(self, artifact_id: str, pinned: bool = True) -> None:
-        await asyncio.to_thread(self._sync.pin, artifact_id, pinned)
-
-    async def record_occurrence(self, a: Artifact, extra_labels: dict | None = None):
-        await asyncio.to_thread(self._sync.record_occurrence, a, extra_labels)

aethergraph/services/memory/distillers/episode.py

@@ -1,116 +0,0 @@
-from __future__ import annotations
-
-import hashlib
-import json
-import time
-from typing import Any
-
-from aethergraph.contracts.services.memory import Distiller, Event, HotLog, Indices, Persistence
-
-
-def _now_iso() -> str:
-    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-
-
-def _stable_event_id(parts: dict[str, Any]) -> str:
-    blob = json.dumps(parts, sort_keys=True, ensure_ascii=False).encode("utf-8")
-    return hashlib.sha256(blob).hexdigest()[:24]
-
-
-def _episode_uri(sessiorun_idn_id: str, tool: str, run_id: str) -> str:
-    safe = tool.replace("/", "_")
-    return f"file://mem/{run_id}/episodes/{safe}/{run_id}.json"
-
-
-class EpisodeSummarizer(Distiller):
-    """
-    Aggregate all events for (tool, run_id) into a compact episode summary:
-      - sources: event_ids
-      - merged metrics (last-write-wins)
-      - notes: last N textual notes
-    Writes a JSON summary artifact and emits a lightweight run_summary event.
-    """
-
-    def __init__(
-        self, *, include_metrics: bool = True, note_limit: int = 8, note_chars: int = 2000
-    ):
-        self.include_metrics = include_metrics
-        self.note_limit = note_limit
-        self.note_chars = note_chars
-
-    async def distill(
-        self,
-        run_id: str,
-        *,
-        hotlog: HotLog,
-        persistence: Persistence,
-        indices: Indices,
-        tool: str,
-        **kw,
-    ) -> dict[str, Any]:
-        # Pull a reasonable window from hot memory; filter in-process.
-        # (If needed later, add a Persistence scan by day.)
-        events = await hotlog.recent(
-            run_id, kinds=["tool_start", "tool_result", "error", "run_summary"], limit=400
-        )
-
-        eps = [e for e in events if e.run_id == run_id and (e.tool or "") == tool]
-        if not eps:
-            return {}
-
-        srcs: list[str] = []
-        notes: list[str] = []
-        metrics: dict[str, float] = {}
-
-        for e in eps:
-            if e.event_id:
-                srcs.append(e.event_id)
-            if self.include_metrics and e.metrics:
-                metrics.update(e.metrics)  # simple merge; last-write-wins
-            if e.text:
-                notes.append(e.text)
-
-        ts = _now_iso()
-        summary = {
-            "kind": "episode_summary",
-            "run_id": run_id,
-            "tool": tool,
-            "ts": ts,
-            "sources": srcs,
-            "metrics": metrics,
-            "notes": notes[-self.note_limit :],
-        }
-
-        uri = _episode_uri(run_id, tool, run_id)
-        await persistence.save_json(uri, summary)
-
-        # Emit a compact run_summary event
-        compact_text = "\n".join(summary["notes"][-self.note_limit :])[: self.note_chars]
-        evt_base = {
-            "run_id": run_id,
-            "tool": tool,
-            "kind": "run_summary",
-            "severity": 1,
-            "tags": ["summary", "episode"],
-        }
-        eid = _stable_event_id(
-            {
-                "ts": ts,
-                "run_id": run_id,
-                "tool": tool,
-                "kind": "run_summary",
-                "text": compact_text[:200],
-            }
-        )
-        evt = Event(
-            event_id=eid,
-            ts=ts,
-            text=compact_text,
-            metrics={"notes": len(notes)},
-            signal=0.5,
-            **evt_base,
-        )
-        await hotlog.append(run_id, evt, ttl_s=7 * 24 * 3600, limit=1000)
-        await persistence.append_event(run_id, evt)
-
-        return {"uri": uri, "sources": srcs, "metrics": metrics}

aethergraph/services/memory/distillers/rolling.py

@@ -1,74 +0,0 @@
-import time
-from typing import Any
-
-from aethergraph.contracts.services.memory import Distiller, Event, HotLog, Indices, Persistence
-
-
-def _now_iso():
-    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
-
-
-def ar_summary_uri(run_id: str, tag: str, ts: str) -> str:
-    # Save summaries under the same base "mem/<run_id>/..." tree as append_event,
-    # but using a file:// URI so FSPersistence can handle it.
-    safe_ts = ts.replace(":", "-")
-    return f"file://mem/{run_id}/summaries/{tag}/{safe_ts}.json"
-
-
-class RollingSummarizer(Distiller):
-    def __init__(
-        self, *, max_turns: int = 20, min_signal: float = 0.25, turn_kinds: list[str] | None = None
-    ):
-        self.max_turns = max_turns
-        self.min_signal = min_signal
-        self.turn_kinds = turn_kinds or ["user_msg", "assistant_msg"]
-
-    async def distill(
-        self, run_id: str, *, hotlog: HotLog, persistence: Persistence, indices: Indices, **kw
-    ) -> dict[str, Any]:
-        turns = await hotlog.recent(run_id, kinds=self.turn_kinds, limit=self.max_turns * 2)
-        kept = [t for t in turns if (t.signal or 0.0) >= self.min_signal]
-        if not kept:
-            return {}
-
-        lines = []
-        srcs: list[str] = []
-        for t in kept[-self.max_turns :]:
-            role = "User" if t.kind == "user_msg" else "Assistant"
-            if t.text:
-                lines.append(f"{role}: {t.text}")
-                srcs.append(t.event_id)
-        digest_text = "\n".join(lines)
-        ts = _now_iso()
-        summary = {
-            "kind": "rolling_summary",
-            "run_id": run_id,
-            "ts": ts,
-            "sources": srcs,
-            "text": digest_text,
-        }
-
-        uri = ar_summary_uri(run_id, "rolling", ts)
-
-        await persistence.save_json(uri=uri, obj=summary)
-
-        evt = Event(
-            event_id="",
-            ts=ts,
-            run_id=run_id,
-            kind="rolling_summary",
-            severity=1,
-            signal=0.5,
-            text=digest_text,
-            metrics={"num_turns": len(kept)},
-            tags=["summary"],
-        )
-
-        from aethergraph.services.memory.facade import stable_event_id
-
-        evt.event_id = stable_event_id(
-            {"ts": ts, "run_id": run_id, "kind": "rolling_summary", "text": digest_text[:200]}
-        )
-        await hotlog.append(run_id, evt, ttl_s=7 * 24 * 3600, limit=1000)
-        await persistence.append_event(run_id, evt)
-        return {"uri": uri, "sources": srcs}