aethergraph 0.1.0a2__py3-none-any.whl → 0.1.0a4__py3-none-any.whl

aethergraph/services/memory/distillers/long_term_v1.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+import time
+from typing import Any
+
+from aethergraph.contracts.services.memory import Distiller, Event, HotLog
+
+# re-use stable_event_id from the MemoryFacade module
+from aethergraph.contracts.storage.doc_store import DocStore
+from aethergraph.services.memory.utils import _summary_doc_id
+
+
+def _now_iso() -> str:
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+
+def ar_summary_uri_by_run_id(run_id: str, tag: str, ts: str) -> str:
+    """
+    NOTE: To deprecate this function in favor of ar_summary_uri below.
+
+    Save summaries under the same base "mem/<run_id>/..." tree as append_event,
+    but using a file:// URI so FSPersistence can handle it.
+    """
+    safe_ts = ts.replace(":", "-")
+    return f"file://mem/{run_id}/summaries/{tag}/{safe_ts}.json"
+
+
+def ar_summary_uri(scope_id: str, tag: str, ts: str) -> str:
+    """
+    Scope summaries by a logical memory scope, not by run_id.
+    In simple setups, scope_id == run_id. For long-lived companions, scope_id
+    might be something like "user:zcliu:persona:companion_v1".
+    """
+    safe_ts = ts.replace(":", "-")
+    return f"file://mem/{scope_id}/summaries/{tag}/{safe_ts}.json"
+
+
+class LongTermSummarizer(Distiller):
+    """
+    Generic long-term summarizer.
+
+    Goal:
+      - Take a slice of recent events (by kind and/or tag).
+      - Build a compact textual digest plus small structured metadata.
+      - Persist the summary as JSON via Persistence.save_json(...).
+      - Emit a summary Event with kind=summary_kind and data["summary_uri"].
+
+    This does NOT call an LLM by itself; it's a structural/logical summarizer.
+    An LLM-based distiller can be layered on top later (using the same URI scheme).
+
+    Typical usage:
+      - Kinds: ["chat_user", "chat_assistant"] or app-specific kinds.
+      - Tag:   "session", "daily", "episode:<id>", etc.
+    """
+
+    def __init__(
+        self,
+        *,
+        summary_kind: str = "long_term_summary",
+        summary_tag: str = "session",
+        include_kinds: list[str] | None = None,
+        include_tags: list[str] | None = None,
+        max_events: int = 200,
+        min_signal: float = 0.0,
+    ):
+        self.summary_kind = summary_kind
+        self.summary_tag = summary_tag
+        self.include_kinds = include_kinds
+        self.include_tags = include_tags
+        self.max_events = max_events
+        self.min_signal = min_signal
+
+    def _filter_events(self, events: Iterable[Event]) -> list[Event]:
+        out: list[Event] = []
+        kinds = set(self.include_kinds) if self.include_kinds else None
+        tags = set(self.include_tags) if self.include_tags else None
+
+        for e in events:
+            if kinds is not None and e.kind not in kinds:
+                continue
+            if tags is not None:
+                if not e.tags:
+                    continue
+                if not tags.issubset(set(e.tags)):
+                    continue
+            if (e.signal or 0.0) < self.min_signal:
+                continue
+            out.append(e)
+        return out
+
+    async def distill(
+        self,
+        run_id: str,
+        timeline_id: str,
+        scope_id: str = None,
+        *,
+        hotlog: HotLog,
+        docs: DocStore,
+        **kw: Any,
+    ) -> dict[str, Any]:
+        """
+        Steps:
+          1) Grab recent events from HotLog for this run.
+          2) Filter by kinds/tags/min_signal.
+          3) Build a digest:
+             - simple text transcript (role: text)
+             - metadata: ts range, num events
+          4) Save JSON summary via DocStore.put(...).
+          5) Log a summary Event to hotlog + persistence, with data.summary_uri.
+        """
+        # 1) fetch more than we might keep to give filter some slack
+        raw = await hotlog.recent(timeline_id, kinds=None, limit=self.max_events * 2)
+        kept = self._filter_events(raw)
+        if not kept:
+            return {}
+
+        # keep only max_events most recent
+        kept = kept[-self.max_events :]
+
+        # 2) Build digest text (simple transcript-like format)
+        lines: list[str] = []
+        src_ids: list[str] = []
+        first_ts = kept[0].ts
+        last_ts = kept[-1].ts
+
+        for e in kept:
+            role = e.stage or e.kind or "event"
+            if e.text:
+                lines.append(f"[{role}] {e.text}")
+                src_ids.append(e.event_id)
+
+        digest_text = "\n".join(lines)
+        ts = _now_iso()
+
+        # 3) Summary JSON shape
+        summary = {
+            "type": self.summary_kind,
+            "version": 1,
+            "run_id": run_id,
+            "scope_id": scope_id or run_id,
+            "summary_tag": self.summary_tag,
+            "ts": ts,
+            "time_window": {
+                "from": first_ts,
+                "to": last_ts,
+            },
+            "num_events": len(kept),
+            "source_event_ids": src_ids,
+            "text": digest_text,
+        }
+
+        # 4) Persist JSON summary via DocStore
+        scope = scope_id or run_id
+        doc_id = _summary_doc_id(scope, self.summary_tag, ts)
+        await docs.put(doc_id, summary)
+
+        # 5) Emit summary Event
+        # NOTE: we only store a preview in text and full summary in data["summary_uri"]
+        preview = digest_text[:2000] + (" …[truncated]" if len(digest_text) > 2000 else "")
+
+        return {
+            "summary_doc_id": doc_id,
+            "summary_kind": self.summary_kind,
+            "summary_tag": self.summary_tag,
+            "time_window": summary["time_window"],
+            "num_events": len(kept),
+            "preview": preview,
+            "ts": ts,
+        }

aethergraph/services/memory/facade/chat.py

@@ -71,6 +71,7 @@ class ChatMixin:
             Event: The fully persisted `Event` object containing the generated ID and timestamp.
         """
         extra_tags = ["chat"]
+
         if tags:
             extra_tags.extend(tags)
         payload: dict[str, Any] = {"role": role, "text": text}
@@ -252,6 +253,7 @@ class ChatMixin:
         *,
         limit: int = 50,
         roles: Sequence[str] | None = None,
+        tags: Sequence[str] | None = None,
     ) -> list[dict[str, Any]]:
         """
         Retrieve the most recent chat turns as a normalized list.
@@ -259,7 +261,9 @@ class ChatMixin:
         This method fetches the last `limit` chat events of type `chat.turn`
         and returns them in a standardized format. Each item in the returned
         list contains the timestamp, role, text, and tags associated with the
-        chat event.
+        chat event. If `tags` is provided, over-fetch and filter because HotLog doesn't filter by tags.
+        Returned messages are chronological, with the most recent last.
+
 
         Examples:
             Fetch the last 10 chat turns:
@@ -286,23 +290,28 @@ class ChatMixin:
                 - "text": The text content of the chat message.
                 - "tags": A list of tags associated with the event.
         """
-        events = await self.recent(kinds=["chat.turn"], limit=limit)
+        fetch_n = limit
+        if tags:
+            fetch_n = max(limit * 5, 100)
+
+        events = await self.recent(kinds=["chat.turn"], limit=fetch_n)
+
+        want = set(tags or [])
         out: list[dict[str, Any]] = []
 
         for e in events:
-            # 1) Resolve role (from stage or data)
+            etags = set(e.tags or [])
+            if want and not want.issubset(etags):
+                continue
+
             role = (
                 getattr(e, "stage", None)
                 or ((e.data or {}).get("role") if getattr(e, "data", None) else None)
                 or "user"
             )
-
             if roles is not None and role not in roles:
                 continue
 
-            # 2) Resolve text:
-            #    - prefer Event.text
-            #    - fall back to data["text"]
             raw_text = getattr(e, "text", "") or ""
             if not raw_text and getattr(e, "data", None):
                 raw_text = (e.data or {}).get("text", "") or ""
@@ -316,7 +325,8 @@ class ChatMixin:
                 }
             )
 
-        return out
+        # IMPORTANT: keep the most recent `limit` messages
+        return out[-limit:] if limit else []
 
     async def chat_history_for_llm(
         self: MemoryFacadeInterface,

aethergraph/services/memory/facade/core.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import logging
+import time
 from typing import Any
 
 from aethergraph.contracts.services.llm import LLMClientProtocol
@@ -9,8 +10,10 @@ from aethergraph.contracts.services.memory import Event, HotLog, Indices, Persis
 from aethergraph.contracts.storage.artifact_store import AsyncArtifactStore
 from aethergraph.contracts.storage.doc_store import DocStore
 from aethergraph.core.runtime.runtime_metering import current_metering
+from aethergraph.services.indices.scoped_indices import ScopedIndices
 from aethergraph.services.rag.facade import RAGFacade
 from aethergraph.services.scope.scope import Scope
+from aethergraph.storage.vector_index.utils import build_index_meta_from_scope
 
 from .chat import ChatMixin
 from .distillation import DistillationMixin
@@ -20,6 +23,57 @@ from .retrieval import RetrievalMixin
 from .utils import now_iso, stable_event_id
 
 
+def _normalize_tags(tags: list[str] | None) -> list[str]:
+    """
+    Normalize a list of tags by stripping whitespace, removing empties,
+    and deduplicating while preserving order.
+    """
+    if not tags:
+        return []
+    seen: set[str] = set()
+    out: list[str] = []
+    for t in tags:
+        if not t:
+            continue
+        tt = t.strip()
+        if not tt or tt in seen:
+            continue
+        seen.add(tt)
+        out.append(tt)
+    return out
+
+
+def derive_timeline_id(
+    *,
+    memory_scope_id: str | None,
+    run_id: str,
+    org_id: str | None = None,
+    sep: str = "|",
+) -> str:
+    """
+    Derive the storage partition key for timeline events.
+
+    - If org_id is present, prefix with `org:{org_id}|...` to prevent cross-org mixing.
+    - Keep fallback behavior: if memory_scope_id is missing, fall back to run_id.
+    - Avoid redundant `org:{org_id}|org:{org_id}` when the bucket is already org-scoped.
+    """
+    bucket = (memory_scope_id or "").strip()
+    if not bucket:
+        bucket = run_id
+
+    if org_id:
+        org_prefix = f"org:{org_id}"
+
+        # If the bucket is already exactly org-scoped, don't double-prefix
+        if bucket == org_prefix:
+            return org_prefix
+
+        return f"{org_prefix}{sep}{bucket}"
+
+    # No org context -> behave like current local mode
+    return bucket
+
+
 class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RAGMixin):
     """
     MemoryFacade coordinates core memory services for a specific run/session.
@@ -36,7 +90,8 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
         scope: Scope | None = None,
         hotlog: HotLog,
         persistence: Persistence,
-        indices: Indices,
+        mem_indices: Indices,
+        scoped_indices: ScopedIndices | None = None,
         docs: DocStore,
         artifact_store: AsyncArtifactStore,
         hot_limit: int = 1000,
@@ -53,7 +108,8 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
         self.scope = scope
         self.hotlog = hotlog
         self.persistence = persistence
-        self.indices = indices
+        self.indices = mem_indices
+        self.scoped_indices = scoped_indices
         self.docs = docs
         self.artifacts = artifact_store
         self.hot_limit = hot_limit
@@ -66,7 +122,11 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
         self.memory_scope_id = (
             self.scope.memory_scope_id() if self.scope else self.session_id or self.run_id
         )
-        self.timeline_id = self.memory_scope_id or self.run_id
+        self.timeline_id = derive_timeline_id(
+            memory_scope_id=self.memory_scope_id,
+            run_id=self.run_id,
+            org_id=self.scope.org_id if self.scope else None,
+        )
 
     async def record_raw(
         self,
@@ -112,7 +172,8 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
         Returns:
             Event: The fully constructed and persisted `Event` object.
         """
-        ts = now_iso()
+        ts_iso = now_iso()
+        ts_num = time.time()  # numeric timestamp for created_at_ts
 
         # Merge Scope dimensions
         dims: dict[str, str] = {}
@@ -123,12 +184,25 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
         session_id = base.get("session_id") or dims.get("session_id") or self.session_id
         scope_id = base.get("scope_id") or self.memory_scope_id or session_id or run_id
 
+        user_id = base.get("user_id") or dims.get("user_id")
+        org_id = base.get("org_id") or dims.get("org_id")
+        client_id = base.get("client_id") or dims.get("client_id")
+        graph_id = base.get("graph_id") or dims.get("graph_id") or self.graph_id
+        node_id = base.get("node_id") or dims.get("node_id") or self.node_id
+
         base.setdefault("run_id", run_id)
         base.setdefault("scope_id", scope_id)
         base.setdefault("session_id", session_id)
-        # ... (populate other fields from dims if needed) ...
-
+        base.setdefault("run_id", run_id)
+        base.setdefault("graph_id", graph_id)
+        base.setdefault("node_id", node_id)
+        base.setdefault("scope_id", scope_id)
+        base.setdefault("user_id", user_id)
+        base.setdefault("org_id", org_id)
+        base.setdefault("client_id", client_id)
+        base.setdefault("session_id", session_id)
         severity = int(base.get("severity", 2))
+
         signal = base.get("signal")
         if signal is None:
             signal = self._estimate_signal(text=text, metrics=metrics, severity=severity)
@@ -137,7 +211,7 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
 
         eid = stable_event_id(
             {
-                "ts": ts,
+                "ts": ts_iso,
                 "run_id": base["run_id"],
                 "kind": kind,
                 "text": (text or "")[:6000],
@@ -147,26 +221,76 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
 
         evt = Event(
             event_id=eid,
-            ts=ts,
+            ts=ts_iso,
             run_id=run_id,
             scope_id=scope_id,
+            user_id=user_id,
+            org_id=org_id,
+            client_id=client_id,
+            session_id=session_id,
             kind=kind,
+            stage=base.get("stage"),
             text=text,
-            data=base.get("data"),
             tags=base.get("tags"),
+            data=base.get("data"),
             metrics=metrics,
+            graph_id=graph_id,
+            node_id=node_id,
             tool=base.get("tool"),
+            topic=base.get("topic"),
             severity=severity,
             signal=signal,
             inputs=base.get("inputs"),
             outputs=base.get("outputs"),
-            # ... pass other fields ...
+            embedding=base.get("embedding"),
+            pii_flags=base.get("pii_flags"),
             version=2,
         )
 
         await self.hotlog.append(self.timeline_id, evt, ttl_s=self.hot_ttl_s, limit=self.hot_limit)
         await self.persistence.append_event(self.timeline_id, evt)
 
+        # wire memory event text into ScopedIndices for searchability
+        if self.scoped_indices is not None and self.scoped_indices.backend is not None:
+            try:
+                kind_val = getattr(evt.kind, "value", str(evt.kind))
+                preview = (text or "")[:500] if text else ""
+
+                extra_meta = {
+                    "run_id": evt.run_id,
+                    "scope_id": evt.scope_id,
+                    "session_id": evt.session_id,
+                    "graph_id": evt.graph_id,
+                    "node_id": evt.node_id,
+                    "stage": evt.stage,
+                    "tags": evt.tags or [],
+                    "severity": evt.severity,
+                    "signal": evt.signal,
+                    "tool": evt.tool,
+                    "topic": evt.topic,
+                    "preview": preview,  # short text preview
+                    "timeline_id": self.timeline_id,
+                }
+
+                meta = build_index_meta_from_scope(
+                    kind=kind_val,
+                    source="memory",
+                    ts=ts_iso,
+                    created_at_ts=ts_num,
+                    extra=extra_meta,
+                )
+
+                await self.scoped_indices.upsert(
+                    corpus="event",
+                    item_id=evt.event_id,
+                    text=evt.text or "",
+                    metadata=meta,
+                )
+
+            except Exception:
+                if self.logger:
+                    self.logger.exception("Error indexing memory event %s", evt.event_id)
+
         # Metering hook
         try:
             meter = current_metering()
@@ -264,6 +388,8 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
             except Exception:
                 data_field = {"repr": repr(data)}
 
+        # 4) normalize tags to remove empties and duplicates
+        tags = _normalize_tags(tags)
         base: dict[str, Any] = dict(
             kind=kind,
             stage=stage,
@@ -298,6 +424,8 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
         include_recent_tools: bool = False,
         tool: str | None = None,
         tool_limit: int = 10,
+        recent_chat_tags: list[str] | None = None,
+        recent_tool_tags: list[str] | None = None,
     ) -> dict[str, Any]:
         """
         Assemble memory context for prompts, including long-term summaries,
@@ -371,25 +499,37 @@ class MemoryFacade(ChatMixin, ResultMixin, RetrievalMixin, DistillationMixin, RA
                 st = s.get("summary") or s.get("text") or s.get("body") or s.get("value") or ""
                 if st:
                     parts.append(st)
-
             if parts:
-                # multiple long-term summaries → concatenate oldest→newest
                 long_term_text = "\n\n".join(parts)
 
-        recent_chat = await self.recent_chat(limit=recent_chat_limit)
+        # 1) Recent chat (delegate tag filtering + correct "last N" to recent_chat)
+        recent_chat = await self.recent_chat(
+            limit=recent_chat_limit,
+            tags=recent_chat_tags,
+        )
 
+        # 2) Recent tools
         recent_tools: list[dict[str, Any]] = []
         if include_recent_tools:
-            events = await self.recent_tool_results(
-                tool=tool,
-                limit=tool_limit,
-            )
+            fetch_n = tool_limit
+            if recent_tool_tags:
+                fetch_n = max(tool_limit * 5, 50)
+
+            events = await self.recent_tool_results(tool=tool, limit=fetch_n)
+
+            if recent_tool_tags:
+                want = set(recent_tool_tags)
+                events = [e for e in events if want.issubset(set(e.tags or []))]
+
+            # IMPORTANT: keep the most recent tool events
+            events = events[-tool_limit:] if tool_limit else []
+
             for e in events:
                 recent_tools.append(
                     {
                         "ts": getattr(e, "ts", None),
-                        "tool": e.tool,
-                        "message": e.text,
+                        "tool": getattr(e, "tool", None),
+                        "message": getattr(e, "text", None),
                         "inputs": getattr(e, "inputs", None),
                         "outputs": getattr(e, "outputs", None),
                         "tags": list(e.tags or []),