citadeldb-ms-agent-framework 2.0.0__py3-none-any.whl

citadeldb_ms_agent_framework/__init__.py

@@ -0,0 +1,14 @@
+"""Microsoft Agent Framework storage backed by Citadel, encrypted at rest."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from .history import CitadelHistoryProvider
+from .memory import CitadelContextProvider
+
+__all__ = ["CitadelContextProvider", "CitadelHistoryProvider", "__version__"]
+
+
+try:
+    __version__ = version("citadeldb-ms-agent-framework")
+except PackageNotFoundError:  # running from a source tree, never installed
+    __version__ = "0+unknown"

citadeldb_ms_agent_framework/history.py

@@ -0,0 +1,164 @@
+"""HistoryProvider over an encrypted Citadel region."""
+from __future__ import annotations
+
+import asyncio
+from typing import Any, ClassVar, Sequence
+
+import citadeldb
+from agent_framework import HistoryProvider, Message
+
+KIND = "message"
+DEFAULT_PATH = "agent_history.cdl"
+DEFAULT_REGION = "history"
+PAGE = 10_000
+
+
+def _page(mem: Any, region: str, session_id: str) -> list[Any]:
+    """Page to the end: one fetch is bounded, and a partial erase must not look whole."""
+    out: list[Any] = []
+    after = None
+    while True:
+        got = mem.fetch(region, KIND, payload_filter={"sid": session_id}, limit=PAGE,
+                        after_id=after)
+        out.extend(got)
+        if len(got) < PAGE:
+            return out
+        after = got[-1].id
+# session_id is optional in the protocol; keep unattributed history together.
+DEFAULT_SESSION = "default"
+
+
+# A Database is pinned to its opening thread, so workers take Memory, not self.
+
+
+def _text_of(message: Message) -> str:
+    """An atom needs text to embed; a textless message falls back to role."""
+    return message.text or str(message.role)
+
+
+def _load(mem: Any, region: str, session_id: str) -> list[Message]:
+    hits = _page(mem, region, session_id)
+    return [Message.from_dict(h.payload["msg"]) for h in hits]
+
+
+def _append(mem: Any, region: str, session_id: str, messages: Sequence[Message]) -> None:
+    atoms = [
+        {
+            "kind": KIND,
+            "text": _text_of(m),
+            "payload": {"sid": session_id, "msg": m.to_dict()},
+        }
+        for m in messages
+    ]
+    if atoms:
+        # One batch draws a contiguous id range, so list order survives.
+        mem.remember_batch(region, atoms)
+
+
+def _forget(mem: Any, region: str, session_id: str) -> int:
+    hits = _page(mem, region, session_id)
+    if not hits:
+        return 0
+    return mem.forget(region, [h.id for h in hits]).erased_count
+
+
+def _recall(
+    mem: Any, region: str, session_id: str, query: str, limit: int
+) -> list[Message]:
+    hits = mem.recall(
+        region,
+        text=query,
+        k=limit,
+        kinds=[KIND],
+        options=citadeldb.RecallOptions(payload_filter={"sid": session_id}),
+    )
+    return [Message.from_dict(h.payload["msg"]) for h in hits]
+
+
+class CitadelHistoryProvider(HistoryProvider):
+    """A `HistoryProvider` backed by one encrypted Citadel region."""
+
+    DEFAULT_SOURCE_ID: ClassVar[str] = "citadel_history"
+
+    def __init__(
+        self,
+        path: str = DEFAULT_PATH,
+        key: str = "",
+        *,
+        source_id: str = DEFAULT_SOURCE_ID,
+        region: str = DEFAULT_REGION,
+        embedder: Any | None = None,
+        load_messages: bool = True,
+        store_inputs: bool = True,
+        store_context_messages: bool = False,
+        store_context_from: set[str] | None = None,
+        store_outputs: bool = True,
+    ) -> None:
+        super().__init__(
+            source_id=source_id,
+            load_messages=load_messages,
+            store_inputs=store_inputs,
+            store_context_messages=store_context_messages,
+            store_context_from=store_context_from,
+            store_outputs=store_outputs,
+        )
+        if not key:
+            raise ValueError("a passphrase is required: transcripts are the payload")
+        try:
+            self._db = citadeldb.connect(path, key=key, region_keys=True)
+        except citadeldb.OperationalError as e:
+            if "locked" not in str(e):
+                raise
+            raise RuntimeError(
+                f"{path} is open in another process. Citadel is embedded, so one "
+                f"process owns the file."
+            ) from e
+        self._mem = self._db.memory()
+        self._region = region
+        # Idempotent for a region of the same width, so a dim clash raises here.
+        self._mem.create_encrypted_region(
+            region, embedder or citadeldb.MockEmbedder(dim=64)
+        )
+
+    # ---- the abstract surface --------------------------------------------
+    # The bindings are sync, so a worker thread keeps the event loop free.
+
+    async def get_messages(
+        self, session_id: str | None, *, state: dict[str, Any] | None = None, **kwargs: Any
+    ) -> list[Message]:
+        return await asyncio.to_thread(
+            _load, self._mem, self._region, session_id or DEFAULT_SESSION
+        )
+
+    async def save_messages(
+        self,
+        session_id: str | None,
+        messages: Sequence[Message],
+        *,
+        state: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        # Appends rather than replaces: history is a transcript, not a set.
+        await asyncio.to_thread(
+            _append,
+            self._mem,
+            self._region,
+            session_id or DEFAULT_SESSION,
+            list(messages),
+        )
+
+    # ---- beyond the protocol ---------------------------------------------
+
+    async def search(
+        self, session_id: str | None, query: str, *, limit: int = 5
+    ) -> list[Message]:
+        """Messages from one session ranked by hybrid recall, best first."""
+        return await asyncio.to_thread(
+            _recall, self._mem, self._region, session_id or DEFAULT_SESSION, query, limit
+        )
+
+    async def forget(self, session_id: str | None) -> int:
+        """Destroy one session's messages, returning the number erased."""
+        return await asyncio.to_thread(
+            _forget, self._mem, self._region, session_id or DEFAULT_SESSION
+        )

citadeldb_ms_agent_framework/memory.py

@@ -0,0 +1,163 @@
+"""ContextProvider over an encrypted Citadel region."""
+from __future__ import annotations
+
+import asyncio
+from typing import Any, ClassVar, Sequence
+
+import citadeldb
+from agent_framework import ContextProvider, Message
+
+KIND = "memory"
+DEFAULT_PATH = "agent_memory.cdl"
+DEFAULT_REGION = "memories"
+PAGE = 10_000
+
+
+def _page(mem: Any, region: str, criterion: dict[str, Any]) -> list[Any]:
+    """Page to the end: one fetch is bounded, and a partial erase must not look whole."""
+    out: list[Any] = []
+    after = None
+    while True:
+        got = mem.fetch(region, KIND, payload_filter=criterion, limit=PAGE, after_id=after)
+        out.extend(got)
+        if len(got) < PAGE:
+            return out
+        after = got[-1].id
+# Roles worth remembering; tool traffic is transcript detail, not knowledge.
+_REMEMBERED_ROLES = ("user", "assistant", "system")
+
+
+# A Database is pinned to its opening thread, so workers take Memory, not self.
+
+
+def _role_of(message: Message) -> str:
+    role = message.role
+    return getattr(role, "value", None) or str(role)
+
+
+def _remember(mem: Any, region: str, scope: str, texts: Sequence[str]) -> None:
+    atoms = [
+        {"kind": KIND, "text": t, "payload": {"scope": scope, "text": t}} for t in texts
+    ]
+    if atoms:
+        mem.remember_batch(region, atoms)
+
+
+def _recall(mem: Any, region: str, scope: str, query: str, limit: int) -> list[str]:
+    """Distinct memories for `scope`, best first.
+
+    after_run stores every turn verbatim, so a fact the user restates is stored
+    once per turn. Those copies are one memory to the model, and asking the
+    engine for `limit` rows would spend the whole budget on them - 30 repeats of
+    one fact deliver one line and hide everything else in the scope. Widen until
+    `limit` distinct texts are found or the scope runs out.
+    """
+    options = citadeldb.RecallOptions(payload_filter={"scope": scope})
+    k = max(limit, 32)
+    while True:
+        hits = mem.recall(region, text=query, k=k, kinds=[KIND], options=options)
+        out: list[str] = []
+        seen: set[str] = set()
+        for h in hits:
+            text = h.payload["text"]
+            if text in seen:
+                continue
+            seen.add(text)
+            out.append(text)
+        if len(out) >= limit or len(hits) < k:
+            return out[:limit]
+        k *= 2
+
+
+def _forget(mem: Any, region: str, scope: str) -> int:
+    hits = _page(mem, region, {"scope": scope})
+    if not hits:
+        return 0
+    return mem.forget(region, [h.id for h in hits]).erased_count
+
+
+class CitadelContextProvider(ContextProvider):
+    """A `ContextProvider` backed by one encrypted Citadel region."""
+
+    DEFAULT_SOURCE_ID: ClassVar[str] = "citadel_memory"
+    DEFAULT_CONTEXT_PROMPT: ClassVar[str] = (
+        "## Memories\nConsider the following memories from earlier conversations:"
+    )
+
+    def __init__(
+        self,
+        path: str = DEFAULT_PATH,
+        key: str = "",
+        *,
+        source_id: str = DEFAULT_SOURCE_ID,
+        scope: str = "default",
+        region: str = DEFAULT_REGION,
+        embedder: Any | None = None,
+        limit: int = 5,
+        context_prompt: str = DEFAULT_CONTEXT_PROMPT,
+    ) -> None:
+        super().__init__(source_id)
+        if not key:
+            raise ValueError("a passphrase is required: memories are the payload")
+        self.scope = scope
+        self.limit = limit
+        self.context_prompt = context_prompt
+        try:
+            self._db = citadeldb.connect(path, key=key, region_keys=True)
+        except citadeldb.OperationalError as e:
+            if "locked" not in str(e):
+                raise
+            raise RuntimeError(
+                f"{path} is open in another process. Citadel is embedded, so one "
+                f"process owns the file."
+            ) from e
+        self._mem = self._db.memory()
+        self._region = region
+        # Idempotent for a region of the same width, so a dim clash raises here.
+        self._mem.create_encrypted_region(
+            region, embedder or citadeldb.MockEmbedder(dim=64)
+        )
+
+    # ---- the pipeline hooks ----------------------------------------------
+
+    async def before_run(
+        self, *, agent: Any, session: Any, context: Any, state: dict[str, Any]
+    ) -> None:
+        """Recall what is relevant to this turn and add it to the context."""
+        query = "\n".join(
+            m.text for m in context.input_messages if m and m.text and m.text.strip()
+        )
+        if not query:
+            return
+        memories = await asyncio.to_thread(
+            _recall, self._mem, self._region, self.scope, query, self.limit
+        )
+        if not memories:
+            return
+        context.extend_messages(
+            self.source_id,
+            [Message("user", [f"{self.context_prompt}\n" + "\n".join(memories)])],
+        )
+
+    async def after_run(
+        self, *, agent: Any, session: Any, context: Any, state: dict[str, Any]
+    ) -> None:
+        """Remember this turn, inputs and response alike."""
+        turn: list[Message] = list(context.input_messages)
+        if context.response and context.response.messages:
+            turn.extend(context.response.messages)
+        texts = [
+            m.text
+            for m in turn
+            if m and m.text and m.text.strip() and _role_of(m) in _REMEMBERED_ROLES
+        ]
+        if texts:
+            await asyncio.to_thread(
+                _remember, self._mem, self._region, self.scope, texts
+            )
+
+    # ---- beyond the pipeline ---------------------------------------------
+
+    async def forget(self) -> int:
+        """Destroy this scope's memories, returning the number erased."""
+        return await asyncio.to_thread(_forget, self._mem, self._region, self.scope)

citadeldb_ms_agent_framework-2.0.0.dist-info/METADATA

@@ -0,0 +1,100 @@
+Metadata-Version: 2.5
+Name: citadeldb-ms-agent-framework
+Version: 2.0.0
+Summary: Microsoft Agent Framework chat history backed by Citadel: encrypted at rest, with deletes that destroy the key
+Project-URL: Homepage, https://citadeldb.dev
+Project-URL: Repository, https://github.com/yp3y5akh0v/citadel
+Author: Yuriy Peysakhov
+License-Expression: Apache-2.0
+Keywords: agent-framework,agents,autogen,chat-history,encryption,memory,microsoft-agent-framework
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: agent-framework-core<2,>=1.13
+Requires-Dist: citadeldb<3,>=2.0
+Provides-Extra: test
+Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
+Requires-Dist: pytest>=8; extra == 'test'
+Description-Content-Type: text/markdown
+
+# citadeldb-ms-agent-framework
+
+[Microsoft Agent Framework](https://github.com/microsoft/agent-framework) storage backed
+by [Citadel](https://citadeldb.dev). Encrypted at rest, embedded in your process, and
+deletes that destroy the key, not just the row.
+
+```
+pip install citadeldb-ms-agent-framework
+```
+
+Two providers for two jobs, matching how the framework's own Redis integration is split:
+
+| Class | Implements | Use when |
+|---|---|---|
+| `CitadelHistoryProvider` | `HistoryProvider` | a session must recover its complete transcript |
+| `CitadelContextProvider` | `ContextProvider` | an agent should recall relevant facts across sessions |
+
+```python
+from agent_framework import Agent
+from citadeldb_ms_agent_framework import CitadelContextProvider, CitadelHistoryProvider
+
+agent = Agent(
+    client=chat_client,          # any agent_framework chat client
+    context_providers=[
+        CitadelHistoryProvider("agent.cdl", key="your-passphrase"),
+        CitadelContextProvider("agent.cdl", key="your-passphrase", scope="user-123"),
+    ],
+)
+```
+
+Both can share one encrypted file: a path already open on this thread, under the same
+passphrase, is shared. Construct them on the same thread.
+
+These are the framework's own extension points, with the file encrypted and a key per
+message. The built-in `FileHistoryProvider` writes plaintext JSONL or MessagePack.
+
+## Deletes destroy the key
+
+```python
+history = CitadelHistoryProvider("agent.cdl", key="your-passphrase")
+memory = CitadelContextProvider("agent.cdl", key="your-passphrase", scope="user-123")
+
+await history.forget("session-42")   # returns the number erased
+await memory.forget()                # this provider's whole scope
+```
+
+Clearing a conversation destroys each message's own key and then deletes its row, so any
+ciphertext surviving elsewhere stays unreadable.
+
+## History provider
+
+Implements `get_messages` and `save_messages`; the base class's `before_run`/`after_run`
+handle loading and storing according to its configuration flags, so an audit-only or
+evaluation-only provider works as documented:
+
+```python
+CitadelHistoryProvider("agent.cdl", key="your-passphrase", load_messages=False)  # stores, never loads
+```
+
+Messages round-trip through the framework's own serialization, so roles, author names,
+multi-part contents and `additional_properties` all survive.
+
+## Context provider
+
+Recalls with Citadel's hybrid search: vector distance, keyword rank and recency, fused
+into one score. The default `MockEmbedder` is lexical; pass `embedder=` a real one to
+match across wording.
+
+```python
+memory = CitadelContextProvider("agent.cdl", key="your-passphrase", scope="user-123", limit=5)
+```
+
+Memories are scoped rather than session-bound, so a later conversation can recall an
+earlier one. `scope` is the boundary an erasure request applies to.
+
+## License
+
+Apache-2.0

citadeldb_ms_agent_framework-2.0.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+citadeldb_ms_agent_framework/__init__.py,sha256=BKdeYcX4OYg0f4oihIjpPKZs4wo_xBYAw39Y4vlj55c,478
+citadeldb_ms_agent_framework/history.py,sha256=9XvFQ70kJ9_-SiSo0pDdt7_y7VZHLGRlvUuluFO73C0,5464
+citadeldb_ms_agent_framework/memory.py,sha256=KQKvsvIlrz0IsHAkIc1JvMq4NDk8WlSD3UuTYemPnWk,5770
+citadeldb_ms_agent_framework-2.0.0.dist-info/METADATA,sha256=YDDb7Id6acv1jwJRN6_lYybTNwl-i8WdZ3rZ2IR3qFo,3801
+citadeldb_ms_agent_framework-2.0.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+citadeldb_ms_agent_framework-2.0.0.dist-info/RECORD,,

citadeldb_ms_agent_framework-2.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any