hindsight-livekit 0.1.0__py3-none-any.whl

hindsight_livekit/__init__.py

@@ -0,0 +1,48 @@
+"""Hindsight-LiveKit: Persistent memory for LiveKit voice agents.
+
+Recalls what Hindsight knows about the user when the agent starts and before
+each LLM reply, and retains the conversation as it happens, one Hindsight
+document per session. Optional tools let the model search and store memories
+itself, which also works with realtime models.
+
+Basic usage::
+
+    from hindsight_livekit import HindsightAgent, HindsightMemory
+
+    @server.rtc_session()
+    async def entrypoint(ctx: JobContext):
+        session = AgentSession(stt=..., llm=..., tts=...)
+
+        memory = HindsightMemory(bank_id="user-123", document_id=ctx.room.name)
+        memory.attach(session)                    # retain the conversation
+        ctx.add_shutdown_callback(memory.aclose)  # flush on hang-up
+
+        await session.start(
+            agent=HindsightAgent(memory=memory, instructions="You are a helpful voice assistant."),
+            room=ctx.room,
+        )
+"""
+
+from .agent import HindsightAgent
+from .config import (
+    HindsightLiveKitConfig,
+    configure,
+    get_config,
+    reset_config,
+)
+from .errors import HindsightLiveKitError
+from .memory import HindsightMemory
+from .tools import create_hindsight_tools
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "HindsightAgent",
+    "HindsightLiveKitConfig",
+    "HindsightLiveKitError",
+    "HindsightMemory",
+    "configure",
+    "create_hindsight_tools",
+    "get_config",
+    "reset_config",
+]

hindsight_livekit/_client.py

@@ -0,0 +1,46 @@
+"""Shared Hindsight client resolution logic."""
+
+from __future__ import annotations
+
+import os
+from importlib import metadata
+from typing import Any
+
+from hindsight_client import Hindsight
+
+from .config import DEFAULT_HINDSIGHT_API_URL, HINDSIGHT_API_KEY_ENV, HINDSIGHT_API_URL_ENV, get_config
+
+try:
+    _VERSION = metadata.version("hindsight-livekit")
+except metadata.PackageNotFoundError:
+    _VERSION = "0.0.0"
+USER_AGENT = f"hindsight-livekit/{_VERSION}"
+
+
+def resolve_client(
+    client: Hindsight | None,
+    hindsight_api_url: str | None,
+    api_key: str | None,
+) -> Hindsight:
+    """Resolve a Hindsight client from explicit args, global config, or env vars.
+
+    Order: explicit arguments, then ``configure()``, then the ``HINDSIGHT_API_URL`` /
+    ``HINDSIGHT_API_KEY`` env vars, then Hindsight Cloud. A LiveKit worker is usually
+    configured through env vars, so they are honoured without a ``configure()`` call.
+    """
+    if client is not None:
+        return client
+
+    config = get_config()
+    url = (
+        hindsight_api_url
+        or (config.hindsight_api_url if config else None)
+        or os.environ.get(HINDSIGHT_API_URL_ENV)
+        or DEFAULT_HINDSIGHT_API_URL
+    )
+    key = api_key or (config.api_key if config else None) or os.environ.get(HINDSIGHT_API_KEY_ENV)
+
+    kwargs: dict[str, Any] = {"base_url": url, "timeout": 30.0, "user_agent": USER_AGENT}
+    if key:
+        kwargs["api_key"] = key
+    return Hindsight(**kwargs)

hindsight_livekit/agent.py

@@ -0,0 +1,85 @@
+"""LiveKit ``Agent`` with Hindsight memory recall."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterable
+from typing import Any
+
+from livekit.agents import Agent, ModelSettings, llm
+
+from .memory import HindsightMemory
+
+
+class HindsightAgent(Agent):
+    """A LiveKit ``Agent`` that recalls Hindsight memories.
+
+    Use it in place of ``Agent`` (or subclass it) and pass the session's
+    :class:`HindsightMemory`::
+
+        class Assistant(HindsightAgent):
+            def __init__(self, memory: HindsightMemory) -> None:
+                super().__init__(memory=memory, instructions="You are a helpful voice assistant.")
+
+    Memory is recalled at two points:
+
+    * **When the agent starts** (``on_enter``): what is known about the user is
+      added to the agent's instructions. This works with STT-LLM-TTS pipelines and
+      realtime models alike. If you override ``on_enter``, call
+      ``await super().on_enter()`` first.
+    * **Before each LLM reply** (``llm_node``): memories relevant to the user's
+      latest message are added to a copy of the chat context for that call only.
+      Realtime models don't use ``llm_node``, so this step only applies to
+      STT-LLM-TTS pipelines; give realtime agents the tools from
+      :func:`~hindsight_livekit.create_hindsight_tools` to look things up mid-call.
+    """
+
+    def __init__(self, *, memory: HindsightMemory, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+        self._memory = memory
+        # Instructions without recalled memories, so re-entering the agent (e.g. after
+        # a handoff back to it) replaces the memory block instead of stacking another.
+        self._base_instructions = self.instructions
+
+    @property
+    def memory(self) -> HindsightMemory:
+        return self._memory
+
+    async def on_enter(self) -> None:
+        memories = await self._memory.recall_session_context()
+        if memories is not None:
+            await self.update_instructions(self._with_memories(memories))
+
+    async def llm_node(
+        self,
+        chat_ctx: llm.ChatContext,
+        tools: list[llm.Tool],
+        model_settings: ModelSettings,
+    ) -> AsyncIterable[llm.ChatChunk | str]:
+        chat_ctx = await self._ensure_session_memories(chat_ctx)
+        chat_ctx = await self._memory.inject_memories(chat_ctx)
+        async for chunk in Agent.default.llm_node(self, chat_ctx, tools, model_settings):
+            yield chunk
+
+    def _with_memories(self, memories: str) -> str:
+        return f"{self._base_instructions}\n\n{memories}"
+
+    async def _ensure_session_memories(self, chat_ctx: llm.ChatContext) -> llm.ChatContext:
+        """Make sure this LLM call sees the session memories.
+
+        LiveKit runs ``on_enter`` in the background and doesn't hold the first reply
+        for it, so a user who speaks first can get a reply built from the instructions
+        as they were before ``on_enter`` added the memories. Wait for the (shared,
+        already running) session recall and, if the instructions here are still the
+        original ones, use the version with memories for this call.
+        """
+        memories = await self._memory.recall_session_context()
+        if memories is None:
+            return chat_ctx
+        for i, item in enumerate(chat_ctx.items):
+            if item.type == "message" and item.role == "system" and item.text_content == self._base_instructions:
+                updated = chat_ctx.copy()
+                # copy() shares item objects with the agent's history, so replace the
+                # item rather than editing it.
+                updated.items[i] = item.model_copy(update={"content": [self._with_memories(memories)]})
+                return updated
+        return chat_ctx

hindsight_livekit/config.py

@@ -0,0 +1,87 @@
+"""Global configuration for Hindsight-LiveKit integration."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Literal
+
+DEFAULT_HINDSIGHT_API_URL = "https://api.hindsight.vectorize.io"
+HINDSIGHT_API_URL_ENV = "HINDSIGHT_API_URL"
+HINDSIGHT_API_KEY_ENV = "HINDSIGHT_API_KEY"
+
+Budget = Literal["low", "mid", "high"]
+TagsMatch = Literal["any", "all", "any_strict", "all_strict"]
+
+# Voice defaults: every recall happens while a caller waits, so they favour speed.
+# Hindsight documents "low" as the budget for fast, simple lookups.
+DEFAULT_RECALL_BUDGET: Budget = "low"
+DEFAULT_RECALL_MAX_TOKENS = 1024
+DEFAULT_RECALL_TIMEOUT = 3.0
+DEFAULT_RECALL_TAGS_MATCH: TagsMatch = "any"
+
+
+@dataclass
+class HindsightLiveKitConfig:
+    """Connection and default settings for the LiveKit integration.
+
+    Attributes:
+        hindsight_api_url: URL of the Hindsight API server.
+        api_key: API key for Hindsight authentication.
+        recall_budget: Default recall budget level (low/mid/high).
+        recall_max_tokens: Default maximum tokens for recall results.
+        recall_timeout: Default seconds to wait for recall before answering without memories.
+    """
+
+    hindsight_api_url: str = DEFAULT_HINDSIGHT_API_URL
+    api_key: str | None = None
+    recall_budget: Budget = DEFAULT_RECALL_BUDGET
+    recall_max_tokens: int = DEFAULT_RECALL_MAX_TOKENS
+    recall_timeout: float = DEFAULT_RECALL_TIMEOUT
+
+
+_global_config: HindsightLiveKitConfig | None = None
+
+
+def configure(
+    hindsight_api_url: str | None = None,
+    api_key: str | None = None,
+    recall_budget: Budget = DEFAULT_RECALL_BUDGET,
+    recall_max_tokens: int = DEFAULT_RECALL_MAX_TOKENS,
+    recall_timeout: float = DEFAULT_RECALL_TIMEOUT,
+) -> HindsightLiveKitConfig:
+    """Configure Hindsight connection and default settings.
+
+    Args:
+        hindsight_api_url: Hindsight API URL. Falls back to HINDSIGHT_API_URL env var,
+            then Hindsight Cloud.
+        api_key: API key. Falls back to HINDSIGHT_API_KEY env var.
+        recall_budget: Default recall budget (low/mid/high).
+        recall_max_tokens: Default max tokens for recall.
+        recall_timeout: Default recall timeout in seconds.
+
+    Returns:
+        The configured HindsightLiveKitConfig.
+    """
+    global _global_config
+
+    _global_config = HindsightLiveKitConfig(
+        hindsight_api_url=hindsight_api_url or os.environ.get(HINDSIGHT_API_URL_ENV) or DEFAULT_HINDSIGHT_API_URL,
+        api_key=api_key or os.environ.get(HINDSIGHT_API_KEY_ENV),
+        recall_budget=recall_budget,
+        recall_max_tokens=recall_max_tokens,
+        recall_timeout=recall_timeout,
+    )
+
+    return _global_config
+
+
+def get_config() -> HindsightLiveKitConfig | None:
+    """Get the current global configuration."""
+    return _global_config
+
+
+def reset_config() -> None:
+    """Reset global configuration to None."""
+    global _global_config
+    _global_config = None

hindsight_livekit/errors.py

@@ -0,0 +1,7 @@
+"""Hindsight-LiveKit error types."""
+
+
+class HindsightLiveKitError(Exception):
+    """Exception raised when the Hindsight-LiveKit integration is misconfigured."""
+
+    pass

hindsight_livekit/memory.py

@@ -0,0 +1,391 @@
+"""Hindsight memory for LiveKit voice agents.
+
+``HindsightMemory`` handles long-term memory for one ``AgentSession``:
+
+* **Session recall** — :meth:`HindsightMemory.recall_session_context` looks up what
+  is known about the user once, when the agent starts. ``HindsightAgent`` adds it to
+  the agent's instructions. Works with STT-LLM-TTS pipelines and realtime models.
+* **Per-turn recall** — :meth:`HindsightMemory.inject_memories` searches Hindsight
+  with the user's latest message and returns a copy of the chat context with the
+  results inserted just before that message. ``HindsightAgent`` calls it from
+  ``llm_node``, which only STT-LLM-TTS pipelines use.
+* **Retain** — :meth:`HindsightMemory.attach` listens to the session's
+  ``conversation_item_added`` events and appends the conversation to one Hindsight
+  document per session (``update_mode="append"``).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+from hindsight_client import Hindsight
+from hindsight_client_api.exceptions import NotFoundException
+from livekit.agents import AgentSession, CloseEvent, ConversationItemAddedEvent
+from livekit.agents.llm import ChatContext, ChatMessage
+
+from ._client import resolve_client
+from .config import (
+    DEFAULT_RECALL_BUDGET,
+    DEFAULT_RECALL_MAX_TOKENS,
+    DEFAULT_RECALL_TAGS_MATCH,
+    DEFAULT_RECALL_TIMEOUT,
+    Budget,
+    TagsMatch,
+    get_config,
+)
+from .errors import HindsightLiveKitError
+
+logger = logging.getLogger(__name__)
+
+_MEMORY_MARKER = "<hindsight_memories>"
+_DEFAULT_MEMORY_PREFIX = (
+    "Relevant memories about the user from past conversations. "
+    "Use them only when they help answer, and never read this list out loud."
+)
+_DEFAULT_SESSION_RECALL_QUERY = "Who is this user? Their name, preferences, and important facts about them."
+
+
+@dataclass
+class _TranscriptLine:
+    role: str
+    created_at: float
+    text: str
+
+
+def _last_user_message_index(chat_ctx: ChatContext) -> int | None:
+    """Return the index of the last user message in ``chat_ctx``, or None."""
+    for i in range(len(chat_ctx.items) - 1, -1, -1):
+        item = chat_ctx.items[i]
+        if item.type == "message" and item.role == "user":
+            return i
+    return None
+
+
+def _to_datetime(created_at: float) -> datetime:
+    return datetime.fromtimestamp(created_at, tz=timezone.utc)
+
+
+class HindsightMemory:
+    """Persistent Hindsight memory for a single LiveKit ``AgentSession``.
+
+    Create one per session, attach it to the session so the conversation is
+    retained, and give it to a :class:`~hindsight_livekit.HindsightAgent` so
+    memories are recalled::
+
+        memory = HindsightMemory(bank_id="user-123", document_id=ctx.room.name)
+        memory.attach(session)
+        ctx.add_shutdown_callback(memory.aclose)
+        await session.start(agent=HindsightAgent(memory=memory, instructions="..."), room=ctx.room)
+
+    Args:
+        bank_id: Hindsight memory bank to read from and write to (usually one per end user).
+        client: Pre-configured Hindsight client. If omitted, one is created from
+            ``hindsight_api_url``/``api_key``, then ``configure()``, then the
+            ``HINDSIGHT_API_URL``/``HINDSIGHT_API_KEY`` env vars.
+        hindsight_api_url: API URL (used if no client provided).
+        api_key: API key (used if no client provided).
+        document_id: Hindsight document that this session's transcript is appended to.
+            Defaults to a random ``livekit-<uuid>`` id; the room name is a good choice.
+        recall_budget: Recall budget level — ``"low"``, ``"mid"``, ``"high"``.
+            Defaults to ``"low"``, which Hindsight recommends for fast lookups.
+        recall_max_tokens: Maximum tokens of memories injected per recall.
+        recall_timeout: Seconds to wait for recall. On timeout the agent continues
+            without memories instead of leaving the caller in silence.
+        recall_types: Memory types to search (``"world"``, ``"experience"``,
+            ``"observation"``). ``None`` searches all of them.
+        tags: Tags attached to retained content.
+        recall_tags: Tags to filter recall by. Defaults to no filter.
+        recall_tags_match: How ``recall_tags`` are matched (any/all/any_strict/all_strict).
+        retain_context: Short label describing the source, passed to Hindsight's
+            fact extraction as ``context``.
+        user_name: Speaker label for the user in the retained transcript.
+        agent_name: Speaker label for the agent in the retained transcript.
+        enable_session_recall: Recall what is known about the user once, when the
+            agent starts, and add it to the agent's instructions.
+        session_recall_query: Query used for session recall.
+        enable_recall: Recall memories relevant to each user message before the
+            LLM replies (STT-LLM-TTS pipelines only).
+        enable_retain: Store the conversation as it happens.
+        memory_prefix: Instruction placed above recalled memories.
+    """
+
+    def __init__(
+        self,
+        bank_id: str,
+        *,
+        client: Hindsight | None = None,
+        hindsight_api_url: str | None = None,
+        api_key: str | None = None,
+        document_id: str | None = None,
+        recall_budget: Budget | None = None,
+        recall_max_tokens: int | None = None,
+        recall_timeout: float | None = None,
+        recall_types: list[str] | None = None,
+        tags: list[str] | None = None,
+        recall_tags: list[str] | None = None,
+        recall_tags_match: TagsMatch = DEFAULT_RECALL_TAGS_MATCH,
+        retain_context: str = "voice conversation",
+        user_name: str = "User",
+        agent_name: str = "Assistant",
+        enable_session_recall: bool = True,
+        session_recall_query: str = _DEFAULT_SESSION_RECALL_QUERY,
+        enable_recall: bool = True,
+        enable_retain: bool = True,
+        memory_prefix: str = _DEFAULT_MEMORY_PREFIX,
+    ) -> None:
+        self._bank_id = bank_id
+        self._owns_client = client is None
+        self._client = resolve_client(client, hindsight_api_url, api_key)
+        config = get_config()
+        self._document_id = document_id or f"livekit-{uuid.uuid4().hex}"
+        self._recall_budget: Budget = recall_budget or (config.recall_budget if config else DEFAULT_RECALL_BUDGET)
+        self._recall_max_tokens = recall_max_tokens or (
+            config.recall_max_tokens if config else DEFAULT_RECALL_MAX_TOKENS
+        )
+        self._recall_timeout = recall_timeout or (config.recall_timeout if config else DEFAULT_RECALL_TIMEOUT)
+        self._recall_types = recall_types
+        self._tags = tags
+        self._recall_tags = recall_tags
+        self._recall_tags_match = recall_tags_match
+        self._retain_context = retain_context
+        self._user_name = user_name
+        self._agent_name = agent_name
+        self._enable_session_recall = enable_session_recall
+        self._session_recall_query = session_recall_query
+        self._enable_recall = enable_recall
+        self._enable_retain = enable_retain
+        self._memory_prefix = memory_prefix
+
+        self._session: AgentSession | None = None
+        # Transcript lines not yet sent to Hindsight. See _on_conversation_item_added
+        # for when they are flushed.
+        self._pending: list[_TranscriptLine] = []
+        # Appends to one document must not overlap (Hindsight rejects concurrent
+        # appends), so they run one at a time, in order. An asyncio.Lock is safe here
+        # because a HindsightMemory belongs to one AgentSession, which runs on one loop.
+        self._retain_lock = asyncio.Lock()
+        self._retain_tasks: set[asyncio.Task[None]] = set()
+        # Recall result for the current user message, shared by every LLM call that
+        # answers it (preemptive generation, tool-call follow-ups). Keyed on message
+        # id and text, so a revised transcript gets a fresh recall.
+        self._recall_message_id: str | None = None
+        self._recall_query: str | None = None
+        self._recall_task: asyncio.Task[str | None] | None = None
+        self._session_recall_task: asyncio.Task[str | None] | None = None
+        self._closed = False
+
+    @property
+    def bank_id(self) -> str:
+        return self._bank_id
+
+    @property
+    def document_id(self) -> str:
+        return self._document_id
+
+    @property
+    def client(self) -> Hindsight:
+        return self._client
+
+    # ------------------------------------------------------------------
+    # Recall
+    # ------------------------------------------------------------------
+
+    async def recall(self, query: str) -> str | None:
+        """Search Hindsight and return a formatted memory block, or None.
+
+        Never raises: errors and timeouts are logged and return None so the
+        conversation continues without memories.
+        """
+        try:
+            response = await asyncio.wait_for(
+                self._client.arecall(
+                    bank_id=self._bank_id,
+                    query=query,
+                    types=self._recall_types,
+                    budget=self._recall_budget,
+                    max_tokens=self._recall_max_tokens,
+                    tags=self._recall_tags,
+                    tags_match=self._recall_tags_match,
+                ),
+                timeout=self._recall_timeout,
+            )
+        except NotFoundException:
+            # A new user's bank doesn't exist until the first retain creates it.
+            logger.debug(f"Hindsight bank {self._bank_id!r} not found yet (no memories to recall)")
+            return None
+        except asyncio.TimeoutError:
+            logger.warning(f"Hindsight recall timed out after {self._recall_timeout}s (continuing without memories)")
+            return None
+        except Exception as e:
+            logger.warning(f"Hindsight recall failed (continuing without memories): {e}")
+            return None
+
+        if not response.results:
+            return None
+        lines = [_MEMORY_MARKER, self._memory_prefix]
+        for i, result in enumerate(response.results, 1):
+            lines.append(f"{i}. {result.text}")
+        lines.append("</hindsight_memories>")
+        return "\n".join(lines)
+
+    async def recall_session_context(self) -> str | None:
+        """Recall what is known about the user, for the start of a session.
+
+        Returns a formatted memory block, or None if session recall is disabled or
+        nothing is found. The lookup runs once per session (``attach()`` starts it
+        early) and later calls return the same result. Never raises.
+        """
+        if not self._enable_session_recall:
+            return None
+        return await asyncio.shield(self._start_session_recall())
+
+    def _start_session_recall(self) -> asyncio.Task[str | None]:
+        if self._session_recall_task is None:
+            self._session_recall_task = asyncio.create_task(self.recall(self._session_recall_query))
+        return self._session_recall_task
+
+    async def inject_memories(self, chat_ctx: ChatContext) -> ChatContext:
+        """Return ``chat_ctx`` with recalled memories inserted before the last user message.
+
+        The input context is not modified, so memories never enter the session's
+        stored history. Recall runs at most once per user message. If there is no
+        user message or nothing relevant is found, ``chat_ctx`` is returned as is.
+        """
+        if not self._enable_recall:
+            return chat_ctx
+        index = _last_user_message_index(chat_ctx)
+        if index is None:
+            return chat_ctx
+        user_message = chat_ctx.items[index]
+        query = user_message.text_content
+        if not query or not query.strip():
+            return chat_ctx
+
+        if self._recall_task is None or self._recall_message_id != user_message.id or self._recall_query != query:
+            self._recall_message_id = user_message.id
+            self._recall_query = query
+            self._recall_task = asyncio.create_task(self.recall(query))
+        # Shield so a cancelled generation (e.g. a discarded preemptive reply)
+        # doesn't cancel the recall the next generation is waiting on.
+        memories = await asyncio.shield(self._recall_task)
+        if memories is None:
+            return chat_ctx
+
+        # LiveKit's RAG example adds looked-up context as an assistant message
+        # before the user's turn (docs.livekit.io/agents/build/external-data).
+        enriched = chat_ctx.copy()
+        enriched.items.insert(
+            index,
+            ChatMessage(
+                role="assistant",
+                content=[memories],
+                created_at=user_message.created_at - 1e-6,
+            ),
+        )
+        return enriched
+
+    # ------------------------------------------------------------------
+    # Retain
+    # ------------------------------------------------------------------
+
+    def attach(self, session: AgentSession) -> None:
+        """Retain this session's conversation in Hindsight as it happens.
+
+        Call once, before ``session.start()``. The transcript is appended to the
+        Hindsight document ``document_id`` in the background. Also starts the
+        session recall early, so it is usually ready by the time the agent starts.
+        """
+        if self._session is session:
+            return
+        if self._session is not None:
+            raise HindsightLiveKitError("HindsightMemory is already attached to another AgentSession")
+        self._session = session
+        session.on("conversation_item_added", self._on_conversation_item_added)
+        session.on("close", self._on_session_close)
+        # LiveKit runs Agent.on_enter in the background and doesn't hold the first
+        # reply for it, so a slow session recall could miss the first turn. Starting
+        # it here overlaps it with the room and model connecting in session.start().
+        if self._enable_session_recall:
+            self._start_session_recall()
+
+    def _on_conversation_item_added(self, event: ConversationItemAddedEvent) -> None:
+        if not self._enable_retain or self._closed:
+            return
+        item = event.item
+        if item.type != "message" or item.role not in ("user", "assistant"):
+            return
+        text = item.text_content
+        if not text or not text.strip():
+            return
+
+        self._pending.append(_TranscriptLine(role=item.role, created_at=item.created_at, text=text.strip()))
+        # Flush once the buffer holds both sides of the conversation. With an
+        # STT-LLM-TTS pipeline the user's message is always added before the reply.
+        # Realtime models may add the user's transcript only after the reply it
+        # prompted, so flushing on every reply would store them out of order; waiting
+        # for both roles and sorting by created_at keeps the transcript in order.
+        roles = {line.role for line in self._pending}
+        if "user" in roles and "assistant" in roles:
+            self._schedule_retain()
+
+    def _on_session_close(self, event: CloseEvent) -> None:
+        self._schedule_retain()
+
+    def _schedule_retain(self) -> None:
+        if not self._pending:
+            return
+        lines = sorted(self._pending, key=lambda line: line.created_at)
+        self._pending = []
+        content = "\n".join(self._format_line(line) for line in lines)
+        task = asyncio.create_task(self._retain(content, _to_datetime(lines[-1].created_at)))
+        self._retain_tasks.add(task)
+        task.add_done_callback(self._retain_tasks.discard)
+
+    def _format_line(self, line: _TranscriptLine) -> str:
+        # "Name (timestamp): text" is the format Hindsight's docs use for conversations,
+        # so facts are attributed to the right speaker and time.
+        speaker = self._user_name if line.role == "user" else self._agent_name
+        timestamp = _to_datetime(line.created_at).isoformat(timespec="seconds")
+        return f"{speaker} ({timestamp}): {line.text}"
+
+    async def _retain(self, content: str, timestamp: datetime) -> None:
+        """Append ``content`` to this session's document (errors are logged and swallowed)."""
+        async with self._retain_lock:
+            try:
+                await self._client.aretain(
+                    bank_id=self._bank_id,
+                    content=content,
+                    timestamp=timestamp,
+                    context=self._retain_context,
+                    document_id=self._document_id,
+                    tags=self._tags,
+                    update_mode="append",
+                )
+            except Exception as e:
+                logger.warning(f"Hindsight retain failed: {e}")
+
+    async def flush(self) -> None:
+        """Send any buffered transcript lines and wait for in-flight retains to finish."""
+        self._schedule_retain()
+        if self._retain_tasks:
+            await asyncio.gather(*self._retain_tasks, return_exceptions=True)
+
+    async def aclose(self) -> None:
+        """Flush pending retains and close the client if this object created it.
+
+        Register it with ``ctx.add_shutdown_callback(memory.aclose)`` so the end
+        of the call is stored before the job exits.
+        """
+        if self._closed:
+            return
+        await self.flush()
+        self._closed = True
+        for task in (self._recall_task, self._session_recall_task):
+            if task is not None and not task.done():
+                task.cancel()
+        if self._owns_client:
+            await self._client.aclose()

hindsight_livekit/tools.py

@@ -0,0 +1,168 @@
+"""LiveKit function tools for Hindsight memory operations.
+
+Provides a factory that creates LiveKit ``FunctionTool`` instances backed by
+Hindsight's retain/recall/reflect APIs. Pass them to ``Agent(tools=[...])``.
+They work with STT-LLM-TTS pipelines and realtime models.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from hindsight_client import Hindsight
+from livekit.agents import RunContext, ToolError, function_tool
+from livekit.agents.llm import FunctionTool
+
+from ._client import resolve_client
+from .config import (
+    DEFAULT_RECALL_BUDGET,
+    DEFAULT_RECALL_MAX_TOKENS,
+    DEFAULT_RECALL_TAGS_MATCH,
+    Budget,
+    TagsMatch,
+    get_config,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def create_hindsight_tools(
+    *,
+    bank_id: str,
+    client: Hindsight | None = None,
+    hindsight_api_url: str | None = None,
+    api_key: str | None = None,
+    budget: Budget | None = None,
+    max_tokens: int | None = None,
+    tags: list[str] | None = None,
+    recall_tags: list[str] | None = None,
+    recall_tags_match: TagsMatch = DEFAULT_RECALL_TAGS_MATCH,
+    recall_types: list[str] | None = None,
+    retain_context: str = "voice conversation",
+    include_retain: bool = True,
+    include_recall: bool = True,
+    include_reflect: bool = True,
+) -> list[FunctionTool]:
+    """Create Hindsight memory tools for a LiveKit agent.
+
+    Returns ``FunctionTool`` instances for ``Agent(tools=[...])``. With a
+    :class:`~hindsight_livekit.HindsightMemory`, share its client and bank::
+
+        tools = create_hindsight_tools(bank_id=memory.bank_id, client=memory.client)
+
+    Errors are raised as LiveKit ``ToolError`` so the model hears what went wrong
+    and can tell the caller, instead of the call failing.
+
+    Args:
+        bank_id: The Hindsight memory bank to operate on.
+        client: Pre-configured Hindsight client (preferred).
+        hindsight_api_url: API URL (used if no client provided).
+        api_key: API key (used if no client provided).
+        budget: Recall/reflect budget level (low/mid/high). Defaults to ``"low"``.
+        max_tokens: Maximum tokens for recall and reflect results.
+        tags: Tags applied when storing memories via retain.
+        recall_tags: Tags to filter by when searching memories.
+        recall_tags_match: Tag matching mode (any/all/any_strict/all_strict).
+        recall_types: Fact types to search (world, experience, observation).
+        retain_context: Source label passed to Hindsight when storing memories.
+        include_retain: Include the ``hindsight_retain`` (store) tool.
+        include_recall: Include the ``hindsight_recall`` (search) tool.
+        include_reflect: Include the ``hindsight_reflect`` (synthesize) tool. Reflect
+            runs an LLM on the Hindsight server, so it is the slowest of the three.
+
+    Returns:
+        List of LiveKit FunctionTool instances.
+    """
+    resolved_client = resolve_client(client, hindsight_api_url, api_key)
+    config = get_config()
+    effective_budget: Budget = budget or (config.recall_budget if config else DEFAULT_RECALL_BUDGET)
+    effective_max_tokens = max_tokens or (config.recall_max_tokens if config else DEFAULT_RECALL_MAX_TOKENS)
+
+    tools: list[FunctionTool] = []
+
+    if include_retain:
+
+        @function_tool
+        async def hindsight_retain(context: RunContext, content: str) -> str:
+            """Store information in long-term memory so it is remembered in future calls.
+
+            Use this when the user shares something worth remembering, such as a
+            preference, a plan, or a fact about themselves.
+
+            Args:
+                content: The information to store, as a complete sentence.
+            """
+            try:
+                await resolved_client.aretain(
+                    bank_id=bank_id,
+                    content=content,
+                    context=retain_context,
+                    tags=tags,
+                )
+            except Exception as e:
+                logger.error(f"Hindsight retain failed: {e}")
+                raise ToolError("Could not save that to memory right now.") from e
+            return "Saved to memory."
+
+        tools.append(hindsight_retain)
+
+    if include_recall:
+
+        @function_tool
+        async def hindsight_recall(context: RunContext, query: str) -> str:
+            """Search long-term memory for things the user said in earlier conversations.
+
+            Use this when the user refers to something from the past, or when knowing
+            their preferences or history would help you answer.
+
+            Args:
+                query: What to look for, as a short question or phrase.
+            """
+            try:
+                response = await resolved_client.arecall(
+                    bank_id=bank_id,
+                    query=query,
+                    types=recall_types,
+                    budget=effective_budget,
+                    max_tokens=effective_max_tokens,
+                    tags=recall_tags,
+                    tags_match=recall_tags_match,
+                )
+            except Exception as e:
+                logger.error(f"Hindsight recall failed: {e}")
+                raise ToolError("Could not search memory right now.") from e
+            if not response.results:
+                return "No relevant memories found."
+            return "\n".join(f"{i}. {result.text}" for i, result in enumerate(response.results, 1))
+
+        tools.append(hindsight_recall)
+
+    if include_reflect:
+
+        @function_tool
+        async def hindsight_reflect(context: RunContext, query: str) -> str:
+            """Answer a question by reasoning over everything in long-term memory.
+
+            Use this for broader questions, like summarising what you know about the
+            user, rather than looking up a single fact.
+
+            Args:
+                query: The question to answer from memory.
+            """
+            try:
+                response = await resolved_client.areflect(
+                    bank_id=bank_id,
+                    query=query,
+                    budget=effective_budget,
+                    max_tokens=effective_max_tokens,
+                    tags=recall_tags,
+                    tags_match=recall_tags_match,
+                )
+            except Exception as e:
+                logger.error(f"Hindsight reflect failed: {e}")
+                raise ToolError("Could not reflect on memory right now.") from e
+            return response.text or "No relevant memories found."
+
+        tools.append(hindsight_reflect)
+
+    return tools

hindsight_livekit-0.1.0.dist-info/METADATA

@@ -0,0 +1,242 @@
+Metadata-Version: 2.5
+Name: hindsight-livekit
+Version: 0.1.0
+Summary: LiveKit Agents integration for Hindsight - persistent memory for voice AI agents
+Project-URL: Homepage, https://github.com/namandidthis/hindsight-livekit
+Project-URL: Repository, https://github.com/namandidthis/hindsight-livekit
+Project-URL: Issues, https://github.com/namandidthis/hindsight-livekit/issues
+Project-URL: Hindsight, https://github.com/vectorize-io/hindsight
+Author-email: Naman <naman6176@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: agents,ai,hindsight,livekit,memory,realtime,voice
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: hindsight-client>=0.6.0
+Requires-Dist: livekit-agents<2.0,>=1.2
+Description-Content-Type: text/markdown
+
+# Hindsight LiveKit Integration
+
+Persistent long-term memory for [LiveKit Agents](https://github.com/livekit/agents) voice AI via [Hindsight](https://vectorize.io/hindsight). Your agent knows what it learned in earlier calls, and stores each new conversation as it happens. Works with STT-LLM-TTS pipelines and realtime (speech-to-speech) models.
+
+## Quick Start
+
+```bash
+pip install hindsight-livekit
+# or the latest from GitHub:
+pip install git+https://github.com/namandidthis/hindsight-livekit
+```
+
+> ✨ **Recommended: [Hindsight Cloud](https://ui.hindsight.vectorize.io)** — free tier, no self-hosting required. Sign up and grab an API key in under a minute.
+
+```python
+from livekit.agents import AgentServer, AgentSession, JobContext, inference
+from hindsight_livekit import HindsightAgent, HindsightMemory
+
+server = AgentServer()
+
+
+@server.rtc_session()
+async def entrypoint(ctx: JobContext):
+    session = AgentSession(stt=..., tts=...)
+
+    memory = HindsightMemory(
+        bank_id="user-123",                    # one memory bank per end user
+        document_id=ctx.room.name,             # one transcript per call
+        hindsight_api_url="https://api.hindsight.vectorize.io",
+        api_key="hsk_...",                     # or set HINDSIGHT_API_KEY
+    )
+    memory.attach(session)                     # retain the conversation
+    ctx.add_shutdown_callback(memory.aclose)   # store the end of the call on hang-up
+
+    await session.start(
+        agent=HindsightAgent(
+            memory=memory,
+            llm=inference.LLM(model="google/gemma-4-31b-it"),
+            instructions="You are a friendly voice assistant.",
+        ),
+        room=ctx.room,
+    )
+    await ctx.connect()
+```
+
+`HindsightAgent` is a regular LiveKit `Agent`: subclass it, add `@function_tool`s, override `on_enter` (call `await super().on_enter()` first), and so on.
+
+Runnable agents built on [agent-starter-python](https://github.com/livekit-examples/agent-starter-python):
+
+- [`examples/voice_agent.py`](./examples/voice_agent.py) — STT-LLM-TTS pipeline
+- [`examples/realtime_agent.py`](./examples/realtime_agent.py) — realtime model with memory tools
+
+### Self-hosting (local development)
+
+```bash
+docker run --rm -p 8888:8888 -e HINDSIGHT_API_LLM_API_KEY=$OPENAI_API_KEY ghcr.io/vectorize-io/hindsight:latest
+```
+
+```python
+memory = HindsightMemory(bank_id="user-123", hindsight_api_url="http://localhost:8888")
+```
+
+See the [Hindsight installation guide](https://hindsight.vectorize.io/developer/installation) for other options. LiveKit itself can be [self-hosted](https://docs.livekit.io/transport/self-hosting/local/) too; the integration works the same either way.
+
+## How It Works
+
+```
+memory.attach(session)
+  └─ Session recall starts: "who is this user?"  (overlaps with session.start)
+
+Agent starts (on_enter)
+  └─ Known facts about the user are added to the agent's instructions
+
+Before each LLM reply (llm_node)                       — STT-LLM-TTS pipelines only
+  └─ Recall memories relevant to the user's latest message
+       └─ Insert them before that message, in a copy of the chat context
+
+As the conversation happens (conversation_item_added)
+  └─ Append "User (time): …" / "Assistant (time): …" lines to the call's Hindsight document
+```
+
+| | STT-LLM-TTS pipeline | Realtime model |
+|---|---|---|
+| Recall when the agent starts | ✅ | ✅ |
+| Recall before each reply | ✅ | — (no step before a realtime reply; use the tools) |
+| Memory tools (`create_hindsight_tools`) | ✅ | ✅ |
+| Retain the conversation | ✅ | ✅ (needs the model's input audio transcription for the user's side) |
+
+**Session recall** runs once per call. `attach()` starts it, so it overlaps with LiveKit connecting the room and the model, and `HindsightAgent.on_enter` adds the result to the instructions. LiveKit runs `on_enter` in the background, so with a pipeline the first `llm_node` call also waits for the lookup and uses the instructions with memories, even if the user speaks first. A realtime model has no such step: a reply that starts before the lookup finishes (bounded by `recall_timeout`) won't include those memories, which is rare in practice since the lookup starts before the call connects.
+
+**Per-turn recall** runs in [`llm_node`](https://docs.livekit.io/agents/build/nodes/), the step every LLM call in a pipeline goes through. That means:
+
+- It works for spoken turns, chat text input and `session.generate_reply(user_input=...)`. The `on_user_turn_completed` hook only runs for spoken turns.
+- It works with [preemptive generation](https://docs.livekit.io/agents/build/audio/#preemptive-generation): recall starts while LiveKit is still confirming the end of the user's turn, and the preemptive reply is kept. Editing the context in `on_user_turn_completed` would make LiveKit discard it.
+- Memories are only added to a copy of the context for that call, so they never pile up in the session history. When the LLM calls a tool, the follow-up call reuses the same recall result.
+
+As in LiveKit's [RAG example](https://docs.livekit.io/agents/build/external-data/), per-turn memories are added as an `assistant` message just before the user's message, wrapped in `<hindsight_memories>` tags with a note telling the model not to read them out.
+
+**Retain** follows Hindsight's [guidance for conversations](https://hindsight.vectorize.io/developer/api/retain#retaining-a-conversation): each call is one document (`document_id`), written as `Name (timestamp): text` lines, grown with [`update_mode="append"`](https://hindsight.vectorize.io/developer/api/retain#update_mode) so Hindsight only extracts facts from the new part. Lines are sent once both the user and the agent have spoken, sorted by time, because realtime models can deliver the user's transcript after the reply it prompted. Appends are sent one at a time, in order, in the background, so they never delay a reply. `aclose()` sends whatever is left when the call ends.
+
+**Failures never break the call.** If recall fails or takes longer than `recall_timeout`, the agent carries on without memories. If retain fails, the error is logged.
+
+## Memory Tools
+
+`create_hindsight_tools()` gives the model `hindsight_retain`, `hindsight_recall` and `hindsight_reflect` tools, like the other Hindsight framework integrations. They are the way for a realtime model to look things up mid-call, and work with pipelines too:
+
+```python
+from hindsight_livekit import HindsightAgent, create_hindsight_tools
+
+agent = HindsightAgent(
+    memory=memory,
+    llm=inference.RealtimeModel(model="openai/gpt-realtime-mini"),
+    tools=create_hindsight_tools(bank_id=memory.bank_id, client=memory.client, include_reflect=False),
+    instructions=(
+        "You have long-term memory of past calls. When the user asks about themselves, "
+        "their past, or their preferences, call hindsight_recall first. When they share "
+        "something worth remembering, call hindsight_retain."
+    ),
+)
+```
+
+Tell the model when to use the tools, as above: models call them far more reliably with explicit guidance. `hindsight_reflect` runs an LLM on the Hindsight server, so it is the slowest; leave it out for snappier calls. Tool errors are raised as LiveKit `ToolError`, so the model hears what went wrong and can tell the caller.
+
+## Latency
+
+Recall happens while a caller waits, so the defaults are tuned for voice:
+
+- `recall_budget="low"`: Hindsight's [fast lookup](https://hindsight.vectorize.io/developer/api/recall#budget) setting.
+- `recall_max_tokens=1024`: keeps the prompt small.
+- `recall_timeout=3.0`: the longest a reply waits on memory before going ahead without it.
+
+With preemptive generation turned on, most per-turn recall time overlaps with end-of-turn detection.
+
+## Configuration
+
+```python
+HindsightMemory(
+    bank_id="user-123",                  # Required: memory bank to use
+    client=None,                         # Pre-built hindsight_client.Hindsight (optional)
+    hindsight_api_url="...",             # Defaults to HINDSIGHT_API_URL, then Hindsight Cloud
+    api_key="hsk_...",                   # Defaults to HINDSIGHT_API_KEY
+    document_id=None,                    # Defaults to a random "livekit-<uuid>"
+    recall_budget="low",                 # "low", "mid", or "high"
+    recall_max_tokens=1024,              # Max tokens of memories per recall
+    recall_timeout=3.0,                  # Seconds before carrying on without memories
+    recall_types=None,                   # e.g. ["world", "experience"]; None = all
+    tags=None,                           # Tags for retained content
+    recall_tags=None,                    # Tags to filter recall by
+    recall_tags_match="any",             # "any", "all", "any_strict", "all_strict"
+    retain_context="voice conversation", # Source label that guides fact extraction
+    user_name="User",                    # Speaker labels in the stored transcript
+    agent_name="Assistant",
+    enable_session_recall=True,          # Recall known facts when the agent starts
+    session_recall_query="Who is this user? ...",
+    enable_recall=True,                  # Recall before each reply (pipelines)
+    enable_retain=True,                  # Store the conversation
+)
+```
+
+### Global configuration
+
+```python
+from hindsight_livekit import configure
+
+configure(
+    hindsight_api_url="https://api.hindsight.vectorize.io",
+    api_key="hsk_...",
+    recall_budget="low",
+)
+
+memory = HindsightMemory(bank_id="user-123")
+```
+
+### Using your own `Agent` or `llm_node`
+
+`memory.attach(session)` works with any `Agent`. To add recall to an agent that already overrides `llm_node` or `on_enter`, call the memory directly instead of subclassing `HindsightAgent`:
+
+```python
+from livekit.agents import Agent
+
+class MyAgent(Agent):
+    def __init__(self, memory: HindsightMemory) -> None:
+        super().__init__(instructions="...")
+        self.memory = memory
+
+    async def on_enter(self):
+        if memories := await self.memory.recall_session_context():
+            await self.update_instructions(f"{self.instructions}\n\n{memories}")
+
+    async def llm_node(self, chat_ctx, tools, model_settings):
+        chat_ctx = await self.memory.inject_memories(chat_ctx)
+        async for chunk in Agent.default.llm_node(self, chat_ctx, tools, model_settings):
+            yield chunk
+```
+
+## Prerequisites
+
+- `livekit-agents` 1.2 or later
+- A running Hindsight instance, self-hosted or [Hindsight Cloud](https://ui.hindsight.vectorize.io)
+
+## Running Tests
+
+```bash
+uv sync
+uv run pytest tests -v
+```
+
+The unit tests run a real LiveKit `AgentSession` with a scripted LLM and a mocked Hindsight client. The live tests in `tests/test_live_integration.py` run against a real Hindsight server when `HINDSIGHT_API_URL` is set, and also against real pipeline and realtime models through LiveKit Inference when LiveKit Cloud credentials are set:
+
+```bash
+docker run --rm -p 8888:8888 -e HINDSIGHT_API_LLM_PROVIDER=none ghcr.io/vectorize-io/hindsight:latest
+HINDSIGHT_API_URL=http://localhost:8888 uv run pytest tests -v
+
+# optional: real models via LiveKit Inference
+LIVEKIT_URL=wss://... LIVEKIT_API_KEY=... LIVEKIT_API_SECRET=... HINDSIGHT_API_URL=http://localhost:8888 uv run pytest tests -v
+```

hindsight_livekit-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+hindsight_livekit/__init__.py,sha256=OA-AvFrvZClGocM7msFcnPdKAVAJVmzsj4nQEb1kxLc,1432
+hindsight_livekit/_client.py,sha256=qeFVo0ql5zjOaGWDoNITaI66-7oLSJef6ih1QPpBm5s,1465
+hindsight_livekit/agent.py,sha256=J1Efha5Z2PkGEJiIUsinh4EfsoV7Y_wrAQ4o7Y3m39I,3747
+hindsight_livekit/config.py,sha256=AEEpG_OFXveOW0WubWqPDgmS0LenxY20tbdZ013KPvc,2950
+hindsight_livekit/errors.py,sha256=gPlcCs_KajuRSsLdT2Fsv_msUnrPbe6xE8KZetBrBq8,173
+hindsight_livekit/memory.py,sha256=XaQUuImN_NlCFrrOtb3YvIUB5ggI8-UsTdHazAF-hxM,17585
+hindsight_livekit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hindsight_livekit/tools.py,sha256=bg67Qg1GFJnYE-_9i_0DMhFoUEB2m8MU71huLrlwkYk,6543
+hindsight_livekit-0.1.0.dist-info/METADATA,sha256=kfCSkT0wDHqmywDUkiph3Ncd_KcVQB59Jp9BNLRL_8Q,12439
+hindsight_livekit-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+hindsight_livekit-0.1.0.dist-info/licenses/LICENSE,sha256=juw1vPPlfLMPts6FbNdNfpe_j3FinPgCJpbFppuHpB4,1062
+hindsight_livekit-0.1.0.dist-info/RECORD,,

hindsight_livekit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

hindsight_livekit-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Naman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.