hermes-memory-pgvector 0.3.0__py3-none-any.whl

pgvector/embed.py

@@ -0,0 +1,103 @@
+"""embed.py — minimal embedding client for the pgvector memory plugin.
+
+Posts to an OpenAI-compatible /v1/embeddings or Ollama native /api/embed
+endpoint and returns a list of floats. No retries beyond a single attempt
+— callers decide what to do with failures (we want fail-soft, not retry
+storms — that was Honcho's mistake).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import urllib.error
+import urllib.request
+from typing import List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingError(Exception):
+    """Raised when the embedding endpoint fails to return a usable vector."""
+
+
+def embed(
+    text: str,
+    *,
+    base_url: str,
+    model: str = "nomic-embed-text",
+    timeout: float = 10.0,
+) -> List[float]:
+    """Return a 768-dim embedding for `text`.
+
+    Tries the OpenAI-compatible `/v1/embeddings` path first; falls back to
+    Ollama's native `/api/embed`. Raises EmbeddingError on any failure.
+    Single attempt — no retries.
+    """
+    if not text or not text.strip():
+        raise EmbeddingError("empty input")
+
+    # Trim to avoid the n_ctx_train=2048 cliff on nomic — at ~4 chars/token
+    # that's ~8000 chars. Keep a safety margin.
+    if len(text) > 6000:
+        text = text[:6000]
+
+    base_url = base_url.rstrip("/")
+
+    # Path A: OpenAI-compatible
+    try:
+        return _post(
+            f"{base_url}/v1/embeddings",
+            {"model": model, "input": text},
+            timeout=timeout,
+            extract=lambda d: d["data"][0]["embedding"],
+        )
+    except EmbeddingError as exc:
+        logger.debug("OpenAI-compat embed failed (%s); trying native", exc)
+
+    # Path B: Ollama native
+    return _post(
+        f"{base_url}/api/embed",
+        {"model": model, "input": text},
+        timeout=timeout,
+        extract=lambda d: (d.get("embeddings") or [d.get("embedding")])[0],
+    )
+
+
+def _post(url: str, body: dict, *, timeout: float, extract) -> List[float]:
+    data = json.dumps(body).encode("utf-8")
+    req = urllib.request.Request(
+        url,
+        data=data,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            payload = json.loads(resp.read())
+    except urllib.error.HTTPError as exc:
+        raise EmbeddingError(f"HTTP {exc.code}: {exc.reason}") from exc
+    except urllib.error.URLError as exc:
+        raise EmbeddingError(f"connection failed: {exc.reason}") from exc
+    except (json.JSONDecodeError, ValueError) as exc:
+        raise EmbeddingError(f"invalid JSON response: {exc}") from exc
+
+    try:
+        vec = extract(payload)
+    except (KeyError, IndexError, TypeError) as exc:
+        raise EmbeddingError(f"unexpected response shape: {exc}") from exc
+
+    if not isinstance(vec, list) or not vec:
+        raise EmbeddingError("response had no embedding array")
+    if len(vec) != 768:
+        raise EmbeddingError(f"expected 768 dims, got {len(vec)}")
+    return vec
+
+
+def to_pgvector_literal(vec: List[float]) -> str:
+    """Render a Python list of floats as a pgvector input literal.
+
+    psycopg can also handle this via type adapters, but the literal form
+    keeps the plugin dependency-light.
+    """
+    return "[" + ",".join(f"{x:.6g}" for x in vec) + "]"

pgvector/migrations/001_schema.sql

@@ -0,0 +1,95 @@
+-- 001_schema.sql — pgvector memory plugin schema.
+--
+-- TWO tables, both in the existing hermes_memory database:
+--   memory_entries  → mirrors hermes-agent's built-in `memory` tool
+--                     (MEMORY.md / USER.md from tools/memory_tool.py)
+--   conversations   → every (user, assistant) chat turn, semantic-searchable
+--                     for cross-session recall of "what did we talk about"
+--
+-- Both scoped per agent_identity (marketing / sales / trading / incident / …),
+-- both with 768-dim embeddings, both with HNSW indexes tuned the same way as
+-- hermes_memory.events.
+--
+-- Apply once:
+--   sudo -u postgres psql -d hermes_memory -f 001_schema.sql
+-- Idempotent (CREATE IF NOT EXISTS everywhere); safe to re-run.
+
+CREATE EXTENSION IF NOT EXISTS vector;
+
+
+-- ---------------------------------------------------------------------------
+-- memory_entries
+-- One row per add/replace from hermes-agent's built-in `memory` tool,
+-- mirrored via the on_memory_write hook. Hard-deleted on `remove`.
+-- ---------------------------------------------------------------------------
+
+CREATE TABLE IF NOT EXISTS memory_entries (
+  id              BIGSERIAL PRIMARY KEY,
+
+  -- Per-agent theme. Maps to hermes-agent's `agent_identity` profile
+  -- (passed to MemoryProvider.initialize via kwargs).
+  -- Examples: 'marketing', 'sales', 'trading', 'incident', 'default'.
+  agent_identity  TEXT NOT NULL DEFAULT 'default',
+
+  -- Mirrors hermes-agent's two built-in stores:
+  --   'memory' → MEMORY.md (env facts, project conventions, tool quirks)
+  --   'user'   → USER.md   (about the user: name, role, preferences)
+  target          TEXT NOT NULL CHECK (target IN ('memory', 'user')),
+
+  content         TEXT NOT NULL,
+  embedding       vector(768),
+
+  created_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+
+  -- Provenance from on_memory_write kwargs: session_id, platform,
+  -- write_origin, tool_name, parent_session_id, etc.
+  metadata        JSONB NOT NULL DEFAULT '{}'::jsonb,
+
+  -- Built-in tool dedupes on (target, exact content match). We replicate
+  -- that within a single agent's scope so add-of-existing is a no-op.
+  CONSTRAINT memory_entries_unique
+    UNIQUE (agent_identity, target, content)
+);
+
+-- Per-agent + target listing (the most common scan).
+CREATE INDEX IF NOT EXISTS ix_memory_entries_agent_target
+  ON memory_entries (agent_identity, target, updated_at DESC);
+
+-- Semantic recall: SELECT … ORDER BY embedding <=> $1 LIMIT $2
+-- Cosine distance via HNSW — same tuning as hermes_memory.events.
+CREATE INDEX IF NOT EXISTS ix_memory_entries_embedding_hnsw
+  ON memory_entries USING hnsw (embedding vector_cosine_ops)
+  WITH (m = 16, ef_construction = 64);
+
+
+-- ---------------------------------------------------------------------------
+-- conversations
+-- One row per substantive chat turn, captured via sync_turn(). Short /
+-- boilerplate turns ("ok", "thanks", < 40 chars) are filtered out at
+-- the plugin layer before INSERT; what lands here is recall-worthy.
+-- ---------------------------------------------------------------------------
+
+CREATE TABLE IF NOT EXISTS conversations (
+  id              BIGSERIAL PRIMARY KEY,
+  session_id      TEXT NOT NULL,
+  agent_identity  TEXT NOT NULL DEFAULT 'default',
+  role            TEXT NOT NULL CHECK (role IN ('user', 'assistant', 'system', 'tool')),
+  content         TEXT NOT NULL,
+  ts              TIMESTAMPTZ NOT NULL DEFAULT now(),
+  embedding       vector(768),
+  metadata        JSONB NOT NULL DEFAULT '{}'::jsonb
+);
+
+-- Per-session timeline (the "recent turns in this conversation" path).
+CREATE INDEX IF NOT EXISTS ix_conversations_session_ts
+  ON conversations (session_id, ts DESC);
+
+-- Per-agent timeline (the "what's marketing been doing lately" path).
+CREATE INDEX IF NOT EXISTS ix_conversations_agent_ts
+  ON conversations (agent_identity, ts DESC);
+
+-- Semantic recall across all turns. Same HNSW tuning as memory_entries.
+CREATE INDEX IF NOT EXISTS ix_conversations_embedding_hnsw
+  ON conversations USING hnsw (embedding vector_cosine_ops)
+  WITH (m = 16, ef_construction = 64);

pgvector/plugin.yaml

@@ -0,0 +1,9 @@
+name: pgvector
+version: 0.3.0
+description: "Postgres + pgvector storage layer for hermes-agent. Mirrors built-in MEMORY.md / USER.md + captures every substantive (user, assistant) chat turn into a conversations table. Per-agent themes (marketing/sales/trading/incident/morning-report/…) via X-Hermes-Session-Key header — every systemd-run minion gets its own scope. 768-dim semantic recall. Async writer + connection pool. No LLM mediation."
+pip_dependencies:
+  - "psycopg[binary]>=3.3.4"
+  - "psycopg-pool>=3.3.1"
+requires_env: []
+hooks:
+  - on_session_end