n3memorycore-free 1.3.1__py3-none-any.whl

n3memorycore/__init__.py

@@ -0,0 +1,7 @@
+"""N3MemoryCore — Free Edition.
+
+A persistent hybrid-search memory store (SQLite + sqlite-vec + FTS5)
+for Claude Code, with hook integration.
+"""
+
+__version__ = "1.3.1"

n3memorycore/core/database.py

@@ -0,0 +1,308 @@
+import sqlite3
+import struct
+import re
+import logging
+from typing import Optional, List, Tuple
+
+logger = logging.getLogger(__name__)
+
+_FTS_PUNCT_RE = re.compile(r'[()[\]{}<>,.!?;:\-—–…\'\"\`~@#$%^&*+=|/\\]')
+_FTS_MAX_TERMS = 30
+
+
+def strip_fts_punctuation(text: str) -> str:
+    # Substitute with space, not empty: dashes between words ("Alpha-9") should
+    # split into ["Alpha", "9"], not concatenate into "Alpha9".
+    cleaned = _FTS_PUNCT_RE.sub(' ', text)
+    return re.sub(r'\s+', ' ', cleaned).strip()
+
+
+def _quote_fts_query(text: str) -> str:
+    stripped = strip_fts_punctuation(text)
+    terms = stripped.split()[:_FTS_MAX_TERMS]
+    return ' '.join(f'"{t}"' for t in terms)
+
+
+def serialize_vector(vec: list) -> bytes:
+    return struct.pack(f'{len(vec)}f', *vec)
+
+
+def deserialize_vector(data: bytes) -> list:
+    n = len(data) // 4
+    return list(struct.unpack(f'{n}f', data))
+
+
+def _load_vec_extension(conn: sqlite3.Connection) -> None:
+    try:
+        conn.enable_load_extension(True)
+        try:
+            import sqlite_vec
+            sqlite_vec.load(conn)
+        except Exception as e1:
+            msg = str(e1).lower()
+            if 'already' in msg or 'duplicate' in msg:
+                return
+            # Fallback: direct load
+            try:
+                conn.load_extension('vec0')
+            except Exception as e2:
+                msg2 = str(e2).lower()
+                if 'already' in msg2 or 'duplicate' in msg2:
+                    return
+                raise e2
+    except Exception as e:
+        msg = str(e).lower()
+        if 'already' in msg or 'duplicate' in msg:
+            return
+        raise
+
+
+def get_connection(db_path: str) -> sqlite3.Connection:
+    conn = sqlite3.connect(db_path, check_same_thread=False)
+    try:
+        _load_vec_extension(conn)
+    except Exception as e:
+        conn.close()
+        raise sqlite3.DatabaseError(
+            f"Failed to load sqlite-vec extension: {e}. "
+            "Recovery: pip install sqlite-vec"
+        ) from e
+    conn.execute("PRAGMA synchronous = FULL")
+    conn.execute("PRAGMA journal_mode = WAL")
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def init_db(conn: sqlite3.Connection) -> None:
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS memories (
+            id         TEXT PRIMARY KEY,
+            content    TEXT NOT NULL,
+            timestamp  TEXT NOT NULL,
+            owner_id   TEXT NOT NULL,
+            local_id   TEXT,
+            agent_name TEXT,
+            session_id TEXT,
+            turn_id    TEXT
+        )
+    """)
+    conn.execute("""
+        CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+            content,
+            tokenize='porter unicode61'
+        )
+    """)
+    conn.execute("""
+        CREATE VIRTUAL TABLE IF NOT EXISTS memories_vec USING vec0(
+            embedding float[768]
+        )
+    """)
+    # Defer the turn_id index to migrate_schema so an old table without the
+    # column does not blow up here.
+    cols = {r[1] for r in conn.execute("PRAGMA table_info(memories)").fetchall()}
+    if 'turn_id' in cols:
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_memories_turn_id ON memories(turn_id)"
+        )
+    conn.commit()
+
+
+def migrate_schema(conn: sqlite3.Connection) -> None:
+    cursor = conn.execute("PRAGMA table_info(memories)")
+    existing_cols = {row[1] for row in cursor.fetchall()}
+
+    for col, typedef in [
+        ("local_id", "TEXT"),
+        ("agent_name", "TEXT"),
+        ("session_id", "TEXT"),
+        ("turn_id", "TEXT"),
+    ]:
+        if col not in existing_cols:
+            try:
+                conn.execute(f"ALTER TABLE memories ADD COLUMN {col} {typedef}")
+                conn.commit()
+            except Exception:
+                pass
+
+    cursor = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='index' AND name='idx_memories_turn_id'"
+    )
+    if not cursor.fetchone():
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_memories_turn_id ON memories(turn_id)")
+        conn.commit()
+
+    # Migrate FTS tokenizer if trigram
+    cursor = conn.execute(
+        "SELECT sql FROM sqlite_master WHERE type='table' AND name='memories_fts'"
+    )
+    row = cursor.fetchone()
+    if row and row[0] and 'trigram' in row[0].lower():
+        cursor2 = conn.execute("SELECT rowid, content FROM memories_fts")
+        existing_fts = cursor2.fetchall()
+        conn.execute("DROP TABLE IF EXISTS memories_fts")
+        conn.execute("""
+            CREATE VIRTUAL TABLE memories_fts USING fts5(
+                content,
+                tokenize='porter unicode61'
+            )
+        """)
+        for r in existing_fts:
+            stripped = strip_fts_punctuation(r[1])
+            conn.execute(
+                "INSERT INTO memories_fts(rowid, content) VALUES (?, ?)",
+                (r[0], stripped)
+            )
+        conn.commit()
+        logger.info("Migrated FTS tokenizer from trigram to porter unicode61")
+
+
+def insert_memory(
+    conn: sqlite3.Connection,
+    id: str,
+    content: str,
+    timestamp: str,
+    owner_id: str,
+    embedding: Optional[list] = None,
+    local_id: Optional[str] = None,
+    agent_name: Optional[str] = None,
+    session_id: Optional[str] = None,
+    turn_id: Optional[str] = None,
+) -> int:
+    conn.execute(
+        """INSERT INTO memories
+           (id, content, timestamp, owner_id, local_id, agent_name, session_id, turn_id)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+        (id, content, timestamp, owner_id, local_id, agent_name, session_id, turn_id),
+    )
+    rowid = conn.execute("SELECT last_insert_rowid()").fetchone()[0]
+    stripped = strip_fts_punctuation(content)
+    conn.execute(
+        "INSERT INTO memories_fts(rowid, content) VALUES (?, ?)", (rowid, stripped)
+    )
+    if embedding is not None:
+        conn.execute(
+            "INSERT INTO memories_vec(rowid, embedding) VALUES (?, ?)",
+            (rowid, serialize_vector(embedding)),
+        )
+    conn.commit()
+    return rowid
+
+
+def search_vector(
+    conn: sqlite3.Connection, embedding: list, k: int = 50
+) -> List[sqlite3.Row]:
+    vec_bytes = serialize_vector(embedding)
+    try:
+        cursor = conn.execute(
+            """SELECT m.id, m.content, m.timestamp, m.owner_id, m.local_id,
+                      m.agent_name, m.session_id, m.turn_id, mv.distance, m.rowid
+               FROM memories_vec mv
+               JOIN memories m ON mv.rowid = m.rowid
+               WHERE mv.embedding MATCH ?
+                 AND k = ?
+               ORDER BY mv.distance""",
+            (vec_bytes, k),
+        )
+        return cursor.fetchall()
+    except Exception as e:
+        logger.warning(f"Vector search failed: {e}")
+        return []
+
+
+def search_fts(
+    conn: sqlite3.Connection, query: str, limit: int = 50
+) -> List[sqlite3.Row]:
+    if len(query.strip()) < 2:
+        return []
+    fts_query = _quote_fts_query(query)
+    if not fts_query.strip():
+        return []
+    try:
+        cursor = conn.execute(
+            """SELECT m.id, m.content, m.timestamp, m.owner_id, m.local_id,
+                      m.agent_name, m.session_id, m.turn_id,
+                      bm25(memories_fts) AS bm25_score, m.rowid
+               FROM memories_fts
+               JOIN memories m ON memories_fts.rowid = m.rowid
+               WHERE memories_fts MATCH ?
+               ORDER BY bm25_score
+               LIMIT ?""",
+            (fts_query, limit),
+        )
+        return cursor.fetchall()
+    except Exception as e:
+        logger.warning(f"FTS search failed: {e}")
+        return []
+
+
+def get_all_memories(conn: sqlite3.Connection) -> List[sqlite3.Row]:
+    cursor = conn.execute(
+        """SELECT id, content, timestamp, owner_id, local_id, agent_name,
+                  session_id, turn_id, rowid
+           FROM memories
+           ORDER BY rowid"""
+    )
+    return cursor.fetchall()
+
+
+def delete_memory(conn: sqlite3.Connection, memory_id: str) -> bool:
+    row = conn.execute(
+        "SELECT rowid FROM memories WHERE id = ?", (memory_id,)
+    ).fetchone()
+    if not row:
+        return False
+    rowid = row[0]
+    try:
+        conn.execute("DELETE FROM memories_fts WHERE rowid = ?", (rowid,))
+        conn.execute("DELETE FROM memories_vec WHERE rowid = ?", (rowid,))
+        conn.execute("DELETE FROM memories WHERE id = ?", (memory_id,))
+        conn.commit()
+        return True
+    except Exception:
+        conn.rollback()
+        raise
+
+
+def count_memories(conn: sqlite3.Connection) -> int:
+    return conn.execute("SELECT COUNT(*) FROM memories").fetchone()[0]
+
+
+def check_exact_duplicate(conn: sqlite3.Connection, content: str) -> bool:
+    row = conn.execute(
+        "SELECT 1 FROM memories WHERE content = ? LIMIT 1", (content,)
+    ).fetchone()
+    return row is not None
+
+
+def find_unindexed_memories(
+    conn: sqlite3.Connection, limit: int = 200, offset: int = 0
+) -> List[sqlite3.Row]:
+    cursor = conn.execute(
+        """SELECT m.id, m.content, m.timestamp, m.owner_id, m.local_id,
+                  m.agent_name, m.session_id, m.turn_id, m.rowid,
+                  mv.rowid AS vec_rowid,
+                  mf.rowid AS fts_rowid
+           FROM memories m
+           LEFT JOIN memories_vec mv ON m.rowid = mv.rowid
+           LEFT JOIN memories_fts mf ON m.rowid = mf.rowid
+           WHERE mv.rowid IS NULL OR mf.rowid IS NULL
+           LIMIT ? OFFSET ?""",
+        (limit, offset),
+    )
+    return cursor.fetchall()
+
+
+def get_memories_by_turn_id(
+    conn: sqlite3.Connection, turn_id: str
+) -> List[sqlite3.Row]:
+    cursor = conn.execute(
+        """SELECT id, content, timestamp, owner_id, local_id, agent_name,
+                  session_id, turn_id, rowid
+           FROM memories
+           WHERE turn_id = ?
+           ORDER BY
+               CASE WHEN content LIKE '[user%' THEN 0 ELSE 1 END,
+               rowid""",
+        (turn_id,),
+    )
+    return cursor.fetchall()

n3memorycore/core/processor.py

@@ -0,0 +1,361 @@
+import os
+import re
+import math
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from .database import (
+    get_connection,
+    init_db,
+    insert_memory,
+    search_vector,
+    search_fts,
+    get_all_memories,
+    delete_memory,
+    count_memories,
+    check_exact_duplicate,
+    find_unindexed_memories,
+    serialize_vector,
+    get_memories_by_turn_id,
+    strip_fts_punctuation,
+)
+
+logger = logging.getLogger(__name__)
+
+_CODE_BLOCK_RE = re.compile(r'```[\s\S]*?```')
+
+# ---------------------------------------------------------------------------
+# Lone-surrogate sanitization (Windows / cp932 data-loss guard)
+# ---------------------------------------------------------------------------
+# Subprocess stdin pipes on Windows can deliver UTF-8 bytes that python's
+# decoder maps to lone UTF-16 surrogate halves (U+D800..U+DFFF). Such strings
+# round-trip through json.loads but blow up at sqlite3.execute() with
+# UnicodeEncodeError, silently dropping the entire write — a direct violation
+# of the complete-preservation contract. We strip these halves before any
+# value is bound to SQLite.
+_LONE_SURROGATE_RE = re.compile(r'[\ud800-\udfff]')
+
+
+def sanitize_surrogates(text):
+    """Strip lone UTF-16 surrogate halves from a string (or pass-through).
+
+    Recursively cleans dicts / lists too so audit-log JSON payloads with
+    surrogates buried inside multimodal content do not break json.dumps.
+    """
+    if text is None:
+        return text
+    if isinstance(text, str):
+        return _LONE_SURROGATE_RE.sub('', text)
+    if isinstance(text, list):
+        return [sanitize_surrogates(x) for x in text]
+    if isinstance(text, dict):
+        return {k: sanitize_surrogates(v) for k, v in text.items()}
+    return text
+
+DEFAULT_EMBED_MODEL = 'intfloat/multilingual-e5-base'
+
+_model = None
+_model_name = None
+
+
+def get_model(name: Optional[str] = None):
+    """Lazy-load and cache the embedding model.
+
+    On the first call (or when `name` differs from the cached model),
+    the model is selected from (in priority order):
+      1. `name` argument
+      2. `$N3MC_EMBED_MODEL` environment variable
+      3. `DEFAULT_EMBED_MODEL` (multilingual baseline)
+
+    Subsequent calls with `name=None` return the cached model without
+    re-resolving — so the model loaded by the server lifespan (with
+    `embed_model` from config.json) wins, and `embed_passage` /
+    `embed_query` reuse it transparently.
+
+    Callers that want to switch to a language-specialised model should
+    set `embed_model` in `config.json` and run `--repair` once after
+    the first save under the new model (see spec §5 vector re-index
+    migration).
+    """
+    global _model, _model_name
+    if name is None and _model is not None:
+        return _model
+    target = name or os.environ.get('N3MC_EMBED_MODEL') or DEFAULT_EMBED_MODEL
+    if _model is None or _model_name != target:
+        from sentence_transformers import SentenceTransformer
+        _model = SentenceTransformer(target)
+        _model_name = target
+    return _model
+
+
+def embed_passage(text: str) -> list:
+    model = get_model()
+    prefixed = "passage: " + text
+    vec = model.encode(prefixed, normalize_embeddings=True)
+    return vec.tolist()
+
+
+def embed_query(text: str) -> list:
+    model = get_model()
+    prefixed = "query: " + text
+    vec = model.encode(prefixed, normalize_embeddings=True)
+    return vec.tolist()
+
+
+def purify_text(text: str) -> str:
+    """Replace closed fenced code blocks with [code omitted] AND strip lone
+    surrogates as a defense-in-depth layer (spec §5).
+
+    Inline backtick spans and unclosed fences are preserved verbatim.
+    """
+    if not text:
+        return text
+    cleaned = sanitize_surrogates(text)
+    return _CODE_BLOCK_RE.sub('[code omitted]', cleaned)
+
+
+def chunk_text(text: str, max_chars: int = 400, overlap: int = 40) -> list:
+    if not text or not text.strip():
+        return []
+    if len(text) <= max_chars:
+        return [text]
+
+    # Paragraph split
+    paragraphs = [p.strip() for p in re.split(r'\n\s*\n', text) if p.strip()]
+    if len(paragraphs) > 1:
+        chunks = _merge_chunks(paragraphs, max_chars, overlap, sep='\n\n')
+        if chunks:
+            return chunks
+
+    # Sentence split
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+    if len(sentences) > 1:
+        chunks = _merge_chunks(sentences, max_chars, overlap, sep=' ')
+        if chunks:
+            return chunks
+
+    # Hard window fallback
+    return _hard_window(text, max_chars, overlap)
+
+
+def _merge_chunks(parts: list, max_chars: int, overlap: int, sep: str) -> list:
+    chunks = []
+    current = ""
+    for part in parts:
+        if len(part) > max_chars:
+            if current:
+                chunks.append(current)
+                current = ""
+            sub = _hard_window(part, max_chars, overlap)
+            chunks.extend(sub)
+        elif current and len(current) + len(sep) + len(part) > max_chars:
+            chunks.append(current)
+            # Overlap: keep tail of previous chunk
+            tail = current[-overlap:] if overlap else ""
+            current = (tail + sep + part) if tail else part
+        else:
+            current = (current + sep + part) if current else part
+    if current:
+        chunks.append(current)
+    return chunks
+
+
+def _hard_window(text: str, max_chars: int, overlap: int) -> list:
+    chunks = []
+    start = 0
+    length = len(text)
+    while start < length:
+        end = min(start + max_chars, length)
+        chunks.append(text[start:end])
+        if end >= length:
+            break
+        start = end - overlap
+        if start <= 0:
+            start = end
+    return chunks
+
+
+def add_chunk_prefixes(chunks: list, role: str) -> list:
+    if not chunks:
+        return []
+    n = len(chunks)
+    if n == 1:
+        return [f"[{role}] {chunks[0]}"]
+    return [f"[{role} {i + 1}/{n}] {chunks[i]}" for i in range(n)]
+
+
+def cosine_sim_from_l2(l2_distance: float) -> float:
+    return max(0.0, 1.0 - (l2_distance ** 2) / 2.0)
+
+
+def time_decay(timestamp_str: str, half_life_days: float) -> float:
+    try:
+        ts = datetime.fromisoformat(timestamp_str)
+        if ts.tzinfo is None:
+            ts = ts.replace(tzinfo=timezone.utc)
+        now = datetime.now(timezone.utc)
+        days_elapsed = (now - ts).total_seconds() / 86400.0
+        return 2.0 ** (-days_elapsed / half_life_days)
+    except Exception:
+        return 1.0
+
+
+def keyword_relevance(bm25_score: float, max_abs_bm25: float, bm25_min_threshold: float) -> float:
+    abs_score = abs(bm25_score)
+    if abs_score < bm25_min_threshold:
+        return 0.0
+    return abs_score / max(1.0, max_abs_bm25)
+
+
+def hybrid_search(
+    db_path: str,
+    query: str,
+    config: dict,
+    current_session_id: Optional[str] = None,
+) -> dict:
+    half_life = config.get('half_life_days', 90)
+    bm25_min = config.get('bm25_min_threshold', 0.1)
+    limit = config.get('search_result_limit', 20)
+    min_score = config.get('min_score', 0.2)
+    max_query = config.get('search_query_max_chars', 2000)
+
+    query = query[:max_query]
+
+    conn = get_connection(db_path)
+    try:
+        # Vector search
+        vec_results = {}
+        if query.strip():
+            try:
+                q_vec = embed_query(query)
+                rows = search_vector(conn, q_vec, k=limit * 3)
+                for row in rows:
+                    rid = row['id']
+                    dist = row['distance']
+                    cs = cosine_sim_from_l2(dist)
+                    vec_results[rid] = {
+                        'id': rid,
+                        'content': row['content'],
+                        'timestamp': row['timestamp'],
+                        'session_id': row['session_id'],
+                        'turn_id': row['turn_id'],
+                        'agent_name': row['agent_name'],
+                        'cos_sim': cs,
+                        'bm25_score': None,
+                    }
+            except Exception as e:
+                logger.warning(f"Vector search error: {e}")
+
+        # FTS search
+        fts_results = {}
+        if query.strip():
+            try:
+                rows = search_fts(conn, query, limit=limit * 3)
+                max_abs = max((abs(r['bm25_score']) for r in rows), default=0.0)
+                for row in rows:
+                    rid = row['id']
+                    fts_results[rid] = {
+                        'id': rid,
+                        'content': row['content'],
+                        'timestamp': row['timestamp'],
+                        'session_id': row['session_id'],
+                        'turn_id': row['turn_id'],
+                        'agent_name': row['agent_name'],
+                        'bm25_score': row['bm25_score'],
+                        'max_abs_bm25': max_abs,
+                        'cos_sim': None,
+                    }
+            except Exception as e:
+                logger.warning(f"FTS search error: {e}")
+
+        # Combine
+        all_ids = set(vec_results.keys()) | set(fts_results.keys())
+        scored = []
+        fts_max_abs = 0.0
+        if fts_results:
+            fts_max_abs = max(abs(v['bm25_score']) for v in fts_results.values())
+
+        for rid in all_ids:
+            vec_r = vec_results.get(rid)
+            fts_r = fts_results.get(rid)
+
+            meta = vec_r or fts_r
+            cs = vec_r['cos_sim'] if vec_r else 0.0
+            bm25 = fts_r['bm25_score'] if fts_r else 0.0
+            kr = keyword_relevance(bm25, fts_max_abs, bm25_min)
+
+            ts = meta['timestamp']
+            decay = time_decay(ts, half_life)
+
+            sess = meta.get('session_id')
+            b_session = 1.0 if (current_session_id and sess == current_session_id) else 0.6
+
+            score = (cs * 0.7 + kr * 0.3) * decay * b_session
+
+            scored.append({
+                'id': rid,
+                'content': meta['content'],
+                'timestamp': ts,
+                'session_id': sess,
+                'turn_id': meta.get('turn_id'),
+                'agent_name': meta.get('agent_name'),
+                'score': score,
+                'cos_sim': cs,
+                'keyword_relevance': kr,
+            })
+
+        scored.sort(key=lambda x: x['score'], reverse=True)
+        filtered = [r for r in scored if r['score'] >= min_score]
+        results = filtered[:limit]
+
+        # Q-A pair collection
+        pairs = {}
+        seen_turn_ids = set()
+        for r in results:
+            tid = r.get('turn_id')
+            if tid and tid not in seen_turn_ids:
+                seen_turn_ids.add(tid)
+                siblings = get_memories_by_turn_id(conn, tid)
+                if siblings:
+                    pairs[tid] = [
+                        {
+                            'id': s['id'],
+                            'content': s['content'],
+                            'timestamp': s['timestamp'],
+                        }
+                        for s in siblings
+                    ]
+
+        return {'results': results, 'pairs': pairs}
+    finally:
+        conn.close()
+
+
+def render_memory_context(search_result: dict, query: str) -> str:
+    results = search_result.get('results', [])
+    pairs = search_result.get('pairs', {})
+
+    lines = [f"# Recalled Memory Context\n検索クエリ: {query}\n"]
+
+    if not results:
+        lines.append("_No relevant memories found._\n")
+    else:
+        lines.append("## Top matches (use these to answer the question)\n")
+        for i, r in enumerate(results, 1):
+            score = r['score']
+            content = r['content']
+            lines.append(f"### [{i}] score={score:.4f}")
+            lines.append(content)
+            lines.append("")
+
+    if pairs:
+        lines.append("\n## Previous matching Q-A exchanges (supplementary context)\n")
+        for tid, siblings in pairs.items():
+            lines.append(f"**Turn:** `{tid}`\n")
+            for s in siblings:
+                lines.append(s.get('content', '[content missing]'))
+                lines.append("")
+            lines.append("---")
+
+    return '\n'.join(lines)