contexara 0.2.1__py3-none-any.whl

contexara/__init__.py

@@ -0,0 +1,11 @@
+"""Contexara public package API."""
+
+from .enhance import enhance
+from .ingest import ingest_turn
+from .main import ask
+from .retrieve import retrieve
+from .store import clear_all, compress_old_memories, consolidate, delete_memory, get_stats, list_all, search_memories, store
+
+__all__ = ["ask", "store", "list_all", "clear_all", "delete_memory", "search_memories", "get_stats", "consolidate", "compress_old_memories", "retrieve", "enhance", "ingest_turn"]
+
+__version__ = "0.2.1"

contexara/__main__.py

@@ -0,0 +1,7 @@
+"""Module entrypoint for `python -m contexara`."""
+
+from .cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

contexara/cli.py

@@ -0,0 +1,236 @@
+"""Command-line interface for Contexara."""
+
+from __future__ import annotations
+
+import argparse
+from typing import Iterable
+
+from . import __version__
+from .main import ask
+from .store import clear_all, consolidate, delete_memory, get_stats, list_all, search_memories, store
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="contexara",
+        description="CLI-first memory engine for LLM context management.",
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+
+    subparsers = parser.add_subparsers(dest="command")
+
+    subparsers.add_parser("chat", help="Start interactive chat mode.").set_defaults(func=_cmd_chat)
+
+    ask_parser = subparsers.add_parser("ask", help="Ask a memory-aware question.")
+    ask_parser.add_argument("query", nargs="+")
+    ask_parser.set_defaults(func=_cmd_ask)
+
+    store_parser = subparsers.add_parser("store", help="Store a memory string.")
+    store_parser.add_argument("content", nargs="+")
+    store_parser.set_defaults(func=_cmd_store)
+
+    subparsers.add_parser("list", help="List all stored memories.").set_defaults(func=_cmd_list)
+
+    delete_parser = subparsers.add_parser("delete", help="Delete a memory by ID.")
+    delete_parser.add_argument("id", type=int, help="Memory ID (from contexara list).")
+    delete_parser.set_defaults(func=_cmd_delete)
+
+    search_parser = subparsers.add_parser("search", help="Semantic search over memories.")
+    search_parser.add_argument("query", nargs="+")
+    search_parser.add_argument("--top", type=int, default=5, help="Number of results (default 5).")
+    search_parser.set_defaults(func=_cmd_search)
+
+    subparsers.add_parser("stats", help="Show memory statistics.").set_defaults(func=_cmd_stats)
+
+    subparsers.add_parser("consolidate", help="LLM-merge overlapping memories.").set_defaults(func=_cmd_consolidate)
+
+    clear_parser = subparsers.add_parser("clear", help="Delete all stored memories.")
+    clear_parser.add_argument("--yes", action="store_true", help="Skip confirmation.")
+    clear_parser.set_defaults(func=_cmd_clear)
+
+    return parser
+
+
+def _join_parts(parts: Iterable[str]) -> str:
+    return " ".join(parts).strip()
+
+
+def _print_memories_table(memories: list[dict]) -> None:
+    if not memories:
+        print("(no memories)")
+        return
+
+    id_w = max(len("ID"), max(len(str(m["id"])) for m in memories))
+    kind_w = max(len("KIND"), max(len(str(m.get("kind", ""))) for m in memories))
+    imp_w = max(len("IMP"), max(len(str(m.get("importance", ""))) for m in memories))
+    src_w = max(len("SRC"), max(len(str(m.get("source", "raw"))) for m in memories))
+    content_w = 60
+
+    header = f"{'ID':<{id_w}}  {'KIND':<{kind_w}}  {'IMP':<{imp_w}}  {'SRC':<{src_w}}  {'CONTENT':<{content_w}}"
+    print(header)
+    print("-" * len(header))
+    for m in memories:
+        content = m["content"]
+        if len(content) > content_w:
+            content = content[: content_w - 3] + "..."
+        print(f"{str(m['id']):<{id_w}}  {str(m.get('kind','')):<{kind_w}}  {str(m.get('importance','')):<{imp_w}}  {str(m.get('source','raw')):<{src_w}}  {content}")
+
+
+def _cmd_chat(_args: argparse.Namespace) -> int:
+    print("\nContexara Chat (type 'exit' to quit)")
+    print("Commands: /store <text>, /list, /show <id>, /delete <id>, /search <query>, /stats, /consolidate, /clear\n")
+
+    while True:
+        user_input = input("You: ").strip()
+        if not user_input:
+            continue
+
+        if user_input.lower() == "exit":
+            print("Goodbye.")
+            return 0
+
+        if user_input.startswith("/store "):
+            store(user_input[7:].strip())
+            print("Memory stored.")
+            continue
+
+        if user_input == "/list":
+            _print_memories_table(list_all())
+            continue
+
+        if user_input.startswith("/delete "):
+            try:
+                mid = int(user_input[8:].strip())
+                if delete_memory(mid):
+                    print(f"Memory {mid} deleted.")
+                else:
+                    print(f"No memory with ID {mid}.")
+            except ValueError:
+                print("Usage: /delete <id>")
+            continue
+
+        if user_input.startswith("/search "):
+            query = user_input[8:].strip()
+            results = search_memories(query)
+            _print_memories_table(results)
+            continue
+
+        if user_input.startswith("/show "):
+            try:
+                mid = int(user_input[6:].strip())
+                match = next((m for m in list_all() if m["id"] == mid), None)
+                if match:
+                    print(f"\nID        : {match['id']}")
+                    print(f"Kind      : {match['kind']}")
+                    print(f"Importance: {match['importance']}")
+                    print(f"Source    : {match.get('source', 'raw')}")
+                    print(f"Level     : {match.get('compression_level', 0)}")
+                    print(f"Created   : {match['created_at']}")
+                    print(f"Content   : {match['content']}\n")
+                else:
+                    print(f"No memory with ID {mid}.")
+            except ValueError:
+                print("Usage: /show <id>")
+            continue
+
+        if user_input == "/consolidate":
+            print("Consolidating memories...")
+            result = consolidate()
+            print(f"Done. {result['before']} memories → {result['after']} memories.")
+            continue
+
+        if user_input == "/stats":
+            _print_stats(get_stats())
+            continue
+
+        if user_input == "/clear":
+            clear_all()
+            print("All memories cleared.")
+            continue
+
+        response = ask(user_input)
+        print(f"\nBot: {response}\n")
+
+    return 0
+
+
+def _print_stats(stats: dict) -> None:
+    print(f"\nTotal memories : {stats['total']}")
+    print(f"Oldest         : {stats['oldest'] or 'N/A'}")
+    print(f"Newest         : {stats['newest'] or 'N/A'}")
+    print("\nBy kind:")
+    for kind, count in stats["by_kind"].items():
+        print(f"  {kind:<12} {count}")
+    print("\nBy source:")
+    for source, count in stats.get("by_source", {}).items():
+        ss = stats.get("source_stats", {}).get(source, {})
+        avg_imp = ss.get("avg_importance", "")
+        avg_age = ss.get("avg_age_days", "")
+        print(f"  {source:<12} {count:<6} avg_importance={avg_imp}  avg_age={avg_age}d")
+    print()
+
+
+def _cmd_ask(args: argparse.Namespace) -> int:
+    print(ask(_join_parts(args.query)))
+    return 0
+
+
+def _cmd_store(args: argparse.Namespace) -> int:
+    store(_join_parts(args.content))
+    print("Memory stored.")
+    return 0
+
+
+def _cmd_list(_args: argparse.Namespace) -> int:
+    _print_memories_table(list_all())
+    return 0
+
+
+def _cmd_delete(args: argparse.Namespace) -> int:
+    if delete_memory(args.id):
+        print(f"Memory {args.id} deleted.")
+    else:
+        print(f"No memory with ID {args.id}.")
+    return 0
+
+
+def _cmd_search(args: argparse.Namespace) -> int:
+    results = search_memories(_join_parts(args.query), top_k=args.top)
+    _print_memories_table(results)
+    return 0
+
+
+def _cmd_stats(_args: argparse.Namespace) -> int:
+    _print_stats(get_stats())
+    return 0
+
+
+def _cmd_consolidate(_args: argparse.Namespace) -> int:
+    print("Consolidating memories...")
+    result = consolidate()
+    print(f"Done. {result['before']} memories → {result['after']} memories.")
+    return 0
+
+
+def _cmd_clear(args: argparse.Namespace) -> int:
+    if not args.yes:
+        confirm = input("Delete all memories? [y/N]: ").strip().lower()
+        if confirm not in {"y", "yes"}:
+            print("Cancelled.")
+            return 0
+    clear_all()
+    print("All memories cleared.")
+    return 0
+
+
+def main() -> int:
+    parser = _build_parser()
+    args = parser.parse_args()
+    if not args.command:
+        parser.print_help()
+        return 0
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

contexara/embedder.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import io
+from functools import lru_cache
+
+import numpy as np
+
+_MODEL_NAME = "all-MiniLM-L6-v2"
+
+
+@lru_cache(maxsize=1)
+def _get_model():
+    import logging
+    from sentence_transformers import SentenceTransformer
+    logging.getLogger("transformers.modeling_utils").setLevel(logging.ERROR)
+    return SentenceTransformer(_MODEL_NAME)
+
+
+def embed(text: str) -> np.ndarray:
+    """Return a normalized float32 embedding vector for text."""
+    model = _get_model()
+    vector = model.encode(text, normalize_embeddings=True)
+    return vector.astype(np.float32)
+
+
+def serialize(vector: np.ndarray) -> bytes:
+    buf = io.BytesIO()
+    np.save(buf, vector)
+    return buf.getvalue()
+
+
+def deserialize(blob: bytes) -> np.ndarray:
+    buf = io.BytesIO(blob)
+    return np.load(buf)
+
+
+def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
+    """Cosine similarity for pre-normalized vectors (dot product suffices)."""
+    return float(np.dot(a, b))

contexara/enhance.py

@@ -0,0 +1,30 @@
+def enhance(query: str, memories: list[str]) -> str:
+    """
+    Build an enhanced prompt by injecting relevant memories.
+    """
+
+    if not query or not query.strip():
+        raise ValueError("Query cannot be empty")
+
+    query = query.strip()
+
+    # Format memory block
+    if memories:
+        memory_lines = "\n".join(f"- {m}" for m in memories)
+
+        context_block = f"""You are an AI assistant with memory.
+
+Relevant context:
+{memory_lines}
+
+"""
+    else:
+        context_block = "You are an AI assistant.\n\n"
+
+    # Final prompt
+    prompt = f"""{context_block}User:
+{query}
+
+Assistant:"""
+
+    return prompt

contexara/extract.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+MEMORY_IMPORTANCE = {
+    "profile": 5,
+    "preference": 4,
+    "constraint": 4,
+    "task": 3,
+}
+
+VALID_KINDS = set(MEMORY_IMPORTANCE.keys()) | {"note"}
+
+LOW_SIGNAL_PREFIXES = (
+    "hi",
+    "hello",
+    "thanks",
+    "thank you",
+    "ok",
+    "okay",
+    "cool",
+    "great",
+)
+
+_EXTRACTION_SYSTEM_PROMPT = """\
+You are a memory extraction engine. Output ONLY a raw JSON array — no markdown, no prose, no explanation.
+
+Extract the most important, distinct, long-term facts about the user from the conversation turn.
+Be selective — prefer 1-4 high-quality memories over many overlapping ones.
+
+Rules:
+- Include: name, role, things built/completed/tested/shipped, stated preferences, hard constraints, current tasks.
+- Skip: vague plans, filler, questions, anything already covered by the existing memories listed below.
+- Do NOT paraphrase an existing memory — if the fact is already stored, skip it entirely.
+- Third-person statements only (e.g. "User built hybrid search with RRF").
+- kind: profile | preference | task | constraint | note
+- importance: integer 1-5 (profile=5, preference/constraint=4, task=3, note=1)
+
+Output format (strictly):
+[{"content": "...", "kind": "...", "importance": N}, ...]
+
+If nothing new is worth remembering output exactly: []
+"""
+
+
+def _call_llm_for_extraction(
+    user_text: str,
+    assistant_text: str,
+    existing_memories: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    """Call Bedrock to extract memories from a conversation turn. Returns [] on any failure."""
+    try:
+        from contexara.llm import BedrockConfig, _get_client
+
+        import json as _json
+
+        existing_block = ""
+        if existing_memories:
+            items = "\n".join(f"- {m}" for m in existing_memories[:20])
+            existing_block = f"\n\nAlready stored memories (do NOT re-extract these):\n{items}\n"
+
+        turn = f"{existing_block}\nConversation turn:\nUser: {user_text.strip()}\nAssistant: {assistant_text.strip()}"
+        payload = {
+            "anthropic_version": "bedrock-2023-05-31",
+            "system": _EXTRACTION_SYSTEM_PROMPT,
+            "messages": [{"role": "user", "content": [{"type": "text", "text": turn}]}],
+            "max_tokens": 512,
+            "temperature": 0.0,
+        }
+
+        config = BedrockConfig.from_env()
+        client = _get_client(config.region)
+        response = client.invoke_model(
+            modelId=config.model_id,
+            body=_json.dumps(payload),
+            contentType="application/json",
+            accept="application/json",
+        )
+        result = _json.loads(response["body"].read())
+        raw_text = result["content"][0]["text"].strip()
+
+        # Strip markdown code fences if model wraps output in ```json ... ```
+        raw_text = re.sub(r"^```(?:json)?\s*", "", raw_text)
+        raw_text = re.sub(r"\s*```$", "", raw_text).strip()
+
+        candidates = json.loads(raw_text)
+        if not isinstance(candidates, list):
+            return []
+
+        memories = []
+        for item in candidates:
+            if not isinstance(item, dict):
+                continue
+            content = str(item.get("content", "")).strip()
+            kind = str(item.get("kind", "note")).strip().lower()
+            importance = item.get("importance", 1)
+
+            if not content:
+                continue
+            if kind not in VALID_KINDS:
+                kind = "note"
+            try:
+                importance = max(1, min(5, int(importance)))
+            except (TypeError, ValueError):
+                importance = MEMORY_IMPORTANCE.get(kind, 1)
+
+            memories.append({"content": content, "kind": kind, "importance": importance, "ttl_days": None})
+
+        return memories
+
+    except Exception:
+        return []
+
+
+# ---------------------------------------------------------------------------
+# Regex fallback (V1 logic — used when LLM extraction fails)
+# ---------------------------------------------------------------------------
+
+def _clean_text(text: str) -> str:
+    return " ".join(text.strip().split())
+
+
+def _split_into_segments(text: str) -> list[str]:
+    if not text:
+        return []
+    normalized = _clean_text(text)
+    punctuation_parts = re.split(r"[.;]\s+|\n+", normalized)
+    segments: list[str] = []
+    for part in punctuation_parts:
+        chunk = part.strip()
+        if not chunk:
+            continue
+        chunk = re.sub(
+            r"\s+and\s+(?=(i\s+(?:am|m|will)|my\s+|first\s+task|tomorrow\b))",
+            " ||| ",
+            chunk,
+            flags=re.IGNORECASE,
+        )
+        sub_parts = [p.strip() for p in chunk.split("|||") if p.strip()]
+        segments.extend(sub_parts)
+    return segments
+
+
+def _extract_profile(text: str) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+    name_match = re.search(r"\bmy name is\s+([A-Za-z][A-Za-z'\-\s]{1,50})", text, re.IGNORECASE)
+    if name_match:
+        name = _clean_text(name_match.group(1)).rstrip(".,!?")
+        items.append({"content": f"User name is {name}", "kind": "profile", "importance": 5, "ttl_days": None})
+    role_match = re.search(r"\bi am\s+(a|an)\s+([A-Za-z][A-Za-z'\-\s]{1,80})", text, re.IGNORECASE)
+    if role_match:
+        role = _clean_text(role_match.group(2)).rstrip(".,!?")
+        if len(role.split()) <= 8:
+            items.append({"content": f"User role is {role}", "kind": "profile", "importance": 5, "ttl_days": None})
+    return items
+
+
+def _extract_preferences(text: str) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+    patterns = [
+        r"\bi prefer\s+(.+?)(?=$|\bbut\b|\bhowever\b)",
+        r"\bi like\s+(.+?)(?=$|\bbut\b|\bhowever\b)",
+        r"\bi love\s+(.+?)(?=$|\bbut\b|\bhowever\b)",
+        r"\bi dislike\s+(.+?)(?=$|\bbut\b|\bhowever\b)",
+        r"\bi hate\s+(.+?)(?=$|\bbut\b|\bhowever\b)",
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, text, re.IGNORECASE)
+        if not match:
+            continue
+        detail = _clean_text(match.group(1)).rstrip(".,!?")
+        if not detail or detail.lower().startswith(LOW_SIGNAL_PREFIXES):
+            continue
+        items.append({"content": f"User preference: {detail}", "kind": "preference", "importance": 4, "ttl_days": None})
+    return items
+
+
+def _extract_tasks(text: str) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+    task_patterns = [
+        r"\bi am building\s+(.+)",
+        r"\bi'?m building\s+(.+)",
+        r"\bi am working on\s+(.+)",
+        r"\bi'?m working on\s+(.+)",
+        r"\bi am creating\s+(.+)",
+        r"\bi'?m creating\s+(.+)",
+        r"\bi (?:just\s+)?built\s+(.+)",
+        r"\bi (?:just\s+)?finished\s+(.+)",
+        r"\bi (?:just\s+)?completed\s+(.+)",
+        r"\bi (?:just\s+)?shipped\s+(.+)",
+        r"\bi(?:'?m|'m| am) (?:now\s+)?testing\s+(.+)",
+        r"\bi will (?:start\s+)?(?:work on|build|implement|create)\s+(.+)",
+        r"\btomorrow(?:\s+i)?\s+will\s+(?:start\s+)?(?:work on|build|implement|create)\s+(.+)",
+        r"\bfirst task(?:\s+tomorrow)?\s+will\s+be\s+(?:implementing|building|creating|working on)\s+(.+)",
+    ]
+    for pattern in task_patterns:
+        match = re.search(pattern, text, re.IGNORECASE)
+        if not match:
+            continue
+        task = _clean_text(match.group(1)).rstrip(".,!?")
+        if task:
+            items.append({"content": f"User is working on {task}", "kind": "task", "importance": 3, "ttl_days": 30})
+    return items
+
+
+def _extract_constraints(text: str) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+    if re.search(r"\b(i need|must|should|do not|don't|cannot|can't)\b", text, re.IGNORECASE):
+        cleaned = _clean_text(text).rstrip(".,!?")
+        if cleaned and not cleaned.endswith("?"):
+            items.append({"content": f"User constraint: {cleaned}", "kind": "constraint", "importance": 4, "ttl_days": 14})
+    return items
+
+
+def _regex_extract(user_text: str) -> list[dict[str, Any]]:
+    extracted: list[dict[str, Any]] = []
+    for segment in _split_into_segments(user_text):
+        cleaned = _clean_text(segment)
+        if not cleaned or cleaned.endswith("?"):
+            continue
+        extracted.extend(_extract_profile(cleaned))
+        extracted.extend(_extract_preferences(cleaned))
+        extracted.extend(_extract_tasks(cleaned))
+        extracted.extend(_extract_constraints(cleaned))
+    return extracted
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def extract_memories(
+    user_text: str,
+    assistant_text: str = "",
+    existing_memories: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    """
+    Extract high-signal memories from a conversation turn.
+
+    Strategy: LLM extraction (Bedrock) analysing both user and assistant text,
+    with existing memories passed as context so the LLM skips already-known facts.
+    Falls back to deterministic regex on user text if LLM call fails.
+    """
+    llm_results = _call_llm_for_extraction(user_text or "", assistant_text or "", existing_memories)
+
+    if llm_results:
+        candidates = llm_results
+    else:
+        candidates = _regex_extract(user_text or "")
+
+    # Deduplicate within this extraction call
+    deduped: list[dict[str, Any]] = []
+    seen: set[tuple[str, str]] = set()
+    for item in candidates:
+        key = (item["kind"], item["content"].lower())
+        if key not in seen:
+            seen.add(key)
+            deduped.append(item)
+
+    return deduped

contexara/ingest.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from .extract import extract_memories
+from .store import list_all, upsert_memory
+
+
+def ingest_turn(user_text: str, assistant_text: str) -> dict[str, int]:
+    """
+    Ingest one conversation turn and persist extracted high-signal memories.
+
+    Passes existing memory contents to the extractor so the LLM skips
+    facts that are already stored (context-aware extraction).
+
+    Returns ingestion stats for observability.
+    """
+    stats = {
+        "extracted": 0,
+        "inserted": 0,
+        "updated": 0,
+        "skipped": 0,
+    }
+
+    existing = [m["content"] for m in list_all()]
+    candidates = extract_memories(
+        user_text=user_text,
+        assistant_text=assistant_text,
+        existing_memories=existing,
+    )
+    stats["extracted"] = len(candidates)
+
+    for memory in candidates:
+        try:
+            action = upsert_memory(
+                content=memory["content"],
+                kind=memory["kind"],
+                importance=memory["importance"],
+                ttl_days=memory.get("ttl_days"),
+            )
+            stats[action] += 1
+        except Exception:
+            stats["skipped"] += 1
+
+    return stats