chip-memory 1.1.0__py3-none-any.whl

chip_memory/api.py

@@ -0,0 +1,628 @@
+"""
+chip_memory.api — the one-line API for any agent.
+
+The point: any agent (yours, mine, a third-party) should be able to
+use chip-memory with three lines of code:
+
+    from chip_memory import api
+    api.remember("the user prefers terse responses")
+    ctx = api.ask("what do I know about polish ecommerce")
+    api.bump(ctx[0]['id'])  # mark a result as useful
+
+That's it. No path wrangling, no search import, no heat math, no
+embedding model awareness. The whole brain is hidden behind three verbs.
+
+Why this is a separate module from search/auto_write/etc:
+  - search.py is the CLI/search primitive (returns raw scored nodes)
+  - api.py is the **agent contract** — return values are pre-formatted
+    to be dropped into a system prompt, errors are caught and returned
+    as `{"error": "..."}` dicts, and the three verbs are stable
+    (their signatures won't change between versions, even if internals do).
+
+Optional env: CHIP_HOME (default ~/chip-memory/brain/), CHIP_AGENT
+(default 'default') — used to namespace feedback per-agent.
+"""
+from __future__ import annotations
+import json
+import logging
+import os
+import sys
+from datetime import datetime, timezone
+from typing import Any
+
+from . import _utils, paths, node as node_mod
+from . import search as search_mod
+
+log = logging.getLogger("chip-memory")
+
+
+# ─────────────────────────────────────────────────────────────────────
+# ask — query the brain, get system-prompt-friendly context back
+# ─────────────────────────────────────────────────────────────────────
+
+def ask(query: str, top_k: int = 5, domain: str | None = None,
+        as_brief: bool = True,
+        use_vector: bool = True, use_bm25: bool = True,
+        try_typed_edges: bool = True) -> list[dict[str, Any]]:
+    """Ask the brain about a topic. Returns a list of search result dicts.
+
+    NOTE: The return shape of ask() may vary — when try_typed_edges matches,
+    it returns a single-item list with a ``typed_edge`` marker dict instead
+    of standard search results. For a stable return shape use ``answer()``
+    instead, which always returns ``{"kind": ..., "results": [...]}``.
+
+    Each result: {"id", "content", "domain", "type", "score", "heat",
+                  "importance", "cosine_similarity", "bm25_score",
+                  "hybrid_score"}
+
+    If try_typed_edges=True (default), first attempts a structured typed-edge
+    query ("who works at acme"). If it matches a known pattern and has results,
+    returns those instead of BM25/vector search. If no edge pattern matches,
+    falls back to standard hybrid search.
+
+    If as_brief=True (default), the results are also sorted/filtered
+    to be useful in a system prompt — skip cold, sort by score, cap
+    at top_k. If as_brief=False, you get the raw ranked list.
+
+    Returns [] if the brain is empty or no results pass the threshold.
+    Never raises — failures return [] (logged via the brain's normal
+    log path; you can also catch exceptions by passing top_k=0).
+    """
+    if not query or not query.strip():
+        return []
+
+    # Try typed-edge query first (structured relations)
+    if try_typed_edges:
+        try:
+            from .query import triple_query as tq
+            edge_result = tq.try_query(query)
+            if edge_result:
+                # Return as a single result with the structured answer
+                return [{
+                    "id": "typed_edge_result",
+                    "content": edge_result,
+                    "domain": "structured",
+                    "type": "typed_edge_answer",
+                    "score": 1.0,
+                    "heat": 1.0,
+                    "importance": 1.0,
+                    "cosine_similarity": 0.0,
+                    "bm25_score": 0.0,
+                    "hybrid_score": 0.0,
+                    "typed_edge": True,
+                }]
+        except Exception:
+            log.warning("typed-edge query failed, falling through to standard search", exc_info=True)
+
+    try:
+        results = search_mod.search(
+            query, top_k=top_k, domain=domain,
+            skip_cold=True, min_score=0.02,
+            use_vector=use_vector, use_bm25=use_bm25,
+        )
+    except Exception:
+        log.exception("ask() search failed")
+        return []
+    if not results:
+        return []
+    # Bump access_count on returned nodes (this is the "hot" feedback).
+    # Locked so concurrent api.ask() calls don't race on access_count writes.
+    with node_mod.brain_lock():
+        now = node_mod.now_iso()
+        for r in results:
+            p = paths.nodes_dir() / f"{r['id']}.json"
+            if not p.exists():
+                continue
+            try:
+                n = json.loads(p.read_text(encoding="utf-8"))
+            except (json.JSONDecodeError, OSError):
+                continue
+            n["access_count"] = n.get("access_count", 0) + 1
+            n["last_accessed"] = now
+            node_mod.atomic_write_json(p, n)
+    # Co-access boost: every time the agent asks and gets back a cluster,
+    # the map learns "these go together." Best-effort — failure doesn't
+    # affect the return value.
+    if len(results) >= 2:
+        try:
+            from . import linker
+            from . import timeline
+            boosted = linker.boost_co_access([r["id"] for r in results])
+            if boosted > 0:
+                # Log the co-access for the timeline. We log a single
+                # "boosted cluster" event rather than N(N-1)/2 edge
+                # events — the timeline would be unreadable otherwise.
+                timeline.log_event(
+                    timeline.EV_LINK_BOOSTED,
+                    cluster=[r["id"] for r in results],
+                    boosts_applied=boosted,
+                    query=query[:80],
+                )
+        except Exception:
+            log.warning("co-access boost failed in api.ask()", exc_info=True)
+    return results
+
+
+def ask_text(query: str, top_k: int = 5) -> str:
+    """ask() but as a pre-formatted string for direct system-prompt use.
+
+    Returns "" if no results. Example output:
+
+        [betting/rule] Kelly criterion with safety: stake = (edge / odds) * 0.25
+        [dropshipping/event] VEVERLO hit 9k EUR/month after 6-7 months
+
+    Grouped by domain, with a 220-char preview per node.
+    """
+    results = ask(query, top_k=top_k, as_brief=True)
+    if not results:
+        return ""
+    by_domain: dict[str, list[dict[str, Any]]] = {}
+    for r in results:
+        by_domain.setdefault(r["domain"], []).append(r)
+    lines = [f"# Brain context for: {query!r}  ({len(results)} nodes)"]
+    for dom, items in by_domain.items():
+        lines.append(f"\n## {dom} ({len(items)})")
+        for r in items:
+            preview = _utils.truncate_preview(r["content"].replace("\n", " ").strip(), 220)
+            lines.append(f"- [{r['type']}] {preview}")
+    return "\n".join(lines)
+
+
+# ─────────────────────────────────────────────────────────────────────
+# answer — stable-shape query result
+# ─────────────────────────────────────────────────────────────────────
+
+
+def answer(query: str, top_k: int = 5, domain: str | None = None,
+           use_vector: bool = True, use_bm25: bool = True,
+           try_typed_edges: bool = True) -> dict[str, Any]:
+    """Answer a query with a stable return shape.
+
+    Unlike ``ask()``, which may return different shapes (a typed-edge result
+    dict, a list of search results, or an empty list), ``answer()`` always
+    returns a dict with a predictable ``kind`` key so callers never need to
+    branch on the presence of a ``typed_edge`` marker.
+
+    Returns:
+        A dict with keys:
+            - kind: ``"typed_edge"`` | ``"search"`` | ``"empty"``
+            - results: list of result dicts (contents vary by kind)
+
+    Example::
+
+        >>> api.answer("who works at acme")
+        {"kind": "typed_edge", "results": [{"content": "...", "type": "typed_edge_answer"}]}
+
+        >>> api.answer("python tips")
+        {"kind": "search", "results": [{"id": "...", "content": "...", ...}]}
+
+        >>> api.answer("zzznothing")
+        {"kind": "empty", "results": []}
+    """
+    if not query or not query.strip():
+        return {"kind": "empty", "results": []}
+
+    # Try typed-edge query first (structured relations)
+    if try_typed_edges:
+        try:
+            from .query import triple_query as tq
+            edge_result = tq.try_query(query)
+            if edge_result:
+                return {
+                    "kind": "typed_edge",
+                    "results": [{
+                        "content": edge_result,
+                        "type": "typed_edge_answer",
+                    }],
+                }
+        except Exception:
+            log.warning(
+                "typed-edge query failed in answer(), falling through",
+                exc_info=True,
+            )
+
+    try:
+        results = search_mod.search(
+            query, top_k=top_k, domain=domain,
+            skip_cold=True, min_score=0.02,
+            use_vector=use_vector, use_bm25=use_bm25,
+        )
+    except Exception:
+        log.exception("answer() search failed")
+        return {"kind": "empty", "results": []}
+
+    if not results:
+        return {"kind": "empty", "results": []}
+
+    # Bump access_count on returned nodes (same as ask()).
+    with node_mod.brain_lock():
+        now = node_mod.now_iso()
+        for r in results:
+            p = paths.nodes_dir() / f"{r['id']}.json"
+            if not p.exists():
+                continue
+            try:
+                n = json.loads(p.read_text(encoding="utf-8"))
+            except (json.JSONDecodeError, OSError):
+                continue
+            n["access_count"] = n.get("access_count", 0) + 1
+            n["last_accessed"] = now
+            node_mod.atomic_write_json(p, n)
+
+    # Co-access boost (same as ask()).
+    if len(results) >= 2:
+        try:
+            from . import linker
+            from . import timeline
+            boosted = linker.boost_co_access([r["id"] for r in results])
+            if boosted > 0:
+                timeline.log_event(
+                    timeline.EV_LINK_BOOSTED,
+                    cluster=[r["id"] for r in results],
+                    boosts_applied=boosted,
+                    query=query[:80],
+                )
+        except Exception:
+            log.warning("co-access boost failed in api.answer()", exc_info=True)
+
+    return {"kind": "search", "results": results}
+
+
+# ─────────────────────────────────────────────────────────────────────
+# remember — write a single node (or several)
+# ─────────────────────────────────────────────────────────────────────
+
+def remember(content: str, type: str = "fact", domain: str = "general",
+             tags: list[str] | None = None, importance: float = 0.5,
+             source: str | None = None,
+             ref: str | None = None,
+             ttl: str | None = None,
+             skip_redact: bool = False,
+             provenance: dict | None = None,
+             trust_confidence: float | None = None) -> dict[str, Any] | None:
+    """Write a single memory node. Returns the new node dict, or None on failure.
+
+    SECURITY: content is redacted by default before storage (Rail 5).
+    Pass skip_redact=True to bypass (only for internal use where content
+    was already redacted upstream — e.g., the absorb pipeline).
+
+    Idempotent: writing the same content twice returns the existing
+    node the second time (same id, same fields). This means agents
+    can re-call remember() with the same content safely — no dedup
+    bookkeeping needed on the caller side.
+
+    `source` defaults to $CHIP_AGENT (so different agents writing to
+    the same brain can be told apart in audit logs).
+
+    `ttl` is a duration string like "30d", "12h", "2w", "90m" — the
+    node auto-expires after that. Use for things that age: prices,
+    project status, contact info. None = never expires.
+    """
+    if not content or len(content.strip()) < 10:
+        return None
+    # Rail 5: redact secrets unless explicitly skipped (absorb pipeline
+    # already redacted upstream). This protects ALL write paths —
+    # MCP, API, CLI, HTTP server — with a single enforcement point.
+    if not skip_redact:
+        from .collect import redact as _redact
+        content = _redact.redact(content)
+    v = node_mod.validate({
+        "content": content, "type": type, "domain": domain,
+        "tags": tags or [], "importance": importance,
+    })
+    if v is None:
+        return None
+    source_str = source or os.environ.get("CHIP_AGENT", "api")
+    v["source"] = source_str
+    if provenance is not None:
+        v["provenance"] = provenance
+    else:
+        v["provenance"] = {"source": source_str, "ref": ref}
+    if trust_confidence is not None:
+        v["trust_confidence"] = trust_confidence
+    else:
+        v["trust_confidence"] = node_mod.compute_trust_confidence(source_str)
+    if ttl:
+        exp = node_mod.parse_ttl(ttl)
+        if exp:
+            v["expires_at"] = exp
+    created, nid = node_mod.write(v)
+    # Re-read so we return the full record (with id, timestamps, defaults).
+    # Note: the timeline hook for "node added" lives in node.write itself,
+    # not here — that way every entry point (api.remember, seed, daemon
+    # cycle) gets logged without us having to hook each one.
+    n = node_mod.read(nid)
+    return n
+
+
+def remember_many(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Bulk version of remember(). Each item is a kwargs-dict.
+
+    Returns only the successfully written nodes. Invalid items are
+    silently skipped (same policy as auto_write).
+    """
+    out = []
+    for item in items:
+        content = item.get("content", "")
+        if not content:
+            continue
+        n = remember(
+            content=content,
+            type=item.get("type", "fact"),
+            domain=item.get("domain", "general"),
+            tags=item.get("tags", []),
+            importance=item.get("importance", 0.5),
+            source=item.get("source"),
+            provenance=item.get("provenance"),
+            trust_confidence=item.get("trust_confidence"),
+        )
+        if n is not None:
+            out.append(n)
+    return out
+
+
+# ─────────────────────────────────────────────────────────────────────
+# bump — feedback on a node (useful / not useful)
+# ─────────────────────────────────────────────────────────────────────
+
+def bump(node_id: str, useful: bool = True, amount: float = 0.05) -> dict[str, Any] | None:
+    """Provide feedback on a node. Adjusts importance by ±`amount`.
+
+    Same semantics as POST /feedback on the server. Returns the updated
+    node, or None if not found.
+
+    Why: this is the *only* way to teach the brain what matters. If
+    a node gets bumped a lot, its heat stays high via access_count.
+    If it gets negative-bumped, its importance sinks and the daemon
+    will eventually archive it. Use this in your agent loop:
+
+        results = api.ask(...)
+        for r in results:
+            if user_liked_this(r):
+                api.bump(r['id'], useful=True)
+    """
+    n = node_mod.read(node_id)
+    if n is None:
+        return None
+    # Lock the read-modify-write so concurrent api.bump / daemon heat
+    # updates don't clobber each other's changes.
+    with node_mod.brain_lock():
+        # Re-read inside the lock to get the latest state
+        n = node_mod.read(node_id)
+        if n is None:
+            return None
+        if useful:
+            n["importance"] = min(1.0, n.get("importance", 0.5) + amount)
+        else:
+            n["importance"] = max(0.0, n.get("importance", 0.5) - (amount * 2))
+        n["last_accessed"] = node_mod.now_iso()
+        n["access_count"] = n.get("access_count", 0) + 1
+        node_mod.write_raw(paths.nodes_dir() / f"{node_id}.json", n)
+    return n
+
+
+# ─────────────────────────────────────────────────────────────────────
+# forget / archive — explicit cleanup
+# ─────────────────────────────────────────────────────────────────────
+
+def forget(node_id: str) -> bool:
+    """Archive a node (never delete). Returns True if it was archived.
+
+    Archived nodes are skipped by search and by health, but kept on
+    disk for audit/recovery. The `chip-memory search --include-archived`
+    flag (not yet implemented) would show them.
+    """
+    n = node_mod.read(node_id)
+    if n is None:
+        return False
+    # Lock so concurrent writes don't clobber each other.
+    with node_mod.brain_lock():
+        n = node_mod.read(node_id)
+        if n is None:
+            return False
+        n["archived"] = True
+        n["archived_at"] = node_mod.now_iso()
+        node_mod.write_raw(paths.nodes_dir() / f"{node_id}.json", n)
+    return True
+
+
+# ─────────────────────────────────────────────────────────────────────
+# stats — one-line brain health
+# ─────────────────────────────────────────────────────────────────────
+
+def stats() -> dict[str, Any]:
+    """Quick brain stats for agent introspection. Returns a dict.
+
+    Keys: nodes, domains, hot, avg_heat, health_score, top_domain,
+    cold_nodes (not accessed in 60+ days), orphan_nodes (zero map
+    connections), hubs (top-5 most-connected nodes, with degree).
+    """
+    nodes = node_mod.load_all()
+    if not nodes:
+        return {"nodes": 0, "domains": 0, "hot": 0, "avg_heat": 0.0,
+                "health_score": 0, "top_domain": None, "cold_nodes": 0,
+                "orphan_nodes": 0, "hubs": []}
+    from collections import Counter
+    domains = Counter(n.get("domain", "unknown") for n in nodes)
+    hot = sum(1 for n in nodes if n.get("heat", 0) >= 0.4)
+    avg_heat = sum(n.get("heat", 0) for n in nodes) / len(nodes)
+    # Lazy import: health not always needed (empty-brain fast path skips it)
+    from . import health as health_mod
+    h = health_mod.compute_health(nodes)
+    cold = len(health_mod.find_stale(nodes))
+    # Map stats (the connection network)
+    from . import linker
+    orphan = linker.orphan_count(nodes)
+    hubs = linker.hub_nodes(nodes, top_n=5)
+    return {
+        "nodes": len(nodes),
+        "domains": len(domains),
+        "hot": hot,
+        "avg_heat": round(avg_heat, 3),
+        "health_score": h["score"],
+        "top_domain": domains.most_common(1)[0][0] if domains else None,
+        "cold_nodes": cold,
+        "orphan_nodes": orphan,
+        "hubs": hubs,
+    }
+
+
+# ─────────────────────────────────────────────────────────────────────
+# why — provenance/usage report for a single node
+# ─────────────────────────────────────────────────────────────────────
+
+
+def why(node_id: str) -> str | None:
+    """Return a formatted provenance/usage report for a single node.
+
+    Displays:
+      - Node id, content preview, type, domain
+      - Importance, heat
+      - Provenance: source, trust_confidence, created_at
+      - Usage: access_count, last_accessed
+      - Links: connection count and top link strengths
+      - Heat history
+
+    Returns None if node not found.
+    """
+    n = node_mod.read(node_id)
+    if n is None:
+        return None
+
+    lines: list[str] = []
+    lines.append(f"Node: {n['id']}")
+    # Full content (the node's stored content)
+    content = n.get("content", "")
+    lines.append(f"Content: {content}")
+    lines.append(f"Type: {n.get('type', '?')} | Domain: {n.get('domain', '?')}")
+    imp = n.get("importance", 0.5)
+    heat = n.get("heat", 0.0)
+    lines.append(f"Importance: {imp:.2f} | Heat: {heat:.2f}")
+    lines.append("")
+
+    # Provenance section
+    prov = n.get("provenance", {})
+    src = prov.get("source", n.get("source", "?"))
+    tc = n.get("trust_confidence", 0.5)
+    lines.append("Provenance:")
+    lines.append(f"  Source: {src}")
+    lines.append(f"  Trust confidence: {tc:.2f}")
+    ref = prov.get("ref")
+    if ref:
+        lines.append(f"  Ref: {ref}")
+    created = n.get("created_at", "?")
+    lines.append(f"  Learned: {created}")
+    lines.append("")
+
+    # Usage section
+    ac = n.get("access_count", 0)
+    la = n.get("last_accessed", "?")
+    lines.append("Usage:")
+    lines.append(f"  Access count: {ac}")
+    lines.append(f"  Last accessed: {la}")
+    lines.append("")
+
+    # Links section
+    conns = n.get("connections") or []
+    strengths = n.get("link_strengths") or []
+    lines.append("Links:")
+    if conns:
+        lines.append(f"  Connected to: {len(conns)} other nodes")
+        if strengths and len(strengths) == len(conns):
+            paired = sorted(zip(conns, strengths), key=lambda x: -x[1])[:5]
+            strength_strs = [f"{cid[:12]} ({s:.2f})" for cid, s in paired]
+            lines.append(f"  Link strengths: {', '.join(strength_strs)}")
+        else:
+            # Show connections when link_strengths isn't available
+            lines.append(f"  Link strengths: {', '.join(c[:12] + ' (1.00)' for c in conns[:5])}")
+    else:
+        lines.append("  Connected to: 0 other nodes")
+    lines.append("")
+
+    # Heat history
+    lines.append("Heat history:")
+    heat_created = n.get("heat", 0.4)
+    lines.append(f"  Created at heat {heat_created:.2f}")
+    # No per-node heat bump tracking exists yet; note it.
+    lines.append("  (No heat bumps recorded)")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def think(query: str, brain_dir: str | None = None,
+          as_of: str | None = None) -> str:
+    """Answer a natural language query with cited evidence + knowledge gaps.
+
+    No LLM calls — pure heuristic answer engine. Uses hybrid search +
+    typed-edge expansion + linker connections, then groups results by
+    entity and reports gaps for unanswerable sub-asks.
+
+    Args:
+        query: Natural language question (e.g. "who works at acme and who founded it").
+        brain_dir: Optional brain directory override. Defaults to CHIP_BRAIN_DIR.
+        as_of: Temporal filter for typed-edge queries. None = current truth,
+               ISO string = valid at that time. Auto-extracted from query
+               if a temporal qualifier is found.
+
+    Returns:
+        Formatted string with headings, cited evidence, and a "Knowledge gaps"
+        section listing sub-asks with no data.
+    """
+    try:
+        from .synthesis.think import think as _think
+        return _think(query, brain_dir=brain_dir, as_of=as_of)
+    except Exception:
+        log.exception("think() failed")
+        return f"## Query: {query}\n\nError: think mode encountered an internal error.\n"
+
+
+def main():
+    """CLI wrapper for the agent contract. Used by `chip-memory remember`.
+
+    Usage:
+        chip-memory remember "the user prefers terse responses" [--domain general] [--importance 0.8]
+    """
+    import argparse
+    ap = argparse.ArgumentParser(description="Remember a fact in the chip-memory brain")
+    ap.add_argument("content", help="The fact to remember")
+    ap.add_argument("--domain", default="general", help="Domain (default: general)")
+    ap.add_argument("--type", dest="node_type", default="fact", help="Node type (default: fact)")
+    ap.add_argument("--importance", type=float, default=0.5, help="Importance 0.0-1.0 (default: 0.5)")
+    ap.add_argument("--tags", default="", help="Comma-separated tags")
+    ap.add_argument("--ttl", default=None, help="Time-to-live, e.g. '30d', '12h', '2w' (default: never)")
+    ap.add_argument("--json", action="store_true", help="Output as JSON")
+    ap.add_argument("--provenance", default=None,
+                    help='Provenance as JSON, e.g. \'{"source":"git_fix","ref":"abc123"}\'')
+    ap.add_argument("--trust-confidence", type=float, default=None,
+                    help="Trust confidence 0.0-1.0 (default: computed from source)")
+    args = ap.parse_args()
+
+    tags = [t.strip() for t in args.tags.split(",") if t.strip()]
+    provenance = args.provenance
+    if provenance:
+        import json as _json
+        provenance = _json.loads(provenance)
+    result = remember(
+        args.content,
+        type=args.node_type,
+        domain=args.domain,
+        importance=args.importance,
+        tags=tags,
+        ttl=args.ttl,
+        provenance=provenance,
+        trust_confidence=args.trust_confidence,
+    )
+    if result is None:
+        print("ERROR: validation failed (content too short/long, bad domain/type, or bad TTL)", file=sys.stderr)
+        sys.exit(1)
+    if args.json:
+        import json as _json
+        print(_json.dumps(result, indent=2), flush=True)
+    else:
+        print(f"remembered: {result['id']}  domain={result['domain']}  importance={result['importance']}", flush=True)
+
+
+if __name__ == "__main__":
+    main()
+