engraphis 0.1.0__py3-none-any.whl

engraphis/__init__.py

@@ -0,0 +1,8 @@
+"""Engraphis — self-hosted AI memory system."""
+
+from importlib.metadata import PackageNotFoundError, version as _dist_version
+
+try:
+    __version__ = _dist_version("engraphis")
+except PackageNotFoundError:  # source tree without an installed distribution
+    __version__ = "0.1.0"

engraphis/analytics.py

@@ -0,0 +1,243 @@
+"""Workspace analytics — the Pro dashboard's data layer.
+
+House style (AGENTS.md §3): the math is a pure, tested function over plain rows
+(:func:`analytics_from_rows`); SQL lives only in the thin :func:`compute_analytics`
+wrapper. stdlib + core only — no new dependency, no LLM, no network.
+
+Gating note: the *license check does not live here*. This module computes for whoever
+calls it; the Inspector's HTTP layer enforces ``require_feature("analytics")``. Keeping
+the gate at the edge means the core stays honestly open (Apache-2.0) and the paid
+surface is exactly one, auditable place.
+"""
+from __future__ import annotations
+
+import html as _html
+import math
+import time
+from typing import Any, Iterable, Optional
+
+from engraphis.core.scoring import retention
+
+#: Retention level treated as "effectively forgotten" — matches the consolidation
+#: sweep's ``archive_below`` default so the forecast predicts what the next sweep does.
+FORGET_THRESHOLD = 0.05
+
+_WEEK = 7 * 86400.0
+_GROWTH_WEEKS = 12
+_HIST_BUCKETS = 5
+
+
+def _days_until_forgotten(stability: float, last_access: Optional[float],
+                          now: float) -> float:
+    """Days until Ebbinghaus retention exp(-Δt/S) crosses :data:`FORGET_THRESHOLD`.
+
+    Solving exp(-dt/S) = T gives dt = S·ln(1/T); subtract days already elapsed.
+    Returns +inf for pinned-style stability outliers only via natural math (S huge).
+    """
+    s = max(stability or 1.0, 1e-3)
+    horizon_days = s * math.log(1.0 / FORGET_THRESHOLD)
+    elapsed_days = max((now - (last_access if last_access is not None else now)) / 86400.0, 0.0)
+    return horizon_days - elapsed_days
+
+
+def analytics_from_rows(mem_rows: Iterable[dict], audit_counts: dict,
+                        entity_rows: Iterable[dict], *, now: Optional[float] = None) -> dict:
+    """Pure aggregation. ``mem_rows`` need: mtype, stability, last_access, ingested_at,
+    importance, pinned, valid_to, expired_at. ``audit_counts`` maps action→count.
+    ``entity_rows`` need: name, etype, n (mention/edge count)."""
+    now = time.time() if now is None else now
+    rows = list(mem_rows)
+    live = [r for r in rows
+            if r.get("expired_at") is None
+            and (r.get("valid_to") is None or now < r["valid_to"])]
+
+    # ── growth: weekly ingest counts, oldest→newest, exactly _GROWTH_WEEKS buckets ──
+    growth = [0] * _GROWTH_WEEKS
+    for r in rows:
+        ts = r.get("ingested_at")
+        if ts is None:
+            continue
+        idx = _GROWTH_WEEKS - 1 - int((now - ts) // _WEEK)
+        if 0 <= idx < _GROWTH_WEEKS:
+            growth[idx] += 1
+
+    # ── retention histogram over live memories ──
+    hist = [0] * _HIST_BUCKETS
+    ret_sum = 0.0
+    for r in live:
+        ret = retention(r.get("stability") or 1.0, r.get("last_access"), now)
+        ret_sum += ret
+        hist[min(int(ret * _HIST_BUCKETS), _HIST_BUCKETS - 1)] += 1
+
+    # ── decay forecast (pinned memories are exempt from decay archival) ──
+    at_risk_7 = at_risk_30 = 0
+    for r in live:
+        if r.get("pinned"):
+            continue
+        days = _days_until_forgotten(r.get("stability") or 1.0, r.get("last_access"), now)
+        if days <= 7:
+            at_risk_7 += 1
+        if days <= 30:
+            at_risk_30 += 1
+
+    by_type: dict = {}
+    for r in live:
+        by_type[r.get("mtype") or "?"] = by_type.get(r.get("mtype") or "?", 0) + 1
+
+    total = len(rows)
+    n_live = len(live)
+    return {
+        "generated_at": now,
+        "totals": {
+            "live": n_live,
+            "all_rows": total,
+            "superseded": total - n_live,
+            "pinned": sum(1 for r in live if r.get("pinned")),
+            "avg_retention": round(ret_sum / n_live, 4) if n_live else 0.0,
+        },
+        "growth_weekly": growth,
+        "retention_histogram": {
+            "buckets": ["0–20%", "20–40%", "40–60%", "60–80%", "80–100%"],
+            "counts": hist,
+        },
+        "decay_forecast": {
+            "threshold": FORGET_THRESHOLD,
+            "at_risk_7d": at_risk_7,
+            "at_risk_30d": at_risk_30,
+        },
+        "by_type": by_type,
+        "resolver_mix": {k: int(v) for k, v in sorted(audit_counts.items())},
+        "top_entities": [
+            {"name": e["name"], "etype": e.get("etype") or "", "n": int(e.get("n") or 0)}
+            for e in entity_rows
+        ],
+    }
+
+
+_REPORT_CSS = """
+body{font:14px/1.5 system-ui,-apple-system,'Segoe UI',Roboto,sans-serif;color:#1f2328;
+  background:#fff;margin:0;padding:32px;max-width:880px}
+h1{font-size:22px;margin:0 0 2px}
+h2{font-size:15px;margin:26px 0 8px;border-bottom:1px solid #d0d7de;padding-bottom:4px}
+.sub{color:#59636e;font-size:13px;margin:0 0 18px}
+table{border-collapse:collapse;width:100%;font-size:13px;margin:6px 0}
+th,td{text-align:left;padding:6px 10px;border-bottom:1px solid #e4e8ec;vertical-align:top}
+th{color:#59636e;font-weight:600;background:#f6f8fa}
+td.num{text-align:right;font-variant-numeric:tabular-nums}
+.bar{background:#dbe6f4;height:12px;border-radius:3px;display:inline-block;
+  vertical-align:middle;min-width:2px}
+.footer{color:#59636e;font-size:12px;margin-top:28px;border-top:1px solid #d0d7de;
+  padding-top:8px}
+""".strip()
+
+
+def _esc(value: Any) -> str:
+    return _html.escape(str(value), quote=True)
+
+
+def _table(headers: list, rows: list) -> str:
+    """rows are lists of (text, is_numeric) cells, already escaped by the caller."""
+    head = "".join("<th>%s</th>" % h for h in headers)
+    body = "".join(
+        "<tr>%s</tr>" % "".join(
+            '<td class="num">%s</td>' % c[0] if c[1] else "<td>%s</td>" % c[0]
+            for c in row)
+        for row in rows)
+    return "<table><thead><tr>%s</tr></thead><tbody>%s</tbody></table>" % (head, body)
+
+
+def _bar_cell(n: int, peak: int) -> str:
+    width = int(round(n / peak * 160)) if peak else 0
+    return '<span class="bar" style="width:%dpx"></span> %d' % (width, n)
+
+
+def render_analytics_html(data: dict, *, workspace: str, version: str = "") -> str:
+    """A self-contained HTML report over the :func:`analytics_from_rows` payload.
+
+    Everything inline (CSS included), zero external requests — the file can be
+    archived, emailed, or opened offline years later and still render. All dynamic
+    text is escaped; nothing from the store reaches the page unescaped."""
+    t = data.get("totals", {})
+    f = data.get("decay_forecast", {})
+    generated = time.strftime("%Y-%m-%d %H:%M:%S UTC",
+                              time.gmtime(data.get("generated_at", time.time())))
+
+    totals_rows = [
+        [("Live memories", False), (_esc(t.get("live", 0)), True)],
+        [("All rows (incl. superseded history)", False), (_esc(t.get("all_rows", 0)), True)],
+        [("Superseded (kept in history)", False), (_esc(t.get("superseded", 0)), True)],
+        [("Pinned (decay-exempt)", False), (_esc(t.get("pinned", 0)), True)],
+        [("Average retention", False),
+         ("%.0f%%" % (float(t.get("avg_retention", 0.0)) * 100), True)],
+        [("Fading within 7 days", False), (_esc(f.get("at_risk_7d", 0)), True)],
+        [("Fading within 30 days", False), (_esc(f.get("at_risk_30d", 0)), True)],
+    ]
+
+    weeks = list(data.get("growth_weekly", []))
+    peak = max(weeks) if weeks else 0
+    growth_rows = [
+        [("now" if back == 0 else "%d week%s ago" % (back, "" if back == 1 else "s"), False),
+         (_bar_cell(n, peak), True)]
+        for back, n in ((len(weeks) - 1 - i, n) for i, n in enumerate(weeks))]
+
+    hist = data.get("retention_histogram", {})
+    counts = list(hist.get("counts", []))
+    hpeak = max(counts) if counts else 0
+    hist_rows = [[(_esc(b), False), (_bar_cell(n, hpeak), True)]
+                 for b, n in zip(hist.get("buckets", []), counts)]
+
+    type_rows = [[(_esc(k), False), (_esc(v), True)]
+                 for k, v in sorted(data.get("by_type", {}).items())]
+    mix_rows = [[(_esc(k), False), (_esc(v), True)]
+                for k, v in sorted(data.get("resolver_mix", {}).items())]
+    entity_rows = [
+        [(_esc(e.get("name", "")), False), (_esc(e.get("etype", "")), False),
+         (_esc(e.get("n", 0)), True)]
+        for e in data.get("top_entities", [])]
+
+    def section(title: str, body: str, empty_note: str = "nothing recorded yet") -> str:
+        return "<h2>%s</h2>%s" % (
+            _esc(title), body or "<p class=\"sub\">%s</p>" % _esc(empty_note))
+
+    parts = [
+        "<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\">",
+        "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">",
+        "<title>Engraphis analytics — %s</title>" % _esc(workspace),
+        "<style>%s</style></head><body>" % _REPORT_CSS,
+        "<h1>Engraphis analytics report</h1>",
+        "<p class=\"sub\">workspace <strong>%s</strong> · generated %s%s</p>" % (
+            _esc(workspace), _esc(generated),
+            " · engraphis v%s" % _esc(version) if version else ""),
+        section("Totals & decay forecast", _table(["Metric", "Value"], totals_rows)),
+        section("Memories written per week", _table(["Week", "Written"], growth_rows)),
+        section("Retention distribution (live memories)",
+                _table(["Retention", "Memories"], hist_rows)),
+        section("Live memories by type", _table(["Type", "Count"], type_rows)
+                if type_rows else ""),
+        section("Write-path resolver activity", _table(["Action", "Count"], mix_rows)
+                if mix_rows else "", "no resolver events yet"),
+        section("Most connected entities",
+                _table(["Entity", "Type", "Connections"], entity_rows)
+                if entity_rows else "", "no entities yet — they appear as the graph grows"),
+        "<p class=\"footer\">Self-contained report — no external assets, no tracking. "
+        "Engraphis Pro analytics%s.</p>" % (" · v%s" % _esc(version) if version else ""),
+        "</body></html>",
+    ]
+    return "".join(parts)
+
+
+def compute_analytics(store: Any, workspace_id: str, *, now: Optional[float] = None) -> dict:
+    """SQL wrapper: fetch rows for one workspace, delegate to the pure function."""
+    conn = store.conn
+    mem_rows = [dict(r) for r in conn.execute(
+        "SELECT mtype, stability, last_access, ingested_at, importance, pinned, "
+        "valid_to, expired_at FROM memories WHERE workspace_id=?", (workspace_id,))]
+    audit_counts = {r["action"]: r["n"] for r in conn.execute(
+        "SELECT a.action, COUNT(*) AS n FROM audit a JOIN memories m ON m.id = a.target "
+        "WHERE m.workspace_id=? GROUP BY a.action", (workspace_id,))}
+    entity_rows = [dict(r) for r in conn.execute(
+        "SELECT e.name, e.etype, COUNT(ed.id) AS n FROM entities e "
+        "LEFT JOIN edges ed ON (ed.src = e.id OR ed.dst = e.id) "
+        "WHERE e.workspace_id=? GROUP BY e.id, e.name, e.etype "
+        "ORDER BY n DESC, e.created_at DESC LIMIT 8", (workspace_id,))]
+    return analytics_from_rows(mem_rows, audit_counts, entity_rows, now=now)

engraphis/app.py

@@ -0,0 +1,228 @@
+"""FastAPI app assembly — mounts all routes, serves dashboard, initializes DB, starts background loop."""
+from __future__ import annotations
+
+import asyncio
+import hmac
+import logging
+import time
+import uuid
+from collections import defaultdict, deque
+from pathlib import Path
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+
+from engraphis import __version__
+from engraphis.config import settings
+from engraphis.engines import reweight, thoughts as thoughts_engine
+from engraphis.logging_setup import configure_logging
+from engraphis.routes.memory import router as memory_router
+from engraphis.routes.vault import router as vault_router
+from engraphis.stores import get_conn, init_db
+
+logger = logging.getLogger("engraphis")
+
+
+def _const_time_eq(a: str, b: str) -> bool:
+    """Constant-time string comparison (avoids token-timing side channels)."""
+    return hmac.compare_digest(a.encode("utf-8"), b.encode("utf-8"))
+
+
+_background_task: asyncio.Task | None = None
+_STATIC_DIR = Path(__file__).resolve().parent / "static"
+# Readiness cache: only a *successful* embedder init is cached, so a transient
+# failure is re-checked on the next probe instead of wedging the pod NotReady.
+_embedder_ok: bool = False
+
+
+def _embedder_ready() -> bool:
+    global _embedder_ok
+    try:
+        from engraphis.backends.embedder_st import get_embedder
+        emb = get_embedder(settings.embed_model or None, settings.embed_dim or 256)
+        _embedder_ok = emb is not None and int(emb.dim) > 0
+    except Exception as e:  # pragma: no cover - defensive; get_embedder falls back itself
+        logger.warning("Readiness: embedder init failed: %s", e)
+        _embedder_ok = False
+    return _embedder_ok
+
+
+def create_app() -> FastAPI:
+    """Build and configure the FastAPI application."""
+    configure_logging()
+
+    app = FastAPI(
+        title="Engraphis",
+        description="Self-hosted AI memory engine for agents — Ebbinghaus decay, "
+                    "interaction-aware recall, bi-temporal facts, and background "
+                    "consolidation. Local-first; you bring the LLM.",
+        version=__version__,
+    )
+
+    # Local-first CORS: loopback by default, override with ENGRAPHIS_CORS_ORIGINS.
+    # Credentials are only allowed when the allow-list is explicit (never with "*").
+    _wildcard = "*" in settings.cors_origins
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.cors_origins,
+        allow_credentials=not _wildcard,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Optional bearer-token auth. Active only when ENGRAPHIS_API_TOKEN is set.
+    # Health-type probes (liveness + readiness) stay unauthenticated by convention.
+    _PUBLIC_PREFIXES = ("/memory/health", "/api/health", "/api/ready",
+                        "/docs", "/openapi.json", "/redoc", "/static")
+
+    @app.middleware("http")
+    async def _require_token(request: Request, call_next):
+        token = settings.api_token
+        if token and request.method != "OPTIONS" and request.url.path != "/" \
+                and not request.url.path.startswith(_PUBLIC_PREFIXES):
+            header = request.headers.get("authorization", "")
+            presented = header[7:].strip() if header.lower().startswith("bearer ") else ""
+            if not _const_time_eq(presented, token):
+                return JSONResponse({"error": "unauthorized"}, status_code=401)
+        return await call_next(request)
+
+    # Optional in-process rate limiting (per-client-IP sliding window). Disabled unless
+    # ENGRAPHIS_RATE_LIMIT > 0. In-memory/per-process — fine for one self-hosted instance;
+    # front it with a reverse proxy for multi-process or distributed limits.
+    if settings.rate_limit > 0:
+        _hits: dict[str, deque] = defaultdict(deque)
+        _PRUNE_EVERY = 60  # seconds between cleanup sweeps
+        _last_prune = time.monotonic()
+
+        @app.middleware("http")
+        async def _rate_limit(request: Request, call_next):
+            nonlocal _last_prune
+            if request.method == "OPTIONS" or request.url.path.startswith(_PUBLIC_PREFIXES):
+                return await call_next(request)
+            client = request.client.host if request.client else "unknown"
+            now = time.monotonic()
+            # Periodically prune stale IP entries to prevent unbounded growth.
+            if now - _last_prune > _PRUNE_EVERY:
+                cutoff_all = now - settings.rate_window
+                stale = [k for k, dq in _hits.items() if not dq or dq[-1] < cutoff_all]
+                for k in stale:
+                    del _hits[k]
+                _last_prune = now
+            dq = _hits[client]
+            cutoff = now - settings.rate_window
+            while dq and dq[0] <= cutoff:
+                dq.popleft()
+            if len(dq) >= settings.rate_limit:
+                retry = int(dq[0] + settings.rate_window - now) + 1
+                return JSONResponse({"error": "rate limit exceeded"}, status_code=429,
+                                    headers={"Retry-After": str(retry)})
+            dq.append(now)
+            return await call_next(request)
+
+    # Request-ID + access log. Defined last so it is the *outermost* middleware and
+    # also covers requests short-circuited by auth/rate-limit above. An incoming
+    # X-Request-ID is propagated (so a fronting proxy's id survives); otherwise one
+    # is assigned. Echoed on the response for client-side correlation.
+    @app.middleware("http")
+    async def _request_log(request: Request, call_next):
+        request_id = request.headers.get("x-request-id", "").strip() or uuid.uuid4().hex
+        start = time.perf_counter()
+        response = await call_next(request)
+        duration_ms = round((time.perf_counter() - start) * 1000, 1)
+        response.headers["X-Request-ID"] = request_id
+        logger.info(
+            "%s %s -> %d (%.1fms)",
+            request.method, request.url.path, response.status_code, duration_ms,
+            extra={"request_id": request_id, "method": request.method,
+                   "path": request.url.path, "status": response.status_code,
+                   "duration_ms": duration_ms},
+        )
+        return response
+
+    # Database initialization deferred to startup event so CLI can set ENGRAPHIS_DB_PATH first
+    @app.on_event("startup")
+    async def _startup_db():
+        init_db()
+
+    app.include_router(memory_router)
+    app.include_router(vault_router)
+
+    # ── probes (unauthenticated; see _PUBLIC_PREFIXES) ──────────────────────────
+    @app.get("/api/health")
+    async def api_health():
+        """Liveness: the process is up and serving. No dependency checks."""
+        return {"status": "ok", "timestamp": time.time(), "service": "engraphis"}
+
+    @app.get("/api/ready")
+    async def api_ready():
+        """Readiness: DB answers a trivial SELECT and the embedder backend
+        initializes. 503 until both hold, so orchestrators hold traffic."""
+        checks = {"db": False, "embedder": False}
+        try:
+            get_conn().execute("SELECT 1").fetchone()
+            checks["db"] = True
+        except Exception as e:
+            logger.warning("Readiness: db check failed: %s", e)
+        checks["embedder"] = _embedder_ready()
+        ready = all(checks.values())
+        return JSONResponse({"ready": ready, "checks": checks, "version": __version__},
+                            status_code=200 if ready else 503)
+
+    @app.on_event("startup")
+    async def _startup() -> None:
+        global _background_task
+        if settings.loop_interval > 0:
+            _background_task = asyncio.create_task(_consciousness_loop())
+            logger.info("Background consciousness loop started (interval=%ds)", settings.loop_interval)
+        else:
+            logger.info("Background loop disabled (ENGRAPHIS_LOOP_INTERVAL=0)")
+
+    @app.on_event("shutdown")
+    async def _shutdown() -> None:
+        global _background_task
+        if _background_task:
+            _background_task.cancel()
+            try:
+                await _background_task
+            except asyncio.CancelledError:
+                pass
+
+    @app.get("/", response_class=HTMLResponse)
+    async def dashboard():
+        """Serve the visual dashboard."""
+        index_path = _STATIC_DIR / "index.html"
+        if index_path.exists():
+            return HTMLResponse(index_path.read_text(encoding="utf-8"))
+        return HTMLResponse("<h1>Dashboard not found</h1><p>Static files missing at: "
+                            f"{_STATIC_DIR}</p>", status_code=404)
+
+    if _STATIC_DIR.exists():
+        app.mount("/static", StaticFiles(directory=str(_STATIC_DIR)), name="static")
+
+    return app
+
+
+async def _consciousness_loop() -> None:
+    """Phase 2 + Phase 4 background cycle: decay → thought synthesis → reweight."""
+    while True:
+        try:
+            await asyncio.sleep(settings.loop_interval)
+            touched = reweight.decay_pass(namespace=None)
+            if touched:
+                logger.info("Decay pass: %d memories reweighted", touched)
+            result = thoughts_engine.synthesize_thoughts(
+                namespace=None,
+                max_chunks=settings.loop_top_k,
+                persist=True,
+            )
+            if result.get("persisted"):
+                logger.info("Thought synthesized: %s", result.get("thought"))
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            logger.error("Consciousness loop error: %s", e)
+
+
+app = create_app()

engraphis/backends/__init__.py

@@ -0,0 +1,16 @@
+"""Pluggable backends implementing the engraphis.core interfaces.
+
+Phase 0 ships *reference* implementations chosen for portability and zero extra
+dependencies so the system runs and is testable anywhere:
+
+* ``NumpyVectorIndex``  — brute-force cosine over the store. Correct, not fast.
+                          Phase 1 replaces it with a ``sqlite-vec`` / LanceDB /
+                          Qdrant backend behind the same ``VectorIndex`` interface.
+* ``DeterministicEmbedder`` — a hashing embedder with no model download, for
+                          offline tests and CI. Production uses a real model
+                          (BGE-M3 / Qwen3 class) behind the same ``Embedder`` interface.
+"""
+from engraphis.backends.embedder_deterministic import DeterministicEmbedder
+from engraphis.backends.vector_numpy import NumpyVectorIndex
+
+__all__ = ["DeterministicEmbedder", "NumpyVectorIndex"]