superlocalmemory 4.1.13 → 4.1.15

package/src/superlocalmemory/server/routes/data_io.py

@@ -1,329 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-"""SuperLocalMemory V3 - Import/Export Routes
- - AGPL-3.0-or-later
-
-Routes: /api/export, /api/import
-"""
-import asyncio
-import io
-import gzip
-import hashlib
-import json
-import logging
-from typing import Any, Optional
-from datetime import datetime, timezone
-
-from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File
-from fastapi.responses import StreamingResponse
-
-from .helpers import (
-    DB_PATH,
-    dict_factory,
-    get_active_profile,
-    get_db_connection,
-    require_engine,
-)
-
-logger = logging.getLogger("superlocalmemory.routes.data_io")
-
-# Hard cap on the total decompressed byte count for gzip imports.
-# Bounding the compressed upload size alone does not prevent a decompression
-# bomb: a few kilobytes of input can expand to gigabytes.  This cap is checked
-# incrementally during streaming decompression so the full expanded content is
-# never materialized before the guard fires.
-_MAX_DECOMPRESSED_BYTES: int = 200 * 1024 * 1024  # 200 MB
-
-
-def _internal_error(detail: str = "Internal server error") -> HTTPException:
-    """SEC-H-02: log full traceback server-side; return a generic message to the client."""
-    logger.exception("data_io route error")
-    return HTTPException(status_code=500, detail=detail)
-
-# WebSocket manager reference (set by ui_server.py at startup)
-ws_manager = None
-
-router = APIRouter()
-
-
-@router.get("/api/export")
-async def export_memories(
-    request: Request,
-    format: str = Query("json", pattern="^(json|jsonl|csv)$"),
-    category: Optional[str] = None,
-    project_name: Optional[str] = None,
-):
-    """Export memories as JSON, JSONL, or CSV."""
-    # Bulk data export. This GET is not covered by the mutation middleware and
-    # is reached both by a plain fetch and a top-level navigation, neither of
-    # which carries a credential header — so gate on the loopback-trusted
-    # mutation boundary: local owner allowed, remote uncredentialed fails closed.
-    from superlocalmemory.server.write_identity import require_http_mutation_actor
-    require_http_mutation_actor(request, getattr(request.app.state, "daemon_descriptor", None),
-                                actor_kind="data-export")
-    try:
-        conn = get_db_connection()
-        conn.row_factory = dict_factory
-        cursor = conn.cursor()
-        active_profile = get_active_profile()
-
-        # Detect schema
-        try:
-            cursor.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name='atomic_facts'",
-            )
-            use_v3 = cursor.fetchone() is not None
-        except Exception:
-            use_v3 = False
-
-        if use_v3:
-            # Withheld summaries are excluded, and this is about IMPORT, not
-            # tidiness. Import re-ingests every record through the normal
-            # pipeline, which mints a fresh memory with quarantined = 0 — so an
-            # export taken here and imported anywhere would resurrect all 1,195
-            # model-written rows as though the owner had written them, on a
-            # machine where nothing had gone wrong. The repair would then have
-            # to run again there.
-            #
-            # Nothing the owner wrote is lost: these are derived artefacts, the
-            # consolidator regenerates them from the facts that ARE exported,
-            # and their text is kept in consolidated_summaries. (That table is
-            # not in this export either — it is a view, not a memory. Worth
-            # revisiting when export covers derived state.)
-            query = "SELECT * FROM atomic_facts WHERE profile_id = ?"
-            if _has_column(cursor, "atomic_facts", "quarantined"):
-                query += " AND COALESCE(quarantined, 0) = 0"
-            params = [active_profile]
-            if category:
-                query += " AND fact_type = ?"
-                params.append(category)
-            if project_name:
-                query += " AND session_id = ?"
-                params.append(project_name)
-            query += " ORDER BY created_at"
-        else:
-            query = "SELECT * FROM memories WHERE profile = ?"
-            params = [active_profile]
-            if category:
-                query += " AND category = ?"
-                params.append(category)
-            if project_name:
-                query += " AND project_name = ?"
-                params.append(project_name)
-            query += " ORDER BY created_at"
-
-        cursor.execute(query, params)
-        memories = cursor.fetchall()
-        conn.close()
-
-        if format == "jsonl":
-            content = "\n".join(json.dumps(m) for m in memories)
-            media_type = "application/x-ndjson"
-        elif format == "csv":
-            import csv
-            import io as _io
-            if memories:
-                buf = _io.StringIO()
-                fieldnames = list(memories[0].keys())
-                writer = csv.DictWriter(
-                    buf, fieldnames=fieldnames, extrasaction="ignore",
-                )
-                writer.writeheader()
-                for m in memories:
-                    writer.writerow({
-                        k: (json.dumps(v) if isinstance(v, (dict, list)) else v)
-                        for k, v in m.items()
-                    })
-                content = buf.getvalue()
-            else:
-                content = ""
-            media_type = "text/csv"
-        else:
-            content = json.dumps({
-                "version": "3.0.0",
-                "exported_at": datetime.now(timezone.utc).isoformat(),
-                "total_memories": len(memories),
-                "filters": {"category": category, "project_name": project_name},
-                "memories": memories,
-            }, indent=2)
-            media_type = "application/json"
-
-        ts = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')
-        if len(content) > 10000:
-            compressed = gzip.compress(content.encode())
-            return StreamingResponse(
-                io.BytesIO(compressed), media_type="application/gzip",
-                headers={
-                    "Content-Disposition": f"attachment; filename=memories_export_{ts}.{format}.gz",
-                },
-            )
-        return StreamingResponse(
-            io.BytesIO(content.encode()), media_type=media_type,
-            headers={
-                "Content-Disposition": f"attachment; filename=memories_export_{ts}.{format}",
-            },
-        )
-
-    except Exception:
-        raise _internal_error("Export error")
-
-
-def _has_column(cursor: Any, table: str, column: str) -> bool:
-    """Whether ``table`` carries ``column`` in this database.
-
-    Presence-guarded because ``quarantined`` arrives with a migration and this
-    route must keep working on a store the engine has not opened.
-    """
-    try:
-        cursor.execute(f"PRAGMA table_info({table})")
-        return any(row[1] == column for row in cursor.fetchall())
-    except Exception:  # noqa: BLE001 -- an export must not fail over a probe
-        return False
-
-
-@router.post("/api/import")
-async def import_memories(request: Request, file: UploadFile = File(...)):
-    """Import memories from JSON file using V3 engine."""
-    try:
-        # Bound the upload so a huge file cannot OOM the daemon (read one byte
-        # past the cap to detect oversize without buffering the whole payload).
-        _MAX_IMPORT_BYTES = 50 * 1024 * 1024
-        content = await file.read(_MAX_IMPORT_BYTES + 1)
-        if len(content) > _MAX_IMPORT_BYTES:
-            raise HTTPException(status_code=413,
-                                detail="Import file exceeds the 50 MB limit")
-        if file.filename and file.filename.endswith('.gz'):
-            # Stream-decompress with an incremental byte counter so the full
-            # expanded payload is never allocated before the guard fires.
-            _chunk_size = 65_536
-            chunks: list[bytes] = []
-            total_decompressed = 0
-            with gzip.GzipFile(fileobj=io.BytesIO(content)) as _gz:
-                while True:
-                    chunk = _gz.read(_chunk_size)
-                    if not chunk:
-                        break
-                    total_decompressed += len(chunk)
-                    if total_decompressed > _MAX_DECOMPRESSED_BYTES:
-                        raise HTTPException(
-                            status_code=413,
-                            detail=(
-                                f"Decompressed content exceeds the "
-                                f"{_MAX_DECOMPRESSED_BYTES // (1024 * 1024)} MB limit"
-                            ),
-                        )
-                    chunks.append(chunk)
-            content = b"".join(chunks)
-
-        try:
-            data = json.loads(content)
-        except json.JSONDecodeError:
-            logger.warning("import: invalid JSON payload")
-            raise HTTPException(status_code=400, detail="Invalid JSON format")
-
-        if isinstance(data, dict) and 'memories' in data:
-            memories = data['memories']
-        elif isinstance(data, list):
-            memories = data
-        else:
-            raise HTTPException(
-                status_code=400, detail="Invalid format: expected 'memories' array",
-            )
-
-        engine = require_engine(request)
-        from superlocalmemory.core.engine_ingestion import (
-            build_engine_ingestion_command,
-        )
-        from superlocalmemory.core.ingestion_command import (
-            IngestionRequest,
-            IngestionState,
-        )
-
-        command = build_engine_ingestion_command(engine)
-        from superlocalmemory.server.write_identity import (
-            authenticated_request_actor,
-        )
-        actor_id = authenticated_request_actor(
-            request,
-            actor_kind="http-import",
-        )
-        file_digest = hashlib.sha256(content).hexdigest()
-        imported = 0
-        skipped = 0
-        errors = []
-        operation_ids: list[str] = []
-
-        for idx, memory in enumerate(memories):
-            try:
-                memory_content = memory.get('content')
-                if not memory_content:
-                    errors.append(f"Memory {idx}: missing 'content' field")
-                    continue
-                # Imported content is untrusted: scrub secrets before it reaches
-                # any durable or queryable store, exactly as the canonical
-                # ingest path does.
-                from superlocalmemory.core.ingest_policy import scrub_secrets_for_ingest
-                _scrub = scrub_secrets_for_ingest(memory_content)
-                if _scrub.redacted:
-                    memory_content = _scrub.content
-
-                metadata = {
-                    "project_name": memory.get('project_name'),
-                    "category": memory.get('category'),
-                    "tags": memory.get('tags', ''),
-                }
-                for _field in (
-                    "fact_type", "confidence", "importance", "entities",
-                    "canonical_entities", "referenced_date", "pinned",
-                ):
-                    if _field in memory:
-                        metadata[_field] = memory[_field]
-                receipt, created = command.submit_with_status(IngestionRequest(
-                    content=memory_content,
-                    profile_id=engine._profile_id,
-                    source_type="http-import",
-                    idempotency_key=f"import:{file_digest}:{idx}",
-                    metadata=metadata,
-                    scope=memory.get("scope") or "personal",
-                    shared_with=tuple(memory.get("shared_with") or ()),
-                    trusted_actor_id=actor_id,
-                    session_id=memory.get('session_id', ''),
-                    session_date=memory.get('session_date') or "",
-                    speaker=memory.get('speaker') or "",
-                    role=memory.get('role') or "user",
-                ))
-                completed = await asyncio.to_thread(command.materialize, receipt.operation_id)
-                if completed.state is not IngestionState.COMPLETE:
-                    raise RuntimeError(
-                        completed.last_error or "canonical import failed"
-                    )
-                operation_ids.append(completed.operation_id)
-                if created:
-                    imported += 1
-                else:
-                    skipped += 1
-
-                if ws_manager:
-                    await ws_manager.broadcast({
-                        "type": "memory_added", "memory_id": imported,
-                        "timestamp": datetime.now(timezone.utc).isoformat(),
-                    })
-
-            except Exception as e:
-                if "UNIQUE constraint failed" in str(e):
-                    skipped += 1
-                else:
-                    logger.warning("import: memory %d failed: %s", idx, e)
-                    errors.append(f"Memory {idx}: import failed")
-
-        return {
-            "success": True, "imported_count": imported,
-            "skipped_count": skipped, "total_processed": len(memories),
-            "errors": errors[:10], "operation_ids": operation_ids,
-        }
-
-    except HTTPException:
-        raise
-    except Exception:
-        raise _internal_error("Import error")

package/src/superlocalmemory/server/routes/entity.py

@@ -1,237 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Entity compilation API routes — view and recompile entity summaries."""
-
-from __future__ import annotations
-
-from fastapi import APIRouter, HTTPException, Query, Request
-
-from .helpers import get_active_profile, get_read_connection, require_engine
-
-router = APIRouter(prefix="/api/entity", tags=["entity"])
-
-
-def _list_entities_sql(where_sql: str) -> str:
-    """Build the page-first entity query after ``where_sql`` is parameterized."""
-    return f"""
-        WITH page_entities AS MATERIALIZED (
-            SELECT ce.entity_id, ce.profile_id, ce.canonical_name,
-                   ce.entity_type, ce.fact_count, ce.first_seen, ce.last_seen
-            FROM canonical_entities ce
-            WHERE {where_sql}
-            ORDER BY ce.fact_count DESC, ce.entity_id ASC
-            LIMIT ? OFFSET ?
-        ),
-        ranked_profiles AS MATERIALIZED (
-            SELECT ep.*,
-                   ROW_NUMBER() OVER (
-                       PARTITION BY ep.entity_id, ep.profile_id
-                       ORDER BY COALESCE(ep.last_compiled_at, '') DESC,
-                                ep.project_name COLLATE NOCASE ASC,
-                                ep.rowid ASC
-                   ) AS summary_rank
-            FROM entity_profiles ep
-            JOIN page_entities page
-              ON page.entity_id = ep.entity_id
-             AND page.profile_id = ep.profile_id
-            WHERE ep.profile_id = ?
-        )
-        SELECT ce.entity_id, ce.canonical_name, ce.entity_type,
-               ce.fact_count, ce.first_seen, ce.last_seen,
-               ep.knowledge_summary, ep.compiled_truth,
-               ep.compilation_confidence, ep.last_compiled_at
-        FROM page_entities ce
-        LEFT JOIN ranked_profiles ep
-          ON ce.entity_id = ep.entity_id
-         AND ep.profile_id = ce.profile_id
-         AND ep.summary_rank = 1
-        ORDER BY ce.fact_count DESC, ce.entity_id ASC
-    """
-
-
-def _require_read(request: Request, profile: str) -> None:
-    """Authorize entity metadata access for the explicitly requested profile."""
-    from superlocalmemory.access.rbac import Permission
-    from superlocalmemory.server.rbac_enforce import require_permission
-
-    require_permission(request, Permission.READ, profile=profile)
-
-
-def _require_manage(request: Request, profile: str) -> None:
-    """Authorize entity recompilation for the explicitly requested profile."""
-    from superlocalmemory.server.rbac_enforce import require_manage
-
-    require_manage(request, profile=profile)
-
-
-@router.get("/list")
-def list_entities(
-    request: Request,
-    profile: str | None = Query(default=None),
-    entity_type: str | None = Query(default=None, alias="type", max_length=80),
-    search: str | None = Query(default=None, max_length=200),
-    limit: int = Query(default=100, ge=1, le=1000),
-    offset: int = Query(default=0, ge=0),
-):
-    """List a profile's entities, filtering before count and pagination."""
-    engine = require_engine(request)
-    # Default to the ACTIVE profile (request runtime truth), never the literal
-    # "default" — otherwise every profile sees the default profile's entities.
-    profile = profile or get_active_profile()
-    _require_read(request, profile)
-
-    conn = get_read_connection(engine._config.db_path)
-    try:
-        # Hide the placeholder that dated facts with no recognised entity
-        # attach their temporal events to (core/store_pipeline.py,
-        # _ensure_unresolved_entity). It is a hook for a foreign key, not a
-        # concept, and it would otherwise appear here as a nameless entity and
-        # be counted in the total the owner reads as "entities I have".
-        #
-        # Only the presentation layer needs this. The other readers of
-        # canonical_entities are unaffected on inspection: graph_pruner uses
-        # `NOT IN (SELECT entity_id ...)` to find orphaned edges, where
-        # including it is correct; community_summary builds from graph edges and
-        # the placeholder has none; scale_engine counts for backend sync, not
-        # for display.
-        where = ["ce.profile_id = ?", "ce.entity_type != 'unresolved'"]
-        params: list[object] = [profile]
-        if entity_type and entity_type.lower() != "all":
-            where.append("ce.entity_type = ? COLLATE NOCASE")
-            params.append(entity_type.strip().lower())
-        if search and search.strip():
-            escaped = (search.strip().lower().replace("\\", "\\\\")
-                       .replace("%", "\\%").replace("_", "\\_"))
-            where.append(
-                "(LOWER(ce.canonical_name) LIKE ? ESCAPE '\\' "
-                "OR LOWER(COALESCE(ce.entity_type, 'unknown')) LIKE ? ESCAPE '\\' "
-                "OR EXISTS ("
-                "SELECT 1 FROM entity_profiles eps "
-                "WHERE eps.entity_id = ce.entity_id "
-                "AND eps.profile_id = ce.profile_id "
-                "AND LOWER(COALESCE(eps.knowledge_summary, '')) "
-                "LIKE ? ESCAPE '\\'))"
-            )
-            params.extend([f"%{escaped}%"] * 3)
-        where_sql = " AND ".join(where)
-
-        total = conn.execute(
-            "SELECT COUNT(*) FROM canonical_entities ce "
-            f"WHERE {where_sql}",
-            params,
-        ).fetchone()[0]
-
-        rows = conn.execute(
-            _list_entities_sql(where_sql),
-            [*params, limit, offset, profile],
-        ).fetchall()
-
-        entities = []
-        for r in rows:
-            summary = r["knowledge_summary"] or ""
-            entities.append({
-                "entity_id": r["entity_id"],
-                "name": r["canonical_name"],
-                "type": r["entity_type"] or "unknown",
-                "fact_count": r["fact_count"] or 0,
-                "first_seen": r["first_seen"],
-                "last_seen": r["last_seen"],
-                "summary_preview": summary[:200] if summary else "",
-                "has_compiled_truth": bool(r["compiled_truth"]),
-                "confidence": r["compilation_confidence"] or 0.5,
-                "last_compiled_at": r["last_compiled_at"],
-            })
-
-        return {
-            "entities": entities,
-            "total": total,
-            "limit": limit,
-            "offset": offset,
-            "has_more": offset + limit < total,
-        }
-    finally:
-        conn.close()
-
-
-@router.get("/{entity_name}")
-def get_entity(
-    entity_name: str,
-    request: Request,
-    profile: str | None = Query(default=None),
-    project: str = Query(default=""),
-):
-    """Get compiled truth + timeline for an entity."""
-    engine = require_engine(request)
-    profile = profile or get_active_profile()
-    _require_read(request, profile)
-
-    import json
-    conn = get_read_connection(engine._config.db_path)
-    try:
-        # Search by canonical_name (case-insensitive)
-        row = conn.execute("""
-            SELECT ep.compiled_truth, ep.timeline, ep.fact_ids_json,
-                   ep.last_compiled_at, ep.compilation_confidence,
-                   ep.knowledge_summary, ce.entity_type
-            FROM entity_profiles ep
-            JOIN canonical_entities ce ON ep.entity_id = ce.entity_id
-            WHERE LOWER(ce.canonical_name) = LOWER(?)
-              AND ep.profile_id = ?
-              AND ep.project_name = ?
-        """, (entity_name, profile, project)).fetchone()
-
-        if not row:
-            raise HTTPException(404, detail=f"Entity '{entity_name}' not found")
-
-        return {
-            "entity_name": entity_name,
-            "entity_type": row["entity_type"],
-            "compiled_truth": row["compiled_truth"] or "",
-            "knowledge_summary": row["knowledge_summary"] or "",
-            "timeline": json.loads(row["timeline"]) if row["timeline"] else [],
-            "source_fact_ids": json.loads(row["fact_ids_json"]) if row["fact_ids_json"] else [],
-            "last_compiled_at": row["last_compiled_at"],
-            "confidence": row["compilation_confidence"],
-        }
-    finally:
-        conn.close()
-
-
-@router.post("/{entity_name}/recompile")
-def recompile_entity(
-    entity_name: str,
-    request: Request,
-    profile: str | None = Query(default=None),
-    project: str = Query(default=""),
-):
-    """Force immediate recompilation of an entity."""
-    engine = require_engine(request)
-    profile = profile or get_active_profile()
-    _require_manage(request, profile)
-
-    import sqlite3
-    conn = sqlite3.connect(str(engine._config.db_path))
-    conn.row_factory = sqlite3.Row
-    try:
-        entity = conn.execute(
-            "SELECT entity_id, canonical_name, entity_type FROM canonical_entities "
-            "WHERE LOWER(canonical_name) = LOWER(?) AND profile_id = ?",
-            (entity_name, profile),
-        ).fetchone()
-
-        if not entity:
-            raise HTTPException(404, detail=f"Entity '{entity_name}' not found")
-
-        from superlocalmemory.learning.entity_compiler import EntityCompiler
-        compiler = EntityCompiler(str(engine._config.db_path), engine._config)
-        result = compiler.compile_entity(
-            profile, project, entity["entity_id"], entity["canonical_name"],
-        )
-
-        if result:
-            return {"ok": True, **result}
-        return {"ok": False, "reason": "no facts to compile"}
-    finally:
-        conn.close()