superlocalmemory 3.6.13 → 3.6.15

package/src/superlocalmemory/optimize/proxy/vertex_surface.py

@@ -0,0 +1,246 @@
+"""vertex_surface.py — Vertex AI passthrough proxy surface (WP-11).
+
+Transparent passthrough: forward Authorization bearer untouched (AC-2),
+cache by body content never by token (SEC), no SSE — always single JSON.
+
+Key design decisions (per LLD §5 STAGE-5 RESOLUTIONS):
+  CRIT-1: Route by FastAPI PATH /v1/projects/{vertex_path:path}, NOT hostname.
+           Upstream host is reconstructed from /locations/{region}/ in the path.
+  CRIT-2: Vertex bodies have no model/messages/system (model in PATH, prompts
+           under 'contents'). The provider=='vertex' branch in CacheManager.build_key
+           extracts these correctly — preventing all-requests-same-key collision.
+  D2: Pin upstream host to https://{location}-aiplatform.googleapis.com from path.
+      IGNORE providers.vertex.base_url — honoring it is an SSRF surface.
+  SECURITY (AC-3): bearer token structurally excluded from cache key, value,
+           logs, and stored ProxyRequest.headers (redacted via _redact_headers).
+
+WP-11a (gemini-native fix) lives in _helpers.py:_GEMINI_NATIVE_FORWARD_HEADERS.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+
+from fastapi.requests import Request
+from fastapi.responses import Response
+
+from superlocalmemory.optimize.proxy._helpers import (
+    _VERTEX_FORWARD_HEADERS,
+    _derive_tenant_id,
+    _fail_open_forward,
+    _filter_response_headers,
+    _redact_headers,
+    _safe_cache_check,
+    _safe_cache_hit_callbacks,
+    _safe_cache_store,
+    capture_passthrough_forward,
+)
+from superlocalmemory.optimize.proxy.capture import capture_enabled
+from superlocalmemory.optimize.proxy.lifecycle import ProviderResponse, ProxyRequest
+
+logger = logging.getLogger("slm.optimize.proxy.vertex")
+
+# ---------------------------------------------------------------------------
+# SSRF guard — LLD §5
+# ---------------------------------------------------------------------------
+
+# Accepts ONLY:
+#   {project}/locations/{location}/publishers/google/models/{model}:{method}
+# Rejects:
+#   - ../ traversal (no dots allowed in segments via character classes)
+#   - unknown methods (countTokens, EXEC, etc.)
+#   - overlong segments
+_VERTEX_PATH_RE = re.compile(
+    r"^(?P<project>[a-zA-Z0-9_\-]{1,63})"
+    r"/locations/(?P<location>[a-z0-9\-]{1,40})"
+    r"/publishers/google/models/(?P<model>[a-zA-Z0-9._\-]{1,128})"
+    r":(?P<method>generateContent|streamGenerateContent)$"
+)
+
+
+def _validate_vertex_path(path: str) -> bool:
+    """Return True iff path matches the Vertex AI path pattern (SSRF guard).
+
+    Rejects ../, traversal characters, unknown methods, and overlong segments.
+    """
+    return bool(_VERTEX_PATH_RE.match(path))
+
+
+def _parse_vertex_path(
+    path: str,
+) -> tuple[str, str, str, str] | None:
+    """Parse a validated Vertex path into (project, location, model, method).
+
+    Returns None on parse failure (caller should return 400).
+    """
+    m = _VERTEX_PATH_RE.match(path)
+    if m is None:
+        return None
+    return (
+        m.group("project"),
+        m.group("location"),
+        m.group("model"),
+        m.group("method"),
+    )
+
+
+def _build_vertex_upstream_url(location: str, full_path: str) -> str:
+    """Build the Vertex upstream URL.
+
+    D2 (LOCKED): pin host to https://{location}-aiplatform.googleapis.com
+    from the path — IGNORE providers.vertex.base_url (honoring it = SSRF).
+    """
+    host = f"https://{location}-aiplatform.googleapis.com"
+    return f"{host}{full_path}"
+
+
+# ---------------------------------------------------------------------------
+# Route handler
+# ---------------------------------------------------------------------------
+
+async def handle_vertex_generative(
+    proxy: object,
+    request: Request,
+    vertex_path: str,
+) -> Response:
+    """Handle POST /v1/projects/{vertex_path:path}.
+
+    Full pipeline: validate → parse → cache check → POST upstream → cache store.
+    Bearer token is NEVER in the cache key (structurally — build_key reads only body).
+    Always returns a single JSON Response — no SSE/StreamingResponse (LLD §1).
+    Fail-open on unexpected exceptions.
+    """
+    request_id = await proxy.next_request_id()
+
+    # ── SSRF guard — validate path BEFORE any upstream contact (AC-3 / LLD §4) ──
+    if not _validate_vertex_path(vertex_path):
+        logger.warning(
+            "[%s] handle_vertex_generative: rejected invalid path=%r (SSRF guard)",
+            request_id, vertex_path,
+        )
+        return Response(
+            content=b'{"error":{"code":400,"message":"Invalid Vertex path",'
+                    b'"status":"INVALID_ARGUMENT"}}',
+            status_code=400,
+            media_type="application/json",
+        )
+
+    parsed = _parse_vertex_path(vertex_path)
+    if parsed is None:
+        # Should not reach here after _validate_vertex_path, but be defensive.
+        logger.error("[%s] handle_vertex_generative: parse failed after validation", request_id)
+        return Response(
+            content=b'{"error":{"code":400,"message":"Path parse error",'
+                    b'"status":"INVALID_ARGUMENT"}}',
+            status_code=400,
+            media_type="application/json",
+        )
+
+    _project, location, _model, _method = parsed
+    upstream_url = _build_vertex_upstream_url(location, str(request.url.path))
+
+    try:
+        body_bytes = await request.body()
+
+        try:
+            body = json.loads(body_bytes) if body_bytes else {}
+        except json.JSONDecodeError:
+            body = {}
+
+        # v3.6.10 shadow-capture: pure passthrough + corpus record.
+        if capture_enabled():
+            return await capture_passthrough_forward(
+                proxy, request,
+                provider="vertex",
+                upstream_url=upstream_url,
+                allowed_headers=_VERTEX_FORWARD_HEADERS,
+                request_id=request_id,
+                model_hint=_model,
+                sse_parser=None,
+                is_stream=False,
+            )
+
+        # SECURITY (AC-3): headers stored in ProxyRequest are redacted.
+        # Bearer token is structurally excluded from the cache key because
+        # build_key reads only body-derived fields (key_builder.py:90-105).
+        #
+        # SECURITY (WP-D): derive tenant BEFORE _redact_headers strips the
+        # bearer token.  Vertex uses Authorization bearer.
+        _raw_key = request.headers.get("authorization")
+        _tenant_id = _derive_tenant_id("vertex", _raw_key)
+
+        ctx = ProxyRequest(
+            provider="vertex",
+            method="POST",
+            path=str(request.url.path),
+            headers=_redact_headers(dict(request.headers)),  # token → [REDACTED]
+            body=body,
+            body_bytes=body_bytes,
+            request_id=request_id,
+            stream=False,
+            has_tools=False,
+        )
+
+        # Build forward headers — Authorization byte-identical (AC-2).
+        fwd_headers = {
+            k: v for k, v in request.headers.items()
+            if k.lower() in _VERTEX_FORWARD_HEADERS
+        }
+
+        # ── Cache check ────────────────────────────────────────────────────
+        cache_result = None
+        if proxy.hooks.cache:
+            cache_result = await _safe_cache_check(proxy.hooks, ctx, tenant_id=_tenant_id)
+            if cache_result and cache_result.hit and cache_result.data:
+                logger.debug(
+                    "[%s] Vertex cache HIT key=%s",
+                    request_id, cache_result.cache_key,
+                )
+                await _safe_cache_hit_callbacks(
+                    proxy.hooks, ctx, cache_result.data, tokens_saved=0
+                )
+                return Response(
+                    content=cache_result.data,
+                    status_code=200,
+                    media_type="application/json",
+                )
+
+        fwd_headers["content-length"] = str(len(body_bytes))
+
+        # ── POST upstream (single JSON — no SSE) ──────────────────────────
+        upstream_resp = await proxy.http_client.post(
+            upstream_url, content=body_bytes, headers=fwd_headers,
+        )
+        resp_bytes = upstream_resp.content
+
+        # ── Cache store on 200 only ────────────────────────────────────────
+        if (
+            upstream_resp.status_code == 200
+            and proxy.hooks.cache
+            and cache_result is not None
+            and cache_result.cache_key
+        ):
+            prov_resp = ProviderResponse(
+                modified=False,
+                body={},
+                body_bytes=resp_bytes,
+                tokens_before=0,
+                tokens_after=0,
+                strategy="none",
+            )
+            await _safe_cache_store(proxy.hooks, ctx, prov_resp, tenant_id=_tenant_id)
+
+        return Response(
+            content=resp_bytes,
+            status_code=upstream_resp.status_code,
+            media_type="application/json",
+            headers=_filter_response_headers(dict(upstream_resp.headers)),
+        )
+
+    except Exception as exc:
+        logger.error(
+            "[%s] handle_vertex_generative exc=%r — fail-open", request_id, exc
+        )
+        return await _fail_open_forward(proxy, request, upstream_url)

package/src/superlocalmemory/optimize/storage/db.py

@@ -947,6 +947,36 @@ class CacheDB:
         except sqlite3.Error as exc:
             logger.warning("CacheDB.ccr_update_compressed failed: %s", exc)
 
+    def ccr_delete(self, ccr_id: str) -> None:
+        """Delete a CCR row by ccr_id. Idempotent — warns on sqlite error, never raises.
+
+        WP-10 D6: defensive infra + sweep parity. Deleting a non-existent row is a no-op.
+        """
+        try:
+            self._db.execute(
+                "DELETE FROM llmcache_ccr_originals WHERE ccr_id = ?",
+                (ccr_id,),
+            )
+        except sqlite3.Error as exc:
+            logger.warning("CacheDB.ccr_delete failed (non-fatal): %s", exc)
+
+    def ccr_count(self) -> int:
+        """Return UNFILTERED count of rows in llmcache_ccr_originals.
+
+        WP-10 CRIT-2: Do NOT reuse TTL-filtered count at :646. A fresh no-expiry row
+        has ttl_expires=None, so the TTL filter returns 0 and D6 orphan tests would
+        falsely pass. This unfiltered count is test infrastructure only.
+        """
+        try:
+            rows = self._db.execute(
+                "SELECT COUNT(*) AS n FROM llmcache_ccr_originals",
+                (),
+            )
+            return int(dict(rows[0])["n"]) if rows else 0
+        except sqlite3.Error as exc:
+            logger.warning("CacheDB.ccr_count failed: %s", exc)
+            return 0
+
     # ---- v2 additions ----
 
     def get_all_vectors(self, tenant_id: str) -> list[tuple[str, bytes, str]]:

package/src/superlocalmemory/retrieval/bm25_channel.py

@@ -86,9 +86,15 @@ class BM25Channel:
             return
 
         token_map = self._db.get_all_bm25_tokens(profile_id)
+        _inc_global = getattr(self, 'include_global', False)
+        _inc_shared = getattr(self, 'include_shared', False)
         if not token_map:
             # Fallback: tokenize facts directly if no pre-stored tokens
-            facts = self._db.get_all_facts(profile_id)
+            facts = self._db.get_all_facts(
+                profile_id,
+                include_global=_inc_global,
+                include_shared=_inc_shared,
+            )
             for fact in facts:
                 if fact.fact_id in self._fact_id_set:
                     continue
@@ -104,7 +110,11 @@ class BM25Channel:
             # Load raw texts for phrase matching (V3.3.12)
             fact_content_map = {}
             try:
-                facts = self._db.get_all_facts(profile_id)
+                facts = self._db.get_all_facts(
+                    profile_id,
+                    include_global=_inc_global,
+                    include_shared=_inc_shared,
+                )
                 fact_content_map = {f.fact_id: f.content for f in facts}
             except Exception:
                 pass

package/src/superlocalmemory/retrieval/engine.py

@@ -16,6 +16,7 @@ from __future__ import annotations
 import logging
 import math
 import re
+import threading
 import time
 from typing import TYPE_CHECKING, Any, Protocol
 
@@ -84,6 +85,12 @@ class RetrievalEngine:
         self._profile_channel = profile_channel
         self._bridge = bridge_discovery
         self._trust_scorer = trust_scorer
+        # v3.6.15: serialise the per-recall scope-flag set + channel execution.
+        # Channel instances are SHARED across concurrent recalls (the daemon runs
+        # several in parallel); without this, recall B's flags could overwrite
+        # recall A's mid-flight on the shared channels. Uncontended for a single
+        # recall (~0 cost); only the channel phase of concurrent recalls serialises.
+        self._scope_lock = threading.Lock()
 
         # V3.3.4: LRU cache for query embeddings (avoids redundant Ollama API calls)
         # V3.4.40 (2026-05-09): bumped 64 -> 512. Each cached embedding is ~3KB
@@ -117,9 +124,15 @@ class RetrievalEngine:
         mode: Mode = Mode.A, limit: int = 20,
         *,
         extra_disabled_channels: set[str] | None = None,
+        include_global: bool = True,
+        include_shared: bool = True,
     ) -> RecallResponse:
         """Full retrieval pipeline: strategy -> channels -> RRF -> rerank.
 
+        Multi-scope: ``include_global`` / ``include_shared`` control which
+        scopes participate in retrieval. Both default to True for backward
+        compatibility (existing data has scope='personal' — no effect).
+
         V3.4.40 (2026-05-09): ``extra_disabled_channels`` allows callers to
         skip specific channels for a single recall (e.g. SpreadingActivation
         for the ``--fast`` CLI flag) without mutating shared config.
@@ -127,6 +140,12 @@ class RetrievalEngine:
         t0 = time.monotonic()
         self._extra_disabled = set(extra_disabled_channels or ())
 
+        # Multi-scope: scope flags are set on the (shared) channel instances +
+        # the channels executed atomically under self._scope_lock — see the
+        # `# 3. Run channels` block below. (profile_channel does not read scope.)
+        self._include_global = include_global
+        self._include_shared = include_shared
+
         # v3.5.0 diagnostic: stage timing inside retrieval (SLM_RECALL_TIMING=1).
         import os as _os_e
         import time as _time_e
@@ -159,8 +178,18 @@ class RetrievalEngine:
         # Dynamic top-k for aggregation queries
         effective_limit = 100 if strat.query_type == "aggregation" else limit
 
-        # 3. Run 4 channels
-        ch_results = self._run_channels(query, profile_id, strat)
+        # 3. Run channels. Set the scope flags on the shared channel instances
+        # and execute them under self._scope_lock so a concurrent recall can't
+        # interleave its scope visibility onto these channels mid-flight. The
+        # worker threads spawned inside _run_channels are joined before the lock
+        # releases, so every channel read sees THIS recall's flags.
+        with self._scope_lock:
+            for ch in (self._semantic, self._bm25, self._entity, self._temporal,
+                       self._hopfield, self._spreading_activation):
+                if ch is not None:
+                    ch.include_global = include_global
+                    ch.include_shared = include_shared
+            ch_results = self._run_channels(query, profile_id, strat)
         _em("run_channels")
         if profile_hits:
             ch_results["profile"] = profile_hits
@@ -657,7 +686,11 @@ class RetrievalEngine:
         needed = [fr.fact_id for fr in fused]
         if not needed:
             return {}
-        facts = self._db.get_facts_by_ids(needed, profile_id)
+        facts = self._db.get_facts_by_ids(
+            needed, profile_id,
+            include_global=getattr(self, '_include_global', True),
+            include_shared=getattr(self, '_include_shared', True),
+        )
         return {f.fact_id: f for f in facts}
 
     # -- Cross-encoder rerank -----------------------------------------------

package/src/superlocalmemory/retrieval/entity_channel.py

@@ -283,7 +283,7 @@ class EntityGraphChannel:
                 for fid in self._entity_to_facts.get(eid, ()):
                     activation[fid] = max(activation[fid], 1.0)
             else:
-                for fact in self._db.get_facts_by_entity(eid, profile_id):
+                for fact in self._db.get_facts_by_entity(eid, profile_id, include_global=getattr(self, 'include_global', False), include_shared=getattr(self, 'include_shared', False)):
                     activation[fact.fact_id] = max(activation[fact.fact_id], 1.0)
 
         # Spreading activation through graph edges (all in-memory O(1) lookups)
@@ -317,7 +317,7 @@ class EntityGraphChannel:
                     # NOTE: SQL fallback path does NOT use graph intelligence (P1/P2/P3).
                     # Graph intelligence is only available on the in-memory cache path.
                     # This fallback exists for mock/test DBs. See Phase 7 LLD H-01.
-                    for edge in self._db.get_edges_for_node(fid, profile_id):
+                    for edge in self._db.get_edges_for_node(fid, profile_id, include_global=getattr(self, 'include_global', False), include_shared=getattr(self, 'include_shared', False)):
                         neighbor = edge.target_id if edge.source_id == fid else edge.source_id
                         propagated = activation[fid] * self._decay
                         if propagated >= self._threshold and propagated > activation.get(neighbor, 0.0):
@@ -342,7 +342,7 @@ class EntityGraphChannel:
                 new_eids_sql = self._discover_entities(frontier, profile_id, visited_entities)
                 for eid in new_eids_sql:
                     visited_entities.add(eid)
-                    for fact in self._db.get_facts_by_entity(eid, profile_id):
+                    for fact in self._db.get_facts_by_entity(eid, profile_id, include_global=getattr(self, 'include_global', False), include_shared=getattr(self, 'include_shared', False)):
                         if hop_decay > activation.get(fact.fact_id, 0.0):
                             activation[fact.fact_id] = hop_decay
                             next_frontier.add(fact.fact_id)
@@ -438,7 +438,7 @@ class EntityGraphChannel:
                 for fid in self._entity_to_facts.get(eid, ()):
                     activation[fid] = max(activation[fid], 1.0)
             else:
-                for fact in self._db.get_facts_by_entity(eid, profile_id):
+                for fact in self._db.get_facts_by_entity(eid, profile_id, include_global=getattr(self, 'include_global', False), include_shared=getattr(self, 'include_shared', False)):
                     activation[fact.fact_id] = max(activation[fact.fact_id], 1.0)
 
         frontier = set(activation.keys())
@@ -628,7 +628,7 @@ class EntityGraphChannel:
             # Map entity scores to fact scores
             fact_scores: list[tuple[str, float]] = []
             for entity_id, score in scored:
-                facts = self._db.get_facts_by_entity(entity_id, profile_id)
+                facts = self._db.get_facts_by_entity(entity_id, profile_id, include_global=getattr(self, 'include_global', False), include_shared=getattr(self, 'include_shared', False))
                 for fact in facts:
                     fact_scores.append((fact.fact_id, score))
 

package/src/superlocalmemory/retrieval/hopfield_channel.py

@@ -248,7 +248,11 @@ class HopfieldChannel:
 
         # Stage 2: Load candidate facts
         candidate_ids = [fid for fid, _ in knn_results]
-        candidates = self._db.get_facts_by_ids(candidate_ids, profile_id)
+        candidates = self._db.get_facts_by_ids(
+            candidate_ids, profile_id,
+            include_global=getattr(self, 'include_global', False),
+            include_shared=getattr(self, 'include_shared', False),
+        )
         if not candidates:
             return []
 
@@ -304,7 +308,11 @@ class HopfieldChannel:
         # Step 2: Load facts (V3.3.12: cap to most recent 5000 to bound memory)
         # memory-bounding-02: push the cap into SQL (LIMIT) so we don't
         # deserialize the whole table just to slice it.
-        facts = self._db.get_all_facts(profile_id, limit=5000)
+        facts = self._db.get_all_facts(
+            profile_id, limit=5000,
+            include_global=getattr(self, 'include_global', False),
+            include_shared=getattr(self, 'include_shared', False),
+        )
         if not facts:
             return (None, [])
 

package/src/superlocalmemory/retrieval/semantic_channel.py

@@ -168,7 +168,11 @@ class SemanticChannel:
         # Step 2: Load only the candidate facts (NOT all facts)
         candidate_ids = [fid for fid, _ in knn_results]
         knn_scores = {fid: score for fid, score in knn_results}
-        facts = self._db.get_facts_by_ids(candidate_ids, profile_id)
+        facts = self._db.get_facts_by_ids(
+            candidate_ids, profile_id,
+            include_global=getattr(self, 'include_global', False),
+            include_shared=getattr(self, 'include_shared', False),
+        )
 
         if not facts:
             return [(fid, score) for fid, score in knn_results[:top_k]]
@@ -230,7 +234,11 @@ class SemanticChannel:
             q_mean = np.array(qm, dtype=np.float32)
             q_var = np.array(qv, dtype=np.float32)
 
-        facts = self._db.get_all_facts(profile_id)
+        facts = self._db.get_all_facts(
+            profile_id,
+            include_global=getattr(self, 'include_global', False),
+            include_shared=getattr(self, 'include_shared', False),
+        )
 
         scored: list[tuple[str, float]] = []
         for fact in facts:

package/src/superlocalmemory/server/recall_serializer.py

@@ -23,6 +23,8 @@ from __future__ import annotations
 import re
 from typing import Any
 
+from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
+
 
 # ---------------------------------------------------------------------------
 # F-2: Per-fact content clamp
@@ -161,7 +163,7 @@ def apply_source_content_discipline(
 def serialize_recall_response(
     response: Any,
     *,
-    limit: int = 10,
+    limit: int = CANONICAL_RECALL_LIMIT,
     memory_map: dict[str, str] | None = None,
     per_fact_max: int = 2400,
     total_max: int = 12000,