synap-git 0.2.0__py3-none-any.whl

synap_git/provider/gemini.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+import urllib.request
+from typing import Any
+
+from synap_git.provider.base import LLMProvider, LLMResponse
+
+
+class GeminiProvider(LLMProvider):
+    """Gemini API provider implementation."""
+
+    def __init__(self, api_key: str, default_model: str = "gemini-1.5-flash") -> None:
+        self.api_key = api_key
+        self.default_model = default_model
+
+    def generate(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        model: str | None = None,
+        max_tokens: int | None = None,
+        temperature: float = 0.2,
+    ) -> LLMResponse:
+        model_name = model or self.default_model
+        # Strip models/ prefix if present
+        if model_name.startswith("models/"):
+            model_name = model_name[7:]
+        url = f"https://generativelanguage.googleapis.com/v1beta/models/{model_name}:generateContent?key={self.api_key}"
+        headers = {
+            "Content-Type": "application/json",
+        }
+        payload: dict[str, Any] = {
+            "contents": [
+                {
+                    "role": "user",
+                    "parts": [{"text": user_prompt}],
+                }
+            ],
+            "systemInstruction": {"parts": [{"text": system_prompt}]},
+            "generationConfig": {
+                "temperature": temperature,
+            },
+        }
+        if max_tokens:
+            payload["generationConfig"]["maxOutputTokens"] = max_tokens
+
+        req = urllib.request.Request(
+            url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST"
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30.0) as resp:  # nosec B310
+                data = json.loads(resp.read().decode("utf-8"))
+            candidates = data.get("candidates", [])
+            if not candidates:
+                raise RuntimeError(f"Gemini API returned no candidates: {data}")
+            text_val = candidates[0]["content"]["parts"][0]["text"]
+            # Gemini does not always provide exact token counts in basic responses, or they might be in usageMetadata
+            usage = data.get("usageMetadata", {})
+            return LLMResponse(
+                content=str(text_val),
+                prompt_tokens=int(usage.get("promptTokenCount", 0)),
+                completion_tokens=int(usage.get("candidatesTokenCount", 0)),
+            )
+        except Exception as exc:
+            raise RuntimeError(f"Gemini generate failed: {exc}") from exc
+
+    def embed(
+        self,
+        text: str,
+        *,
+        model: str | None = None,
+    ) -> list[float]:
+        model_name = model or "text-embedding-004"
+        if not model_name.startswith("models/"):
+            model_name = f"models/{model_name}"
+        url = f"https://generativelanguage.googleapis.com/v1beta/{model_name}:embedContent?key={self.api_key}"
+        headers = {
+            "Content-Type": "application/json",
+        }
+        payload = {"model": model_name, "content": {"parts": [{"text": text}]}}
+        req = urllib.request.Request(
+            url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST"
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30.0) as resp:  # nosec B310
+                data = json.loads(resp.read().decode("utf-8"))
+            embedding = data["embedding"]["values"]
+            return [float(x) for x in embedding]
+        except Exception as exc:
+            raise RuntimeError(f"Gemini embed failed: {exc}") from exc
+
+    def count_tokens(self, text: str) -> int:
+        import tiktoken
+
+        encoding = tiktoken.get_encoding("cl100k_base")
+        return len(encoding.encode(text, disallowed_special=()))

synap_git/provider/mock.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import hashlib
+
+from synap_git.provider.base import LLMProvider, LLMResponse
+
+
+class MockLLMProvider(LLMProvider):
+    """Mock LLM provider returning deterministic replies."""
+
+    def __init__(self, preset_replies: dict[str, str] | None = None) -> None:
+        self.preset_replies = preset_replies or {}
+
+    def generate(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        model: str | None = None,
+        max_tokens: int | None = None,
+        temperature: float = 0.2,
+    ) -> LLMResponse:
+        _ = (model, max_tokens, temperature)
+        # Match preset replies if any substring matches
+        combined = f"{system_prompt}\n{user_prompt}".lower()
+        for pattern, reply in self.preset_replies.items():
+            if pattern.lower() in combined:
+                return LLMResponse(content=reply, prompt_tokens=10, completion_tokens=10)
+
+        # Default fallback responses based on keywords in prompt
+        if "explain" in combined or "how does" in combined:
+            content = (
+                "Mock Explanation: This subsystem handles core state management. "
+                "It uses an append-only event log stored in SQLite and serializes active graph nodes."
+            )
+        elif "overlay" in combined or "annotate" in combined:
+            content = "Mock Overlay: This module is stable and optimized. It implements caching to prevent graph explosion."
+        else:
+            content = f"Mock LLM generated response for query. Model: {model or 'mock-model'}."
+
+        return LLMResponse(
+            content=content,
+            prompt_tokens=20,
+            completion_tokens=30,
+        )
+
+    def embed(
+        self,
+        text: str,
+        *,
+        model: str | None = None,
+    ) -> list[float]:
+        _ = model
+        # Deterministic pseudo-embedding based on string md5 hash
+        hasher = hashlib.md5(text.encode("utf-8"), usedforsecurity=False)  # noqa: S324
+        digest = hasher.digest()
+        embedding = []
+        for i in range(16):
+            # Normalize to [-1.0, 1.0] range
+            val = (digest[i] - 128.0) / 128.0
+            embedding.append(val)
+        # Pad or repeat to match embedding dimensions (e.g. 128 dimensions)
+        return embedding * 8
+
+    def count_tokens(self, text: str) -> int:
+        return len(text.split())

synap_git/provider/ollama.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+import urllib.request
+from typing import Any
+
+from synap_git.provider.base import LLMProvider, LLMResponse
+
+
+class OllamaProvider(LLMProvider):
+    """Ollama API provider implementation for local running models."""
+
+    def __init__(
+        self, base_url: str = "http://127.0.0.1:11434", default_model: str = "llama3"
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.default_model = default_model
+
+    def generate(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        model: str | None = None,
+        max_tokens: int | None = None,
+        temperature: float = 0.2,
+    ) -> LLMResponse:
+        model_name = model or self.default_model
+        url = f"{self.base_url}/api/chat"
+        headers = {
+            "Content-Type": "application/json",
+        }
+        payload: dict[str, Any] = {
+            "model": model_name,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+            "options": {
+                "temperature": temperature,
+            },
+            "stream": False,
+        }
+        if max_tokens:
+            payload["options"]["num_predict"] = max_tokens
+
+        req = urllib.request.Request(
+            url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST"
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=300.0) as resp:  # nosec B310
+                data = json.loads(resp.read().decode("utf-8"))
+            message = data["message"]
+            # Ollama provides prompt_eval_count and eval_count
+            prompt_tokens = int(data.get("prompt_eval_count", 0))
+            completion_tokens = int(data.get("eval_count", 0))
+            return LLMResponse(
+                content=str(message["content"]),
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+            )
+        except urllib.error.HTTPError as exc:
+            err_body = exc.read().decode("utf-8")
+            raise RuntimeError(f"Ollama generate HTTP error {exc.code}: {err_body}") from exc
+        except Exception as exc:
+            raise RuntimeError(f"Ollama generate failed: {exc}") from exc
+
+    def embed(
+        self,
+        text: str,
+        *,
+        model: str | None = None,
+    ) -> list[float]:
+        model_name = model or self.default_model
+        url = f"{self.base_url}/api/embeddings"
+        headers = {
+            "Content-Type": "application/json",
+        }
+        payload = {"model": model_name, "prompt": text}
+        req = urllib.request.Request(
+            url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST"
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=60.0) as resp:  # nosec B310
+                data = json.loads(resp.read().decode("utf-8"))
+            embedding = data["embedding"]
+            return [float(x) for x in embedding]
+        except urllib.error.HTTPError as exc:
+            err_body = exc.read().decode("utf-8")
+            raise RuntimeError(f"Ollama embed HTTP error {exc.code}: {err_body}") from exc
+        except Exception as exc:
+            raise RuntimeError(f"Ollama embed failed: {exc}") from exc
+
+    def count_tokens(self, text: str) -> int:
+        import tiktoken
+
+        encoding = tiktoken.get_encoding("cl100k_base")
+        return len(encoding.encode(text, disallowed_special=()))

synap_git/provider/openai.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import json
+import urllib.request
+from typing import Any
+
+from synap_git.provider.base import LLMProvider, LLMResponse
+
+
+class OpenAIProvider(LLMProvider):
+    """OpenAI API provider implementation."""
+
+    def __init__(self, api_key: str, default_model: str = "gpt-4o-mini") -> None:
+        self.api_key = api_key
+        self.default_model = default_model
+
+    def generate(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        model: str | None = None,
+        max_tokens: int | None = None,
+        temperature: float = 0.2,
+    ) -> LLMResponse:
+        model_name = model or self.default_model
+        url = "https://api.openai.com/v1/chat/completions"
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+        }
+        payload: dict[str, Any] = {
+            "model": model_name,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+            "temperature": temperature,
+        }
+        if max_tokens:
+            payload["max_tokens"] = max_tokens
+
+        req = urllib.request.Request(  # noqa: S310
+            url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST"
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30.0) as resp:  # nosec B310
+                data = json.loads(resp.read().decode("utf-8"))
+            choice = data["choices"][0]["message"]
+            usage = data.get("usage", {})
+            return LLMResponse(
+                content=str(choice["content"]),
+                prompt_tokens=int(usage.get("prompt_tokens", 0)),
+                completion_tokens=int(usage.get("completion_tokens", 0)),
+            )
+        except Exception as exc:
+            raise RuntimeError(f"OpenAI generate failed: {exc}") from exc
+
+    def embed(
+        self,
+        text: str,
+        *,
+        model: str | None = None,
+    ) -> list[float]:
+        model_name = model or "text-embedding-3-small"
+        url = "https://api.openai.com/v1/embeddings"
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+        }
+        payload = {"input": text, "model": model_name}
+        req = urllib.request.Request(  # noqa: S310
+            url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST"
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30.0) as resp:  # nosec B310
+                data = json.loads(resp.read().decode("utf-8"))
+            embedding = data["data"][0]["embedding"]
+            return [float(x) for x in embedding]
+        except Exception as exc:
+            raise RuntimeError(f"OpenAI embed failed: {exc}") from exc
+
+    def count_tokens(self, text: str) -> int:
+        import tiktoken
+
+        try:
+            encoding = tiktoken.encoding_for_model(self.default_model)
+        except KeyError:
+            encoding = tiktoken.get_encoding("cl100k_base")
+        return len(encoding.encode(text, disallowed_special=()))

synap_git/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561 compliance

synap_git/retrieval/engine.py

@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+from uuid import uuid4
+
+import tiktoken
+
+from synap_git.provider.base import LLMProvider
+from synap_git.storage.sqlite import SynapStore
+
+
+@dataclass(frozen=True)
+class TraceElement:
+    stable_id: str
+    name: str
+    path: str
+    score: float
+    reason: str
+    tokens: int
+
+
+class HybridRetrievalEngine:
+    """Production-grade hybrid retrieval with exact token budgeting and traces."""
+
+    def __init__(
+        self,
+        *,
+        store: SynapStore,
+        llm_provider: LLMProvider | None,
+        trace_store: Any | None = None,
+    ) -> None:
+        self.store = store
+        self.llm_provider = llm_provider
+        # Use tiktoken for exact budgeting
+        self.tokenizer = tiktoken.get_encoding("cl100k_base")
+        self.max_expansion_depth = 2
+        self.trace_store = trace_store
+
+    def retrieve(
+        self,
+        query: str,
+        *,
+        max_tokens: int = 4000,
+        is_dirty: bool = False,
+    ) -> tuple[str, list[dict[str, Any]], dict[str, Any]]:
+        """Perform deterministic 4-stage retrieval: Temporal -> Structural -> Lexical -> Semantic."""
+        import time
+
+        t_start = time.perf_counter()
+        trace_id = str(uuid4())
+        start_time = datetime.now(UTC)
+
+        # 1. Temporal Filter: Handled by SynapStore methods which operate on current state
+        # (In a more advanced version, we'd filter by specific commit/branch here)
+
+        # 2. & 3. Lexical + Structural
+        query_words = {word.lower().strip() for word in query.split() if len(word) > 2}
+
+        # Start with Lexical matches
+        lexical_candidates: dict[str, dict[str, Any]] = {}
+        for word in query_words:
+            symbols = self.store.get_symbols_by_name(word)
+            for sym in symbols:
+                sid = sym["symbol_id"]
+                if sid not in lexical_candidates:
+                    lexical_candidates[sid] = {**sym, "reason": f"lexical:'{word}'"}
+
+        t_lexical = time.perf_counter()
+
+        # Expand to Structural neighborhood
+        structural_candidates: dict[str, dict[str, Any]] = {}
+        structural_hops = []
+        for sid in list(lexical_candidates.keys()):
+            neighbors = self.store.get_neighborhood(sid, depth=self.max_expansion_depth)
+            for n in neighbors:
+                nid = n["symbol_id"]
+                dist = n.get("distance", 0)
+                structural_hops.append(
+                    {
+                        "from_symbol": lexical_candidates[sid]["name"],
+                        "to_symbol": n["name"],
+                        "distance": dist,
+                    }
+                )
+                if nid not in structural_candidates:
+                    structural_candidates[nid] = {**n, "reason": f"structural:dist={dist}"}
+
+        t_structural = time.perf_counter()
+
+        # Combine and Rank
+        combined = {**structural_candidates, **lexical_candidates}
+
+        # 4. Semantic Ranking (simplified for recovery)
+        ranked = self._rank_candidates(query, list(combined.values()))
+
+        t_ranking = time.perf_counter()
+
+        # Context Packing with exact token budgeting
+        packed_blocks = []
+        grounding_sources = []
+        trace_elements = []
+        tokens_used = 0
+
+        # Reserve buffer for system/user prompts
+        reserved_buffer = 600
+        retrieval_budget = max_tokens - reserved_buffer
+
+        for cand, score in ranked:
+            path = cand["source_path"]
+            name = cand["name"]
+            kind = cand["kind"]
+            lines = f"{cand['start_line']}-{cand['end_line']}"
+            reason = cand["reason"]
+
+            block = f"### File: {path}\nSymbol: {name} ({kind})\nLines: {lines}\nReason: {reason}\n"
+            block_tokens = len(self.tokenizer.encode(block))
+
+            if tokens_used + block_tokens > retrieval_budget:
+                remaining = retrieval_budget - tokens_used
+                trace_elements.append(
+                    TraceElement(
+                        stable_id=cand["symbol_id"],
+                        name=name,
+                        path=path,
+                        score=score,
+                        reason=f"truncated:over_budget. Requires {block_tokens} tokens, but only {remaining} remain.",
+                        tokens=block_tokens,
+                    )
+                )
+                continue
+
+            packed_blocks.append(block)
+            grounding_sources.append(cand)
+            tokens_used += block_tokens
+
+            trace_elements.append(
+                TraceElement(
+                    stable_id=cand["symbol_id"],
+                    name=name,
+                    path=path,
+                    score=score,
+                    reason=reason,
+                    tokens=block_tokens,
+                )
+            )
+
+        context_str = "\n".join(packed_blocks)
+
+        # Gating: Only approved lessons affect retrieval
+        approved_lessons = self.store.get_lessons("approved")
+        if approved_lessons:
+            lesson_blocks = ["# APPROVED SYSTEM MEMORY (CRITICAL: MUST ADHERE)"]
+            for i, lesson in enumerate(approved_lessons, 1):
+                lesson_blocks.append(
+                    f"[{i}] DO NOT DO THIS: {lesson['what_failed']} - BECAUSE: {lesson['why_failed']}"
+                )
+            context_str = "\n".join(lesson_blocks) + "\n\n" + context_str
+
+        system_prompt = (
+            "You are an AI coding assistant with deep structural repository understanding.\n"
+            "Answer the user query based ONLY on the provided grounded structural context.\n"
+            "Cite source files and symbol names in your explanation.\n"
+        )
+        user_prompt = f"Repository Context:\n{context_str}\n\nUser Question: {query}\n"
+
+        t_packing = time.perf_counter()
+
+        if self.llm_provider:
+            response = self.llm_provider.generate(
+                system_prompt=system_prompt,
+                user_prompt=user_prompt,
+                temperature=0.1,
+            )
+            answer_content = response.content
+            try:
+                provider_name = self.llm_provider.__class__.__name__.replace("Provider", "").lower()
+                model_name = getattr(self.llm_provider, "default_model", "unknown")
+                self.store.put_llm_call(
+                    provider=provider_name,
+                    model=model_name,
+                    input_tokens=response.prompt_tokens,
+                    output_tokens=response.completion_tokens,
+                    purpose="retrieval",
+                )
+            except Exception:
+                pass
+        else:
+            answer_content = (
+                "Mode A (Structural Only): Context retrieved, but LLM generation is disabled."
+            )
+
+        t_llm = time.perf_counter()
+
+        token_allocation = {
+            "max_tokens": max_tokens,
+            "reserved_buffer": reserved_buffer,
+            "retrieval_budget": retrieval_budget,
+            "tokens_used": tokens_used,
+            "remaining": retrieval_budget - tokens_used,
+        }
+
+        timeline = {
+            "lexical_search_ms": (t_lexical - t_start) * 1000,
+            "structural_expansion_ms": (t_structural - t_lexical) * 1000,
+            "semantic_ranking_ms": (t_ranking - t_structural) * 1000,
+            "context_packing_ms": (t_packing - t_ranking) * 1000,
+            "llm_generation_ms": (t_llm - t_packing) * 1000,
+            "total_ms": (t_llm - t_start) * 1000,
+        }
+
+        trace = {
+            "trace_id": trace_id,
+            "query": query,
+            "nodes_explored": len(combined),
+            "tokens_used": tokens_used,
+            "token_allocation": token_allocation,
+            "timeline": timeline,
+            "elements": [e.__dict__ for e in trace_elements[:40]],
+            "structural_hops": structural_hops[:20],
+            "dirty_tree_warning": is_dirty,
+            "latency_ms": (t_llm - t_start) * 1000,
+        }
+
+        if self.trace_store:
+            self.trace_store.record_trace(
+                trace_type="retrieval",
+                summary=f"Retrieval for query: {query}",
+                details=trace,
+            )
+
+        return answer_content, grounding_sources, trace
+
+    def _rank_candidates(
+        self, query: str, candidates: list[dict[str, Any]]
+    ) -> list[tuple[dict[str, Any], float]]:
+        """Rank based on priority: Lexical > Structural > Distance."""
+        ranked = []
+        for cand in candidates:
+            # Base score from reason
+            reason = cand["reason"]
+            if reason.startswith("lexical"):
+                score = 1.0
+            elif reason.startswith("structural"):
+                dist = int(reason.split("=")[-1])
+                score = 0.8**dist
+            else:
+                score = 0.5
+
+            # Name match boost
+            if any(w in cand["name"].lower() for w in query.lower().split()):
+                score += 0.2
+
+            ranked.append((cand, score))
+
+        ranked.sort(key=lambda x: x[1], reverse=True)
+        return ranked