cherry-docs 0.2.0__py3-none-any.whl

app/services/memory_profiles.py

@@ -0,0 +1,88 @@
+"""Prompt profiles for the CherryDocs internal memory distiller."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+MemoryPromptProfileName = Literal["default", "noise_strict", "verification_first"]
+
+
+@dataclass(frozen=True)
+class MemoryPromptProfile:
+    name: str
+    description: str
+    system_focus: str
+    keep_rules: tuple[str, ...]
+    extra_rules: tuple[str, ...]
+    extraction_bias: tuple[str, ...]
+
+
+_PROMPT_PROFILES: dict[str, MemoryPromptProfile] = {
+    "default": MemoryPromptProfile(
+        name="default",
+        description="Balanced default profile for general engineering continuity.",
+        system_focus="Keep only information that would be expensive for another AI to rediscover later.",
+        keep_rules=(
+            "Do not preserve generic chatter, routine status updates, or cheap file-edit summaries.",
+            "Do not preserve secrets, passwords, API keys, tokens, or obviously sensitive content.",
+            "Never extract test counts, lint error counts, CI status, or PR review status as facts — these change daily.",
+        ),
+        extra_rules=(
+            "Prefer the strongest concrete decision or warning over multiple paraphrases of the same idea.",
+            "Avoid meta commentary about model choice or architecture quality unless it is itself a lasting project decision.",
+        ),
+        extraction_bias=(
+            "what was attempted",
+            "what failed and why",
+            "what decision was made and why",
+            "what tradeoff mattered",
+            "what reusable procedure was discovered",
+            "what warning another AI should not repeat",
+            "whether an explicit verification/test step passed or failed",
+        ),
+    ),
+    "noise_strict": MemoryPromptProfile(
+        name="noise_strict",
+        description="More aggressive profile for cutting broad chatter and weak notes.",
+        system_focus="Keep only high-signal memories with clear rediscovery cost; aggressively drop broad planning chatter.",
+        keep_rules=(
+            "Discard routine planning updates, generic progress notes, and repeated summaries unless they record a concrete decision, failure, or constraint.",
+            "Do not preserve secrets, passwords, API keys, tokens, or obviously sensitive content.",
+        ),
+        extra_rules=(
+            "If two candidates overlap, keep only the more concrete one.",
+            "When uncertain whether something is durable memory or chatter, prefer noise unless there is explicit evidence of a decision, failure, or verification result.",
+        ),
+        extraction_bias=(
+            "what failed and why",
+            "what hidden constraint or warning matters later",
+            "what exact decision changed the direction of work",
+            "what verification/test result invalidated or confirmed an approach",
+        ),
+    ),
+    "verification_first": MemoryPromptProfile(
+        name="verification_first",
+        description="Bias toward preserving test and verification outcomes alongside the reasoning they proved.",
+        system_focus="Prefer memories that connect decisions or warnings to explicit verification outcomes.",
+        keep_rules=(
+            "Preserve decisions, warnings, and conclusions that were confirmed or falsified by tests or verification runs.",
+            "Do not preserve secrets, passwords, API keys, tokens, or obviously sensitive content.",
+        ),
+        extra_rules=(
+            "When a verification step passed or failed, connect that outcome to the decision, warning, or conclusion it supports.",
+            "Avoid procedural memory unless the trace clearly shows a repeatable verification workflow another AI should reuse.",
+        ),
+        extraction_bias=(
+            "which verification/test step passed or failed",
+            "what decision or warning that verification supports",
+            "what failure mode was proven by a test result",
+            "what workflow reliably verifies the same thing again",
+        ),
+    ),
+}
+
+
+def resolve_memory_prompt_profile(profile: MemoryPromptProfileName | str | None = None) -> MemoryPromptProfile:
+    key = " ".join(str(profile or "").split()).strip().lower() or "default"
+    return _PROMPT_PROFILES.get(key, _PROMPT_PROFILES["default"])

app/services/memory_providers.py

@@ -0,0 +1,113 @@
+"""LLM provider implementations for memory distillation.
+
+Priority when no provider is specified:
+  1. ANTHROPIC_API_KEY set → AnthropicMemoryProvider (claude-haiku — best JSON quality)
+  2. Fallback → OllamaMemoryProvider (local, no API key needed)
+
+Override with CHERRY_DISTILL_PROVIDER=anthropic|ollama.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any
+
+import httpx
+
+_OLLAMA_CHAT_URL = os.getenv("OLLAMA_CHAT_URL", "http://127.0.0.1:11434/api/chat")
+_DEFAULT_OLLAMA_MODEL = os.getenv("CHERRY_OLLAMA_MODEL", "qwen2.5:7b-instruct")
+_DEFAULT_ANTHROPIC_MODEL = os.getenv("CHERRY_ANTHROPIC_MODEL", "claude-3-5-haiku-20241022")
+
+
+class OllamaMemoryProvider:
+    """Local Ollama provider — no API key required. Good for dev/offline use."""
+
+    def __init__(
+        self,
+        *,
+        model: str = _DEFAULT_OLLAMA_MODEL,
+        url: str = _OLLAMA_CHAT_URL,
+        timeout: float = 60.0,
+    ) -> None:
+        self.model = model
+        self.url = url
+        self.timeout = timeout
+
+    def extract(self, prompt: str) -> dict[str, Any]:
+        payload = {
+            "model": self.model,
+            "messages": [{"role": "user", "content": prompt}],
+            "stream": False,
+            "format": "json",
+            "options": {"temperature": 0.1},
+        }
+        with httpx.Client(timeout=self.timeout) as client:
+            response = client.post(self.url, json=payload)
+            response.raise_for_status()
+        data = response.json()
+        content = ((data.get("message") or {}).get("content") or "").strip()
+        if not content:
+            return {}
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError:
+            return {}
+
+
+class AnthropicMemoryProvider:
+    """Anthropic Claude provider — best JSON extraction quality.
+
+    Requires ANTHROPIC_API_KEY or explicit api_key param.
+    Default model: claude-3-5-haiku-20241022 (fast, cheap, excellent schema adherence).
+    Override with CHERRY_ANTHROPIC_MODEL env var.
+    """
+
+    def __init__(
+        self,
+        *,
+        model: str = _DEFAULT_ANTHROPIC_MODEL,
+        api_key: str | None = None,
+        timeout: float = 60.0,
+    ) -> None:
+        self.model = model
+        self.api_key = api_key  # None → SDK reads ANTHROPIC_API_KEY from env
+        self.timeout = timeout
+
+    def extract(self, prompt: str) -> dict[str, Any]:
+        try:
+            import anthropic as _anthropic
+        except ImportError as exc:
+            raise ImportError(
+                "anthropic package required: pip install 'cherry-docs[anthropic]'"
+            ) from exc
+
+        client = _anthropic.Anthropic(api_key=self.api_key, timeout=self.timeout)
+        response = client.messages.create(
+            model=self.model,
+            max_tokens=1024,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        content = (response.content[0].text if response.content else "").strip()
+        if not content:
+            return {}
+        # Strip markdown fences Claude sometimes wraps around JSON
+        content = re.sub(r"^```(?:json)?\s*\n?", "", content)
+        content = re.sub(r"\n?```\s*$", "", content)
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError:
+            return {}
+
+
+def resolve_provider() -> OllamaMemoryProvider | AnthropicMemoryProvider:
+    """Return the best available provider based on environment variables.
+
+    Explicit override: CHERRY_DISTILL_PROVIDER=anthropic|ollama
+    Auto-detect: ANTHROPIC_API_KEY set → Anthropic, else → Ollama
+    """
+    explicit = os.getenv("CHERRY_DISTILL_PROVIDER", "").strip().lower()
+    if explicit == "anthropic" or (not explicit and os.getenv("ANTHROPIC_API_KEY")):
+        return AnthropicMemoryProvider()
+    return OllamaMemoryProvider()

app/services/memory_retrieval.py

@@ -0,0 +1,327 @@
+"""Deterministic retrieval over promoted memory plus raw captured evidence."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.services.memory_lifecycle import MemoryRecord, build_event_evidence_id
+from app.services.memory_retrieval_scoring import (
+    _EVENT_TYPE_RANK,
+    _KIND_BONUS_BY_QUESTION,
+    _compute_trust_score,
+    _compute_utility_score,
+    _parse_timestamp,
+    _recentness_bonus,
+    _specificity_bonus,
+)
+from app.services.memory_retrieval_text import (
+    _file_overlap,
+    _file_query_hint,
+    _question_mode,
+    _score_overlap,
+    _trim_text,
+    _wants_recent_context,
+)
+
+
+class EvidenceSnippet(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    evidence_id: str
+    event_type: str
+    timestamp: str | None = None
+    text: str = ""
+    files: list[str] = Field(default_factory=list)
+    score: float = 0.0
+
+
+class RetrievedMemory(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    memory: MemoryRecord
+    score: float
+    trust_score: float = 0.0
+    utility_score: float = 0.0
+    relevance_score: float = 0.0
+    age_days: int = 0
+    reasons: list[str] = Field(default_factory=list)
+
+
+class RetrievalAnswer(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    question: str
+    answer: str
+    summary: str
+    memories: list[RetrievedMemory] = Field(default_factory=list)
+    evidence: list[EvidenceSnippet] = Field(default_factory=list)
+    gaps: list[str] = Field(default_factory=list)
+    stale: bool = False
+
+
+def _event_by_id(events: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
+    return {build_event_evidence_id(event): event for event in events}
+
+
+def collect_event_evidence(
+    question: str,
+    events: list[dict[str, Any]],
+    *,
+    memory_hint: str | None = None,
+    preferred_evidence_ids: list[str] | None = None,
+    file_path: str | None = None,
+    limit: int = 4,
+) -> list[EvidenceSnippet]:
+    event_index = _event_by_id(events)
+    file_hint = _file_query_hint(file_path)
+    anchor_text = f"{question} {memory_hint or ''} {file_hint}".strip()
+    wants_recent = _wants_recent_context(question)
+    now = datetime.now(UTC)
+
+    def _score_event(event: dict, evidence_id: str, base: float, min_overlap: float) -> EvidenceSnippet | None:
+        text = str(event.get("text") or "").strip()
+        if not text:
+            return None
+        overlap = _score_overlap(anchor_text, text)
+        ev_files = [str(p) for p in (event.get("files") or []) if str(p).strip()]
+        file_score = _file_overlap(file_path, ev_files)
+        if overlap < min_overlap and file_score <= 0:
+            return None
+        event_type = str(event.get("event_type") or "unknown")
+        timestamp = str(event.get("timestamp") or "")
+        recent_bonus = 0.0
+        parsed_ts = _parse_timestamp(timestamp)
+        if wants_recent and parsed_ts is not None:
+            recent_bonus = _recentness_bonus(
+                max(0, (now - parsed_ts).days), strong_window_days=7, medium_window_days=30
+            )
+        return EvidenceSnippet(
+            evidence_id=evidence_id,
+            event_type=event_type,
+            timestamp=timestamp,
+            text=_trim_text(text),
+            files=ev_files,
+            score=base + overlap + (file_score * 1.25) + _EVENT_TYPE_RANK.get(event_type, 0.5) + recent_bonus,
+        )
+
+    preferred: list[EvidenceSnippet] = []
+    seen_ids: set[str] = set()
+    for evidence_id in preferred_evidence_ids or []:
+        event = event_index.get(evidence_id)
+        if not event:
+            continue
+        snippet = _score_event(event, evidence_id, base=2.5, min_overlap=0.2)
+        if snippet:
+            preferred.append(snippet)
+            seen_ids.add(evidence_id)
+
+    ranked: list[EvidenceSnippet] = []
+    for event in events:
+        evidence_id = build_event_evidence_id(event)
+        if evidence_id in seen_ids:
+            continue
+        snippet = _score_event(event, evidence_id, base=0.0, min_overlap=0.01)
+        if snippet:
+            ranked.append(snippet)
+
+    def _sort_key(item: EvidenceSnippet) -> tuple:
+        return (item.score, _parse_timestamp(item.timestamp or "") or datetime.min.replace(tzinfo=UTC))
+
+    ranked.sort(key=_sort_key, reverse=True)
+    if preferred:
+        preferred.sort(key=_sort_key, reverse=True)
+        return preferred[: max(1, limit)]
+    return [*preferred, *ranked][: max(1, limit)]
+
+
+def rank_memories(
+    question: str,
+    records: list[MemoryRecord],
+    *,
+    branch: str | None = None,
+    file_path: str | None = None,
+    limit: int = 5,
+) -> list[RetrievedMemory]:
+    mode = _question_mode(question)
+    wants_recent = _wants_recent_context(question)
+    now = datetime.now(UTC)
+    file_hint = _file_query_hint(file_path)
+    ranked: list[RetrievedMemory] = []
+    for record in records:
+        if record.status == "archived":
+            continue
+        text = f"{record.kind} {record.memory_type} {record.summary} {record.rationale} {record.topic_key}".strip()
+        overlap = _score_overlap(f"{question} {file_hint}".strip(), text)
+        branch_score = 0.3 if branch and record.branch and branch == record.branch else 0.0
+        file_score = _file_overlap(file_path, record.files) * 1.25
+        if overlap <= 0 and file_score <= 0:
+            continue
+        confidence_score = max(0.0, min(1.0, record.confidence))
+        kind_bonus = _KIND_BONUS_BY_QUESTION.get(mode, {}).get(record.kind, 0.0)
+        try:
+            updated_at = datetime.fromisoformat(record.updated_at)
+            age_days = max(0, (now - updated_at).days)
+        except Exception:
+            age_days = 0
+        freshness_penalty = min(0.75, age_days / 3650)
+        recent_bonus = _recentness_bonus(age_days) if wants_recent else 0.0
+        trust_score = _compute_trust_score(record, age_days=age_days)
+        utility_score = _compute_utility_score(record)
+        specificity_bonus = _specificity_bonus(
+            record, wants_recent=wants_recent, file_path=file_path, file_score=file_score, overlap=overlap,
+        )
+        relevance_score = overlap + branch_score + file_score + kind_bonus + recent_bonus + specificity_bonus
+        score = relevance_score + trust_score + (utility_score * 0.35) + confidence_score - freshness_penalty
+        reasons = [f"status={record.status}", f"overlap={overlap:.2f}"]
+        if branch_score:
+            reasons.append("branch_match")
+        if file_score:
+            reasons.append(f"file_overlap=+{file_score:.2f}")
+        if kind_bonus:
+            reasons.append(f"kind_bonus={kind_bonus:+.1f}")
+        if freshness_penalty:
+            reasons.append(f"freshness_penalty=-{freshness_penalty:.2f}")
+        if recent_bonus:
+            reasons.append(f"recent_bonus=+{recent_bonus:.2f}")
+        if specificity_bonus:
+            reasons.append(f"specificity_bonus={specificity_bonus:+.2f}")
+        if record.supersedes:
+            reasons.append("supersedes_older_memory")
+        if record.retrieval_count:
+            reasons.append(f"retrieval_count={record.retrieval_count}")
+        if record.evidence_hit_count:
+            reasons.append(f"evidence_hits={record.evidence_hit_count}")
+        ranked.append(RetrievedMemory(
+            memory=record, score=score, trust_score=trust_score,
+            utility_score=utility_score, relevance_score=relevance_score,
+            age_days=age_days, reasons=reasons,
+        ))
+    ranked.sort(
+        key=lambda item: (
+            item.score,
+            _parse_timestamp(item.memory.updated_at) or datetime.min.replace(tzinfo=UTC),
+        ),
+        reverse=True,
+    )
+    return ranked[: max(1, limit)]
+
+
+def apply_retrieval_feedback(
+    records: list[MemoryRecord],
+    retrieved: list[RetrievedMemory],
+    evidence: list[EvidenceSnippet],
+    *,
+    now: datetime | None = None,
+) -> list[MemoryRecord]:
+    if not retrieved:
+        return records
+    reference = (now or datetime.now(UTC)).isoformat()
+    retrieved_ids = {item.memory.memory_id for item in retrieved}
+    evidence_ids = {item.evidence_id for item in evidence}
+    updated: list[MemoryRecord] = []
+    for record in records:
+        if record.memory_id not in retrieved_ids:
+            updated.append(record)
+            continue
+        evidence_hits = sum(1 for evidence_id in record.evidence if evidence_id in evidence_ids)
+        update_payload: dict[str, Any] = {
+            "retrieval_count": max(0, record.retrieval_count) + 1,
+            "last_retrieved_at": reference,
+        }
+        if evidence_hits:
+            update_payload["evidence_hit_count"] = max(0, record.evidence_hit_count) + evidence_hits
+            update_payload["last_evidence_hit_at"] = reference
+        updated.append(record.model_copy(update=update_payload))
+    return updated
+
+
+def synthesize_retrieval_answer(
+    question: str,
+    records: list[MemoryRecord],
+    events: list[dict[str, Any]],
+    *,
+    branch: str | None = None,
+    file_path: str | None = None,
+) -> RetrievalAnswer:
+    memories = rank_memories(question, records, branch=branch, file_path=file_path)
+    memory_hint = ""
+    if memories:
+        top = memories[0].memory
+        memory_hint = f"{top.summary} {top.rationale}".strip()
+    evidence = collect_event_evidence(
+        question, events, memory_hint=memory_hint,
+        preferred_evidence_ids=memories[0].memory.evidence if memories else None,
+        file_path=file_path,
+    )
+    if not memories:
+        if evidence:
+            top_evidence = evidence[0]
+            if file_path:
+                answer = (
+                    f"No promoted memory matched `{file_path}` yet. "
+                    f"Closest raw evidence for that file is from `{top_evidence.event_type}`: {top_evidence.text}"
+                )
+            else:
+                answer = f"Closest captured evidence for this question is from `{top_evidence.event_type}`: {top_evidence.text}"
+            return RetrievalAnswer(
+                question=question, answer=answer, summary="Evidence-only fallback answer.",
+                memories=[], evidence=evidence,
+                gaps=(
+                    ["No promoted memory matched yet; this answer is evidence-only.",
+                     f"Used file hint `{file_path}` to rank raw evidence."]
+                    if file_path
+                    else ["No promoted memory matched yet; this answer is evidence-only."]
+                ),
+            )
+        return RetrievalAnswer(
+            question=question, answer="I could not find a strong memory-backed answer yet.",
+            summary="No strong matching memory found.", memories=[], evidence=evidence,
+            gaps=["Need more captured evidence or promoted memory for this topic."],
+        )
+
+    top = memories[0].memory
+    lines = [top.summary]
+    if top.rationale:
+        lines.append(f"Why: {top.rationale}")
+    if top.status != "active":
+        lines.append(f"Status: {top.status}")
+    if top.confidence < 0.8:
+        lines.append(f"Confidence: {top.confidence:.2f}")
+    if top.supersedes:
+        lines.append(f"Supersedes: {', '.join(top.supersedes)}")
+
+    summary = top.summary
+    if len(memories) > 1:
+        summary += f" (+{len(memories) - 1} related memory matches)"
+
+    top_match = memories[0]
+    stale = top_match.age_days > 30
+    gaps: list[str] = []
+    if not evidence:
+        gaps.append("No direct raw-evidence snippet matched this question.")
+    if top_match.memory.status != "active":
+        gaps.append("Top memory is not currently active.")
+    if top_match.trust_score < 1.0:
+        gaps.append("Top memory is relatively weak or provisional; verify against current code or newer evidence.")
+    if stale:
+        gaps.append(f"Top memory is {top_match.age_days} days old — verify it still reflects current code.")
+
+    return RetrievalAnswer(
+        question=question, answer="\n".join(lines), summary=summary,
+        memories=memories, evidence=evidence, gaps=gaps, stale=stale,
+    )
+
+
+__all__ = [
+    "EvidenceSnippet",
+    "RetrievalAnswer",
+    "RetrievedMemory",
+    "apply_retrieval_feedback",
+    "build_event_evidence_id",
+    "collect_event_evidence",
+    "rank_memories",
+    "synthesize_retrieval_answer",
+]

app/services/memory_retrieval_scoring.py

@@ -0,0 +1,106 @@
+"""Memory ranking scoring helpers — trust, utility, freshness, and specificity."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from app.services.memory_lifecycle import MemoryRecord
+
+_STATUS_RANK = {
+    "active": 4.0,
+    "tentative": 2.5,
+    "stale": 2.0,
+    "contradicted": 1.5,
+    "superseded": 1.0,
+    "archived": 0.5,
+}
+_EVENT_TYPE_RANK = {
+    "assistant_output": 1.5,
+    "remember": 1.4,
+    "tool_result": 1.2,
+    "shell_result": 1.1,
+    "test_result": 1.3,
+    "user_prompt": 0.8,
+}
+_KIND_BONUS_BY_QUESTION = {
+    "why": {"decision": 0.8, "warning": 0.4, "pattern": 0.2, "procedure": -0.2},
+    "what": {"fact": 0.4, "conclusion": 0.3, "decision": 0.1, "warning": 0.1, "attempt": -0.1},
+    "what_failed": {"attempt": 0.8, "warning": 0.5, "decision": 0.1},
+    "how": {"procedure": 0.8, "warning": 0.2, "decision": 0.1},
+}
+
+
+def _parse_timestamp(value: str | None) -> datetime | None:
+    text = (value or "").strip().replace("Z", "+00:00")
+    if not text:
+        return None
+    try:
+        return datetime.fromisoformat(text)
+    except Exception:
+        return None
+
+
+def _recentness_bonus(age_days: int, *, strong_window_days: int = 14, medium_window_days: int = 45) -> float:
+    if age_days <= 1:
+        return 0.5
+    if age_days <= strong_window_days:
+        return 0.35
+    if age_days <= medium_window_days:
+        return 0.18
+    if age_days <= 120:
+        return 0.08
+    return 0.0
+
+
+def _compute_trust_score(record: MemoryRecord, *, age_days: int) -> float:
+    score = _STATUS_RANK.get(record.status, 1.0) / max(_STATUS_RANK.values())
+    score += max(0.0, min(1.0, record.confidence)) * 0.35
+    if record.commit:
+        score += 0.1
+    if record.evidence:
+        score += min(0.15, 0.03 * len(record.evidence))
+    if age_days:
+        score -= min(0.2, age_days / 3650)
+    return max(0.0, min(1.5, score))
+
+
+def _compute_utility_score(record: MemoryRecord) -> float:
+    retrieval_signal = min(0.6, 0.08 * max(0, record.retrieval_count))
+    evidence_signal = min(0.4, 0.06 * max(0, record.evidence_hit_count))
+    supersede_signal = 0.08 if record.supersedes else 0.0
+    return max(0.0, min(1.25, retrieval_signal + evidence_signal + supersede_signal))
+
+
+def _specificity_bonus(
+    record: MemoryRecord,
+    *,
+    wants_recent: bool,
+    file_path: str | None,
+    file_score: float,
+    overlap: float,
+) -> float:
+    bonus = 0.0
+    file_count = len([path for path in record.files if str(path).strip()])
+    evidence_count = len([item for item in record.evidence if str(item).strip()])
+
+    if file_path and file_score >= 1.0:
+        if file_count <= 1:
+            bonus += 0.45
+        elif file_count <= 3:
+            bonus += 0.22
+        elif file_count >= 6:
+            bonus -= 0.15
+
+    if wants_recent and overlap >= 0.25:
+        if file_count <= 2:
+            bonus += 0.2
+        elif file_count >= 6:
+            bonus -= 0.12
+        if evidence_count <= 2:
+            bonus += 0.12
+        elif evidence_count >= 6:
+            bonus -= 0.08
+
+    return bonus