superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/summaries/daily_reflection.py

@@ -1,340 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Daily Reflection — issue #113 bounded summary.
-
-Produces a human-readable summary of all facts recorded on a specific date,
-grouped by time of day and sorted by importance.
-
-MEASURED DATA REALITY
----------------------
-On a real 3,294-fact store there are 48 distinct active days.
-Per-day volumes range from 4 to 454 facts.  Daily reflections are
-structurally viable for this store.
-
-DETERMINISTIC FALLBACK
------------------------
-The extractive path groups facts by topic entities and summarises the top-N
-by importance.  It never calls an LLM and never fails to return a result.
-Mode B/C enrichment is attempted when configured, but falls back to extractive
-on any failure — silence is not acceptable.
-
-HOT PATH EXCLUSION
--------------------
-This module must NEVER be imported from core/recall_pipeline.py or
-core/store_pipeline.py.
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-from datetime import date
-from pathlib import Path
-
-from .base import (
-    clean_llm_summary,
-    format_highlight,
-    SUMMARY_SYSTEM_PROMPT,
-    COVERAGE_FULL,
-    COVERAGE_INSUFFICIENT,
-    COVERAGE_UNAVAILABLE,
-    GENERATED_BY_EXTRACTIVE,
-    GENERATED_BY_LLM_B,
-    GENERATED_BY_LLM_C,
-    SummaryResult,
-    get_mode_str,
-)
-
-logger = logging.getLogger("superlocalmemory.summaries.daily")
-
-_MIN_FACTS = 2          # Below this threshold, coverage=insufficient
-_BODY_FACTS = 10        # Top-N facts shown in extractive summary
-_MAX_FACT_CHARS = 300   # Per-fact character cap in the body
-
-
-def generate_daily_reflection(
-    db_path: str | Path,
-    target_date: str | date,
-    profile_id: str = "default",
-    config: object | None = None,
-) -> SummaryResult:
-    """Generate a Daily Reflection for a specific date.
-
-    Args:
-        db_path:     Path to memory.db.
-        target_date: The date to reflect on.  Accepts ``date`` objects or
-                     ISO-format strings ("2026-08-17").
-        profile_id:  Profile scope — never mix profiles.
-        config:      Optional SLMConfig for LLM enrichment.  None = Mode A
-                     (extractive, always deterministic).
-
-    Returns:
-        SummaryResult with source_fact_ids for every contributing fact.
-        Never returns None.
-    """
-    db_path = Path(db_path)
-    date_str = target_date.isoformat() if isinstance(target_date, date) else str(target_date)
-
-    # ── query ────────────────────────────────────────────────────────────────
-    try:
-        conn = sqlite3.connect(str(db_path), timeout=5.0)
-        conn.row_factory = sqlite3.Row
-        conn.execute("PRAGMA query_only=ON")
-        try:
-            rows = conn.execute(
-                """
-                SELECT fact_id, content, created_at, importance,
-                       canonical_entities_json, lifecycle
-                FROM   atomic_facts
-                WHERE  profile_id = ?
-                  AND  DATE(created_at) = ?
-                  AND  lifecycle != 'archived'
-                ORDER  BY importance DESC, created_at ASC
-                """,
-                (profile_id, date_str),
-            ).fetchall()
-        finally:
-            conn.close()
-    except Exception as exc:
-        logger.warning("daily reflection query failed for %s: %s", date_str, exc)
-        return SummaryResult(
-            kind="daily",
-            profile_id=profile_id,
-            content=f"Daily reflection for {date_str} is unavailable: data access error.",
-            source_fact_ids=[],
-            coverage=COVERAGE_UNAVAILABLE,
-            generated_by=GENERATED_BY_EXTRACTIVE,
-            metadata={"date": date_str, "error": str(exc)},
-        )
-
-    # ── coverage decision ─────────────────────────────────────────────────────
-    if not rows:
-        return SummaryResult(
-            kind="daily",
-            profile_id=profile_id,
-            content=f"No facts recorded on {date_str}.",
-            source_fact_ids=[],
-            coverage=COVERAGE_INSUFFICIENT,
-            generated_by=GENERATED_BY_EXTRACTIVE,
-            metadata={"date": date_str, "fact_count": 0},
-        )
-
-    facts = [dict(r) for r in rows]
-    source_fact_ids = [f["fact_id"] for f in facts]
-    fact_count = len(facts)
-
-    if fact_count < _MIN_FACTS:
-        coverage = COVERAGE_INSUFFICIENT
-    else:
-        coverage = COVERAGE_FULL
-
-    # ── extractive summary (deterministic, always available) ──────────────────
-    extractive_content = _build_extractive_content(date_str, facts, fact_count, db_path, profile_id)
-
-    # ── LLM enrichment (optional) ─────────────────────────────────────────────
-    mode = get_mode_str(config)
-    if fact_count >= _MIN_FACTS and mode in ("b", "c"):
-        llm_content, llm_mode = _try_llm(date_str, facts, config, mode)
-        if llm_content:
-            return SummaryResult(
-                kind="daily",
-                profile_id=profile_id,
-                content=llm_content,
-                source_fact_ids=source_fact_ids,
-                coverage=coverage,
-                generated_by=llm_mode,
-                metadata={"date": date_str, "fact_count": fact_count},
-            )
-
-    return SummaryResult(
-        kind="daily",
-        profile_id=profile_id,
-        content=extractive_content,
-        source_fact_ids=source_fact_ids,
-        coverage=coverage,
-        generated_by=GENERATED_BY_EXTRACTIVE,
-        metadata={"date": date_str, "fact_count": fact_count},
-    )
-
-
-# ── helpers ──────────────────────────────────────────────────────────────────
-
-def _build_extractive_content(
-    date_str: str,
-    facts: list[dict],
-    fact_count: int,
-    db_path: str | Path,
-    profile_id: str,
-) -> str:
-    """Build a deterministic extractive daily reflection.
-
-    ``db_path`` / ``profile_id`` are needed only to turn entity IDs into names
-    for the "Active entities" line — the facts themselves are already loaded.
-    """
-    import json
-
-    lines = [
-        f"Daily reflection: {date_str}",
-        f"Total facts recorded: {fact_count}",
-        "",
-        "Highlights:",
-    ]
-    for f in facts[:_BODY_FACTS]:
-        content = f.get("content", "")
-        content = format_highlight(content)
-        lines.append(f"  - {content}")
-    if fact_count > _BODY_FACTS:
-        lines.append(f"  ... and {fact_count - _BODY_FACTS} additional facts.")
-
-    # Entity summary: which entities appeared most
-    entity_counts: dict[str, int] = {}
-    for f in facts:
-        raw = f.get("canonical_entities_json") or "[]"
-        try:
-            entities = json.loads(raw)
-        except (ValueError, TypeError):
-            entities = []
-        for e in entities:
-            entity_counts[e] = entity_counts.get(e, 0) + 1
-
-    if entity_counts:
-        top_entities = sorted(entity_counts.items(), key=lambda x: x[1], reverse=True)[:5]
-        # Resolve to names. canonical_entities_json stores entity IDs, so this
-        # line previously read "Active entities: 1666abc512904473 (127),
-        # 84288f2dde994afe (124)" — five 16-hex identifiers, which tell a reader
-        # nothing. They resolve to 'Fixed', 'Gateway', 'REVISED' and so on. This
-        # is the same defect 4.0.6 fixed in the Living Brain, where source
-        # quality listed internal identifiers; it survived here because this
-        # generator was never reachable to be looked at.
-        names = _entity_names(db_path, [e for e, _ in top_entities], profile_id)
-        labelled = [
-            (names.get(eid) or eid, count) for eid, count in top_entities
-        ]
-        entity_str = ", ".join(f"{name} ({c})" for name, c in labelled)
-        lines.append("")
-        lines.append(f"Active entities: {entity_str}")
-
-    return "\n".join(lines)
-
-
-def _entity_names(
-    db_path: str | Path, entity_ids: list[str], profile_id: str
-) -> dict[str, str]:
-    """Map entity_id → canonical_name. Missing or unreadable rows are omitted.
-
-    Fail-open: a summary is still useful with a raw id in it, so a query problem
-    must not lose the whole line. The caller falls back to the id per entity.
-    """
-    if not entity_ids:
-        return {}
-    try:
-        conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
-        try:
-            placeholders = ",".join("?" for _ in entity_ids)
-            rows = conn.execute(
-                f"SELECT entity_id, canonical_name FROM canonical_entities "
-                f"WHERE entity_id IN ({placeholders}) AND profile_id = ?",
-                (*entity_ids, profile_id),
-            ).fetchall()
-        finally:
-            conn.close()
-        return {r[0]: r[1] for r in rows if r[1]}
-    except sqlite3.Error:
-        return {}
-
-
-def _try_llm(
-    date_str: str,
-    facts: list[dict],
-    config: object | None,
-    mode: str,
-) -> tuple[str | None, str]:
-    """Attempt LLM-based daily reflection.  Returns (content, generated_by)."""
-    prompt = (
-        f"Write a concise daily reflection for {date_str} based on these facts. "
-        f"Focus on main themes and accomplishments in 3-5 sentences."
-    )
-    if mode == "c":
-        result = _call_cloud_llm(prompt, facts, config)
-        if result:
-            return result, GENERATED_BY_LLM_C
-    if mode in ("b", "c"):
-        result = _call_ollama(prompt, facts, config)
-        if result:
-            return result, GENERATED_BY_LLM_B
-    return None, GENERATED_BY_EXTRACTIVE
-
-
-def _call_ollama(
-    prompt: str,
-    facts: list[dict],
-    config: object | None,
-) -> str | None:
-    """Mode B: call Ollama.  Returns None on any failure (extractive fallback)."""
-    try:
-        import json
-        import urllib.request
-
-        api_base = "http://localhost:11434"
-        model = "llama3.2"
-        timeout = 30
-        if config and hasattr(config, "llm"):
-            api_base = getattr(config.llm, "api_base", api_base) or api_base
-            model = getattr(config.llm, "model", model) or model
-            timeout = (
-                getattr(config.llm, "timeout_seconds", None)
-                or getattr(config.llm, "timeout", None)
-                or timeout
-            )
-
-        fact_texts = "\n".join(f"- {f['content']}" for f in facts[:15])
-        full_prompt = f"{prompt}\n\nFacts from {len(facts)} recorded:\n{fact_texts}"
-        payload = json.dumps({
-            "model": model,
-            "prompt": full_prompt,
-            "system": SUMMARY_SYSTEM_PROMPT,
-            "stream": False,
-            "options": {"num_predict": 300},
-        }).encode()
-        req = urllib.request.Request(
-            f"{api_base}/api/generate",
-            data=payload,
-            headers={"Content-Type": "application/json"},
-        )
-        resp = urllib.request.urlopen(req, timeout=timeout)
-        data = json.loads(resp.read().decode())
-        text = clean_llm_summary(data.get("response", ""))
-        return text if text and len(text) > 20 else None
-    except Exception as exc:
-        logger.debug("Ollama daily reflection failed: %s", exc)
-        return None
-
-
-def _call_cloud_llm(
-    prompt: str,
-    facts: list[dict],
-    config: object | None,
-) -> str | None:
-    """Mode C: call the configured cloud LLM.  Returns None on any failure."""
-    if not config or not hasattr(config, "llm"):
-        return None
-    try:
-        from superlocalmemory.llm.backbone import LLMBackbone
-        llm = LLMBackbone(config.llm)
-        if not llm.is_available():
-            return None
-        fact_texts = "\n".join(f"- {f['content']}" for f in facts[:15])
-        full_prompt = f"{prompt}\n\nFacts:\n{fact_texts}"
-        text = llm.generate(
-            prompt=full_prompt,
-            system=SUMMARY_SYSTEM_PROMPT,
-            max_tokens=300,
-            temperature=0.1,
-        )
-        cleaned = clean_llm_summary(text or "")
-        return cleaned if len(cleaned) > 20 else None
-    except Exception as exc:
-        logger.debug("Cloud LLM daily reflection failed: %s", exc)
-        return None

package/src/superlocalmemory/summaries/non_answer.py

@@ -1,223 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-
-"""Refuse to store a model's non-answer as if it were a memory.
-
-A summarizer is handed a cluster of facts and asked to merge them. When the
-cluster has nothing in common the model does not fail — it answers the question
-it was asked, in prose, and that prose is indistinguishable from a summary to
-any caller that only checks ``if summary:``. On the author's own store the
-result was rows reading
-
-    "Unfortunately, there is no information available about 'Gateway', 'State',
-     'Bounded', or 'Claude' in the provided text."
-
-sitting at ranks 1, 2 and 3 for "what am I working on".
-
-WHAT THIS IS AND IS NOT
------------------------
-This is a **forward guard**: it stops the next such row from being written. It
-is deliberately NOT the repair for rows already stored, because it cannot be.
-Measured across the 307 retrieval-eligible consolidated rows on the author's
-store, this predicate rejects **34** and lets **273** through — because those
-273 are fluent, plausible, entirely generic prose ("The Pro and
-SuperLocalMemory (SLM) projects have made significant progress...") that no
-honest content predicate can separate from a real summary. Repairing by content
-would have cleared a ninth of the problem and declared victory. Existing rows
-are handled by provenance instead.
-
-Measured the other way, on 3,894 genuine facts, it rejects 70 — and all 70 are
-real defects, not false positives: 68 memories carry raw tool-call markup
-scraped in from a transcript, and 2 are a model's refusal that was stored as
-though it were a memory ("I cannot verify when ... ended a session. Can I help
-you with something else?"). Zero legitimate memories are rejected.
-
-So the bar here is: catch text that is *addressed to the prompt* rather than
-*about the facts*, and nothing else. Everything is anchored to the start of the
-text or to a whole leading sentence, because a genuine memory may legitimately
-contain "there is no" in the middle of a sentence.
-
-Companion to ``clean_llm_summary`` in :mod:`superlocalmemory.summaries.base`,
-which strips scaffolding *around* an answer. This one rejects text that is
-scaffolding *all the way through*. Run the stripper first: "Here is a concise
-summary paragraph: <real content>" is salvageable and must not be discarded.
-"""
-
-from __future__ import annotations
-
-import re
-
-__all__ = ["is_non_answer", "NON_ANSWER_PATTERNS", "MIN_USEFUL_CHARS"]
-
-
-#: A *merged summary* this short did not merge anything. The summarizers already
-#: refuse model output under 50 characters; this is the same floor applied to
-#: text that arrived by another route (extractive mode, or a pre-computed summary
-#: handed in by a caller).
-#:
-#: It is NOT a floor for memories in general, and ``is_non_answer`` therefore
-#: does not apply it unless a caller asks. Measured on the author's store, 730
-#: of 3,894 genuine facts are under 50 characters and every sampled one is a
-#: real memory — "2026-05-02 is the date when the session ended",
-#: "This is the case for keeping AMS." Baking this floor into the default would
-#: have made the guard reject a fifth of a user's memory as junk. The floor is
-#: the *caller's* policy about its own output, not a fact about text.
-MIN_USEFUL_CHARS = 50
-
-
-#: Each entry is (regex, why-it-is-not-a-memory). The reason travels with the
-#: pattern so a future reader can tell whether a new false positive means the
-#: pattern is wrong or the input genuinely is a non-answer.
-_PATTERN_SOURCES: tuple[tuple[str, str], ...] = (
-    (
-        r"^\W*(?:unfortunately|regrettably|sadly)\b[^.!?]*\bno\b",
-        "opens by apologising for having nothing to say",
-    ),
-    (
-        r"^\W*there\s+(?:is|are)\s+no\s+"
-        r"(?:information|mention|reference|facts?|details?|data|content)\b",
-        "states the absence of input rather than summarising input",
-    ),
-    (
-        r"\bno\s+(?:information|facts?|details?|data)\s+"
-        r"(?:is|are|was|were)?\s*(?:available|provided|given|present)\b",
-        "reports an empty input set",
-    ),
-    (
-        r"^\W*(?:i\s+(?:cannot|can't|can\s+not|am\s+unable\s+to)|"
-        r"it\s+is\s+not\s+possible\s+to)\b",
-        "declines the task",
-    ),
-    (
-        r"^\W*(?:i\s+don'?t|i\s+do\s+not)\s+(?:have|see|find)\b",
-        "declines the task in the first person",
-    ),
-    (
-        r"\bthe\s+(?:provided|given|above|following)\s+"
-        r"(?:text|facts?|context|input|information)\b",
-        "refers to the prompt, so it is talking to the asker, not about the memory",
-    ),
-    (
-        r"^\W*(?:as\s+an?\s+(?:ai|language\s+model)|i'?m\s+an?\s+ai)\b",
-        "identifies itself as a model",
-    ),
-    (
-        r"^\W*(?:please\s+)?(?:provide|share|give)\s+(?:me\s+)?"
-        r"(?:more|the|some|additional)\b",
-        "asks the user for input instead of answering",
-    ),
-    # The five below were added after running the first four against all 1,035
-    # summaries stored on the author's machine. They catch 14 rows the original
-    # set let through -- every one a measured string from that store, not a
-    # guess about what a model might say.
-    (
-        r"^\W*i\s+(?:did\s*n[o']?t|didn'?t)\s+receive\b",
-        "says it was given nothing to summarise",
-    ),
-    (
-        r"\bthere\s+(?:is|are|was|were)\s+n(?:o|ot)\s+\d+\s+facts?\b",
-        "argues with the number of facts it was asked to merge",
-    ),
-    (
-        r"\bthe\s+text\s+(?:snippet\s+)?"
-        r"(?:appears|seems|does\s+not|doesn'?t|is\s+not)\b",
-        "describes the prompt instead of summarising it",
-    ),
-    (
-        r"\bin\s+the\s+text\s+(?:provided|given|above|supplied)\b"
-        r"|\bthe\s+text\s+(?:provided|supplied)\b",
-        "refers to the prompt (word order the earlier rule missed)",
-    ),
-    (
-        r"^\W*i\s+must\s+point\s+out\b",
-        "opens with meta-commentary about the request",
-    ),
-    (
-        # "This appears to be a detailed log of progress in writing..." — the
-        # model describing the shape of what it was shown rather than saying
-        # what it says. Six of the first twelve rows the dashboard rendered
-        # opened this way. Deliberately narrower than it could be: "This is a
-        # summary of an audit session" is clumsy but it does summarise, so it
-        # is left alone.
-        r"^\W*this\s+(?:appears|seems)\s+to\s+be\b"
-        r"|^\W*this\s+text\s+(?:is|appears|seems)\b",
-        "describes the shape of the input instead of its content",
-    ),
-)
-
-#: Compiled once. ``re.IGNORECASE`` throughout — the casing of a refusal is not
-#: information. ``re.DOTALL`` is deliberately NOT set: ``[^.!?]*`` and the
-#: leading anchors are meant to stay within the opening sentence.
-NON_ANSWER_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = tuple(
-    (re.compile(src, re.IGNORECASE), why) for src, why in _PATTERN_SOURCES
-)
-
-#: Tool-call and markup fragments. A memory whose content carries these was
-#: assembled from a transcript that still had its plumbing attached; the user
-#: sees raw XML in their own memory list. Matched anywhere, not anchored,
-#: because the fragment can appear at any offset in a spliced transcript.
-_MARKUP = re.compile(
-    r"</(?:content|antml:parameter|parameter|invoke|function_calls|thinking)>"
-    r"|<(?:antml:)?(?:parameter|invoke|function_calls)\b"
-    r"|<\|(?:im_start|im_end|endoftext)\|>",
-    re.IGNORECASE,
-)
-
-
-def is_non_answer(
-    text: str | None,
-    *,
-    min_chars: int = 0,
-    model_written: bool = True,
-) -> tuple[bool, str]:
-    """Whether ``text`` is a model talking about the prompt, not a memory.
-
-    Returns ``(rejected, reason)``. ``reason`` is empty when the text is
-    acceptable, and otherwise names which rule fired — callers log it, so a
-    silent rejection never happens and a false positive is diagnosable from
-    normal logs rather than needing a repro.
-
-    ``min_chars`` defaults to **no floor**. A caller that knows its own output
-    should be long — a summarizer merging three or more facts — passes
-    :data:`MIN_USEFUL_CHARS`. See that constant for why this is not the default.
-
-    ``model_written=False`` applies only the rules that hold for text no model
-    composed: the length floor and the markup check. Every other rule here
-    describes a MODEL's behaviour — refusing, apologising, discussing the
-    prompt — and an extractive summary is the owner's own sentences reassembled,
-    so a memory that happens to say "the facts provided" would be rejected for
-    having phrased itself like a chatbot. Nothing is lost by exempting it: an
-    extractive summary cannot refuse, because there is nobody there to refuse.
-
-    Cheap and side-effect free: safe to call on every candidate write.
-
-    >>> is_non_answer("Unfortunately, there is no information available.")[0]
-    True
-    >>> is_non_answer("Varun ships SLM 4.0.10 with an auto-repair migration.")
-    (False, '')
-    >>> is_non_answer("The team has no information silo left to dismantle.")[0]
-    False
-    >>> is_non_answer("2026-05-02 is the date the session ended.")
-    (False, '')
-    >>> is_non_answer("Too short.", min_chars=MIN_USEFUL_CHARS)[0]
-    True
-    """
-    if text is None:
-        return True, "empty"
-    stripped = text.strip()
-    if not stripped:
-        return True, "empty"
-    if min_chars and len(stripped) < min_chars:
-        return True, f"shorter than {min_chars} characters"
-
-    if _MARKUP.search(stripped):
-        return True, "contains tool-call or chat-template markup"
-
-    if not model_written:
-        return False, ""
-
-    for pattern, why in NON_ANSWER_PATTERNS:
-        if pattern.search(stripped):
-            return True, why
-
-    return False, ""