superlocalmemory 4.0.9 → 4.0.10

package/src/superlocalmemory/summaries/non_answer.py

@@ -0,0 +1,223 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Refuse to store a model's non-answer as if it were a memory.
+
+A summarizer is handed a cluster of facts and asked to merge them. When the
+cluster has nothing in common the model does not fail — it answers the question
+it was asked, in prose, and that prose is indistinguishable from a summary to
+any caller that only checks ``if summary:``. On the author's own store the
+result was rows reading
+
+    "Unfortunately, there is no information available about 'Gateway', 'State',
+     'Bounded', or 'Claude' in the provided text."
+
+sitting at ranks 1, 2 and 3 for "what am I working on".
+
+WHAT THIS IS AND IS NOT
+-----------------------
+This is a **forward guard**: it stops the next such row from being written. It
+is deliberately NOT the repair for rows already stored, because it cannot be.
+Measured across the 307 retrieval-eligible consolidated rows on the author's
+store, this predicate rejects **34** and lets **273** through — because those
+273 are fluent, plausible, entirely generic prose ("The Pro and
+SuperLocalMemory (SLM) projects have made significant progress...") that no
+honest content predicate can separate from a real summary. Repairing by content
+would have cleared a ninth of the problem and declared victory. Existing rows
+are handled by provenance instead.
+
+Measured the other way, on 3,894 genuine facts, it rejects 70 — and all 70 are
+real defects, not false positives: 68 memories carry raw tool-call markup
+scraped in from a transcript, and 2 are a model's refusal that was stored as
+though it were a memory ("I cannot verify when ... ended a session. Can I help
+you with something else?"). Zero legitimate memories are rejected.
+
+So the bar here is: catch text that is *addressed to the prompt* rather than
+*about the facts*, and nothing else. Everything is anchored to the start of the
+text or to a whole leading sentence, because a genuine memory may legitimately
+contain "there is no" in the middle of a sentence.
+
+Companion to ``clean_llm_summary`` in :mod:`superlocalmemory.summaries.base`,
+which strips scaffolding *around* an answer. This one rejects text that is
+scaffolding *all the way through*. Run the stripper first: "Here is a concise
+summary paragraph: <real content>" is salvageable and must not be discarded.
+"""
+
+from __future__ import annotations
+
+import re
+
+__all__ = ["is_non_answer", "NON_ANSWER_PATTERNS", "MIN_USEFUL_CHARS"]
+
+
+#: A *merged summary* this short did not merge anything. The summarizers already
+#: refuse model output under 50 characters; this is the same floor applied to
+#: text that arrived by another route (extractive mode, or a pre-computed summary
+#: handed in by a caller).
+#:
+#: It is NOT a floor for memories in general, and ``is_non_answer`` therefore
+#: does not apply it unless a caller asks. Measured on the author's store, 730
+#: of 3,894 genuine facts are under 50 characters and every sampled one is a
+#: real memory — "2026-05-02 is the date when the session ended",
+#: "This is the case for keeping AMS." Baking this floor into the default would
+#: have made the guard reject a fifth of a user's memory as junk. The floor is
+#: the *caller's* policy about its own output, not a fact about text.
+MIN_USEFUL_CHARS = 50
+
+
+#: Each entry is (regex, why-it-is-not-a-memory). The reason travels with the
+#: pattern so a future reader can tell whether a new false positive means the
+#: pattern is wrong or the input genuinely is a non-answer.
+_PATTERN_SOURCES: tuple[tuple[str, str], ...] = (
+    (
+        r"^\W*(?:unfortunately|regrettably|sadly)\b[^.!?]*\bno\b",
+        "opens by apologising for having nothing to say",
+    ),
+    (
+        r"^\W*there\s+(?:is|are)\s+no\s+"
+        r"(?:information|mention|reference|facts?|details?|data|content)\b",
+        "states the absence of input rather than summarising input",
+    ),
+    (
+        r"\bno\s+(?:information|facts?|details?|data)\s+"
+        r"(?:is|are|was|were)?\s*(?:available|provided|given|present)\b",
+        "reports an empty input set",
+    ),
+    (
+        r"^\W*(?:i\s+(?:cannot|can't|can\s+not|am\s+unable\s+to)|"
+        r"it\s+is\s+not\s+possible\s+to)\b",
+        "declines the task",
+    ),
+    (
+        r"^\W*(?:i\s+don'?t|i\s+do\s+not)\s+(?:have|see|find)\b",
+        "declines the task in the first person",
+    ),
+    (
+        r"\bthe\s+(?:provided|given|above|following)\s+"
+        r"(?:text|facts?|context|input|information)\b",
+        "refers to the prompt, so it is talking to the asker, not about the memory",
+    ),
+    (
+        r"^\W*(?:as\s+an?\s+(?:ai|language\s+model)|i'?m\s+an?\s+ai)\b",
+        "identifies itself as a model",
+    ),
+    (
+        r"^\W*(?:please\s+)?(?:provide|share|give)\s+(?:me\s+)?"
+        r"(?:more|the|some|additional)\b",
+        "asks the user for input instead of answering",
+    ),
+    # The five below were added after running the first four against all 1,035
+    # summaries stored on the author's machine. They catch 14 rows the original
+    # set let through -- every one a measured string from that store, not a
+    # guess about what a model might say.
+    (
+        r"^\W*i\s+(?:did\s*n[o']?t|didn'?t)\s+receive\b",
+        "says it was given nothing to summarise",
+    ),
+    (
+        r"\bthere\s+(?:is|are|was|were)\s+n(?:o|ot)\s+\d+\s+facts?\b",
+        "argues with the number of facts it was asked to merge",
+    ),
+    (
+        r"\bthe\s+text\s+(?:snippet\s+)?"
+        r"(?:appears|seems|does\s+not|doesn'?t|is\s+not)\b",
+        "describes the prompt instead of summarising it",
+    ),
+    (
+        r"\bin\s+the\s+text\s+(?:provided|given|above|supplied)\b"
+        r"|\bthe\s+text\s+(?:provided|supplied)\b",
+        "refers to the prompt (word order the earlier rule missed)",
+    ),
+    (
+        r"^\W*i\s+must\s+point\s+out\b",
+        "opens with meta-commentary about the request",
+    ),
+    (
+        # "This appears to be a detailed log of progress in writing..." — the
+        # model describing the shape of what it was shown rather than saying
+        # what it says. Six of the first twelve rows the dashboard rendered
+        # opened this way. Deliberately narrower than it could be: "This is a
+        # summary of an audit session" is clumsy but it does summarise, so it
+        # is left alone.
+        r"^\W*this\s+(?:appears|seems)\s+to\s+be\b"
+        r"|^\W*this\s+text\s+(?:is|appears|seems)\b",
+        "describes the shape of the input instead of its content",
+    ),
+)
+
+#: Compiled once. ``re.IGNORECASE`` throughout — the casing of a refusal is not
+#: information. ``re.DOTALL`` is deliberately NOT set: ``[^.!?]*`` and the
+#: leading anchors are meant to stay within the opening sentence.
+NON_ANSWER_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = tuple(
+    (re.compile(src, re.IGNORECASE), why) for src, why in _PATTERN_SOURCES
+)
+
+#: Tool-call and markup fragments. A memory whose content carries these was
+#: assembled from a transcript that still had its plumbing attached; the user
+#: sees raw XML in their own memory list. Matched anywhere, not anchored,
+#: because the fragment can appear at any offset in a spliced transcript.
+_MARKUP = re.compile(
+    r"</(?:content|antml:parameter|parameter|invoke|function_calls|thinking)>"
+    r"|<(?:antml:)?(?:parameter|invoke|function_calls)\b"
+    r"|<\|(?:im_start|im_end|endoftext)\|>",
+    re.IGNORECASE,
+)
+
+
+def is_non_answer(
+    text: str | None,
+    *,
+    min_chars: int = 0,
+    model_written: bool = True,
+) -> tuple[bool, str]:
+    """Whether ``text`` is a model talking about the prompt, not a memory.
+
+    Returns ``(rejected, reason)``. ``reason`` is empty when the text is
+    acceptable, and otherwise names which rule fired — callers log it, so a
+    silent rejection never happens and a false positive is diagnosable from
+    normal logs rather than needing a repro.
+
+    ``min_chars`` defaults to **no floor**. A caller that knows its own output
+    should be long — a summarizer merging three or more facts — passes
+    :data:`MIN_USEFUL_CHARS`. See that constant for why this is not the default.
+
+    ``model_written=False`` applies only the rules that hold for text no model
+    composed: the length floor and the markup check. Every other rule here
+    describes a MODEL's behaviour — refusing, apologising, discussing the
+    prompt — and an extractive summary is the owner's own sentences reassembled,
+    so a memory that happens to say "the facts provided" would be rejected for
+    having phrased itself like a chatbot. Nothing is lost by exempting it: an
+    extractive summary cannot refuse, because there is nobody there to refuse.
+
+    Cheap and side-effect free: safe to call on every candidate write.
+
+    >>> is_non_answer("Unfortunately, there is no information available.")[0]
+    True
+    >>> is_non_answer("Varun ships SLM 4.0.10 with an auto-repair migration.")
+    (False, '')
+    >>> is_non_answer("The team has no information silo left to dismantle.")[0]
+    False
+    >>> is_non_answer("2026-05-02 is the date the session ended.")
+    (False, '')
+    >>> is_non_answer("Too short.", min_chars=MIN_USEFUL_CHARS)[0]
+    True
+    """
+    if text is None:
+        return True, "empty"
+    stripped = text.strip()
+    if not stripped:
+        return True, "empty"
+    if min_chars and len(stripped) < min_chars:
+        return True, f"shorter than {min_chars} characters"
+
+    if _MARKUP.search(stripped):
+        return True, "contains tool-call or chat-template markup"
+
+    if not model_written:
+        return False, ""
+
+    for pattern, why in NON_ANSWER_PATTERNS:
+        if pattern.search(stripped):
+            return True, why
+
+    return False, ""

package/src/superlocalmemory/ui/index.html

@@ -1604,7 +1604,7 @@
     <script src="static/js/od-ops-health.js?v=400"></script>
     <script src="static/js/od-team.js?v=379"></script>
     <script src="static/js/od-graph.js?v=6812bf6c"></script>
-    <script src="static/js/od-memories.js?v=022ff653"></script>
+    <script src="static/js/od-memories.js?v=997f1674"></script>
     <script src="static/js/od-entities.js?v=379"></script>
     <!-- Multi-Agent Memory pane (v3.8.0): visualises memory written by multiple agents -->
     <script src="static/js/od-agents.js?v=c75ff3c2"></script>

package/src/superlocalmemory/ui/js/od-memories.js

@@ -189,6 +189,18 @@
             '</select>' +
           '</div>' +
           '<div id="' + id + '-sum-out" style="font-size:13px;color:var(--fg-2)">Pick a summary above.</div>' +
+          // Knowledge Overview.
+          //
+          // The Today/Yesterday buttons above are a date filter over
+          // atomic_facts and have never had any link to a cluster, so the
+          // Summaries tab could not show what the store actually knows ABOUT
+          // anything. This card fills that: cluster summaries, read from the
+          // display-only table, which is where they belong now that they are
+          // out of the retrieval corpus.
+          '<div id="' + id + '-kover" style="margin-top:26px;' +
+            'border-top:1px solid var(--border);padding-top:20px">' +
+            _loading('Loading what your memory knows…') +
+          '</div>' +
         '</div>' +
         // Recall Lab pane — exact IDs required by recall-lab.js:
         //   #recall-lab-query (input), #recall-lab-search (button — click check),
@@ -363,7 +375,7 @@
     if (pane) pane.classList.add('active');
     if (tab === 'timeline') _loadTimeline(id);
     if (tab === 'clusters')  _loadClusters(id);
-    if (tab === 'summary')   _loadProjectOptions(id);
+    if (tab === 'summary') { _loadProjectOptions(id); _loadKnowledgeOverview(id); }
   }
 
   /* Populate the project picker from projects SLM has actually recorded.
@@ -815,6 +827,183 @@
 
   // ── Clusters fetch & render ──────────────────────────────────────────────────
 
+  /* Knowledge Overview — cluster summaries, plus an honest health line.
+   *
+   * Two rules this card exists to obey.
+   *
+   * IT READS ONLY THE DISPLAY TABLE. These summaries were in the retrieval
+   * corpus until 4.0.10, where they out-ranked the user's own words. Moving
+   * them out is only worth anything if exactly one surface shows them, and
+   * this is that surface.
+   *
+   * IT DOES NOT DRESS UP JUNK AS INSIGHT. Some stored summaries are a model's
+   * non-answer -- "Unfortunately, there is no information available about
+   * 'State'..." -- and rendering those as a heading called Knowledge Overview
+   * makes the product look broken. The endpoint labels each row's quality; the
+   * usable ones are shown and the rest are counted in one plain sentence. Not
+   * hidden: hiding them would hide the very problem the release fixes, and a
+   * reader who is told "3 could not be generated" trusts the other 47 more.
+   */
+  function _loadKnowledgeOverview(id) {
+    var box = document.getElementById(id + '-kover');
+    if (!box) return;
+    // Guarded on the element, not a module flag — same reason as
+    // _loadProjectOptions: a stale "already loaded" boolean outlives the DOM
+    // it described and leaves a permanent spinner after a re-render.
+    if (box.dataset.loaded === '1') return;
+
+    Promise.all([
+      fetch('/api/v3/abstraction/consolidated?limit=24')
+        .then(function (r) { return r.ok ? r.json() : { summaries: [], unusable: 0 }; })
+        .catch(function () { return { summaries: [], unusable: 0 }; }),
+      fetch('/api/v3/abstraction/health')
+        .then(function (r) { return r.ok ? r.json() : null; })
+        .catch(function () { return null; })
+    ]).then(function (res) {
+      box.dataset.loaded = '1';
+      box.innerHTML = _koverHtml(res[0] || {}, res[1]);
+    });
+  }
+
+  function _koverHtml(data, health) {
+    var all = data.summaries || [];
+    var usable = all.filter(function (s) { return s.quality === 'ok'; });
+    var unusable = typeof data.unusable === 'number' ? data.unusable
+                 : (all.length - usable.length);
+
+    var html = '<h3 style="font-size:15px;margin:0 0 4px">What your memory knows</h3>' +
+      '<p style="font-size:12px;color:var(--fg-2);margin:0 0 14px">' +
+        'Written by the summarizer from groups of your memories. These are a ' +
+        'view of your memory, not memories themselves — they are never returned ' +
+        'as answers.' +
+      '</p>';
+
+    if (health) html += _koverHealth(health);
+
+    if (usable.length === 0) {
+      html += '<div style="padding:20px;color:var(--fg-2);font-size:13px">' +
+        (all.length === 0
+          ? 'No summaries yet. They are written in the background as related ' +
+            'memories accumulate.'
+          : 'None of the ' + all.length + ' stored summaries came back usable. ' +
+            'That usually means the summarizer had nothing in common to merge.') +
+        '</div>';
+      return html;
+    }
+
+    html += '<div style="display:grid;gap:12px;' +
+      'grid-template-columns:repeat(auto-fill,minmax(280px,1fr))">';
+    usable.forEach(function (s) { html += _koverCard(s); });
+    html += '</div>';
+
+    var notes = [];
+    if (unusable > 0) {
+      notes.push(unusable + (unusable === 1 ? ' summary' : ' summaries') +
+        ' could not be generated and are not shown. The memories they were ' +
+        'built from are unaffected.');
+    }
+    if (data.near_duplicates > 0) {
+      notes.push(data.near_duplicates + ' near-identical ' +
+        (data.near_duplicates === 1 ? 'summary was' : 'summaries were') +
+        ' collapsed into the ones above.');
+    }
+    if (notes.length) {
+      html += '<p style="font-size:12px;color:var(--fg-2);margin-top:12px">' +
+        notes.map(_esc).join(' ') + '</p>';
+    }
+    return html;
+  }
+
+  function _koverCard(s) {
+    // Every field is escaped. Anything in a memory can reach this screen: the
+    // summarizer merges fact content verbatim, so a memory containing markup
+    // would otherwise render as markup.
+    var body = _koverPlain(s.content);
+    var clipped = body.length > 320;
+    var shown = clipped ? body.slice(0, 320).replace(/\s+\S*$/, '') + '…' : body;
+
+    var meta = [];
+    if (s.source_count) {
+      meta.push(s.source_count + (s.source_count === 1 ? ' memory' : ' memories'));
+    }
+    var span = _koverSpan(s.source_earliest, s.source_latest);
+    if (span) meta.push(span);
+
+    return '<div style="background:var(--card-2);border:1px solid var(--border);' +
+        'border-radius:var(--r-md);padding:14px">' +
+      (s.entity_name
+        ? '<div style="font-size:13px;font-weight:600;margin-bottom:6px">' +
+            _esc(s.entity_name) + '</div>'
+        : '') +
+      '<div style="font-size:13px;line-height:1.5">' + _esc(shown) + '</div>' +
+      (meta.length
+        ? '<div style="margin-top:10px;display:flex;gap:6px;flex-wrap:wrap">' +
+            meta.map(_koverChip).join('') +
+          '</div>'
+        : '') +
+    '</div>';
+  }
+
+  /* Flatten a summary to plain prose for display.
+   *
+   * Mode B/C summaries come back with markdown in them — "**Audit Round**",
+   * "**Findings**:" — because a chat-tuned model formats for a chat window.
+   * Rendered as text those asterisks read as broken output to exactly the
+   * reader this card is for. Escaping is still what protects the page; this
+   * only removes the emphasis marks that carry no meaning here.
+   */
+  function _koverPlain(text) {
+    return String(text || '')
+      .replace(/```[\s\S]*?```/g, ' ')     // fenced blocks: no room for them
+      .replace(/`([^`]*)`/g, '$1')          // inline code marks
+      .replace(/\*\*([^*]+)\*\*/g, '$1')   // bold
+      .replace(/(^|\s)\*([^*\s][^*]*)\*/g, '$1$2')  // italic, not bullets
+      .replace(/^#{1,6}\s+/gm, '')          // headings
+      .replace(/^\s*[-*+]\s+/gm, '')        // list bullets
+      .replace(/\s+/g, ' ')                 // collapse whitespace + newlines
+      .trim();
+  }
+
+  /* A chip, not a keyword dump.
+   *
+   * The neighbouring community-summary API returns its `summary` field as
+   * "Topics: CCQ, Resume, P07…", which printed verbatim reads as a broken
+   * page. Counts and date spans are rendered as discrete labels instead.
+   */
+  function _koverChip(text) {
+    return '<span style="font-size:11px;padding:2px 8px;border-radius:999px;' +
+      'background:var(--card);border:1px solid var(--border);color:var(--fg-2)">' +
+      _esc(text) + '</span>';
+  }
+
+  function _koverSpan(earliest, latest) {
+    var a = String(earliest || '').slice(0, 10);
+    var b = String(latest || '').slice(0, 10);
+    if (!a && !b) return '';
+    if (!a || !b || a === b) return a || b;
+    return a + ' to ' + b;
+  }
+
+  /* Whether the memories behind all this can actually be found.
+   *
+   * The one question no screen answered before. A machine ran for months with
+   * 43.7% of its store unreachable by meaning while every status line it showed
+   * said healthy, because nothing counted the reachable share.
+   */
+  function _koverHealth(h) {
+    var lines = h.summary || [];
+    if (!lines.length) return '';
+    var bad = h.healthy === false;
+    return '<div style="margin:0 0 16px;padding:12px 14px;border-radius:var(--r-md);' +
+        'background:var(--card-2);border-left:3px solid ' +
+        (bad ? 'var(--warn)' : 'var(--ok)') + '">' +
+      lines.map(function (l) {
+        return '<div style="font-size:12px;color:var(--fg-2);line-height:1.5">' +
+          _esc(l) + '</div>';
+      }).join('') +
+    '</div>';
+  }
+
   function _loadClusters(id) {
     if (_st.cluLoaded) return;
     var grid = document.getElementById(id + '-clu-grid');

package/src/superlocalmemory/ui/js/od-ops-health.js

@@ -1,4 +1,4 @@
-/* od-ops-health.js — Wave-3 Admin Operations & Health panel
+/* od-ops-health.js — Admin Operations & Health panel
  * Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar — AGPL-3.0-or-later
  *
  * Injects an "Admin: Ops Health" tab into the Governance pane tab bar