Brain4Agent 0.0.1__py3-none-any.whl

brain4agent/evidence.py

@@ -0,0 +1,320 @@
+"""Evidence for the caller to judge, instead of a verdict computed here.
+
+Every cheap abstention signal was measured and all of them failed (see
+docs/decisions.md D-008, D-012). At 95% of answerable queries kept, refusal
+tops out around 10% for BM25 score, IDF coverage or unseen-word ratio, and 21%
+for a logistic regression over thirteen features. A model reaches 89-100%, but
+the caller is an agent that already has one, and paying for a second inference
+to answer "is this relevant" is duplicate spend on the critical path.
+
+So this module computes no verdict. It retrieves and reports what it can check
+for free, including the one signal that correlates with an unanswerable
+question: **a term the question treats as central that the corpus never
+contains.** The caller decides what to do with that.
+
+``missing_terms`` is the useful field, and it is a list of words rather than a
+score on purpose: a caller can show it, log it, or ignore it, and a human can
+read it without knowing what a BM25 score of 13.4 means.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Protocol
+
+from brain4agent.tokenize import load_jieba, word_tokens
+
+# Question terms that are structural rather than about the answer. Reporting
+# "什么" as missing would be true of every question and useless.
+_INTERROGATIVE = frozenset(
+    {
+        "什么",
+        "怎么",
+        "怎样",
+        "哪个",
+        "哪些",
+        "哪里",
+        "多少",
+        "几次",
+        "多久",
+        "为什么",
+        "是否",
+        "有没有",
+        "如何",
+    }
+)
+
+_FILLER = frozenset("的了是在和与就都而及或吧呢吗啊呀哦嗯我你他她它这那一个们")
+
+# Part-of-speech tags worth reporting as missing.
+#
+# Without this filter the top-1 keep rate was 0.325 -- 27 false refusals out of
+# 40 answerable questions -- because "直接", "应该", "不是", "哪几个" and "最终"
+# are in plenty of *questions* and in no *corpus*. They are the grammar of
+# asking, not the content of an answer. Only nominals and things that look like
+# names can plausibly be the thing a question is missing.
+_CONTENT_POS = frozenset(
+    {
+        "n",  # 名词
+        "nr",  # 人名
+        "ns",  # 地名
+        "nt",  # 机构团体
+        "nz",  # 其他专名
+        "ng",
+        "nl",
+        "eng",  # 英文
+        "vn",  # 动词性名词：测试、查询
+        "an",  # 形容词性名词
+    }
+)
+
+
+class _MemoryLike(Protocol):
+    id: str
+    session_id: str
+    content: str
+
+
+@dataclass(frozen=True, slots=True)
+class Evidence:
+    """One retrieved session, with the checks a caller can act on.
+
+    ``coverage`` is the share of the question's IDF-weighted terms present in
+    this session. ``missing_terms`` is what *this session* does not contain --
+    which explains why it is ranked where it is, but is not by itself evidence
+    that the memory is absent. A session missing 用户 is normal; the corpus
+    knowing 用户 somewhere else means the concept is not unknown.
+    """
+
+    memory: _MemoryLike
+    score: float
+    coverage: float
+    missing_terms: list[str] = field(default_factory=list)
+    # Memories whose text is close enough to this one that they may be saying the
+    # same thing. Reported, not resolved: the caller has a model and can read both
+    # and decide. Silently merging would be irreversible and this store has no
+    # standing to make that call.
+    possible_duplicates: list[str] = field(default_factory=list)
+
+    @property
+    def session_id(self) -> str:
+        return self.memory.session_id
+
+
+@dataclass(frozen=True, slots=True)
+class SearchResult:
+    """What the memory system hands back.
+
+    ``corpus_missing`` is the strong signal: terms the question treats as
+    central that appear nowhere in the entire stored corpus. A session
+    missing them is irrelevant; the brain missing them is an answer.
+
+    Keeping this separate from ``Evidence.missing_terms`` matters. Conflating
+    the two produced 16 false refusals out of 40, because 用户 and 记忆 are
+    absent from most individual sessions and present in the corpus.
+    """
+
+    query: str
+    evidence: list[Evidence]
+    corpus_terms: int
+    sessions_searched: int
+    corpus_missing: list[str] = field(default_factory=list)
+
+    @property
+    def empty(self) -> bool:
+        return not self.evidence
+
+    def best(self) -> Evidence | None:
+        return self.evidence[0] if self.evidence else None
+
+
+@dataclass(frozen=True, slots=True)
+class JudgedResult:
+    """A search plus a hosted judge's opinion, with the distinction kept.
+
+    ``verdict`` is one of:
+
+    - ``found``     -- the judge accepted a session; ``found`` returns it
+    - ``not_found`` -- every checked session was rejected
+    - ``uncertain`` -- the judge failed for at least one; the answer is unknown
+
+    The third state exists because a judge that cannot be reached is not the
+    same as a memory that does not exist. A caller that treats them the same
+    will tell a user "I never recorded this" during an outage.
+    """
+
+    result: SearchResult
+    kept: list[Evidence]
+    verdict: str
+    checks: int
+
+    @property
+    def found(self) -> Evidence | None:
+        return self.kept[0] if self.kept else None
+
+    @property
+    def certainty(self) -> str:
+        return self.verdict
+
+
+def _is_entity_like(term: str) -> bool:
+    """Fallback heuristic when jieba is absent.
+
+    No dictionary means no POS tags, so length and filler content are all there
+    is to go on. Noisier than the jieba path, and documented as such rather than
+    quietly degraded.
+    """
+    return len(term) > 1 and term not in _INTERROGATIVE and not all(ch in _FILLER for ch in term)
+
+
+def content_words(text: str) -> list[str]:
+    """Nominals and names from the question, in order.
+
+    Uses jieba's POS tagger when available. A long digit-bearing token is kept
+    even though it tags as ``m``: an IP address or a version number is exactly
+    the kind of thing whose absence is worth reporting.
+    """
+    jieba = load_jieba()
+    if jieba is None:
+        return [t for t in word_tokens(text) if _is_entity_like(t)]
+
+    import jieba.posseg as pseg
+
+    out: list[str] = []
+    for word, flag in pseg.cut(text):
+        w = word.strip().lower()
+        if not w or w.isspace():
+            continue
+        long_number = flag == "m" and len(w) >= 3 and any(ch.isdigit() for ch in w)
+        if flag in _CONTENT_POS or long_number:
+            out.append(w)
+    return out
+
+
+# Backwards-compatible alias: the function was private before the graph
+# needed it from another module.
+_content_words = content_words
+
+
+def question_terms(query: str) -> list[str]:
+    """Content terms of the question, longest first.
+
+    Longest first because the more specific term is the more informative
+    absence: for 数据库 what matters is that no session contains it, not that
+    数据 appears somewhere.
+    """
+    terms = [t for t in content_words(query) if t not in _INTERROGATIVE and t not in _FILLER]
+    seen: set[str] = set()
+    unique = [t for t in terms if not (t in seen or seen.add(t))]
+    return sorted(unique, key=len, reverse=True)
+
+
+def build_evidence(
+    *,
+    query: str,
+    hits: list[tuple[str, float]],
+    by_session: dict[str, list[_MemoryLike]],
+    session_text: dict[str, str],
+    id_to_session: dict[str, str],
+    sessions_searched: int,
+    top_k: int,
+) -> SearchResult:
+    """Turn ranked memory ids into ranked session evidence.
+
+    Presence is checked by substring, not by token set. For the question this
+    answers -- does the corpus contain the word 数据库 -- substring matching is
+    both exact and cheap, and it avoids tokenising the whole corpus on every
+    search. Tokenising for presence would also be wrong in the other direction:
+    a session containing 数据 and 存量 separately is not a session about
+    数据库, but a bag of bigrams says it is.
+    """
+    q_terms = question_terms(query)
+
+    # Computed before the empty-hits early return. "Nothing matched" and "the
+    # corpus has never contained 数据库" are different answers, and the second is
+    # what explains the first -- dropping it exactly when nothing matched threw
+    # away the only useful part of an empty result.
+    #
+    # Checked once against the joined corpus, not per session: 用户 missing from
+    # one session says nothing, 用户 missing from every session says everything.
+    texts = list(session_text.values())
+    joined = "\n".join(texts)
+    corpus_missing = [t for t in q_terms if t not in joined]
+
+    if not hits:
+        return SearchResult(
+            query=query,
+            evidence=[],
+            corpus_terms=0,
+            sessions_searched=sessions_searched,
+            corpus_missing=corpus_missing,
+        )
+
+    n_sessions = len(by_session) or 1
+
+    # Memoised. idf(term) is called once per query term for the weight total and
+    # again for every evidence row's coverage, so the same term was rescanned up
+    # to top_k+1 times over the whole corpus. Measured: build_evidence was 10.9 ms
+    # of a 13.8 ms query, and almost all of it was re-walking texts looking for
+    # words it had already looked for.
+    _df_cache: dict[str, int] = {}
+
+    def idf(term: str) -> float:
+        d = _df_cache.get(term)
+        if d is None:
+            d = sum(1 for t in texts if term in t)
+            _df_cache[term] = d
+        return math.log(1.0 + (n_sessions - d + 0.5) / (d + 0.5))
+
+    total_weight = sum(idf(t) for t in q_terms) or 1.0
+
+    evidence: list[Evidence] = []
+    seen: set[str] = set()
+    for memory_id, score in hits:
+        sid = id_to_session.get(memory_id)
+        if sid is None or sid in seen:
+            continue
+        seen.add(sid)
+
+        text = session_text.get(sid, "")
+        present = [t for t in q_terms if t in text]
+        missing = [t for t in q_terms if t not in text]
+        covered = sum(idf(t) for t in present)
+
+        first = by_session[sid][0]
+        evidence.append(
+            Evidence(
+                memory=first,
+                score=score,
+                coverage=round(covered / total_weight, 4),
+                missing_terms=missing,
+            )
+        )
+        if len(evidence) >= top_k:
+            break
+
+    corpus_terms = sum(1 for t in q_terms if any(t in x for x in texts))
+    return SearchResult(
+        query=query,
+        evidence=evidence,
+        corpus_terms=corpus_terms,
+        sessions_searched=sessions_searched,
+        corpus_missing=corpus_missing,
+    )
+
+
+def describe(result: SearchResult) -> str:
+    """Human-readable summary of a search."""
+    if result.empty:
+        line = f"no evidence for {result.query!r}"
+        if result.corpus_missing:
+            line += f"\n  corpus has no: {', '.join(result.corpus_missing)}"
+        return line
+    lines = [f"{len(result.evidence)} sessions for {result.query!r}"]
+    if result.corpus_missing:
+        lines.append(f"  corpus has no: {', '.join(result.corpus_missing)}")
+    for e in result.evidence:
+        missing = f"  session lacks: {', '.join(e.missing_terms)}" if e.missing_terms else ""
+        lines.append(f"  [{e.score:5.2f} cov={e.coverage:.2f}] {e.session_id}{missing}")
+    return "\n".join(lines)

brain4agent/extract.py

@@ -0,0 +1,230 @@
+"""Turn a conversation turn into durable memories.
+
+This is the piece that makes the store a memory system rather than a log.
+Storing every turn verbatim is what produced the noise in D-020: a hundred
+exchanges about the same subsystem, none of which can be told apart. Extracting
+one self-contained fact per thing worth keeping is how a real memory layer
+avoids that — and it is exactly what neural-memory does with fibers.
+
+Optional by design. Without a key, ``Extractor.from_env()`` returns None and the
+store falls back to keeping raw turns, which still works.
+
+What "durable" means here, because it is the whole judgement call:
+
+- a fact about how the system is
+- a decision and its reason
+- an error and its cause
+- a preference about how things should be done
+
+Not durable: acknowledgements, restatements of the question, plans that were
+never executed, and anything a reader would find obvious from the code.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+
+from brain4agent.llm import LLM
+
+KINDS = frozenset({"fact", "decision", "error", "preference", "context"})
+
+MAX_INPUT_CHARS = 6000
+MAX_CONTENT_CHARS = 500
+MAX_ENTITIES = 10
+DEFAULT_MAX_MEMORIES = 5
+
+SYSTEM = """You extract durable memories from one exchange in a working conversation.
+
+A durable memory is something a colleague would need weeks later: a fact about
+how the system is, a decision and its reason, an error and its cause, or a
+stated preference.
+
+Rules:
+- One memory per entry. Each must be self-contained: a reader with no context
+  must understand it. Replace pronouns and deictic references ("it", "that file",
+  "上次那个") with what they actually name.
+- **Keep the question's vocabulary.** A future search will use the words of the
+  person asking, not yours. If the question asks "哪个文件哪一行", the memory
+  must contain 文件 and 行. Rewriting it as only "extraction/keywords.py 第 443 行"
+  makes it unfindable by the very question it answers.
+- Keep specifics verbatim: file names, line numbers, addresses, versions, commit
+  hashes, error messages.
+- Drop acknowledgements, pure pleasantries, and plans never executed.
+- If the exchange has nothing durable, return an empty list. Returning nothing
+  is a correct and common answer.
+
+Output strict JSON and nothing else:
+{"memories":[{"kind":"fact","content":"...","entities":["..."],"importance":0.7}]}
+
+kind is one of: fact, decision, error, preference, context.
+importance is 0.0-1.0: how likely a future question depends on this entry."""
+
+USER = """Exchange:
+---
+{exchange}
+---"""
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractedMemory:
+    """One durable item distilled from a turn."""
+
+    kind: str
+    content: str
+    entities: tuple[str, ...] = ()
+    importance: float = 0.5
+
+
+@dataclass(frozen=True, slots=True)
+class ExtractionResult:
+    """Outcome of one extraction.
+
+    The distinction between an empty result and a failed one decides what the
+    store does: empty means "nothing durable here, drop the turn", failed means
+    "the model was unavailable, keep the raw text". Collapsing them would
+    either lose data on an outage or keep chit-chat when the model says it is
+    chit-chat.
+    """
+
+    memories: tuple[ExtractedMemory, ...] = ()
+    ok: bool = True
+
+
+class Extractor:
+    """Distils a turn into zero or more :class:`ExtractedMemory`.
+
+    Every public method returns a list, never raises for model problems. A
+    failed extraction must degrade to "store the raw turn", which the caller
+    decides; it must not take the write down with it.
+    """
+
+    def __init__(self, llm: LLM, *, max_memories: int = DEFAULT_MAX_MEMORIES) -> None:
+        self.llm = llm
+        self.max_memories = max_memories
+        self.calls = 0
+        self.failures = 0
+        self.empty = 0
+
+    @classmethod
+    def from_env(cls) -> Extractor | None:
+        llm = LLM.from_env()
+        return cls(llm) if llm is not None else None
+
+    def extract(
+        self,
+        text: str,
+        *,
+        question: str | None = None,
+        max_memories: int | None = None,
+    ) -> ExtractionResult:
+        """Distil one turn, or one question/answer pair.
+
+        Passing ``question`` is strongly preferred when the text is an answer:
+        the question carries the words a future search will use, and dropping
+        them was measured to halve hit@1 (D-023).
+        """
+        body = text.strip()
+        if not body:
+            return ExtractionResult((), ok=True)
+
+        if question and question.strip():
+            exchange = f"question: {question.strip()}\nanswer: {body}"
+        else:
+            exchange = body
+
+        limit = max_memories or self.max_memories
+        self.calls += 1
+        raw = self.llm.chat(
+            SYSTEM, USER.format(exchange=exchange[:MAX_INPUT_CHARS]), max_tokens=1024
+        )
+        if raw is None:
+            self.failures += 1
+            return ExtractionResult((), ok=False)
+
+        memories = parse(raw, limit=limit)
+        if not memories:
+            # "Nothing durable here" is a legitimate answer, not a failure, and
+            # must not count against health.
+            self.empty += 1
+        return ExtractionResult(tuple(memories), ok=True)
+
+    def stats(self) -> dict[str, object]:
+        return {
+            "calls": self.calls,
+            "failures": self.failures,
+            "empty": self.empty,
+            **self.llm.stats(),
+        }
+
+
+def parse(raw: str, *, limit: int = DEFAULT_MAX_MEMORIES) -> list[ExtractedMemory]:
+    """Parse a model response into memories, dropping anything malformed.
+
+    Models wrap JSON in prose or fences often enough that a strict
+    ``json.loads`` on the whole string is not enough. Anything that does not
+    survive validation is dropped rather than repaired: a guessed memory is
+    worse than a missing one.
+    """
+    blob = _find_json(raw)
+    if blob is None:
+        return []
+
+    items = blob.get("memories")
+    if not isinstance(items, list):
+        return []
+
+    out: list[ExtractedMemory] = []
+    for item in items[:limit]:
+        if not isinstance(item, dict):
+            continue
+
+        kind = str(item.get("kind", "")).strip().lower()
+        if kind not in KINDS:
+            continue
+
+        content = str(item.get("content", "")).strip()
+        if not content:
+            continue
+
+        raw_entities = item.get("entities")
+        entities: list[str] = []
+        if isinstance(raw_entities, list):
+            for value in raw_entities:
+                text = str(value).strip()
+                if text and text not in entities:
+                    entities.append(text)
+        entities = entities[:MAX_ENTITIES]
+
+        try:
+            importance = float(item.get("importance", 0.5))
+        except (TypeError, ValueError):
+            importance = 0.5
+        importance = min(1.0, max(0.0, importance))
+
+        out.append(
+            ExtractedMemory(
+                kind=kind,
+                content=content[:MAX_CONTENT_CHARS],
+                entities=tuple(entities),
+                importance=importance,
+            )
+        )
+    return out
+
+
+_FENCE = re.compile(r"^```[a-zA-Z]*\s*|\s*```$")
+
+
+def _find_json(raw: str) -> dict | None:
+    text = _FENCE.sub("", raw.strip())
+    start = text.find("{")
+    end = text.rfind("}")
+    if start < 0 or end <= start:
+        return None
+    try:
+        value = json.loads(text[start : end + 1])
+    except json.JSONDecodeError:
+        return None
+    return value if isinstance(value, dict) else None

brain4agent/graph.py

@@ -0,0 +1,100 @@
+"""Entity co-occurrence graph.
+
+D-023 left a specific gap. Distilled memories recall well (hit@10 1.0) but rank
+poorly, because every fact is a clean, topic-focused sentence and facts about
+the same subject share most of their vocabulary. IDF falls, and the top result
+becomes a coin flip between memories that are all "about" the question.
+
+Word matching cannot fix that, because the information is not in the words — it
+is in which memory is *connected* to the question through what it is about.
+
+This module supplies that connection cheaply and without a model:
+
+- each memory records its entities (nouns and names, extracted at write time)
+- an entity shared by two memories is an edge between them
+- retrieval expands from the best word-matches along those edges, with decay,
+  so a memory that shares a specific entity with a strong hit gets a boost even
+  if it does not share the query's exact wording
+
+Deliberately not Personalized PageRank. PPR needs a converged power iteration
+over the whole graph per query, and the measured problem here is one hop deep:
+"the fact I want shares an entity with a fact that already matched". One hop
+plus decay is measurable, explainable, and fast. If a two-hop gain ever shows up
+in a benchmark, this is the place to extend.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+
+# An entity appearing in more than this share of memories carries no signal --
+# it is the corpus's background, not a distinction. Keeping it would let every
+# memory reach every other and turn the boost into a constant.
+COMMON_ENTITY_RATIO = 0.25
+
+
+@dataclass(slots=True)
+class EntityGraph:
+    """Memory <-> entity index with one-hop expansion."""
+
+    _by_entity: dict[str, set[str]] = field(default_factory=dict)
+    _by_memory: dict[str, set[str]] = field(default_factory=dict)
+    _n: int = 0
+
+    def add(self, memory_id: str, entities: list[str] | tuple[str, ...]) -> None:
+        if not entities:
+            return
+        unique = {e for e in entities if e}
+        if not unique:
+            return
+        self._by_memory[memory_id] = unique
+        for entity in unique:
+            self._by_entity.setdefault(entity, set()).add(memory_id)
+
+    def finish(self) -> None:
+        """Record corpus size so the common-entity cut-off can be applied."""
+        self._n = len(self._by_memory)
+
+    def _too_common(self, entity: str) -> bool:
+        if self._n < 8:  # too small to have a meaningful background
+            return False
+        return len(self._by_entity.get(entity, ())) > max(8, self._n * COMMON_ENTITY_RATIO)
+
+    def expand(
+        self,
+        seeds: list[tuple[str, float]],
+        *,
+        decay: float = 0.35,
+        max_seeds: int = 5,
+    ) -> dict[str, float]:
+        """Boost memories that share a specific entity with a strong hit.
+
+        ``seeds`` is the ranked BM25 output. Only the head of it is expanded:
+        propagating from a weak match would spread noise rather than signal.
+
+        Returns memory_id -> boost, excluding the seeds themselves.
+        """
+        boosts: dict[str, float] = {}
+        seed_ids = {mid for mid, _ in seeds}
+
+        for memory_id, score in seeds[:max_seeds]:
+            for entity in self._by_memory.get(memory_id, ()):
+                if self._too_common(entity):
+                    continue
+                for other in self._by_entity.get(entity, ()):
+                    if other in seed_ids:
+                        continue
+                    candidate = score * decay
+                    if candidate > boosts.get(other, 0.0):
+                        boosts[other] = candidate
+        return boosts
+
+    def stats(self) -> dict[str, int]:
+        sizes = Counter(len(v) for v in self._by_memory.values())
+        return {
+            "memories": len(self._by_memory),
+            "entities": len(self._by_entity),
+            "edges": sum(len(v) - 1 for v in self._by_entity.values() if len(v) > 1),
+            "isolated": sizes.get(0, 0) + sum(1 for v in self._by_memory.values() if len(v) == 1),
+        }