structuremappingmemory 1.0.0__py3-none-any.whl

sma/eval/baselines/longcontext_llm.py

@@ -0,0 +1,124 @@
+"""B6 long-context LLM baseline (blueprint section 8.1 B6).
+
+Controls for "maybe you don't need retrieval at all": stuff the query session
+plus its top-20 BM25 candidate precedents (labels included) into one prompt
+and ask deepseek-chat (temperature 0, max_tokens 10) to label the query.
+
+API key comes from SMA_DEEPSEEK_API_KEY or the repo .env, via the same lookup
+the agent layer uses (sma.agent.llm._env_key). Errors get exactly one retry;
+a second failure (or an unparseable reply) marks the row failed.
+"""
+
+from __future__ import annotations
+
+import time
+
+from sma.agent.llm import DEEPSEEK_BASE_URL, DEEPSEEK_KEY_ENV, DEEPSEEK_MODEL, _env_key
+
+CANDIDATE_CHARS = 800
+QUERY_CHARS = 1600
+
+SYSTEM_PROMPT = (
+    "You are an incident triage assistant. You label log sessions as Anomaly or "
+    "Normal by analogy to labeled precedent sessions. Reply with exactly one word: "
+    "Anomaly or Normal."
+)
+
+
+def build_prompt(query_text: str, precedents: list[tuple[str, str]]) -> str:
+    """precedents: list of (label, session_text), already ranked."""
+    lines = [
+        "Query session (label unknown):",
+        query_text[:QUERY_CHARS],
+        "",
+        "Labeled precedent sessions (retrieved by lexical similarity, most similar first):",
+    ]
+    for i, (label, text) in enumerate(precedents, start=1):
+        lines.append(f"[{i}] ({label}) {text[:CANDIDATE_CHARS]}")
+    lines.append("")
+    lines.append(
+        "Based on these precedents, is the query session Anomaly or Normal? "
+        "Answer with exactly one word."
+    )
+    return "\n".join(lines)
+
+
+def parse_label(content: str) -> str | None:
+    lowered = content.lower()
+    has_anomaly = "anomal" in lowered
+    has_normal = "normal" in lowered
+    if has_anomaly and not has_normal:
+        return "Anomaly"
+    if has_normal and not has_anomaly:
+        return "Normal"
+    if has_anomaly and has_normal:  # ambiguous reply
+        first_a = lowered.find("anomal")
+        first_n = lowered.find("normal")
+        return "Anomaly" if first_a < first_n else "Normal"
+    return None
+
+
+class LongContextDeepSeek:
+    def __init__(
+        self,
+        model: str = DEEPSEEK_MODEL,
+        api_key: str | None = None,
+        timeout: float = 60.0,
+        retry_sleep: float = 2.0,
+    ):
+        self.model = model
+        self.api_key = api_key or _env_key(DEEPSEEK_KEY_ENV)
+        self.timeout = timeout
+        self.retry_sleep = retry_sleep
+        self.calls = 0
+        self.failures: list[str] = []
+        self.total_prompt_tokens = 0
+        self.total_completion_tokens = 0
+
+    def _call_once(self, prompt: str) -> str:
+        import httpx
+
+        response = httpx.post(
+            f"{DEEPSEEK_BASE_URL}/chat/completions",
+            headers={"Authorization": f"Bearer {self.api_key}"},
+            json={
+                "model": self.model,
+                "messages": [
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": prompt},
+                ],
+                "temperature": 0,
+                "max_tokens": 10,
+            },
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        payload = response.json()
+        self.calls += 1
+        usage = payload.get("usage", {})
+        self.total_prompt_tokens += int(usage.get("prompt_tokens", 0))
+        self.total_completion_tokens += int(usage.get("completion_tokens", 0))
+        return (payload["choices"][0]["message"]["content"] or "").strip()
+
+    def classify(
+        self, query_id: str, query_text: str, precedents: list[tuple[str, str]]
+    ) -> str | None:
+        """Return 'Anomaly'/'Normal', or None when the row failed (after one retry)."""
+        if not self.api_key:
+            self.failures.append(f"{query_id}: {DEEPSEEK_KEY_ENV} not set")
+            return None
+        prompt = build_prompt(query_text, precedents)
+        last_error = ""
+        for attempt in range(2):  # one initial call + one retry
+            try:
+                content = self._call_once(prompt)
+                label = parse_label(content)
+                if label is not None:
+                    return label
+                last_error = f"unparseable reply: {content!r}"
+            except Exception as exc:  # noqa: BLE001 - any transport/API error retries once
+                last_error = f"{type(exc).__name__}: {exc}"
+            if attempt == 0:
+                time.sleep(self.retry_sleep)
+        self.failures.append(f"{query_id}: {last_error}")
+        return None

sma/eval/baselines/rerank.py

@@ -0,0 +1,41 @@
+"""Cross-encoder reranking (cross-encoder/ms-marco-MiniLM-L-6-v2).
+
+Reranks a candidate pool (here: top-20 of Hybrid-RRF) and returns the top-k.
+Raw model outputs are logits; we pass them through a sigmoid so the scores
+are positive and usable by the protocol's weighted top-5 label vote (a
+negative weight would flip a vote, which is not what 'weighted' means there).
+Candidate texts are truncated to ``max_chars`` before pairing; the tokenizer
+truncates to 512 wordpieces anyway, so this only bounds tokenizer cost.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+MODEL_NAME = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+
+
+class CrossEncoderReranker:
+    def __init__(self, model_name: str = MODEL_NAME, max_chars: int = 2000):
+        from sentence_transformers import CrossEncoder
+
+        self.model = CrossEncoder(model_name, device="cpu", max_length=512)
+        self.max_chars = max_chars
+
+    def rerank(
+        self,
+        query_text: str,
+        candidates: list[tuple[str, str]],
+        top_k: int = 10,
+    ) -> list[tuple[str, float]]:
+        if not candidates:
+            return []
+        q = query_text[: self.max_chars]
+        pairs = [(q, text[: self.max_chars]) for _, text in candidates]
+        logits = self.model.predict(pairs, show_progress_bar=False)
+        scores = 1.0 / (1.0 + np.exp(-np.asarray(logits, dtype=np.float64)))
+        ranked = sorted(
+            zip((doc_id for doc_id, _ in candidates), map(float, scores)),
+            key=lambda row: (-row[1], row[0]),
+        )
+        return ranked[:top_k]

sma/eval/baselines/splade.py

@@ -0,0 +1,77 @@
+"""SPLADE learned sparse retrieval (naver/splade-cocondenser-ensembledistil).
+
+CPU-only. Standard SPLADE document/query representation:
+    rep = max over token positions of log(1 + relu(MLM logits)),
+masked by attention. Documents are batch-encoded once into a scipy CSR
+matrix; query scoring is a sparse dot product. Per-query latency includes
+the query forward pass (same convention as the other neural baselines).
+
+Inputs are truncated to ``max_length`` wordpieces (default 256) to keep the
+CPU forward pass tractable on long log sessions; the same truncation applies
+to documents and queries, so no method gets privileged context.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from scipy import sparse
+
+MODEL_NAME = "naver/splade-cocondenser-ensembledistil"
+
+
+class SpladeRetriever:
+    def __init__(
+        self,
+        model_name: str = MODEL_NAME,
+        max_length: int = 256,
+        batch_size: int = 8,
+    ):
+        import torch
+        from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+        self.torch = torch
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.model = AutoModelForMaskedLM.from_pretrained(model_name)
+        self.model.eval()
+        self.max_length = max_length
+        self.batch_size = batch_size
+        self.doc_ids: list[str] = []
+        self.doc_matrix: sparse.csr_matrix | None = None
+
+    def _encode_batch(self, texts: list[str]) -> np.ndarray:
+        torch = self.torch
+        tokens = self.tokenizer(
+            texts,
+            return_tensors="pt",
+            padding=True,
+            truncation=True,
+            max_length=self.max_length,
+        )
+        with torch.no_grad():
+            logits = self.model(**tokens).logits  # (B, T, V)
+        # log(1 + relu(logits)), max-pooled over valid token positions.
+        weights = torch.log1p(torch.relu(logits))
+        mask = tokens["attention_mask"].unsqueeze(-1)
+        reps = (weights * mask).max(dim=1).values  # (B, V)
+        return reps.numpy()
+
+    def encode(self, texts: list[str]) -> sparse.csr_matrix:
+        rows = []
+        for start in range(0, len(texts), self.batch_size):
+            batch = texts[start : start + self.batch_size]
+            rows.append(sparse.csr_matrix(self._encode_batch(batch)))
+        return sparse.vstack(rows) if rows else sparse.csr_matrix((0, 0))
+
+    def build(self, documents: list[tuple[str, str]]) -> None:
+        self.doc_ids = [doc_id for doc_id, _ in documents]
+        self.doc_matrix = self.encode([text for _, text in documents])
+
+    def retrieve(self, query_text: str, k: int = 10) -> list[tuple[str, float]]:
+        if self.doc_matrix is None:
+            return []
+        q = sparse.csr_matrix(self._encode_batch([query_text]))
+        scores = np.asarray((self.doc_matrix @ q.T).todense()).ravel()
+        ranked = sorted(
+            zip(self.doc_ids, map(float, scores)), key=lambda row: (-row[1], row[0])
+        )
+        return ranked[:k]

sma/eval/baselines/wl_kernel.py

@@ -0,0 +1,163 @@
+"""Weisfeiler-Leman graph-kernel control over SMA's own Tier-0 cases.
+
+The internal control for the ladder: does *generic* graph similarity computed
+on the SAME deterministic extraction (sma.encoders get_encoder("logs")) match
+structure-mapping retrieval? If yes, SMA's edge is the extraction; if no, the
+edge is the mapping mathematics.
+
+Graph construction (per case):
+  - nodes  = unique expressions (keyed by canonical s-expression) plus unique
+    entities (keyed by (name, type));
+  - edges  = statement -> argument, position-annotated.
+
+Node labels (iteration 0):
+  - statements: the functor (exactly what SMA's own content vectors see);
+  - entities whose names are arbitrary per-session identifiers (types
+    "event", "session") are labeled by TYPE ONLY. This is not a kindness, it
+    is a necessity: the session entity touches nearly every statement, so a
+    case-unique session name would make every refined label case-unique after
+    one iteration and the kernel would degenerate to zero similarity.
+  - all other entities (components, event_type tokens, integer counts) keep
+    "type:name" -- their names are shared vocabulary, i.e. real content.
+
+Refinement: 2 WL iterations; new label = hash(own label, position-sorted child
+labels, sorted parent labels). Similarity = cosine over the concatenated label
+histograms of iterations 0..2 (Shervashidze et al. 2011, WL subtree kernel,
+normalized).
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections import Counter
+
+import numpy as np
+from scipy import sparse
+
+from sma.ir.schema import Case, Entity, Statement
+from sma.ir.sexpr import dumps_statement
+
+# Entity types whose names are arbitrary per-session identifiers.
+_ID_LIKE_TYPES = frozenset({"event", "session"})
+
+WL_ITERATIONS = 2
+
+
+def _entity_label(ent: Entity) -> str:
+    if ent.type in _ID_LIKE_TYPES:
+        return f"ent:{ent.type}"
+    return f"ent:{ent.type}:{ent.name}"
+
+
+def _hash_label(payload: str, memo: dict[str, str]) -> str:
+    cached = memo.get(payload)
+    if cached is None:
+        cached = hashlib.blake2b(payload.encode("utf-8"), digest_size=8).hexdigest()
+        memo[payload] = cached
+    return cached
+
+
+def wl_histogram(case: Case, iterations: int = WL_ITERATIONS) -> Counter[str]:
+    """Concatenated WL label histogram (iterations 0..n) for one case."""
+    # --- build the node/edge structure ---------------------------------
+    node_labels: list[str] = []
+    children: list[list[tuple[int, int]]] = []  # node -> [(arg_pos, child_node)]
+    parents: list[list[int]] = []
+    stmt_idx: dict[str, int] = {}
+    ent_idx: dict[tuple[str, str], int] = {}
+
+    def add_node(label: str) -> int:
+        node_labels.append(label)
+        children.append([])
+        parents.append([])
+        return len(node_labels) - 1
+
+    def visit(node: Statement | Entity) -> int:
+        if isinstance(node, Entity):
+            key = (node.name, node.type)
+            idx = ent_idx.get(key)
+            if idx is None:
+                idx = add_node(_entity_label(node))
+                ent_idx[key] = idx
+            return idx
+        skey = dumps_statement(node)
+        idx = stmt_idx.get(skey)
+        if idx is not None:
+            return idx
+        idx = add_node(f"f:{node.functor}")
+        stmt_idx[skey] = idx
+        for pos, arg in enumerate(node.args):
+            child = visit(arg)
+            children[idx].append((pos, child))
+            parents[child].append(idx)
+        return idx
+
+    for statement in case.statements:
+        visit(statement)
+
+    # --- WL refinement ---------------------------------------------------
+    memo: dict[str, str] = {}
+    histogram: Counter[str] = Counter()
+    labels = list(node_labels)
+    for node_label in labels:
+        histogram[f"wl0:{node_label}"] += 1
+    for it in range(1, iterations + 1):
+        new_labels = []
+        for idx, own in enumerate(labels):
+            child_part = sorted(f"c{pos}:{labels[child]}" for pos, child in children[idx])
+            parent_part = sorted(f"p:{labels[p]}" for p in parents[idx])
+            payload = own + "|" + ",".join(child_part) + "|" + ",".join(parent_part)
+            new_labels.append(_hash_label(payload, memo))
+        labels = new_labels
+        for node_label in labels:
+            histogram[f"wl{it}:{node_label}"] += 1
+    return histogram
+
+
+class WLKernelRetriever:
+    """Cosine-normalized WL subtree kernel retrieval over encoded cases."""
+
+    def __init__(self, iterations: int = WL_ITERATIONS):
+        self.iterations = iterations
+        self.doc_ids: list[str] = []
+        self.feature_index: dict[str, int] = {}
+        self.doc_matrix: sparse.csr_matrix | None = None  # rows L2-normalized
+
+    def build(self, cases: list[Case]) -> None:
+        histograms = [wl_histogram(c, self.iterations) for c in cases]
+        self.doc_ids = [c.case_id for c in cases]
+        self.feature_index = {}
+        for hist in histograms:
+            for feat in hist:
+                if feat not in self.feature_index:
+                    self.feature_index[feat] = len(self.feature_index)
+        rows, cols, vals = [], [], []
+        for row, hist in enumerate(histograms):
+            for feat, count in hist.items():
+                rows.append(row)
+                cols.append(self.feature_index[feat])
+                vals.append(float(count))
+        matrix = sparse.csr_matrix(
+            (vals, (rows, cols)), shape=(len(histograms), max(len(self.feature_index), 1))
+        )
+        norms = np.sqrt(matrix.multiply(matrix).sum(axis=1)).A.ravel()
+        norms[norms == 0] = 1.0
+        self.doc_matrix = sparse.diags(1.0 / norms) @ matrix
+
+    def retrieve(self, query_case: Case, k: int = 10) -> list[tuple[str, float]]:
+        if self.doc_matrix is None:
+            return []
+        hist = wl_histogram(query_case, self.iterations)
+        # Query norm uses the FULL histogram (including features absent from
+        # the index vocabulary) so cosine is honest, not inflated.
+        q_norm = float(np.sqrt(sum(v * v for v in hist.values()))) or 1.0
+        q = np.zeros(self.doc_matrix.shape[1])
+        for feat, count in hist.items():
+            col = self.feature_index.get(feat)
+            if col is not None:
+                q[col] = count / q_norm
+        scores = self.doc_matrix @ q
+        ranked = sorted(
+            zip(self.doc_ids, map(float, scores)), key=lambda row: (-row[1], row[0])
+        )
+        return ranked[:k]