infinity-context 0.1.0__py3-none-any.whl

infinity_context_server/memory_comparison_mem0_http_observation.py

@@ -0,0 +1,296 @@
+"""Exact LoCoMo observations at the mem0 OSS wrapper HTTP boundary."""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Mapping
+from threading import RLock
+from typing import NamedTuple
+
+import httpx
+
+from infinity_context_server.memory_comparison_locomo_expected_turn import (
+    ExpectedOfficialLocomoTurn,
+    canonical_locomo_source_id,
+)
+from infinity_context_server.memory_comparison_locomo_transport import (
+    LOCOMO_OFFICIAL_INGEST_MODE,
+    LocomoOfficialTurnsTransportRequest,
+    LocomoTimestampTransportEvidence,
+    RunScopedLocomoTransportEvidenceKey,
+)
+from infinity_context_server.public_benchmark_models import PublicBenchmarkCase
+
+MEM0_HTTP_OBSERVATION_BOUNDARY = "mem0_oss_wrapper_http_client_transport"
+MEM0_HTTP_OBSERVED_REPRESENTATION = "decoded_json_canonical_projection"
+_EXPECTED_REQUEST_KEYS = {
+    "messages",
+    "user_id",
+    "run_id",
+    "metadata",
+    "timestamp",
+}
+_OBSERVATION_CONTEXT_EXTENSION = "infinity_context.mem0_observation_context"
+_EVIDENCE_EXTENSION = "infinity_context.mem0_locomo_transport_evidence"
+_MANAGED_TRIGGER_CASE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$")
+
+
+class _RequestObservationContext(NamedTuple):
+    run_id: str
+    generation: int
+    verifier: RunScopedLocomoTransportEvidenceKey
+    expected_turn: ExpectedOfficialLocomoTurn
+    public_trigger_case_id: str | None
+
+
+class Mem0HttpObservationRecorder:
+    """Own thread-safe run observations and reset-fenced HTTP completions."""
+
+    def __init__(self) -> None:
+        self._lock = RLock()
+        self._generations: dict[str, int] = {}
+        self._verifiers: dict[str, RunScopedLocomoTransportEvidenceKey] = {}
+        self._evidence: dict[str, list[LocomoTimestampTransportEvidence]] = {}
+
+    def reset(self, *, run_id: str) -> None:
+        with self._lock:
+            self._generations[run_id] = self._generations.get(run_id, 0) + 1
+            self._verifiers.pop(run_id, None)
+            self._evidence.pop(run_id, None)
+
+    def prepare_request(
+        self,
+        request: httpx.Request,
+        *,
+        run_id: str,
+        expected_turn: ExpectedOfficialLocomoTurn,
+        public_trigger_case_id: str | None = None,
+    ) -> None:
+        public_trigger = _validated_public_trigger_case_id(public_trigger_case_id)
+        with self._lock:
+            generation = self._generations.get(run_id, 0)
+            verifier = self._verifiers.get(run_id)
+            if verifier is None:
+                verifier = RunScopedLocomoTransportEvidenceKey.generate(run_id=run_id)
+                self._verifiers[run_id] = verifier
+            request.extensions[_OBSERVATION_CONTEXT_EXTENSION] = _RequestObservationContext(
+                run_id=run_id,
+                generation=generation,
+                verifier=verifier,
+                expected_turn=expected_turn,
+                public_trigger_case_id=public_trigger,
+            )
+
+    def observe_at_transport_boundary(self, request: httpx.Request) -> None:
+        context = request.extensions.get(_OBSERVATION_CONTEXT_EXTENSION)
+        if context is None:
+            return
+        if type(context) is not _RequestObservationContext:
+            raise ValueError("mem0 HTTP observation context is invalid")
+        request.extensions[_EVIDENCE_EXTENSION] = observe_mem0_add_request(
+            request,
+            expected_turn=context.expected_turn,
+            verifier=context.verifier,
+            public_trigger_case_id=context.public_trigger_case_id,
+        )
+
+    def record_completed_request(self, request: httpx.Request, *, run_id: str) -> bool:
+        context = request.extensions.pop(_OBSERVATION_CONTEXT_EXTENSION, None)
+        evidence = request.extensions.pop(_EVIDENCE_EXTENSION, None)
+        if context is None and evidence is None:
+            return False
+        if (
+            type(context) is not _RequestObservationContext
+            or type(evidence) is not LocomoTimestampTransportEvidence
+            or context.run_id != run_id
+        ):
+            raise ValueError("mem0 HTTP transport completion context is invalid")
+        with self._lock:
+            current_generation = self._generations.get(run_id, 0)
+            current_verifier = self._verifiers.get(run_id)
+            if context.generation != current_generation or context.verifier is not current_verifier:
+                raise ValueError("stale mem0 HTTP transport completion after reset")
+            self._evidence.setdefault(run_id, []).append(evidence)
+            return True
+
+    def verifier(self, *, run_id: str) -> RunScopedLocomoTransportEvidenceKey | None:
+        with self._lock:
+            return self._verifiers.get(run_id)
+
+    def evidence(
+        self,
+        *,
+        run_id: str,
+    ) -> tuple[LocomoTimestampTransportEvidence, ...]:
+        with self._lock:
+            return tuple(self._evidence.get(run_id, ()))
+
+    def evidence_count(self, *, run_id: str) -> int:
+        with self._lock:
+            return len(self._evidence.get(run_id, ()))
+
+
+def expected_official_locomo_turn_for_group(
+    case: PublicBenchmarkCase,
+    *,
+    group_index: int,
+    run_id: str,
+    corpus_key: str,
+) -> ExpectedOfficialLocomoTurn | None:
+    """Build the sealed provider-neutral loader projection before HTTP mapping."""
+
+    if case.metadata.get("locomo_ingest_mode") != LOCOMO_OFFICIAL_INGEST_MODE:
+        return None
+    if case.benchmark != "locomo":
+        raise ValueError("official-turns ingestion requires the locomo benchmark")
+    if case.conversations or case.documents:
+        raise ValueError("official-turns ingestion must contain only turn memories")
+    if type(group_index) is not int or not 1 <= group_index <= len(case.memories):
+        raise ValueError("official-turns group does not map to exactly one loader turn")
+
+    memory = case.memories[group_index - 1]
+    metadata = memory.metadata
+    source_external_id = memory.source_external_id
+    if type(source_external_id) is not str:
+        raise ValueError("official LoCoMo turn requires source_external_id")
+    timestamp = metadata.get("timestamp")
+    if type(timestamp) is not int:
+        raise ValueError("official LoCoMo turn requires an exact integer timestamp")
+    return ExpectedOfficialLocomoTurn.create(
+        run_id=run_id,
+        corpus_key=corpus_key,
+        source_external_id=source_external_id,
+        source_id=canonical_locomo_source_id(source_external_id),
+        session_key=_exact_string(metadata, "session_key"),
+        dia_id=_exact_string(metadata, "dia_id"),
+        speaker=_exact_string(metadata, "speaker"),
+        session_date=_exact_string(metadata, "session_date"),
+        trigger_case_id=case.case_id,
+        role=_exact_string(metadata, "role"),
+        content=memory.text,
+        timestamp=timestamp,
+    )
+
+
+def observe_mem0_add_request(
+    request: httpx.Request,
+    *,
+    expected_turn: ExpectedOfficialLocomoTurn,
+    verifier: RunScopedLocomoTransportEvidenceKey,
+    public_trigger_case_id: str | None = None,
+) -> LocomoTimestampTransportEvidence:
+    """Observe wrapper HTTP JSON, without claiming downstream SDK wire bytes."""
+
+    if request.method != "POST" or request.url.path != "/memories":
+        raise ValueError("expected the mem0 OSS wrapper POST /memories boundary")
+    content_type = request.headers.get("Content-Type", "").partition(";")[0].strip().lower()
+    if content_type != "application/json":
+        raise ValueError("mem0 AddRequest must use application/json")
+    payload = _exact_json_object(request.content)
+    if set(payload) != _EXPECTED_REQUEST_KEYS:
+        raise ValueError("observed mem0 AddRequest fields are not exact")
+
+    messages = payload["messages"]
+    metadata = payload["metadata"]
+    idempotency_key = request.headers.get("Idempotency-Key")
+    if type(messages) is not list or any(type(message) is not dict for message in messages):
+        raise ValueError("observed mem0 messages must be exact JSON objects")
+    if type(metadata) is not dict:
+        raise ValueError("observed mem0 metadata must be an exact JSON object")
+    if type(idempotency_key) is not str:
+        raise ValueError("observed mem0 AddRequest requires Idempotency-Key")
+
+    projection = LocomoOfficialTurnsTransportRequest.create(
+        messages=messages,
+        user_id=_payload_string(payload, "user_id"),
+        run_id=_payload_string(payload, "run_id"),
+        metadata=metadata,
+        timestamp=_payload_int(payload, "timestamp"),
+        idempotency_key=idempotency_key,
+    )
+    return verifier.issue(
+        projection,
+        expected_turn=expected_turn,
+        public_trigger_case_id=_validated_public_trigger_case_id(public_trigger_case_id),
+    )
+
+
+def _validated_public_trigger_case_id(value: object) -> str | None:
+    if value is None:
+        return None
+    if type(value) is not str or _MANAGED_TRIGGER_CASE_ID.fullmatch(value) is None:
+        raise ValueError("managed public trigger case_id is invalid")
+    return value
+
+
+def mem0_http_observation_metadata(
+    *,
+    required: bool,
+    evidence_count: int,
+) -> dict[str, object]:
+    """Describe exactly what runtime evidence does and does not attest."""
+
+    if type(required) is not bool:
+        raise ValueError("required must be an exact boolean")
+    if type(evidence_count) is not int or evidence_count < 0:
+        raise ValueError("evidence_count must be a non-negative exact integer")
+    return {
+        "required": required,
+        "evidence_count": evidence_count,
+        "boundary": MEM0_HTTP_OBSERVATION_BOUNDARY,
+        "observed_representation": MEM0_HTTP_OBSERVED_REPRESENTATION,
+        "downstream_provider_sdk_wire_bytes_observed": False,
+    }
+
+
+def _exact_json_object(payload: object) -> dict[str, object]:
+    if type(payload) is not bytes or not payload:
+        raise ValueError("observed mem0 AddRequest body must be non-empty exact bytes")
+    try:
+        parsed = json.loads(payload, object_pairs_hook=_exact_object_pairs)
+    except (TypeError, ValueError, UnicodeDecodeError) as error:
+        raise ValueError("observed mem0 AddRequest body is malformed") from error
+    if type(parsed) is not dict:
+        raise ValueError("observed mem0 AddRequest body must be an exact JSON object")
+    return parsed
+
+
+def _exact_object_pairs(pairs: list[tuple[str, object]]) -> dict[str, object]:
+    result: dict[str, object] = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError("observed mem0 AddRequest contains a duplicate field")
+        result[key] = value
+    return result
+
+
+def _exact_string(metadata: Mapping[str, object], key: str) -> str:
+    value = metadata.get(key)
+    if type(value) is not str:
+        raise ValueError(f"official LoCoMo turn requires exact {key}")
+    return value
+
+
+def _payload_string(payload: dict[str, object], key: str) -> str:
+    value = payload[key]
+    if type(value) is not str:
+        raise ValueError(f"observed mem0 AddRequest {key} must be an exact string")
+    return value
+
+
+def _payload_int(payload: dict[str, object], key: str) -> int:
+    value = payload[key]
+    if type(value) is not int:
+        raise ValueError(f"observed mem0 AddRequest {key} must be an exact integer")
+    return value
+
+
+__all__ = (
+    "MEM0_HTTP_OBSERVATION_BOUNDARY",
+    "MEM0_HTTP_OBSERVED_REPRESENTATION",
+    "Mem0HttpObservationRecorder",
+    "expected_official_locomo_turn_for_group",
+    "mem0_http_observation_metadata",
+    "observe_mem0_add_request",
+)

infinity_context_server/memory_comparison_mem0_locomo_prompts.py

@@ -0,0 +1,328 @@
+# SPDX-License-Identifier: Apache-2.0
+# ruff: noqa
+# Vendored from mem0ai/memory-benchmarks at commit
+# 4b61c5d31b9c668a12b4f5e78064248a02c82d2b.
+# Upstream path: benchmarks/locomo/prompts.py
+
+"""
+LOCOMO Benchmark Prompts
+========================
+
+Answer generation and category-specific judge prompts for the LOCOMO
+benchmark (Snap Research, ACL 2024). Uses the industry-standard J-score
+methodology: binary LLM judge (CORRECT/WRONG) on categories 1-4.
+
+Category mapping:
+    1 = multi-hop (282 questions)
+    2 = temporal reasoning (321 questions)
+    3 = open-domain (96 questions)
+    4 = single-hop (841 questions)
+    5 = adversarial (446 questions) — excluded from scoring
+"""
+
+from datetime import datetime as _datetime
+from typing import List
+
+
+# ===============================================================================
+# CATEGORY NAME MAPPING
+# ===============================================================================
+
+CATEGORY_NAMES = {
+    1: "multi-hop",
+    2: "temporal",
+    3: "open-domain",
+    4: "single-hop",
+    5: "adversarial",
+}
+
+CATEGORIES_TO_EVALUATE = [1, 2, 3, 4]
+
+
+# ===============================================================================
+# ANSWER GENERATION PROMPT
+# ===============================================================================
+
+ANSWER_GENERATION_PROMPT = """You are answering a question using retrieved memories from past conversations. Follow these reasoning steps IN ORDER.
+
+## Step 1: SCAN ALL MEMORIES
+Read EVERY memory below from first to last. For each one that contains information relevant to the question, note it. Do NOT stop after finding the first relevant memory — important details are often scattered across many memories, including ones far down the list. Give equal weight to ALL memories regardless of position — a memory near the end is just as likely to contain the answer as one near the beginning. In these memories, "User" refers to the main person whose memories these are.
+
+## Step 2: ENTITY VERIFICATION
+Confirm each relevant memory is about the correct person/entity. If the question asks "What does Person A like?" and a memory says "Person B likes X", do NOT use that memory to answer about Person A. In two-person conversations, both speakers' actions are relevant — if the question asks about person A and a memory attributes an action to person B (the other speaker), that information is still valid evidence from their shared conversations, but always check the attribution is correct.
+
+## Step 3: COMBINE AND CROSS-REFERENCE
+- COMBINE facts from multiple memories about the same topic. If one memory says "won first place" and another says "performed a piece titled X," those describe the same event — connect them.
+- For listing/counting questions, extract EVERY distinct item from ALL memories. A single memory may contain multiple items. Think about what CATEGORIES of answers the question could have, then re-scan specifically for each category.
+- For counting questions ("how many times", "how many X"), enumerate each distinct instance explicitly with its date or context BEFORE giving a final count. Do not estimate — list them out, then count the list.
+- DECOMPOSE complex sentences: "an immersive X with Y, enjoys Z" contains multiple distinct facts. Each could be the answer.
+- Connect related facts across memories: if one says "nearby lake" and another says "Lake Tahoe is great for kayaking", the nearby lake IS Lake Tahoe. If one says "bought X in Paris", infer the country is France.
+
+## Step 4: SELECT THE BEST ANSWER
+- Do NOT assume the highest-ranked memory is correct. Multiple memories may describe different events for the same topic. Compare each candidate's relevance to the SPECIFIC question, not its retrieval score. A lower-ranked memory that directly answers the question beats a higher-ranked one that is only tangentially related.
+- ALWAYS choose the MOST SPECIFIC detail available. A proper name, title, or number beats a generic description. Rate each candidate as HIGH specificity (name, title, number, specific activity) or LOW (generic description), and prefer HIGH.
+- Report what someone actually DID, not what was offered or available to them. "Has not tried X yet" means X was NOT done — disqualify it. "Joined X" or "has done X" means it WAS done — prefer it.
+- When multiple memories repeat the same generic fact, that repetition does NOT make it more correct than a single memory with a more specific answer.
+- Photos depict what was IN the photo, not facts about someone's daily life. Prefer direct statements over photo descriptions for inferences.
+- Re-read the question carefully before answering. If it asks "what aspect/type/kind", answer with the specific aspect. If it asks "what did they discover they both enjoy", answer with the specific thing, not the setting.
+
+## Step 5: TEMPORAL GROUNDING
+These conversations took place around {reference_date}. All events occurred in 2022-2024.
+- Calculate time relative to this date, NOT today. Never output 2025 or 2026.
+- Use dates explicitly stated in memory text. Do not invent or estimate dates.
+- When a question asks what someone "shared" or "mentioned" on a date, that date is when they TALKED about it — look for events shortly BEFORE that date.
+- For "how long" questions, find the start and end dates explicitly, then compute the duration. Do not guess.
+- TEMPORAL DISAMBIGUATION: When you find MULTIPLE instances of similar events at different dates, enumerate them all with their dates before picking. If the question uses past tense + "the" → select the instance closest to (and before) the reference date. If future tense ("plans to", "going to") → select the earliest planned date. NEVER default to the first-mentioned or highest-scored instance — the DATE determines the answer.
+
+## Step 6: INCLUSION CHECK (for lists and counts)
+If you found items during reasoning that you're tempted to exclude from your answer — STOP. Include them unless you have STRONG evidence they are wrong. The most common mistake is finding relevant items but then dropping them due to overly strict filtering. More items is better than fewer when there is supporting evidence.
+- For counting: after enumerating, re-verify each item. Check for duplicates (same event described differently) and ensure you haven't missed items from memories late in the list.
+- The question assumes something happened. Find WHAT happened, don't say nothing happened.
+
+## Step 7: COMMIT AND ANSWER
+Give a direct, specific answer. NEVER say "not specified", "not mentioned", "no record", or "the memories don't say" — if ANY memory contains relevant information, give the best answer from available evidence. No hedging, no caveats. If the question asks for a list, include ALL items found. NEVER return an empty answer when relevant memories exist.
+- NEVER generate specific names, titles, places, or dates that do not appear in any memory above. If no memory contains the specific detail the question asks for, answer with what the memories DO contain rather than guessing.
+- For open-domain/opinion questions ("Would X do Y?", "Is X considered Z?"):
+  * Follow the DIRECT causal reasoning in the memories. Do NOT construct elaborate counter-arguments.
+  * "Would X still do Y without Z?" — If memories show X does Y BECAUSE of Z, then without Z, answer "likely no."
+  * "Would X do Y again soon?" — If the most recent attempt involved a bad experience (accident, scare, trauma), answer "likely no." A recent negative experience outweighs historical positive patterns.
+  * For trait questions ("Is X considered Z?"): weigh ALL evidence including symbolic/indirect references. If there is SOME but not strong evidence, answer with a qualified degree ("somewhat") rather than flat "no."
+
+# Instructions
+
+## Misc
+
+1. Make reasonable deductions based on your memories. Memory shows store with a lot of working people -> store employs a lot of people
+2. If a memory describes something recognizable (e.g., "romantic drama about memory and relationships"), you may name it (e.g., "Eternal Sunshine of the Spotless Mind").
+3. Use domain knowledge to connect facts: a game exclusive to one platform implies ownership of that platform. An unnamed company deal can be linked to a previously expressed brand preference.
+
+{memories}
+
+Question: {question}
+
+Work through Steps 1-7, then give your final answer after "ANSWER:".
+"""
+
+
+ANSWERER_MEMORY_LIMIT = 200
+
+
+def _to_human_date(iso_str: str) -> str:
+    """Convert ISO 8601 timestamp to human-readable date (e.g., 'May 7, 2023')."""
+    for fmt in ("%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
+        try:
+            return _datetime.strptime(iso_str[:26].rstrip("Z"), fmt.replace("%z", "")).strftime("%A, %B %d, %Y")
+        except ValueError:
+            continue
+    # Last resort: just show the date portion
+    return iso_str[:10]
+
+
+def _format_user_profile(user_profile: dict) -> str:
+    """Format a user profile dict as readable key-value pairs for the prompt.
+
+    Omits keys with null/empty values. Formats lists as comma-separated strings.
+    """
+    lines = ["## User Profile"]
+    for key, value in user_profile.items():
+        if value is None:
+            continue
+        # Format key: snake_case -> Title Case
+        display_key = key.replace("_", " ").title()
+        if isinstance(value, list):
+            if not value:
+                continue
+            display_value = ", ".join(str(v) for v in value)
+        elif isinstance(value, str):
+            if not value.strip():
+                continue
+            display_value = value
+        else:
+            display_value = str(value)
+        lines.append(f"{display_key}: {display_value}")
+    # Only return if we have at least one field beyond the header
+    if len(lines) <= 1:
+        return ""
+    return "\n".join(lines)
+
+
+def get_answer_generation_prompt(
+    question: str,
+    search_results: list,
+    reference_date: str = None,
+    user_profile: dict = None,
+) -> str:
+    """Build the answer generation prompt from search results.
+
+    Shows the top ANSWERER_MEMORY_LIMIT memories sorted chronologically
+    (oldest first) with human-readable dates. No rank numbers or scores
+    are shown to avoid anchoring bias.
+
+    Args:
+        question: The question to answer.
+        search_results: List of memory dicts from search.
+        reference_date: Human-readable date string (e.g., "January 04, 2024")
+            representing when the conversations took place. Used for temporal
+            reasoning. Defaults to "2023" if not provided.
+        user_profile: Optional dict of user profile data. When provided, a
+            User Profile section is added to the prompt before the memories.
+    """
+    if reference_date is None:
+        reference_date = "2023"
+
+    if not search_results:
+        memories_text = "(No relevant memories found)"
+    else:
+        top_results = search_results[:ANSWERER_MEMORY_LIMIT]
+        # Sort chronologically (oldest first) to present as a narrative
+        sorted_results = sorted(top_results, key=lambda x: x.get("created_at", ""))
+        lines = ["The following memories are presented in chronological order (oldest to newest).", ""]
+        for result in sorted_results:
+            memory = result.get("memory", "")
+            created_at = result.get("created_at", "")
+            if created_at:
+                date_str = _to_human_date(created_at)
+                lines.append(f"({date_str}) {memory}")
+            else:
+                lines.append(f"(unknown date) {memory}")
+        memories_text = "\n".join(lines)
+
+    # Optionally prepend user profile section before memories
+    profile_section = ""
+    if user_profile:
+        profile_section = _format_user_profile(user_profile)
+        if profile_section:
+            profile_section = profile_section + "\n\n"
+
+    return ANSWER_GENERATION_PROMPT.format(
+        memories=profile_section + memories_text,
+        question=question,
+        reference_date=reference_date,
+    )
+
+
+# ===============================================================================
+# JUDGE PROMPT (unified — evidence is an optional add-on)
+# ===============================================================================
+
+# ── Shared judge system prompt ──
+JUDGE_SYSTEM_PROMPT = "You are evaluating conversational AI memory recall. Return JSON only with the format requested."
+
+# ── Evidence chunk (injected when evidence is available) ──
+_EVIDENCE_CHUNK = """
+## Evidence (actual conversation messages containing the answer)
+{evidence_context}
+"""
+
+_EVIDENCE_RULE = """
+5. **EVIDENCE SUPPORTS ANSWER**: If the evidence corroborates the generated answer, mark CORRECT — even when the generated answer diverges from the gold answer. The gold answer may be wrong or oversimplified; if the generated answer provides a more accurate or better-supported conclusion based on the evidence, that is acceptable. Use evidence only to ACCEPT answers, never to reject them more strictly.
+"""
+
+_EVIDENCE_WRONG_CLAUSE = " AND is not supported by evidence"
+
+# ── Unified judge prompt template ──
+_JUDGE_TEMPLATE = """Label the generated answer as CORRECT or WRONG.
+{evidence_section}
+## Rules
+
+1. **PARTIAL CREDIT**: If the generated answer includes AT LEAST ONE correct item from the gold answer's list, mark CORRECT. Getting 1 out of 2, 2 out of 4, etc. is always acceptable. Only mark WRONG if NONE of the gold answer items appear.
+
+2. **PARAPHRASES COUNT**: Same concept in different words is CORRECT. "Chocolate raspberry tart" = "chocolate cake with raspberries". "Shelter meal service" = "volunteering at a homeless shelter". Emotions and sentiments in the same positive/negative family count as paraphrases: "proud" = "fulfilled" = "accomplished"; "huge success" = "relieved" = "thrilled" (all express positive achievement). Judge semantic meaning, not exact wording.
+
+3. **EXTRA DETAIL IS FINE**: A longer answer that includes the gold answer's key facts plus additional information is CORRECT. Never penalize for being more detailed or specific. If the generated answer adds extra descriptive details beyond the gold answer while still referencing the same core entity or concept, mark CORRECT.
+
+4. **DATE TOLERANCE**: Dates within 14 days of each other are CORRECT. Durations within 50% are CORRECT (e.g., "5 months" matches "six months"; "19 days" matches "two weeks"). Relative dates ("few days before November") match specific dates in the same window. A specific date (e.g., "February 2020") that is consistent with a vague reference (e.g., "a few years ago" relative to 2023) is CORRECT. Converting "last year" to the actual year (e.g., "2022" when conversations are in 2023) is CORRECT.
+{evidence_rule}
+5. **SEMANTIC OVERLAP**: Judge whether the generated answer addresses the same topic and captures the core idea of the gold answer. Different wording, phrasing, or level of detail should not result in WRONG if the underlying concept matches. For EMOTIONS and FEELINGS questions, answers expressing sentiments in the same valence (positive/negative) about the same event are CORRECT — do not require the exact same emotion word.
+
+6. **SAME REFERENT**: If the generated answer mentions or references the same named entity, character, person, or concept as the gold answer, mark CORRECT — even if the generated answer provides a different physical description or includes additional details. The key question is: does the generated answer identify the same core entity? If yes, it is CORRECT.
+
+7. **FOCUS ON KNOWLEDGE, NOT WORDING**: The goal is to assess whether the system recalled the right fact. Minor differences in specificity, phrasing, or scope should not result in WRONG. Only mark WRONG when the generated answer demonstrates a genuinely different or incorrect understanding.
+
+## ONLY mark WRONG if:
+- The generated answer contains ZERO correct items from the gold answer{evidence_wrong_clause}
+- The answer addresses a completely different topic
+
+## Question
+Question: {{question}}
+Gold answer: {{answer}}
+Generated answer: {{response}}
+
+Return JSON with "reasoning" (one sentence) and "label" (CORRECT or WRONG). Do NOT include both labels."""
+
+
+def _build_judge_prompt(evidence_context: str = None) -> str:
+    """Build the judge prompt template, with or without evidence."""
+    if evidence_context:
+        prompt = _JUDGE_TEMPLATE.format(
+            evidence_section=_EVIDENCE_CHUNK.format(evidence_context=evidence_context),
+            evidence_rule=_EVIDENCE_RULE,
+            evidence_wrong_clause=_EVIDENCE_WRONG_CLAUSE,
+        )
+        # Renumber rules when evidence rule is inserted (5→6, 6→7, 7→8)
+        prompt = prompt.replace("\n5. **SEMANTIC OVERLAP", "\n6. **SEMANTIC OVERLAP")
+        prompt = prompt.replace("\n6. **SAME REFERENT", "\n7. **SAME REFERENT")
+        prompt = prompt.replace("\n7. **FOCUS ON KNOWLEDGE", "\n8. **FOCUS ON KNOWLEDGE")
+    else:
+        prompt = _JUDGE_TEMPLATE.format(
+            evidence_section="",
+            evidence_rule="",
+            evidence_wrong_clause="",
+        )
+    return prompt
+
+
+# Keep JUDGE_PROMPT as a module-level constant for backward compat (no evidence)
+JUDGE_PROMPT = _build_judge_prompt(evidence_context=None)
+
+
+# ===============================================================================
+# DISPATCH
+# ===============================================================================
+
+def get_judge_prompt(
+    category: int,
+    question: str,
+    answer: str,
+    response: str,
+) -> str:
+    """Return the formatted unified judge prompt (no evidence).
+
+    Accepts category for backwards compatibility but uses the same
+    unified prompt for all categories.
+    """
+    return JUDGE_PROMPT.format(
+        question=question,
+        answer=answer,
+        response=response,
+    )
+
+
+def get_judge_prompt_with_evidence(
+    category: int,
+    question: str,
+    answer: str,
+    response: str,
+    evidence_context: str,
+) -> str:
+    """Return the formatted judge prompt with evidence context.
+
+    Same prompt as get_judge_prompt but with the evidence section
+    and evidence-specific rule injected.
+    """
+    prompt = _build_judge_prompt(evidence_context=evidence_context)
+    return prompt.format(
+        question=question,
+        answer=answer,
+        response=response,
+    )
+
+
+def preprocess_answer(category: int, answer: str) -> str:
+    """Preprocess ground truth answer based on category.
+
+    Category 3 (open-domain): use only the first part before semicolon.
+    """
+    if category == 3 and ";" in answer:
+        return answer.split(";")[0].strip()
+    return answer