trace-score 0.1.0__py3-none-any.whl

trace_score/__init__.py

@@ -0,0 +1,9 @@
+from .trace import compute_TRACE, TRACEEvaluator, WEIGHT_PRESETS
+from .components.temporal   import compute_T
+from .components.reliability import compute_R
+from .components.adaptive   import compute_A
+from .components.coherence  import compute_C
+from .components.epistemic  import compute_E
+
+__version__ = "0.1.0"
+__author__  = "Girinath V"

trace_score/components/__init__.py

@@ -0,0 +1,5 @@
+from .temporal   import compute_T
+from .reliability import compute_R
+from .adaptive   import compute_A
+from .coherence  import compute_C
+from .epistemic  import compute_E

trace_score/components/adaptive.py

@@ -0,0 +1,111 @@
+import re
+import numpy as np
+from typing import List, Tuple, Dict, Optional
+CORRECTION_MARKERS = [
+    r"\bactually\b",       r"\bno[,\s]",
+    r"\bnot exactly\b",    r"\bi said\b",
+    r"\bi mentioned\b",    r"\bi told you\b",
+    r"\bplease avoid\b",   r"\bplease don'?t\b",
+    r"\bdon'?t forget\b",  r"\bi already said\b",
+    r"\bcorrection\b",     r"\bwait[,\s]",
+    r"\bi meant\b",        r"\bthat'?s wrong\b",
+    r"\bnot quite\b",      r"\byou forgot\b",
+    r"\byou missed\b",     r"\bremember[,\s]",
+]
+def has_correction_marker(text: str) -> bool:
+    text_lower = text.lower()
+    return any(re.search(p, text_lower) for p in CORRECTION_MARKERS)
+def is_correction_turn(
+    user_text        : str,
+    prev_asst_text   : Optional[str],
+    nli_model        = None
+) -> bool:
+    if not has_correction_marker(user_text):
+        return False
+    if prev_asst_text is None or nli_model is None:
+        return True
+    scores = nli_model.predict(
+        [[prev_asst_text, user_text]],
+        apply_softmax=True
+    )
+    return float(scores[0][0]) >= 0.35 or has_correction_marker(user_text)
+def compute_A(
+    conversation:List[Tuple[str, str]],
+    sbert_model=None,
+    nli_model=None,
+    retention_threshold: float = 0.55,
+    gamma:float = 0.80
+) -> Dict:
+    if sbert_model is None:
+        from sentence_transformers import SentenceTransformer
+        sbert_model = SentenceTransformer("all-MiniLM-L6-v2")
+    if nli_model is None:
+        from sentence_transformers import CrossEncoder
+        nli_model = CrossEncoder(
+            "cross-encoder/nli-deberta-v3-small",
+            max_length=512
+        )
+    corrections    = []
+    prev_assistant = None
+    for i, (role, text) in enumerate(conversation):
+        if role == "assistant":
+            prev_assistant = text
+        elif role == "user":
+            if is_correction_turn(text, prev_assistant, nli_model):
+                corrections.append({"turn_index": i, "text": text})
+    K = len(corrections)
+    if K == 0:
+        return {
+            "score": 1.0,
+            "per_turn_scores": [],
+            "decay_weights": [],
+            "corrections_found": 0,
+            "retained_count": 0,
+            "details": [],
+            "explanation": "No correction turns detected. A = 1.0 (vacuously true)."
+        }
+    per_turn_scores = []
+    details = []
+    for k, correction in enumerate(corrections):
+        turn_idx = correction["turn_index"]
+        c_text   = correction["text"]
+        subsequent = [
+            text for i, (role, text) in enumerate(conversation)
+            if role == "assistant" and i > turn_idx
+        ]
+        if not subsequent:
+            A_k  = 1.0
+            max_sim = 1.0
+        else:
+            c_emb   = sbert_model.encode(
+                [c_text], convert_to_numpy=True, normalize_embeddings=True
+            )
+            s_embs  = sbert_model.encode(
+                subsequent, convert_to_numpy=True, normalize_embeddings=True
+            )
+            sims    = np.dot(c_emb, s_embs.T)[0]
+            max_sim = float(sims.max())
+            A_k     = 1.0 if max_sim >= retention_threshold else 0.0
+        per_turn_scores.append(A_k)
+        details.append({
+            "turn_index":   turn_idx,
+            "text":         c_text[:100],
+            "retained":     bool(A_k),
+            "max_sim":      round(max_sim, 4),
+        })
+    decay_weights = [gamma ** (K - 1 - k) for k in range(K)]
+    Z = sum(decay_weights)
+    A_score = sum(w * s for w, s in zip(decay_weights, per_turn_scores)) / Z
+    return {
+        "score": round(A_score, 4),
+        "per_turn_scores": per_turn_scores,
+        "decay_weights": [round(w, 4) for w in decay_weights],
+        "corrections_found": K,
+        "retained_count": int(sum(per_turn_scores)),
+        "details": details,
+        "explanation": (
+            f"Found {K} correction(s). "
+            f"Per-correction scores: {per_turn_scores}. "
+            f"A = {A_score:.4f}."
+        )
+    }

trace_score/components/coherence.py

@@ -0,0 +1,46 @@
+import numpy as np
+from typing import List, Tuple, Dict
+def compute_C(
+    conversation : List[Tuple[str, str]],
+    sbert_model  = None,
+    gamma : float = 0.80
+) -> Dict:
+    if sbert_model is None:
+        from sentence_transformers import SentenceTransformer
+        sbert_model = SentenceTransformer("all-MiniLM-L6-v2")
+    all_texts = [text for _, text in conversation]
+    N = len(all_texts)
+    if N <= 1:
+        return {
+            "score": 1.0,
+            "per_turn_sims": [],
+            "decay_weights": [],
+            "mean_drift":0.0,
+            "explanation":f"Only {N} turn(s). C = 1.0."
+        }
+    embeddings = sbert_model.encode(
+        all_texts,
+        convert_to_numpy=True,
+        normalize_embeddings=True,
+        batch_size=64,
+        show_progress_bar=False
+    )
+    sim_vector = np.sum(embeddings[:-1] * embeddings[1:], axis=1)
+    sim_vector = np.clip(sim_vector, 0.0, 1.0)
+    per_turn_sims = [round(float(s), 4) for s in sim_vector]
+    M = len(per_turn_sims) 
+    decay_weights = [gamma ** (M - 1 - t) for t in range(M)]
+    Z = sum(decay_weights)
+    C_score = sum(w * s for w, s in zip(decay_weights, per_turn_sims)) / Z
+    mean_drift = 1.0 - float(np.mean(sim_vector))
+    return {
+        "score": round(C_score, 4),
+        "per_turn_sims": per_turn_sims,
+        "decay_weights": [round(w, 4) for w in decay_weights],
+        "mean_drift": round(mean_drift, 4),
+        "explanation": (
+            f"Encoded {N} turns via SBERT. "
+            f"Adjacent-pair sims: {per_turn_sims}. "
+            f"C = {C_score:.4f}, mean drift = {mean_drift:.4f}."
+        )
+    }

trace_score/components/epistemic.py

@@ -0,0 +1,110 @@
+import numpy as np
+from typing import List, Tuple, Dict
+CONFIDENCE_LEXICON = {
+    "definitely": 1.00,"certainly": 1.00,
+    "absolutely": 1.00,"undoubtedly": 0.98,
+    "without a doubt": 0.97,"for sure": 0.95,
+    "clearly": 0.94,"obviously": 0.93,
+    "i am certain": 0.95,"i am confident": 0.92,
+    "i know": 0.90,"i believe": 0.80,
+    "i think": 0.75,"likely": 0.77,
+    "probably": 0.74,"it seems": 0.72,
+    "generally": 0.76,"typically": 0.75,
+    "usually": 0.74,"might": 0.52,
+    "could": 0.51,"may": 0.56,
+    "possibly": 0.46,"i suppose": 0.51,
+    "i guess": 0.46,"sometimes": 0.56,
+    "perhaps": 0.31,"maybe": 0.31,
+    "i am not sure": 0.20,"i'm not sure": 0.20,
+    "uncertain": 0.26,"unclear": 0.26,
+    "it depends": 0.31,"hard to say": 0.20,
+    "i doubt": 0.22,"i cannot say": 0.15,
+}
+DEFAULT_CONFIDENCE = 0.68
+HIGH_CONF_ANCHOR   = "I am absolutely certain and confident about this."
+LOW_CONF_ANCHOR    = "I am not sure and quite uncertain about this perhaps."
+def lexicon_confidence(text: str) -> float:
+    text_lower   = text.lower()
+    found_scores = [
+        score for marker, score in sorted(
+            CONFIDENCE_LEXICON.items(), key=lambda x: len(x[0]), reverse=True
+        )
+        if marker in text_lower
+    ]
+    return float(np.mean(found_scores)) if found_scores else DEFAULT_CONFIDENCE
+def sbert_anchor_confidence(text, sbert_model, high_emb, low_emb) -> float:
+    text_emb = sbert_model.encode(
+        [text], convert_to_numpy=True, normalize_embeddings=True
+    )[0]
+    sim_high = float(np.dot(text_emb, high_emb))
+    sim_low  = float(np.dot(text_emb, low_emb))
+    total    = sim_high + sim_low
+    return float(sim_high / total) if total > 0 else DEFAULT_CONFIDENCE
+def compute_E(
+    conversation    : List[Tuple[str, str]],
+    sbert_model     = None,
+    gamma           : float = 0.80,
+    lexicon_weight  : float = 0.60,
+    sbert_weight    : float = 0.40
+) -> Dict:
+    if sbert_model is None:
+        from sentence_transformers import SentenceTransformer
+        sbert_model = SentenceTransformer("all-MiniLM-L6-v2")
+    anchors  = sbert_model.encode(
+        [HIGH_CONF_ANCHOR, LOW_CONF_ANCHOR],
+        convert_to_numpy=True,
+        normalize_embeddings=True
+    )
+    high_emb = anchors[0]
+    low_emb  = anchors[1]
+    assistant_texts = [text for role, text in conversation if role == "assistant"]
+    N = len(assistant_texts)
+    if N == 0:
+        return {
+            "score": 1.0,
+            "variance_penalty": 0.0,
+            "turn_confidences": [],
+            "decay_weights": [],
+            "mean_confidence":  DEFAULT_CONFIDENCE,
+            "raw_variance": 0.0,
+            "explanation": "No assistant turns. E = 1.0, V = 0.0."
+        }
+    if N == 1:
+        conf = (
+            lexicon_weight * lexicon_confidence(assistant_texts[0]) +
+            sbert_weight   * sbert_anchor_confidence(assistant_texts[0], sbert_model, high_emb, low_emb)
+        )
+        return {
+            "score": round(conf, 4),
+            "variance_penalty": 0.0,
+            "turn_confidences": [round(conf, 4)],
+            "decay_weights": [1.0],
+            "mean_confidence": round(conf, 4),
+            "raw_variance": 0.0,
+            "explanation": f"Single assistant turn. E = {conf:.4f}, V = 0.0."
+        }
+    turn_confidences = [
+        lexicon_weight * lexicon_confidence(t) +
+        sbert_weight   * sbert_anchor_confidence(t, sbert_model, high_emb, low_emb)
+        for t in assistant_texts
+    ]
+    conf_array = np.array(turn_confidences)
+    decay_weights = [gamma ** (N - 1 - t) for t in range(N)]
+    Z = sum(decay_weights)
+    E_score = sum(w * c for w, c in zip(decay_weights, turn_confidences)) / Z
+    mu = float(conf_array.mean())
+    raw_variance = float(np.mean((conf_array - mu) ** 2))
+    V_norm = min(raw_variance / 0.25, 1.0)
+    return {
+        "score": round(E_score, 4),
+        "variance_penalty": round(V_norm, 4),
+        "turn_confidences": [round(c, 4) for c in turn_confidences],
+        "decay_weights": [round(w, 4) for w in decay_weights],
+        "mean_confidence": round(mu, 4),
+        "raw_variance": round(raw_variance, 6),
+        "explanation": (
+            f"Analyzed {N} assistant turns. "
+            f"Confidences: {[round(c,3) for c in turn_confidences]}. "
+            f"E = {E_score:.4f}, V_norm = {V_norm:.4f}."
+        )
+    }

trace_score/components/reliability.py

@@ -0,0 +1,87 @@
+from itertools import combinations
+from typing import List, Tuple, Dict
+import numpy as np
+def compute_R(
+    conversation  : List[Tuple[str, str]],
+    nli_model     = None,
+    threshold     : float = 0.75,
+    gamma         : float = 0.80,
+    bidirectional : bool  = True
+) -> Dict:
+    if nli_model is None:
+        from sentence_transformers import CrossEncoder
+        nli_model = CrossEncoder(
+            "cross-encoder/nli-deberta-v3-small",
+            max_length=512
+        )
+    assistant_turns = [
+        (i, text)
+        for i, (role, text) in enumerate(conversation)
+        if role == "assistant"
+    ]
+    N = len(assistant_turns)
+    if N < 2:
+        return {
+            "score":               1.0,
+            "penalty":             0.0,
+            "per_turn_scores":     [1.0] * N,
+            "decay_weights":       [1.0] * N,
+            "total_pairs":         0,
+            "contradiction_count": 0,
+            "contradictions":      [],
+            "explanation":         f"Only {N} assistant turn(s). R = 1.0, P = 0.0."
+        }
+    all_pairs = list(combinations(range(N), 2))
+    forward_inputs = [
+        [assistant_turns[i][1], assistant_turns[j][1]]
+        for i, j in all_pairs
+    ]
+    forward_scores = nli_model.predict(forward_inputs, apply_softmax=True)
+    forward_probs  = forward_scores[:, 0]
+    if bidirectional:
+        backward_inputs = [
+            [assistant_turns[j][1], assistant_turns[i][1]]
+            for i, j in all_pairs
+        ]
+        backward_scores = nli_model.predict(backward_inputs, apply_softmax=True)
+        backward_probs  = backward_scores[:, 0]
+        contradiction_probs = np.maximum(forward_probs, backward_probs)
+    else:
+        contradiction_probs = forward_probs
+    contradictions   = []
+    contradicted_set = set()   
+    for k, (i, j) in enumerate(all_pairs):
+        prob = float(contradiction_probs[k])
+        if prob >= threshold:
+            contradictions.append({
+                "turn_i":                    assistant_turns[i][0],
+                "turn_j":                    assistant_turns[j][0],
+                "contradiction_probability": round(prob, 4),
+                "text_i":                    assistant_turns[i][1][:100],
+                "text_j":                    assistant_turns[j][1][:100],
+            })
+            contradicted_set.add(i)
+            contradicted_set.add(j)
+    per_turn_scores = [
+        0.0 if t in contradicted_set else 1.0
+        for t in range(N)
+    ]
+    decay_weights = [gamma ** (N - 1 - t) for t in range(N)]
+    Z = sum(decay_weights)
+    R_score = sum(w * s for w, s in zip(decay_weights, per_turn_scores)) / Z
+    total_pairs = len(all_pairs)
+    P = len(contradictions) / total_pairs if total_pairs > 0 else 0.0
+    return {
+        "score":               round(R_score, 4),
+        "penalty":             round(P, 4),
+        "per_turn_scores":     per_turn_scores,
+        "decay_weights":       [round(w, 4) for w in decay_weights],
+        "total_pairs":         total_pairs,
+        "contradiction_count": len(contradictions),
+        "contradictions":      contradictions,
+        "explanation":         (
+            f"Checked {total_pairs} pairs across {N} assistant turns. "
+            f"Contradictions: {len(contradictions)}. "
+            f"R = {R_score:.4f}, P (penalty) = {P:.4f}."
+        )
+    }

trace_score/components/temporal.py

@@ -0,0 +1,94 @@
+import re
+import numpy as np
+from typing import List, Tuple, Dict
+FACT_PATTERNS = [
+    (r"i(?:'m| am) ([\w\s]+?)(?:\s+and\s+|\s*,\s*|\.|\?|$)",
+     "user is {}"),
+    (r"i have ([\w\s]+?)(?:\s+and\s+|\s*,\s*|\.|\?|$)",
+     "user has {}"),
+    (r"i (hate|love|prefer|avoid|dislike|like|eat|drink) ([\w\s]+?)(?:\s+and\s+|\s*,\s*|\.|\?|$)",
+     "user {}s {}"),
+    (r"allergic to ([\w\s]+?)(?:\s+and\s+|\s*,\s*|\.|\?|$)",
+     "user is allergic to {}"),
+    (r"my ([\w\s]+?) is ([\w\s\d]+?)(?:\s+and\s+|\s*,\s*|\.|\?|$)",
+     "user's {} is {}"),
+]
+def extract_atomic_facts(text: str) -> List[str]:
+    facts      = []
+    text_lower = text.lower().strip()
+    for pattern, template in FACT_PATTERNS:
+        for match in re.findall(pattern, text_lower):
+            if isinstance(match, tuple):
+                fact = template.format(*[m.strip() for m in match])
+            else:
+                fact = template.format(match.strip())
+            if len(fact.split()) >= 3:
+                facts.append(fact)
+    return list(set(facts))
+def compute_T(
+    conversation : List[Tuple[str, str]],
+    sbert_model  = None,
+    threshold    : float = 0.60,
+    gamma        : float = 0.80
+) -> Dict:
+    if sbert_model is None:
+        from sentence_transformers import SentenceTransformer
+        sbert_model = SentenceTransformer("all-MiniLM-L6-v2")
+    cumulative_facts = []
+    assistant_turns  = [] 
+    for role, text in conversation:
+        if role == "user":
+            new_facts = extract_atomic_facts(text)
+            for f in new_facts:
+                if f not in cumulative_facts:
+                    cumulative_facts.append(f)
+        elif role == "assistant":
+            assistant_turns.append({
+                "text":   text,
+                "facts":  list(cumulative_facts)  
+            })
+    N = len(assistant_turns)
+    if N == 0:
+        return {
+            "score": 1.0,
+            "per_turn_scores": [],
+            "decay_weights":   [],
+            "user_facts":      cumulative_facts,
+            "explanation":     "No assistant turns found. T = 1.0."
+        }
+    per_turn_scores = []
+    for turn_data in assistant_turns:
+        facts = turn_data["facts"]
+        text  = turn_data["text"]
+        if not facts:
+            per_turn_scores.append(1.0)
+            continue
+        fact_embs = sbert_model.encode(
+            facts,
+            convert_to_numpy=True,
+            normalize_embeddings=True
+        )
+        turn_emb = sbert_model.encode(
+            [text],
+            convert_to_numpy=True,
+            normalize_embeddings=True
+        )
+        sims     = np.dot(fact_embs, turn_emb.T).flatten()
+        recalled = int(np.sum(sims >= threshold))
+        T_t      = recalled / len(facts)
+        per_turn_scores.append(round(T_t, 4))
+    decay_weights = [gamma ** (N - 1 - t) for t in range(N)]
+    Z = sum(decay_weights)
+    T_score = sum(w * s for w, s in zip(decay_weights, per_turn_scores)) / Z
+    return {
+        "score":           round(T_score, 4),
+        "per_turn_scores": per_turn_scores,
+        "decay_weights":   [round(w, 4) for w in decay_weights],
+        "user_facts":      cumulative_facts,
+        "explanation":     (
+            f"Computed T across {N} assistant turns with gamma={gamma}. "
+            f"Per-turn scores: {per_turn_scores}. "
+            f"Decay weights: {[round(w,3) for w in decay_weights]}. "
+            f"T = {T_score:.4f}."
+        )
+    }

trace_score/trace.py

@@ -0,0 +1,166 @@
+from typing import List, Tuple, Dict, Optional
+from .components.temporal    import compute_T
+from .components.reliability import compute_R
+from .components.adaptive    import compute_A
+from .components.coherence   import compute_C
+from .components.epistemic   import compute_E
+WEIGHT_PRESETS = {
+    "equal":            {"w_T": 0.20, "w_R": 0.20, "w_A": 0.20, "w_C": 0.20, "w_E": 0.20},
+    "customer_service": {"w_T": 0.30, "w_R": 0.20, "w_A": 0.30, "w_C": 0.10, "w_E": 0.10},
+    "technical_qa":     {"w_T": 0.20, "w_R": 0.30, "w_A": 0.10, "w_C": 0.10, "w_E": 0.30},
+    "medical_chatbot":  {"w_T": 0.30, "w_R": 0.30, "w_A": 0.20, "w_C": 0.10, "w_E": 0.10},
+    "education_tutor":  {"w_T": 0.20, "w_R": 0.10, "w_A": 0.30, "w_C": 0.30, "w_E": 0.10},
+}
+INTERPRETATION_THRESHOLDS = [
+    (0.85, "Excellent — conversation is highly consistent"),
+    (0.70, "Good — minor consistency issues"),
+    (0.55, "Moderate — notable consistency failures"),
+    (0.40, "Poor — significant consistency problems"),
+    (0.00, "Very poor — conversation is highly inconsistent"),
+]
+class TRACEEvaluator:
+    def __init__(self):
+        self._sbert = None
+        self._nli   = None
+    def _load_models(self):
+        if self._sbert is None:
+            from sentence_transformers import SentenceTransformer
+            self._sbert = SentenceTransformer("all-MiniLM-L6-v2")
+        if self._nli is None:
+            from sentence_transformers import CrossEncoder
+            self._nli = CrossEncoder(
+                "cross-encoder/nli-deberta-v3-small",
+                max_length=512
+            )
+    def evaluate(
+        self,
+        conversation : List[Tuple[str, str]],
+        weights      : Optional[Dict[str, float]] = None,
+        preset       : str   = "equal",
+        gamma        : float = 0.80,
+        lambda_      : float = 0.15,
+        delta        : float = 0.10,
+        alpha        : float = 0.05,
+        beta         : float = 0.05,
+        verbose      : bool  = False
+    ) -> Dict:
+        self._load_models()
+        if not conversation:
+            return self._empty_result(weights or WEIGHT_PRESETS[preset])
+        for role, text in conversation:
+            if role not in ("user", "assistant"):
+                raise ValueError(f"Invalid role '{role}'. Must be 'user' or 'assistant'.")
+            if not text or not text.strip():
+                raise ValueError("Empty turn text found in conversation.")
+        w = self._resolve_weights(weights, preset)
+        t_result = compute_T(conversation, sbert_model=self._sbert, gamma=gamma)
+        r_result = compute_R(conversation, nli_model=self._nli,     gamma=gamma)
+        a_result = compute_A(conversation, sbert_model=self._sbert,
+                             nli_model=self._nli, gamma=gamma)
+        c_result = compute_C(conversation, sbert_model=self._sbert, gamma=gamma)
+        e_result = compute_E(conversation, sbert_model=self._sbert, gamma=gamma)
+        T = t_result["score"]
+        R = r_result["score"]
+        A = a_result["score"]
+        C = c_result["score"]
+        E = e_result["score"]
+        P = r_result["penalty"]           
+        V = e_result["variance_penalty"] 
+        base_score = (
+            w["w_T"] * T +
+            w["w_R"] * R +
+            w["w_A"] * A +
+            w["w_C"] * C +
+            w["w_E"] * E
+        )
+        penalty_term = lambda_ * P + delta * V
+        interaction_term = alpha * (T * C) + beta * (A * R)
+        raw_trace = base_score - penalty_term + interaction_term
+        trace_score = round(max(0.0, min(1.0, raw_trace)), 4)
+        formula_breakdown = (
+            f"TRACE = [{w['w_T']}×{T:.3f} + {w['w_R']}×{R:.3f} + "
+            f"{w['w_A']}×{A:.3f} + {w['w_C']}×{C:.3f} + {w['w_E']}×{E:.3f}]"
+            f" - [{lambda_}×{P:.3f} + {delta}×{V:.3f}]"
+            f" + [{alpha}×({T:.3f}×{C:.3f}) + {beta}×({A:.3f}×{R:.3f})]"
+            f" = {base_score:.4f} - {penalty_term:.4f} + {interaction_term:.4f}"
+            f" = {raw_trace:.4f} → clamped → {trace_score:.4f}"
+        )
+        result = {
+            "trace_score":       trace_score,
+            "base_score":        round(base_score, 4),
+            "penalty_term":      round(penalty_term, 4),
+            "interaction_term":  round(interaction_term, 4),
+            "T": T, "R": R, "A": A, "C": C, "E": E,
+            "P": P, "V": V,
+            "weights":           w,
+            "preset":            preset if weights is None else "custom",
+            "gamma":             gamma,
+            "lambda":            lambda_,
+            "delta":             delta,
+            "alpha":             alpha,
+            "beta":              beta,
+            "interpretation":    self._interpret(trace_score),
+            "formula_breakdown": formula_breakdown,
+            "num_turns":         len(conversation),
+        }
+        if verbose:
+            result["details"] = {
+                "T": t_result,
+                "R": r_result,
+                "A": a_result,
+                "C": c_result,
+                "E": e_result,
+            }
+        return result
+    def _resolve_weights(self, weights, preset):
+        if weights is not None:
+            total = sum(weights.values())
+            if abs(total - 1.0) > 0.01:
+                raise ValueError(f"Weights must sum to 1.0, got {total:.4f}.")
+            return weights
+        if preset not in WEIGHT_PRESETS:
+            raise ValueError(
+                f"Unknown preset '{preset}'. "
+                f"Available: {list(WEIGHT_PRESETS.keys())}"
+            )
+        return WEIGHT_PRESETS[preset]
+    def _interpret(self, score: float) -> str:
+        for threshold, label in INTERPRETATION_THRESHOLDS:
+            if score >= threshold:
+                return label
+        return INTERPRETATION_THRESHOLDS[-1][1]
+    def _empty_result(self, w):
+        return {
+            "trace_score": 1.0,
+            "base_score":  1.0,
+            "penalty_term": 0.0,
+            "interaction_term": 0.0,
+            "T": 1.0, "R": 1.0, "A": 1.0, "C": 1.0, "E": 1.0,
+            "P": 0.0, "V": 0.0,
+            "weights": w,
+            "interpretation": "Empty conversation. TRACE = 1.0.",
+            "num_turns": 0,
+        }
+def compute_TRACE(
+    conversation : List[Tuple[str, str]],
+    weights      : Optional[Dict[str, float]] = None,
+    preset       : str   = "equal",
+    gamma        : float = 0.80,
+    lambda_      : float = 0.15,
+    delta        : float = 0.10,
+    alpha        : float = 0.05,
+    beta         : float = 0.05,
+    verbose      : bool  = False
+) -> Dict:
+    evaluator = TRACEEvaluator()
+    return evaluator.evaluate(
+        conversation,
+        weights=weights,
+        preset=preset,
+        gamma=gamma,
+        lambda_=lambda_,
+        delta=delta,
+        alpha=alpha,
+        beta=beta,
+        verbose=verbose
+    )

trace_score-0.1.0.dist-info/METADATA

@@ -0,0 +1,270 @@
+Metadata-Version: 2.4
+Name: trace-score
+Version: 0.1.0
+Summary: Multi-turn LLM Conversation Consistency Metric
+Home-page: https://github.com/Giri530/trace-score
+Author: Girinath V
+Author-email: Girinath V <your-email@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Girinath V
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/Giri530/trace-score
+Project-URL: Repository, https://github.com/Giri530/trace-score
+Keywords: nlp,llm,evaluation,multi-turn,consistency,trace-score
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: numpy>=1.21.0
+Requires-Dist: torch>=1.11.0
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# TRACE Score
+
+**Multi-turn LLM Conversation Consistency Metric**
+
+> The first unified, deterministic, reference-free evaluation metric for
+> multi-turn conversational consistency in Large Language Models.
+
+[![PyPI version](https://badge.fury.io/py/trace-score.svg)](https://pypi.org/project/trace-score/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+
+---
+
+## The Problem
+
+Existing metrics (BLEU, ROUGE, BERTScore, RAGAS) evaluate each conversation
+turn **in isolation**. They cannot detect failures that only become visible
+**across multiple turns**:
+
+| Failure Type | Example | BLEU | ROUGE | BERTScore | TRACE |
+|---|---|---|---|---|---|
+| Fact forgotten | User says "I am diabetic" → model recommends sugar-rich food 5 turns later | Miss | Miss | Miss | **Catch** |
+| Correction ignored | User corrects model → model reverts to old behavior | Miss | Miss | Miss | **Catch** |
+| Self-contradiction | Model says X at turn 2, contradicts X at turn 7 | Miss | Miss | Miss | **Catch** |
+| Topic drift | Conversation gradually drifts off-topic | Miss | Miss | Miss | **Catch** |
+| Confidence drift | Model says "definitely" then "perhaps" about same claim | Miss | Miss | Miss | **Catch** |
+
+---
+
+## Formula
+
+```
+TRACE(C) = Σ(wᵢ · Sᵢ) − λ·P − δ·V + α·(T·C) + β·(A·R)
+```
+
+Each component uses **time-decay aggregation** — recent turns weighted more:
+
+```
+Sᵢ = (1/Z) · Σ γ^(N-t) · Sᵢ,ₜ
+Z  = Σ γ^(N-t)
+```
+
+| Symbol | Component | Measures |
+|--------|-----------|---------|
+| **T** | Temporal Retention | Did assistant remember user-stated facts? |
+| **R** | Reliability Consistency | Did assistant contradict itself? |
+| **A** | Adaptive Correction | Did assistant retain user corrections? |
+| **C** | Context Coherence | Did conversation stay on topic? |
+| **E** | Epistemic Stability | Did confidence stay calibrated? |
+| P | Contradiction penalty | Global contradiction rate |
+| V | Variance penalty | Confidence variance |
+| γ | Time decay factor | Default: 0.80 |
+| λ | Contradiction weight | Default: 0.15 |
+| δ | Variance weight | Default: 0.10 |
+| α | T·C interaction | Default: 0.05 |
+| β | A·R interaction | Default: 0.05 |
+
+---
+
+## Install
+
+```bash
+pip install trace-score
+```
+
+---
+
+## Quick Start
+
+```python
+from trace_score import compute_TRACE
+
+conversation = [
+    ("user",      "I am diabetic and hate spicy food"),
+    ("assistant", "I will suggest low sugar mild options."),
+    ("user",      "Actually I eat fish too. I am pescatarian."),
+    ("assistant", "Spicy chicken with cashews!"),   # failure turn
+]
+
+result = compute_TRACE(conversation, verbose=True)
+
+print(result["trace_score"])        # 0.41 — catches failures
+print(result["T"])                  # 0.50 — forgot user facts
+print(result["A"])                  # 0.00 — ignored correction
+print(result["formula_breakdown"])  # full formula with values
+print(result["interpretation"])     # "Poor consistency"
+```
+
+---
+
+## Batch Evaluation
+
+```python
+from trace_score import TRACEEvaluator
+
+# Models loaded once, reused across all calls — much faster
+evaluator = TRACEEvaluator()
+results   = [evaluator.evaluate(conv) for conv in conversations]
+```
+
+---
+
+## Adaptive Weights
+
+```python
+# Equal weights (default)
+result = compute_TRACE(conv, preset="equal")
+
+# Medical chatbot — memory and reliability weighted more
+result = compute_TRACE(conv, preset="medical_chatbot")
+
+# Custom weights — must sum to 1.0
+result = compute_TRACE(conv, weights={
+    "w_T": 0.35, "w_R": 0.25,
+    "w_A": 0.20, "w_C": 0.10, "w_E": 0.10
+})
+```
+
+Available presets: `equal`, `customer_service`, `technical_qa`,
+`medical_chatbot`, `education_tutor`
+
+---
+
+## Benchmark Results
+
+Evaluated on **30 multi-turn conversations** across 3 categories
+(Fact Memory, Correction Retention, Contradiction Detection).
+Conversations generated by **Llama-3.1-8B via Groq API**.
+
+### Overall Metric Comparison
+
+| Metric | Overall | Fact Memory | Correction | Contradiction |
+|--------|---------|-------------|------------|---------------|
+| **TRACE** | **0.699** | **0.703** | **0.550** | **0.843** |
+| BLEU | 0.102 | 0.046 | 0.149 | 0.110 |
+| ROUGE-L | 0.239 | 0.177 | 0.301 | 0.239 |
+| BERTScore | 0.822 | 0.800 | 0.842 | 0.823 |
+
+**Key finding:** BLEU and ROUGE-L show similar low scores across all categories
+— they cannot distinguish between different types of consistency failures.
+BERTScore appears high but provides no diagnostic breakdown.
+**TRACE clearly separates Correction (0.550) from Contradiction (0.843)**,
+revealing that Llama-3.1-8B struggles most with retaining user corrections.
+
+---
+
+### TRACE Component Breakdown by Category
+
+| Category | T | R | A | C | E |
+|----------|---|---|---|---|---|
+| Fact Memory | 0.137 | 0.955 | **1.000** | 0.503 | 0.697 |
+| Correction | 0.491 | 0.927 | **0.144** | 0.465 | 0.712 |
+| Contradiction | **0.973** | 0.875 | 0.900 | 0.510 | 0.696 |
+
+**Diagnostic insight:**
+
+- Fact Memory: T=0.137 — model **forgets user-stated facts** (A=1.0 means
+  no corrections were needed, so A is vacuously true here)
+- Correction: A=0.144 — model **ignores user corrections** (critical failure)
+- Contradiction: T=0.973, A=0.900 — model handles these well
+
+No existing metric (BLEU, ROUGE, BERTScore) can produce this breakdown.
+
+---
+
+### The Gap TRACE Reveals — BERTScore vs TRACE
+
+Conversations where **BERTScore is high but TRACE is low**
+(failures invisible to BERTScore, caught by TRACE):
+
+| Conversation | Category | TRACE | BERTScore | Gap |
+|---|---|---|---|---|
+| CR_006 | Correction | 0.314 | 0.876 | +0.562 |
+| CR_009 | Correction | 0.381 | 0.861 | +0.480 |
+| CR_004 | Correction | 0.535 | 0.884 | +0.349 |
+| CR_003 | Correction | 0.494 | 0.864 | +0.370 |
+| CR_002 | Correction | 0.442 | 0.822 | +0.380 |
+
+**In all 5 cases:** BERTScore ≥ 0.82 (looks good), TRACE < 0.55 (failures detected).
+The A component reveals why — user corrections completely ignored (A=0.00).
+This is invisible to any per-turn metric.
+
+---
+
+## Why TRACE?
+
+| Metric | Multi-turn | Reference-free | Deterministic | Time-decay | Diagnostic |
+|--------|-----------|----------------|---------------|-----------|-----------|
+| BLEU | No | No | Yes | No | No |
+| ROUGE | No | No | Yes | No | No |
+| BERTScore | No | No | Yes | No | No |
+| RAGAS | No | Yes | No | No | Partial |
+| **TRACE** | **Yes** | **Yes** | **Yes** | **Yes** | **Yes** |
+
+---
+
+## Models Used
+
+| Model | Purpose | Size |
+|-------|---------|------|
+| `all-MiniLM-L6-v2` | Semantic similarity (T, A, C, E) | 80MB |
+| `cross-encoder/nli-deberta-v3-small` | Contradiction detection (R, A) | 184MB |
+
+Models downloaded automatically on first use (~264MB total).
+CPU-friendly — no GPU required.
+
+---
+
+## Citation
+
+```bibtex
+@article{girinathv2026trace,
+  title   = {TRACE: A Unified Deterministic Metric for Multi-turn
+             Conversational Consistency in Large Language Models},
+  author  = {Girinath, V},
+  year    = {2026}
+}
+```
+
+---
+
+*Author: Girinath V*
+*GitHub: https://github.com/Giri530/trace-score*

trace_score-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+trace_score/__init__.py,sha256=--Nr7FRCszGBpz1D-CMH8cxsAwcbsS9UYo67X0U0iog,341
+trace_score/trace.py,sha256=60MxMgmPkV04CLnuOZqWJnRqVGtJJX5gpjn02pL4ZXw,6858
+trace_score/components/__init__.py,sha256=HPkrl-R3_AmDdcVjYnZC6VlhKarNAeDrYsYQ03TB9fc,171
+trace_score/components/adaptive.py,sha256=yfA9I02o9gRfVkymTFfDkUSv8q23VbhBTGJukrxhSIY,3984
+trace_score/components/coherence.py,sha256=JNsVH0dqkST_p2gTv9riKjFZ0DALkAk1YsQ5LZjac1o,1618
+trace_score/components/epistemic.py,sha256=4qOKEQdHXj-EOl9jtNBO1jSFjjysj8isX3xNNanbPw0,4352
+trace_score/components/reliability.py,sha256=nJjXwjWIEl5ZItXW_U8l9Gy_BtyimV83N7U9FAldAm0,3365
+trace_score/components/temporal.py,sha256=OCQw4lZEUh4UepelbDzV1pAIpZIz9oFh-MJgdjaZh7Y,3442
+trace_score-0.1.0.dist-info/licenses/LICENSE,sha256=FWXZwUInhOZvTxKqwnCPYbVby_65dHGXRCLrdAFhtJ8,1067
+trace_score-0.1.0.dist-info/METADATA,sha256=PxZfpV6eSf-yS9nAWwKLFcUziNLmeZ76UlYf3hqfDlI,9294
+trace_score-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+trace_score-0.1.0.dist-info/top_level.txt,sha256=QNr7AmluwDQlIJJgmyz35K5ACpW9gEiOPEDECwrft_4,12
+trace_score-0.1.0.dist-info/RECORD,,

trace_score-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

trace_score-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Girinath V
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

trace_score-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+trace_score