trace-score 0.1.0__tar.gz

trace_score-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Girinath V
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

trace_score-0.1.0/PKG-INFO

@@ -0,0 +1,270 @@
+Metadata-Version: 2.4
+Name: trace-score
+Version: 0.1.0
+Summary: Multi-turn LLM Conversation Consistency Metric
+Home-page: https://github.com/Giri530/trace-score
+Author: Girinath V
+Author-email: Girinath V <your-email@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Girinath V
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/Giri530/trace-score
+Project-URL: Repository, https://github.com/Giri530/trace-score
+Keywords: nlp,llm,evaluation,multi-turn,consistency,trace-score
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: numpy>=1.21.0
+Requires-Dist: torch>=1.11.0
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# TRACE Score
+
+**Multi-turn LLM Conversation Consistency Metric**
+
+> The first unified, deterministic, reference-free evaluation metric for
+> multi-turn conversational consistency in Large Language Models.
+
+[![PyPI version](https://badge.fury.io/py/trace-score.svg)](https://pypi.org/project/trace-score/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+
+---
+
+## The Problem
+
+Existing metrics (BLEU, ROUGE, BERTScore, RAGAS) evaluate each conversation
+turn **in isolation**. They cannot detect failures that only become visible
+**across multiple turns**:
+
+| Failure Type | Example | BLEU | ROUGE | BERTScore | TRACE |
+|---|---|---|---|---|---|
+| Fact forgotten | User says "I am diabetic" → model recommends sugar-rich food 5 turns later | Miss | Miss | Miss | **Catch** |
+| Correction ignored | User corrects model → model reverts to old behavior | Miss | Miss | Miss | **Catch** |
+| Self-contradiction | Model says X at turn 2, contradicts X at turn 7 | Miss | Miss | Miss | **Catch** |
+| Topic drift | Conversation gradually drifts off-topic | Miss | Miss | Miss | **Catch** |
+| Confidence drift | Model says "definitely" then "perhaps" about same claim | Miss | Miss | Miss | **Catch** |
+
+---
+
+## Formula
+
+```
+TRACE(C) = Σ(wᵢ · Sᵢ) − λ·P − δ·V + α·(T·C) + β·(A·R)
+```
+
+Each component uses **time-decay aggregation** — recent turns weighted more:
+
+```
+Sᵢ = (1/Z) · Σ γ^(N-t) · Sᵢ,ₜ
+Z  = Σ γ^(N-t)
+```
+
+| Symbol | Component | Measures |
+|--------|-----------|---------|
+| **T** | Temporal Retention | Did assistant remember user-stated facts? |
+| **R** | Reliability Consistency | Did assistant contradict itself? |
+| **A** | Adaptive Correction | Did assistant retain user corrections? |
+| **C** | Context Coherence | Did conversation stay on topic? |
+| **E** | Epistemic Stability | Did confidence stay calibrated? |
+| P | Contradiction penalty | Global contradiction rate |
+| V | Variance penalty | Confidence variance |
+| γ | Time decay factor | Default: 0.80 |
+| λ | Contradiction weight | Default: 0.15 |
+| δ | Variance weight | Default: 0.10 |
+| α | T·C interaction | Default: 0.05 |
+| β | A·R interaction | Default: 0.05 |
+
+---
+
+## Install
+
+```bash
+pip install trace-score
+```
+
+---
+
+## Quick Start
+
+```python
+from trace_score import compute_TRACE
+
+conversation = [
+    ("user",      "I am diabetic and hate spicy food"),
+    ("assistant", "I will suggest low sugar mild options."),
+    ("user",      "Actually I eat fish too. I am pescatarian."),
+    ("assistant", "Spicy chicken with cashews!"),   # failure turn
+]
+
+result = compute_TRACE(conversation, verbose=True)
+
+print(result["trace_score"])        # 0.41 — catches failures
+print(result["T"])                  # 0.50 — forgot user facts
+print(result["A"])                  # 0.00 — ignored correction
+print(result["formula_breakdown"])  # full formula with values
+print(result["interpretation"])     # "Poor consistency"
+```
+
+---
+
+## Batch Evaluation
+
+```python
+from trace_score import TRACEEvaluator
+
+# Models loaded once, reused across all calls — much faster
+evaluator = TRACEEvaluator()
+results   = [evaluator.evaluate(conv) for conv in conversations]
+```
+
+---
+
+## Adaptive Weights
+
+```python
+# Equal weights (default)
+result = compute_TRACE(conv, preset="equal")
+
+# Medical chatbot — memory and reliability weighted more
+result = compute_TRACE(conv, preset="medical_chatbot")
+
+# Custom weights — must sum to 1.0
+result = compute_TRACE(conv, weights={
+    "w_T": 0.35, "w_R": 0.25,
+    "w_A": 0.20, "w_C": 0.10, "w_E": 0.10
+})
+```
+
+Available presets: `equal`, `customer_service`, `technical_qa`,
+`medical_chatbot`, `education_tutor`
+
+---
+
+## Benchmark Results
+
+Evaluated on **30 multi-turn conversations** across 3 categories
+(Fact Memory, Correction Retention, Contradiction Detection).
+Conversations generated by **Llama-3.1-8B via Groq API**.
+
+### Overall Metric Comparison
+
+| Metric | Overall | Fact Memory | Correction | Contradiction |
+|--------|---------|-------------|------------|---------------|
+| **TRACE** | **0.699** | **0.703** | **0.550** | **0.843** |
+| BLEU | 0.102 | 0.046 | 0.149 | 0.110 |
+| ROUGE-L | 0.239 | 0.177 | 0.301 | 0.239 |
+| BERTScore | 0.822 | 0.800 | 0.842 | 0.823 |
+
+**Key finding:** BLEU and ROUGE-L show similar low scores across all categories
+— they cannot distinguish between different types of consistency failures.
+BERTScore appears high but provides no diagnostic breakdown.
+**TRACE clearly separates Correction (0.550) from Contradiction (0.843)**,
+revealing that Llama-3.1-8B struggles most with retaining user corrections.
+
+---
+
+### TRACE Component Breakdown by Category
+
+| Category | T | R | A | C | E |
+|----------|---|---|---|---|---|
+| Fact Memory | 0.137 | 0.955 | **1.000** | 0.503 | 0.697 |
+| Correction | 0.491 | 0.927 | **0.144** | 0.465 | 0.712 |
+| Contradiction | **0.973** | 0.875 | 0.900 | 0.510 | 0.696 |
+
+**Diagnostic insight:**
+
+- Fact Memory: T=0.137 — model **forgets user-stated facts** (A=1.0 means
+  no corrections were needed, so A is vacuously true here)
+- Correction: A=0.144 — model **ignores user corrections** (critical failure)
+- Contradiction: T=0.973, A=0.900 — model handles these well
+
+No existing metric (BLEU, ROUGE, BERTScore) can produce this breakdown.
+
+---
+
+### The Gap TRACE Reveals — BERTScore vs TRACE
+
+Conversations where **BERTScore is high but TRACE is low**
+(failures invisible to BERTScore, caught by TRACE):
+
+| Conversation | Category | TRACE | BERTScore | Gap |
+|---|---|---|---|---|
+| CR_006 | Correction | 0.314 | 0.876 | +0.562 |
+| CR_009 | Correction | 0.381 | 0.861 | +0.480 |
+| CR_004 | Correction | 0.535 | 0.884 | +0.349 |
+| CR_003 | Correction | 0.494 | 0.864 | +0.370 |
+| CR_002 | Correction | 0.442 | 0.822 | +0.380 |
+
+**In all 5 cases:** BERTScore ≥ 0.82 (looks good), TRACE < 0.55 (failures detected).
+The A component reveals why — user corrections completely ignored (A=0.00).
+This is invisible to any per-turn metric.
+
+---
+
+## Why TRACE?
+
+| Metric | Multi-turn | Reference-free | Deterministic | Time-decay | Diagnostic |
+|--------|-----------|----------------|---------------|-----------|-----------|
+| BLEU | No | No | Yes | No | No |
+| ROUGE | No | No | Yes | No | No |
+| BERTScore | No | No | Yes | No | No |
+| RAGAS | No | Yes | No | No | Partial |
+| **TRACE** | **Yes** | **Yes** | **Yes** | **Yes** | **Yes** |
+
+---
+
+## Models Used
+
+| Model | Purpose | Size |
+|-------|---------|------|
+| `all-MiniLM-L6-v2` | Semantic similarity (T, A, C, E) | 80MB |
+| `cross-encoder/nli-deberta-v3-small` | Contradiction detection (R, A) | 184MB |
+
+Models downloaded automatically on first use (~264MB total).
+CPU-friendly — no GPU required.
+
+---
+
+## Citation
+
+```bibtex
+@article{girinathv2026trace,
+  title   = {TRACE: A Unified Deterministic Metric for Multi-turn
+             Conversational Consistency in Large Language Models},
+  author  = {Girinath, V},
+  year    = {2026}
+}
+```
+
+---
+
+*Author: Girinath V*
+*GitHub: https://github.com/Giri530/trace-score*

trace_score-0.1.0/README.md

@@ -0,0 +1,223 @@
+# TRACE Score
+
+**Multi-turn LLM Conversation Consistency Metric**
+
+> The first unified, deterministic, reference-free evaluation metric for
+> multi-turn conversational consistency in Large Language Models.
+
+[![PyPI version](https://badge.fury.io/py/trace-score.svg)](https://pypi.org/project/trace-score/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/)
+
+---
+
+## The Problem
+
+Existing metrics (BLEU, ROUGE, BERTScore, RAGAS) evaluate each conversation
+turn **in isolation**. They cannot detect failures that only become visible
+**across multiple turns**:
+
+| Failure Type | Example | BLEU | ROUGE | BERTScore | TRACE |
+|---|---|---|---|---|---|
+| Fact forgotten | User says "I am diabetic" → model recommends sugar-rich food 5 turns later | Miss | Miss | Miss | **Catch** |
+| Correction ignored | User corrects model → model reverts to old behavior | Miss | Miss | Miss | **Catch** |
+| Self-contradiction | Model says X at turn 2, contradicts X at turn 7 | Miss | Miss | Miss | **Catch** |
+| Topic drift | Conversation gradually drifts off-topic | Miss | Miss | Miss | **Catch** |
+| Confidence drift | Model says "definitely" then "perhaps" about same claim | Miss | Miss | Miss | **Catch** |
+
+---
+
+## Formula
+
+```
+TRACE(C) = Σ(wᵢ · Sᵢ) − λ·P − δ·V + α·(T·C) + β·(A·R)
+```
+
+Each component uses **time-decay aggregation** — recent turns weighted more:
+
+```
+Sᵢ = (1/Z) · Σ γ^(N-t) · Sᵢ,ₜ
+Z  = Σ γ^(N-t)
+```
+
+| Symbol | Component | Measures |
+|--------|-----------|---------|
+| **T** | Temporal Retention | Did assistant remember user-stated facts? |
+| **R** | Reliability Consistency | Did assistant contradict itself? |
+| **A** | Adaptive Correction | Did assistant retain user corrections? |
+| **C** | Context Coherence | Did conversation stay on topic? |
+| **E** | Epistemic Stability | Did confidence stay calibrated? |
+| P | Contradiction penalty | Global contradiction rate |
+| V | Variance penalty | Confidence variance |
+| γ | Time decay factor | Default: 0.80 |
+| λ | Contradiction weight | Default: 0.15 |
+| δ | Variance weight | Default: 0.10 |
+| α | T·C interaction | Default: 0.05 |
+| β | A·R interaction | Default: 0.05 |
+
+---
+
+## Install
+
+```bash
+pip install trace-score
+```
+
+---
+
+## Quick Start
+
+```python
+from trace_score import compute_TRACE
+
+conversation = [
+    ("user",      "I am diabetic and hate spicy food"),
+    ("assistant", "I will suggest low sugar mild options."),
+    ("user",      "Actually I eat fish too. I am pescatarian."),
+    ("assistant", "Spicy chicken with cashews!"),   # failure turn
+]
+
+result = compute_TRACE(conversation, verbose=True)
+
+print(result["trace_score"])        # 0.41 — catches failures
+print(result["T"])                  # 0.50 — forgot user facts
+print(result["A"])                  # 0.00 — ignored correction
+print(result["formula_breakdown"])  # full formula with values
+print(result["interpretation"])     # "Poor consistency"
+```
+
+---
+
+## Batch Evaluation
+
+```python
+from trace_score import TRACEEvaluator
+
+# Models loaded once, reused across all calls — much faster
+evaluator = TRACEEvaluator()
+results   = [evaluator.evaluate(conv) for conv in conversations]
+```
+
+---
+
+## Adaptive Weights
+
+```python
+# Equal weights (default)
+result = compute_TRACE(conv, preset="equal")
+
+# Medical chatbot — memory and reliability weighted more
+result = compute_TRACE(conv, preset="medical_chatbot")
+
+# Custom weights — must sum to 1.0
+result = compute_TRACE(conv, weights={
+    "w_T": 0.35, "w_R": 0.25,
+    "w_A": 0.20, "w_C": 0.10, "w_E": 0.10
+})
+```
+
+Available presets: `equal`, `customer_service`, `technical_qa`,
+`medical_chatbot`, `education_tutor`
+
+---
+
+## Benchmark Results
+
+Evaluated on **30 multi-turn conversations** across 3 categories
+(Fact Memory, Correction Retention, Contradiction Detection).
+Conversations generated by **Llama-3.1-8B via Groq API**.
+
+### Overall Metric Comparison
+
+| Metric | Overall | Fact Memory | Correction | Contradiction |
+|--------|---------|-------------|------------|---------------|
+| **TRACE** | **0.699** | **0.703** | **0.550** | **0.843** |
+| BLEU | 0.102 | 0.046 | 0.149 | 0.110 |
+| ROUGE-L | 0.239 | 0.177 | 0.301 | 0.239 |
+| BERTScore | 0.822 | 0.800 | 0.842 | 0.823 |
+
+**Key finding:** BLEU and ROUGE-L show similar low scores across all categories
+— they cannot distinguish between different types of consistency failures.
+BERTScore appears high but provides no diagnostic breakdown.
+**TRACE clearly separates Correction (0.550) from Contradiction (0.843)**,
+revealing that Llama-3.1-8B struggles most with retaining user corrections.
+
+---
+
+### TRACE Component Breakdown by Category
+
+| Category | T | R | A | C | E |
+|----------|---|---|---|---|---|
+| Fact Memory | 0.137 | 0.955 | **1.000** | 0.503 | 0.697 |
+| Correction | 0.491 | 0.927 | **0.144** | 0.465 | 0.712 |
+| Contradiction | **0.973** | 0.875 | 0.900 | 0.510 | 0.696 |
+
+**Diagnostic insight:**
+
+- Fact Memory: T=0.137 — model **forgets user-stated facts** (A=1.0 means
+  no corrections were needed, so A is vacuously true here)
+- Correction: A=0.144 — model **ignores user corrections** (critical failure)
+- Contradiction: T=0.973, A=0.900 — model handles these well
+
+No existing metric (BLEU, ROUGE, BERTScore) can produce this breakdown.
+
+---
+
+### The Gap TRACE Reveals — BERTScore vs TRACE
+
+Conversations where **BERTScore is high but TRACE is low**
+(failures invisible to BERTScore, caught by TRACE):
+
+| Conversation | Category | TRACE | BERTScore | Gap |
+|---|---|---|---|---|
+| CR_006 | Correction | 0.314 | 0.876 | +0.562 |
+| CR_009 | Correction | 0.381 | 0.861 | +0.480 |
+| CR_004 | Correction | 0.535 | 0.884 | +0.349 |
+| CR_003 | Correction | 0.494 | 0.864 | +0.370 |
+| CR_002 | Correction | 0.442 | 0.822 | +0.380 |
+
+**In all 5 cases:** BERTScore ≥ 0.82 (looks good), TRACE < 0.55 (failures detected).
+The A component reveals why — user corrections completely ignored (A=0.00).
+This is invisible to any per-turn metric.
+
+---
+
+## Why TRACE?
+
+| Metric | Multi-turn | Reference-free | Deterministic | Time-decay | Diagnostic |
+|--------|-----------|----------------|---------------|-----------|-----------|
+| BLEU | No | No | Yes | No | No |
+| ROUGE | No | No | Yes | No | No |
+| BERTScore | No | No | Yes | No | No |
+| RAGAS | No | Yes | No | No | Partial |
+| **TRACE** | **Yes** | **Yes** | **Yes** | **Yes** | **Yes** |
+
+---
+
+## Models Used
+
+| Model | Purpose | Size |
+|-------|---------|------|
+| `all-MiniLM-L6-v2` | Semantic similarity (T, A, C, E) | 80MB |
+| `cross-encoder/nli-deberta-v3-small` | Contradiction detection (R, A) | 184MB |
+
+Models downloaded automatically on first use (~264MB total).
+CPU-friendly — no GPU required.
+
+---
+
+## Citation
+
+```bibtex
+@article{girinathv2026trace,
+  title   = {TRACE: A Unified Deterministic Metric for Multi-turn
+             Conversational Consistency in Large Language Models},
+  author  = {Girinath, V},
+  year    = {2026}
+}
+```
+
+---
+
+*Author: Girinath V*
+*GitHub: https://github.com/Giri530/trace-score*

trace_score-0.1.0/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires      = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name            = "trace-score"
+version         = "0.1.0"
+description     = "Multi-turn LLM Conversation Consistency Metric"
+readme          = "README.md"
+license         = {file = "LICENSE"}
+requires-python = ">=3.8"
+authors         = [
+    {name = "Girinath V", email = "your-email@gmail.com"}
+]
+dependencies = [
+    "sentence-transformers>=2.2.0",
+    "numpy>=1.21.0",
+    "torch>=1.11.0",
+]
+keywords = [
+    "nlp", "llm", "evaluation",
+    "multi-turn", "consistency", "trace-score"
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+]
+
+[project.urls]
+Homepage   = "https://github.com/Giri530/trace-score"
+Repository = "https://github.com/Giri530/trace-score"

trace_score-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

trace_score-0.1.0/setup.py

@@ -0,0 +1,36 @@
+from setuptools import setup, find_packages
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+setup(
+    name  = "trace-score",
+    version="0.1.0",
+    author  = "Girinath V",
+    author_email  = "girinathv48@gmail.com",
+    description   = "Multi-turn LLM Conversation Consistency Metric",
+    long_description = long_description,
+    long_description_content_type = "text/markdown",
+    url = "https://github.com/Giri530/trace-score",
+    packages  = find_packages(),
+    python_requires  = ">=3.8",
+    install_requires = [
+        "sentence-transformers>=2.2.0",
+        "numpy>=1.21.0",
+        "torch>=1.11.0",
+    ],
+    classifiers = [
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Science/Research",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+    ],
+    keywords = [
+        "nlp", "llm", "evaluation", "metrics",
+        "multi-turn", "consistency", "dialogue",
+        "trace-score", "conversational-ai",
+    ],
+)

trace_score-0.1.0/trace_score/__init__.py

@@ -0,0 +1,9 @@
+from .trace import compute_TRACE, TRACEEvaluator, WEIGHT_PRESETS
+from .components.temporal   import compute_T
+from .components.reliability import compute_R
+from .components.adaptive   import compute_A
+from .components.coherence  import compute_C
+from .components.epistemic  import compute_E
+
+__version__ = "0.1.0"
+__author__  = "Girinath V"

trace_score-0.1.0/trace_score/components/__init__.py

@@ -0,0 +1,5 @@
+from .temporal   import compute_T
+from .reliability import compute_R
+from .adaptive   import compute_A
+from .coherence  import compute_C
+from .epistemic  import compute_E