contexttrace 0.1.0__tar.gz

contexttrace-0.1.0/MANIFEST.in

@@ -0,0 +1,18 @@
+include README.md
+include pyproject.toml
+include setup.py
+include contexttrace/py.typed
+recursive-include contexttrace *.py
+prune build
+prune dist
+prune tests
+prune .contexttrace
+prune .pytest_cache
+prune contexttrace.egg-info
+global-exclude __pycache__/*
+global-exclude *.py[cod]
+global-exclude *.db
+global-exclude *.sqlite
+global-exclude *.sqlite3
+global-exclude *.log
+global-exclude .env

contexttrace-0.1.0/PKG-INFO

@@ -0,0 +1,154 @@
+Metadata-Version: 2.1
+Name: contexttrace
+Version: 0.1.0
+Summary: Local-first SDK and CLI for RAG and agent reliability tracing, citation checks, and failure diagnosis.
+Author: ContextTrace contributors
+License: MIT
+Project-URL: Homepage, https://github.com/samarth1412/Context-Trace
+Project-URL: Documentation, https://github.com/samarth1412/Context-Trace/tree/main/docs
+Project-URL: Repository, https://github.com/samarth1412/Context-Trace
+Project-URL: Issues, https://github.com/samarth1412/Context-Trace/issues
+Project-URL: Changelog, https://github.com/samarth1412/Context-Trace/blob/main/CHANGELOG.md
+Keywords: rag,llm,retrieval-augmented-generation,citations,evaluation,observability,agents,cli,sqlite
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.1
+Requires-Dist: httpx>=0.27
+Requires-Dist: typing-extensions>=4.9
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.2; extra == "langchain"
+Provides-Extra: llamaindex
+Requires-Dist: llama-index-core>=0.10; extra == "llamaindex"
+Provides-Extra: local
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.110; extra == "fastapi"
+Provides-Extra: langgraph
+Requires-Dist: langgraph>=0.2; extra == "langgraph"
+Provides-Extra: otel
+Requires-Dist: opentelemetry-api>=1.24; extra == "otel"
+Provides-Extra: opentelemetry
+Requires-Dist: opentelemetry-api>=1.24; extra == "opentelemetry"
+Provides-Extra: integrations
+Requires-Dist: fastapi>=0.110; extra == "integrations"
+Requires-Dist: langchain-core>=0.2; extra == "integrations"
+Requires-Dist: langgraph>=0.2; extra == "integrations"
+Requires-Dist: llama-index-core>=0.10; extra == "integrations"
+Requires-Dist: opentelemetry-api>=1.24; extra == "integrations"
+Provides-Extra: all
+Requires-Dist: fastapi>=0.110; extra == "all"
+Requires-Dist: langchain-core>=0.2; extra == "all"
+Requires-Dist: langgraph>=0.2; extra == "all"
+Requires-Dist: llama-index-core>=0.10; extra == "all"
+Requires-Dist: opentelemetry-api>=1.24; extra == "all"
+Provides-Extra: test
+Requires-Dist: pytest>=8.0; extra == "test"
+
+# ContextTrace
+
+**Debug RAG failures before users find them.**
+
+ContextTrace is a local-first Python SDK and CLI for evaluating existing RAG and AI agent systems. It records retrieved chunks, selected context, answer claims, citations, token usage, latency, and agent events, then writes local traces and HTML reports without requiring a hosted dashboard.
+
+## Install
+
+```bash
+pip install contexttrace
+contexttrace --version
+contexttrace init
+```
+
+Optional integrations:
+
+```bash
+pip install "contexttrace[langchain]"
+pip install "contexttrace[llamaindex]"
+pip install "contexttrace[fastapi]"
+pip install "contexttrace[langgraph]"
+pip install "contexttrace[otel]"
+pip install "contexttrace[all]"
+```
+
+## Quickstart
+
+```bash
+contexttrace init
+contexttrace demo --dataset refund_policy
+contexttrace report --last
+contexttrace doctor
+```
+
+By default, traces are stored locally in:
+
+```text
+.contexttrace/contexttrace.db
+```
+
+## SDK Example
+
+```python
+from contexttrace import ContextTrace
+
+ct = ContextTrace(project="support-rag")
+
+with ct.trace(query="What is the refund policy?") as trace:
+    chunks = retriever.search("What is the refund policy?")
+    trace.log_retrieval(chunks)
+    trace.log_context(chunks[:5])
+
+    answer = llm.generate("What is the refund policy?", chunks[:5])
+    trace.log_answer(answer, usage={"total_tokens": 1200})
+    trace.log_citations([
+        {"claim": "Refunds are available within 30 days.", "source_chunk_id": "chunk_12"}
+    ])
+
+    result = trace.evaluate()
+    print(result["failure"]["failure_type"])
+```
+
+## BYO RAG Endpoint
+
+Evaluate a running local or hosted RAG API without adding SDK code:
+
+```bash
+contexttrace eval \
+  --dataset evals/questions.json \
+  --endpoint http://localhost:8000/query \
+  --method POST \
+  --input-key question \
+  --answer-path $.answer \
+  --contexts-path $.contexts \
+  --citations-path $.citations \
+  --fail-on "failure_rate>0.25"
+```
+
+## What It Catches
+
+- `retrieval_miss`
+- `citation_mismatch`
+- `unsupported_answer`
+- `contradicted_answer`
+- `conflicting_sources`
+- `should_have_abstained`
+- agent failures such as `stale_memory_used` and `tool_error`
+
+## Privacy
+
+Local mode is the default. ContextTrace makes no network calls unless you configure an LLM judge provider or evaluate a RAG endpoint you provide.
+
+## Links
+
+- Repository: https://github.com/samarth1412/Context-Trace
+- Documentation: https://github.com/samarth1412/Context-Trace/tree/main/docs
+- Issues: https://github.com/samarth1412/Context-Trace/issues

contexttrace-0.1.0/README.md

@@ -0,0 +1,97 @@
+# ContextTrace
+
+**Debug RAG failures before users find them.**
+
+ContextTrace is a local-first Python SDK and CLI for evaluating existing RAG and AI agent systems. It records retrieved chunks, selected context, answer claims, citations, token usage, latency, and agent events, then writes local traces and HTML reports without requiring a hosted dashboard.
+
+## Install
+
+```bash
+pip install contexttrace
+contexttrace --version
+contexttrace init
+```
+
+Optional integrations:
+
+```bash
+pip install "contexttrace[langchain]"
+pip install "contexttrace[llamaindex]"
+pip install "contexttrace[fastapi]"
+pip install "contexttrace[langgraph]"
+pip install "contexttrace[otel]"
+pip install "contexttrace[all]"
+```
+
+## Quickstart
+
+```bash
+contexttrace init
+contexttrace demo --dataset refund_policy
+contexttrace report --last
+contexttrace doctor
+```
+
+By default, traces are stored locally in:
+
+```text
+.contexttrace/contexttrace.db
+```
+
+## SDK Example
+
+```python
+from contexttrace import ContextTrace
+
+ct = ContextTrace(project="support-rag")
+
+with ct.trace(query="What is the refund policy?") as trace:
+    chunks = retriever.search("What is the refund policy?")
+    trace.log_retrieval(chunks)
+    trace.log_context(chunks[:5])
+
+    answer = llm.generate("What is the refund policy?", chunks[:5])
+    trace.log_answer(answer, usage={"total_tokens": 1200})
+    trace.log_citations([
+        {"claim": "Refunds are available within 30 days.", "source_chunk_id": "chunk_12"}
+    ])
+
+    result = trace.evaluate()
+    print(result["failure"]["failure_type"])
+```
+
+## BYO RAG Endpoint
+
+Evaluate a running local or hosted RAG API without adding SDK code:
+
+```bash
+contexttrace eval \
+  --dataset evals/questions.json \
+  --endpoint http://localhost:8000/query \
+  --method POST \
+  --input-key question \
+  --answer-path $.answer \
+  --contexts-path $.contexts \
+  --citations-path $.citations \
+  --fail-on "failure_rate>0.25"
+```
+
+## What It Catches
+
+- `retrieval_miss`
+- `citation_mismatch`
+- `unsupported_answer`
+- `contradicted_answer`
+- `conflicting_sources`
+- `should_have_abstained`
+- agent failures such as `stale_memory_used` and `tool_error`
+
+## Privacy
+
+Local mode is the default. ContextTrace makes no network calls unless you configure an LLM judge provider or evaluate a RAG endpoint you provide.
+
+## Links
+
+- Repository: https://github.com/samarth1412/Context-Trace
+- Documentation: https://github.com/samarth1412/Context-Trace/tree/main/docs
+- Issues: https://github.com/samarth1412/Context-Trace/issues

contexttrace-0.1.0/contexttrace/__init__.py

@@ -0,0 +1,36 @@
+from contexttrace._version import __version__
+from contexttrace.client import AsyncContextTrace, ContextTrace
+from contexttrace.config import ContextTraceConfig
+from contexttrace.errors import (
+    ContextTraceConfigError,
+    ContextTraceError,
+    ContextTraceHTTPError,
+    ContextTraceLocalError,
+)
+from contexttrace.integrations.fastapi import ContextTraceFastAPIMiddleware
+from contexttrace.integrations.langchain import ContextTraceCallbackHandler
+from contexttrace.integrations.langgraph import ContextTraceLangGraphTracer
+from contexttrace.integrations.llamaindex import ContextTraceLlamaIndexCallbackHandler
+from contexttrace.integrations.opentelemetry import OpenTelemetryExporter, export_contexttrace_trace
+from contexttrace.reliability import ReliabilityScore, ReliabilityScorer
+from contexttrace.report import ReportGenerator
+
+__all__ = [
+    "AsyncContextTrace",
+    "ContextTrace",
+    "ContextTraceConfig",
+    "ContextTraceConfigError",
+    "ContextTraceCallbackHandler",
+    "ContextTraceError",
+    "ContextTraceFastAPIMiddleware",
+    "ContextTraceHTTPError",
+    "ContextTraceLocalError",
+    "ContextTraceLangGraphTracer",
+    "ContextTraceLlamaIndexCallbackHandler",
+    "OpenTelemetryExporter",
+    "ReliabilityScore",
+    "ReliabilityScorer",
+    "ReportGenerator",
+    "export_contexttrace_trace",
+    "__version__",
+]

contexttrace-0.1.0/contexttrace/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"