contexttrace 0.1.0__py3-none-any.whl

contexttrace/transport.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any, Optional, Protocol
+
+import httpx
+
+from contexttrace.errors import ContextTraceHTTPError
+
+logger = logging.getLogger("contexttrace")
+
+
+class Transport(Protocol):
+    def post(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+        ...
+
+    def get(self, path: str) -> dict[str, Any]:
+        ...
+
+
+class HttpTransport:
+    def __init__(
+        self,
+        *,
+        base_url: str,
+        api_key: str,
+        timeout: float = 30.0,
+        retries: int = 2,
+        debug: bool = False,
+        client: Optional[httpx.Client] = None,
+    ) -> None:
+        self.retries = max(0, retries)
+        self.debug = debug
+        self._client = client or httpx.Client(
+            base_url=base_url.rstrip("/"),
+            timeout=timeout,
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "User-Agent": "contexttrace-python/0.1.0",
+            },
+        )
+
+    def post(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+        return self._request("POST", path, json=payload or {})
+
+    def get(self, path: str) -> dict[str, Any]:
+        return self._request("GET", path)
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
+        last_error: Optional[BaseException] = None
+        attempts = self.retries + 1
+
+        for attempt in range(attempts):
+            try:
+                if self.debug:
+                    logger.debug("%s %s", method, path)
+                response = self._client.request(method, path, **kwargs)
+                if response.status_code >= 500 and attempt < attempts - 1:
+                    last_error = _http_error(response)
+                    _sleep_before_retry(attempt)
+                    continue
+                return _json_or_raise(response)
+            except (httpx.TimeoutException, httpx.NetworkError, httpx.TransportError) as exc:
+                last_error = exc
+                if attempt >= attempts - 1:
+                    break
+                _sleep_before_retry(attempt)
+
+        raise ContextTraceHTTPError(
+            "ContextTrace API request failed after %s attempt(s): %s %s: %s"
+            % (attempts, method, path, last_error)
+        )
+
+    def close(self) -> None:
+        self._client.close()
+
+
+class AsyncTransport(Protocol):
+    async def post(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+        ...
+
+    async def get(self, path: str) -> dict[str, Any]:
+        ...
+
+
+class AsyncHttpTransport:
+    def __init__(
+        self,
+        *,
+        base_url: str,
+        api_key: str,
+        timeout: float = 30.0,
+        retries: int = 2,
+        debug: bool = False,
+        client: Optional[httpx.AsyncClient] = None,
+    ) -> None:
+        self.retries = max(0, retries)
+        self.debug = debug
+        self._client = client or httpx.AsyncClient(
+            base_url=base_url.rstrip("/"),
+            timeout=timeout,
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "User-Agent": "contexttrace-python/0.1.0",
+            },
+        )
+
+    async def post(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
+        return await self._request("POST", path, json=payload or {})
+
+    async def get(self, path: str) -> dict[str, Any]:
+        return await self._request("GET", path)
+
+    async def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
+        last_error: Optional[BaseException] = None
+        attempts = self.retries + 1
+
+        for attempt in range(attempts):
+            try:
+                if self.debug:
+                    logger.debug("%s %s", method, path)
+                response = await self._client.request(method, path, **kwargs)
+                if response.status_code >= 500 and attempt < attempts - 1:
+                    last_error = _http_error(response)
+                    await _async_sleep_before_retry(attempt)
+                    continue
+                return _json_or_raise(response)
+            except (httpx.TimeoutException, httpx.NetworkError, httpx.TransportError) as exc:
+                last_error = exc
+                if attempt >= attempts - 1:
+                    break
+                await _async_sleep_before_retry(attempt)
+
+        raise ContextTraceHTTPError(
+            "ContextTrace API request failed after %s attempt(s): %s %s: %s"
+            % (attempts, method, path, last_error)
+        )
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+
+def _json_or_raise(response: httpx.Response) -> dict[str, Any]:
+    try:
+        response.raise_for_status()
+    except httpx.HTTPStatusError as exc:
+        raise _http_error(response) from exc
+
+    try:
+        data = response.json()
+    except ValueError as exc:
+        raise ContextTraceHTTPError(
+            "ContextTrace API returned invalid JSON for %s %s."
+            % (response.request.method, response.request.url)
+        ) from exc
+
+    if not isinstance(data, dict):
+        raise ContextTraceHTTPError("ContextTrace API returned a non-object JSON response.")
+    return data
+
+
+def _http_error(response: httpx.Response) -> ContextTraceHTTPError:
+    detail = response.text
+    try:
+        body = response.json()
+        if isinstance(body, dict):
+            detail = str(body.get("detail") or body.get("error") or body)
+    except ValueError:
+        pass
+    return ContextTraceHTTPError(
+        "ContextTrace API request failed with HTTP %s for %s %s: %s"
+        % (response.status_code, response.request.method, response.request.url, detail)
+    )
+
+
+def _sleep_before_retry(attempt: int) -> None:
+    time.sleep(min(0.5, 0.1 * (2 ** attempt)))
+
+
+async def _async_sleep_before_retry(attempt: int) -> None:
+    await asyncio.sleep(min(0.5, 0.1 * (2 ** attempt)))

contexttrace/viewer.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+from html import escape
+from pathlib import Path
+from typing import Any, Callable
+from wsgiref.simple_server import make_server
+
+from contexttrace.report import ReportGenerator
+from contexttrace.storage import SQLiteTraceStore
+
+
+def create_viewer_app(storage_path: str = ".contexttrace/contexttrace.db") -> Callable:
+    store = SQLiteTraceStore(storage_path)
+
+    def app(environ: dict[str, Any], start_response: Callable) -> list[bytes]:
+        path = environ.get("PATH_INFO") or "/"
+        try:
+            status = "200 OK"
+            body = _route(path, store)
+        except Exception as exc:
+            status = "500 Internal Server Error"
+            body = _page("ContextTrace Viewer Error", "<p>%s</p>" % escape(str(exc)))
+        start_response(status, [("Content-Type", "text/html; charset=utf-8")])
+        return [body.encode("utf-8")]
+
+    return app
+
+
+def serve_viewer(
+    *,
+    storage_path: str = ".contexttrace/contexttrace.db",
+    host: str = "127.0.0.1",
+    port: int = 8765,
+) -> None:
+    app = create_viewer_app(storage_path)
+    with make_server(host, port, app) as server:
+        print("ContextTrace local viewer: http://%s:%s" % (host, port))
+        server.serve_forever()
+
+
+def _route(path: str, store: SQLiteTraceStore) -> str:
+    if path == "/":
+        status = store.trace_count()
+        last_eval = store.last_eval_run()
+        return _page(
+            "ContextTrace Local Viewer",
+            """
+            <section class="hero">
+              <h1>ContextTrace Local Viewer</h1>
+              <p>Inspect local RAG and agent traces stored in SQLite.</p>
+              <div class="metrics">
+                <div><dt>Traces</dt><dd>{trace_count}</dd></div>
+                <div><dt>Last Eval Run</dt><dd>{last_eval}</dd></div>
+              </div>
+            </section>
+            """.format(
+                trace_count=status,
+                last_eval=escape(str((last_eval or {}).get("id") or "None")),
+            ),
+        )
+    if path == "/traces":
+        rows = []
+        for trace in store.list_traces(limit=100):
+            failure = ((trace.get("evaluation") or {}).get("failure") or {}).get("failure_type") or "not_evaluated"
+            score = ((trace.get("evaluation") or {}).get("scores") or {}).get("citation_support", "")
+            rows.append(
+                "<tr><td><a href=\"/traces/{id}\">{id}</a></td><td>{query}</td><td>{failure}</td><td>{score}</td><td>{created}</td></tr>".format(
+                    id=escape(str(trace.get("id"))),
+                    query=escape(str(trace.get("query") or "")[:140]),
+                    failure=escape(str(failure)),
+                    score=escape(str(score)),
+                    created=escape(str(trace.get("created_at") or "")),
+                )
+            )
+        return _page(
+            "Traces",
+            "<h1>Traces</h1><table><thead><tr><th>Trace ID</th><th>Query</th><th>Failure</th><th>Citation Support</th><th>Created</th></tr></thead><tbody>%s</tbody></table>"
+            % ("\n".join(rows) or "<tr><td colspan=\"5\">No traces found.</td></tr>"),
+        )
+    if path.startswith("/traces/"):
+        trace_id = path.rsplit("/", 1)[-1]
+        trace = store.get_trace(trace_id)
+        return ReportGenerator().render(trace)
+    if path == "/eval-runs":
+        rows = []
+        for run in store.list_eval_runs(limit=100):
+            summary = run.get("summary") or {}
+            rows.append(
+                "<tr><td>{id}</td><td>{dataset}</td><td>{endpoint}</td><td>{score}</td><td>{created}</td></tr>".format(
+                    id=escape(str(run.get("id"))),
+                    dataset=escape(str(run.get("dataset") or "")),
+                    endpoint=escape(str(run.get("endpoint") or "")),
+                    score=escape(str(summary.get("reliability_score", ""))),
+                    created=escape(str(run.get("created_at") or "")),
+                )
+            )
+        return _page(
+            "Eval Runs",
+            "<h1>Eval Runs</h1><table><thead><tr><th>ID</th><th>Dataset</th><th>Endpoint</th><th>Reliability</th><th>Created</th></tr></thead><tbody>%s</tbody></table>"
+            % ("\n".join(rows) or "<tr><td colspan=\"5\">No eval runs found.</td></tr>"),
+        )
+    if path == "/reports":
+        report_dir = Path(".contexttrace") / "reports"
+        items = []
+        for report in sorted(report_dir.glob("*.html")) if report_dir.exists() else []:
+            items.append("<li>%s</li>" % escape(str(report)))
+        return _page("Reports", "<h1>Reports</h1><ul>%s</ul>" % ("\n".join(items) or "<li>No reports found.</li>"))
+    return _page("Not Found", "<h1>Not Found</h1><p>No local viewer page exists for this path.</p>")
+
+
+def _page(title: str, body: str) -> str:
+    return """<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>{title}</title>
+  <style>
+    :root {{ --bg: #f7f8fa; --panel: #fff; --text: #1f2933; --muted: #697386; --line: #d8dee8; --accent: #2458d3; }}
+    * {{ box-sizing: border-box; }}
+    body {{ margin: 0; background: var(--bg); color: var(--text); font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }}
+    main {{ max-width: 1120px; margin: 0 auto; padding: 32px 20px 56px; }}
+    nav {{ display: flex; gap: 16px; margin-bottom: 22px; }}
+    nav a {{ color: var(--accent); text-decoration: none; font-weight: 700; }}
+    h1 {{ margin-top: 0; }}
+    section, table {{ background: var(--panel); border: 1px solid var(--line); border-radius: 8px; }}
+    section {{ padding: 20px; }}
+    table {{ width: 100%; border-collapse: collapse; overflow: hidden; }}
+    th, td {{ border-bottom: 1px solid var(--line); padding: 10px; text-align: left; vertical-align: top; }}
+    th {{ color: var(--muted); font-size: 12px; text-transform: uppercase; }}
+    .metrics {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; margin-top: 16px; }}
+    .metrics div {{ border: 1px solid var(--line); border-radius: 8px; padding: 12px; background: #fbfcfe; }}
+    dt {{ color: var(--muted); font-size: 12px; font-weight: 700; text-transform: uppercase; }}
+    dd {{ margin: 4px 0 0; font-size: 20px; }}
+  </style>
+</head>
+<body>
+  <main>
+    <nav>
+      <a href="/">Overview</a>
+      <a href="/traces">Traces</a>
+      <a href="/eval-runs">Eval Runs</a>
+      <a href="/reports">Reports</a>
+    </nav>
+    {body}
+  </main>
+</body>
+</html>""".format(title=escape(title), body=body)

contexttrace-0.1.0.dist-info/METADATA

@@ -0,0 +1,154 @@
+Metadata-Version: 2.1
+Name: contexttrace
+Version: 0.1.0
+Summary: Local-first SDK and CLI for RAG and agent reliability tracing, citation checks, and failure diagnosis.
+Author: ContextTrace contributors
+License: MIT
+Project-URL: Homepage, https://github.com/samarth1412/Context-Trace
+Project-URL: Documentation, https://github.com/samarth1412/Context-Trace/tree/main/docs
+Project-URL: Repository, https://github.com/samarth1412/Context-Trace
+Project-URL: Issues, https://github.com/samarth1412/Context-Trace/issues
+Project-URL: Changelog, https://github.com/samarth1412/Context-Trace/blob/main/CHANGELOG.md
+Keywords: rag,llm,retrieval-augmented-generation,citations,evaluation,observability,agents,cli,sqlite
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.1
+Requires-Dist: httpx>=0.27
+Requires-Dist: typing-extensions>=4.9
+Provides-Extra: all
+Requires-Dist: fastapi>=0.110; extra == "all"
+Requires-Dist: langchain-core>=0.2; extra == "all"
+Requires-Dist: langgraph>=0.2; extra == "all"
+Requires-Dist: llama-index-core>=0.10; extra == "all"
+Requires-Dist: opentelemetry-api>=1.24; extra == "all"
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.110; extra == "fastapi"
+Provides-Extra: integrations
+Requires-Dist: fastapi>=0.110; extra == "integrations"
+Requires-Dist: langchain-core>=0.2; extra == "integrations"
+Requires-Dist: langgraph>=0.2; extra == "integrations"
+Requires-Dist: llama-index-core>=0.10; extra == "integrations"
+Requires-Dist: opentelemetry-api>=1.24; extra == "integrations"
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.2; extra == "langchain"
+Provides-Extra: langgraph
+Requires-Dist: langgraph>=0.2; extra == "langgraph"
+Provides-Extra: llamaindex
+Requires-Dist: llama-index-core>=0.10; extra == "llamaindex"
+Provides-Extra: local
+Provides-Extra: opentelemetry
+Requires-Dist: opentelemetry-api>=1.24; extra == "opentelemetry"
+Provides-Extra: otel
+Requires-Dist: opentelemetry-api>=1.24; extra == "otel"
+Provides-Extra: test
+Requires-Dist: pytest>=8.0; extra == "test"
+
+# ContextTrace
+
+**Debug RAG failures before users find them.**
+
+ContextTrace is a local-first Python SDK and CLI for evaluating existing RAG and AI agent systems. It records retrieved chunks, selected context, answer claims, citations, token usage, latency, and agent events, then writes local traces and HTML reports without requiring a hosted dashboard.
+
+## Install
+
+```bash
+pip install contexttrace
+contexttrace --version
+contexttrace init
+```
+
+Optional integrations:
+
+```bash
+pip install "contexttrace[langchain]"
+pip install "contexttrace[llamaindex]"
+pip install "contexttrace[fastapi]"
+pip install "contexttrace[langgraph]"
+pip install "contexttrace[otel]"
+pip install "contexttrace[all]"
+```
+
+## Quickstart
+
+```bash
+contexttrace init
+contexttrace demo --dataset refund_policy
+contexttrace report --last
+contexttrace doctor
+```
+
+By default, traces are stored locally in:
+
+```text
+.contexttrace/contexttrace.db
+```
+
+## SDK Example
+
+```python
+from contexttrace import ContextTrace
+
+ct = ContextTrace(project="support-rag")
+
+with ct.trace(query="What is the refund policy?") as trace:
+    chunks = retriever.search("What is the refund policy?")
+    trace.log_retrieval(chunks)
+    trace.log_context(chunks[:5])
+
+    answer = llm.generate("What is the refund policy?", chunks[:5])
+    trace.log_answer(answer, usage={"total_tokens": 1200})
+    trace.log_citations([
+        {"claim": "Refunds are available within 30 days.", "source_chunk_id": "chunk_12"}
+    ])
+
+    result = trace.evaluate()
+    print(result["failure"]["failure_type"])
+```
+
+## BYO RAG Endpoint
+
+Evaluate a running local or hosted RAG API without adding SDK code:
+
+```bash
+contexttrace eval \
+  --dataset evals/questions.json \
+  --endpoint http://localhost:8000/query \
+  --method POST \
+  --input-key question \
+  --answer-path $.answer \
+  --contexts-path $.contexts \
+  --citations-path $.citations \
+  --fail-on "failure_rate>0.25"
+```
+
+## What It Catches
+
+- `retrieval_miss`
+- `citation_mismatch`
+- `unsupported_answer`
+- `contradicted_answer`
+- `conflicting_sources`
+- `should_have_abstained`
+- agent failures such as `stale_memory_used` and `tool_error`
+
+## Privacy
+
+Local mode is the default. ContextTrace makes no network calls unless you configure an LLM judge provider or evaluate a RAG endpoint you provide.
+
+## Links
+
+- Repository: https://github.com/samarth1412/Context-Trace
+- Documentation: https://github.com/samarth1412/Context-Trace/tree/main/docs
+- Issues: https://github.com/samarth1412/Context-Trace/issues

contexttrace-0.1.0.dist-info/RECORD

@@ -0,0 +1,31 @@
+contexttrace/__init__.py,sha256=PYE5v4eQyRQj85tm4r1QU4vD50Ha_sMy2y_Vq0oP5V0,1347
+contexttrace/_version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
+contexttrace/cli.py,sha256=HwPlpP89ZlMjVdU-tjIFD_Z4cwgxOWtG-Xgn389P_sI,19751
+contexttrace/client.py,sha256=i-fFRG3ESXgfIIy2_wmmQpgWK1F8VmVDx8K5KpDdQdk,34333
+contexttrace/config.py,sha256=inyY1xhYB8Py9T5COmI6Rj_WcLLR6ItP1Hs7T2XtjTk,7401
+contexttrace/demo.py,sha256=COxHRNnyYySgujBrBrwe-z1ti19VZzubkDNrhm79358,11445
+contexttrace/demo_data.py,sha256=cJjprUipIqjorF_JAscOqONpqeSqdEUFx4WlLqcFmAc,14069
+contexttrace/endpoint_eval.py,sha256=0WZLYnTs03GXiJoHOnXyj_ATeYpkLPtZTYaeU_Mb4Qs,12242
+contexttrace/errors.py,sha256=WsqJGLoP02pjheLTZEDL1R9N52vA3A0_QhlJBOGDr-A,393
+contexttrace/evaluator.py,sha256=bQhHvnysDZuwBGmU1GfPLB9ae7_Gaquh3aFP3Zc9HCU,14414
+contexttrace/local.py,sha256=k6_6gvbzQEg_ZETkTC4gzN1vXitYSFus7d8GtNj3A58,12723
+contexttrace/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+contexttrace/regression.py,sha256=mexwqdv8H-HUNqyoOUT6oBQWiApDKtNvRjNn9N0i77E,4494
+contexttrace/reliability.py,sha256=MYxgwNOsHhErnjTDJkAYsAhNPjgFqcvLbE5pkannxYw,11130
+contexttrace/report.py,sha256=AwUIweDMy1WINaxSYOM2bj-LkVwlHo23NlunuqLBF7A,17667
+contexttrace/thresholds.py,sha256=J6yrMD3dPRtqx7JKnLFYqvchd4paiyXBPm5KhfLKs1Q,1572
+contexttrace/transport.py,sha256=ra4GIgSjM5gPkS_VGHQXFmgeP-JswYHO9JSJqMNnhgc,5938
+contexttrace/viewer.py,sha256=GERrf61SMvVNZ8wt-RDOMHiqJU5uqvX0OJshAEEMO2U,6599
+contexttrace/integrations/__init__.py,sha256=EhuHHsPlRfxzMm3L_1BSV1VJj7BCf0KmVE8MG8xbbEM,645
+contexttrace/integrations/fastapi.py,sha256=uDL2NfTGHbw8Dq58Ubx83mKqpDmsANOUeTYquX3cDeA,11417
+contexttrace/integrations/langchain.py,sha256=w3FoqlrvBojpOfIcTAXSkX7ZaWPaJ-lqM1o6S0csh0M,15584
+contexttrace/integrations/langgraph.py,sha256=9RkqTwmf0dIIbEIe_JsYTFCa7hKATj_kBLjBKWyezbw,6775
+contexttrace/integrations/llamaindex.py,sha256=J3bEUDZmCBKJ6skWsHO-o9vVYu_2tKzRJNUohalPv_U,14446
+contexttrace/integrations/opentelemetry.py,sha256=9hlMvnFiz3XykFWb3kw3oH5fLesV8t7yXw_UFAAbF1E,4179
+contexttrace/storage/__init__.py,sha256=VcZgNDrQXc9SWt37nJ6F2LYOyMQryXYBWpedLQkK2vQ,95
+contexttrace/storage/sqlite_store.py,sha256=hg117peJvWoCPe4pQ_8zMohq6otIF0Zowa80WvFjcnk,23807
+contexttrace-0.1.0.dist-info/METADATA,sha256=P-fJL8KKplGOYuv6F3r77xihSlplWWAlTMU-KOAm80I,5291
+contexttrace-0.1.0.dist-info/WHEEL,sha256=BNRMDyzLkkcmlv0J8ppDQkk2VED33SesJDynr9ED1gc,91
+contexttrace-0.1.0.dist-info/entry_points.txt,sha256=RJDO0pz-10mTUe9HZwgtBSRWsyz4JtSNvzvt0T0_T5k,55
+contexttrace-0.1.0.dist-info/top_level.txt,sha256=r1Az9kxVpsJuPfnrMOPOp2U_6ceBO4CQt6C5tTnGaLM,13
+contexttrace-0.1.0.dist-info/RECORD,,

contexttrace-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.3.4)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

contexttrace-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+contexttrace = contexttrace.cli:main

contexttrace-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+contexttrace