traceroai 0.1.0__tar.gz

traceroai-0.1.0/.gitignore

@@ -0,0 +1,45 @@
+# Environment files
+.env
+.env.local
+.env.*.local
+
+# Node / Next.js
+node_modules/
+.next/
+out/
+build/
+coverage/
+.vercel/
+*.tsbuildinfo
+next-env.d.ts
+
+# Python
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+# Logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+pnpm-debug.log*
+
+# OS / editor
+.DS_Store
+Thumbs.db
+.vscode/
+.idea/
+
+# Secrets / certificates
+*.pem
+*.key
+
+.venv/
+venv/
+docs/reference-traces/
+# Python packaging
+*.egg-info/

traceroai-0.1.0/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.4
+Name: traceroai
+Version: 0.1.0
+Summary: Python SDK for sending RAG traces to TraceroAI
+Project-URL: Homepage, https://github.com/chinmai-sd-123/TraceroAI
+Project-URL: Repository, https://github.com/chinmai-sd-123/TraceroAI
+Author: chinmai-sd-123
+License: MIT
+Keywords: evaluation,llm,observability,rag,tracing
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27.0
+Description-Content-Type: text/markdown
+
+# TraceroAI Python SDK
+
+Send RAG traces to [TraceroAI](https://github.com/chinmai-sd-123/TraceroAI) — a
+RAG observability and evaluation platform. Instrument any RAG pipeline
+(LangChain, LlamaIndex, or your own) and every answer becomes a debuggable trace.
+
+## Install
+
+```bash
+pip install traceroai
+```
+
+## Usage
+
+### Context manager (recommended)
+
+Times the block and sends the trace automatically:
+
+```python
+from traceroai import TraceroClient
+
+client = TraceroClient(base_url="http://localhost:8000")
+
+with client.trace("How long does a refund take?") as t:
+    t.log_retrieval(chunks, strategy="hybrid", config={"final_top_k": 3})
+    t.log_prompt(prompt_text, version="grounded_v1")
+    t.log_generation(answer, model="gpt-4o-mini")
+
+print(t.trace_id)
+```
+
+### Decorator
+
+For a function that returns `(answer, chunks)`:
+
+```python
+@client.traced(model="gpt-4o-mini", strategy="hybrid")
+def answer(query: str):
+    chunks = retrieve(query)
+    return generate(query, chunks), chunks
+
+answer("What is the maximum file upload size?")  # traced automatically
+```
+
+### Low-level
+
+```python
+client.log_trace(
+    query={"original": question},
+    retrieval={"strategy": "hybrid", "chunks": chunks},
+    generation={"model": "gpt-4o-mini", "answer": answer},
+)
+```
+
+## Authentication (multi-tenant)
+
+Pass your project API key; the server attributes traces to your project:
+
+```python
+client = TraceroClient(base_url="https://api.traceroai.example", api_key="key_acme")
+```

traceroai-0.1.0/README.md

@@ -0,0 +1,61 @@
+# TraceroAI Python SDK
+
+Send RAG traces to [TraceroAI](https://github.com/chinmai-sd-123/TraceroAI) — a
+RAG observability and evaluation platform. Instrument any RAG pipeline
+(LangChain, LlamaIndex, or your own) and every answer becomes a debuggable trace.
+
+## Install
+
+```bash
+pip install traceroai
+```
+
+## Usage
+
+### Context manager (recommended)
+
+Times the block and sends the trace automatically:
+
+```python
+from traceroai import TraceroClient
+
+client = TraceroClient(base_url="http://localhost:8000")
+
+with client.trace("How long does a refund take?") as t:
+    t.log_retrieval(chunks, strategy="hybrid", config={"final_top_k": 3})
+    t.log_prompt(prompt_text, version="grounded_v1")
+    t.log_generation(answer, model="gpt-4o-mini")
+
+print(t.trace_id)
+```
+
+### Decorator
+
+For a function that returns `(answer, chunks)`:
+
+```python
+@client.traced(model="gpt-4o-mini", strategy="hybrid")
+def answer(query: str):
+    chunks = retrieve(query)
+    return generate(query, chunks), chunks
+
+answer("What is the maximum file upload size?")  # traced automatically
+```
+
+### Low-level
+
+```python
+client.log_trace(
+    query={"original": question},
+    retrieval={"strategy": "hybrid", "chunks": chunks},
+    generation={"model": "gpt-4o-mini", "answer": answer},
+)
+```
+
+## Authentication (multi-tenant)
+
+Pass your project API key; the server attributes traces to your project:
+
+```python
+client = TraceroClient(base_url="https://api.traceroai.example", api_key="key_acme")
+```

traceroai-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "traceroai"
+version = "0.1.0"
+description = "Python SDK for sending RAG traces to TraceroAI"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [{ name = "chinmai-sd-123" }]
+keywords = ["rag", "observability", "evaluation", "llm", "tracing"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+  "httpx>=0.27.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/chinmai-sd-123/TraceroAI"
+Repository = "https://github.com/chinmai-sd-123/TraceroAI"
+
+[tool.hatch.build.targets.wheel]
+packages = ["traceroai"]
+
+[tool.pytest.ini_options]
+pythonpath = ["."]

traceroai-0.1.0/test_sdk_ergonomics.py

@@ -0,0 +1,70 @@
+"""Lightweight tests for the SDK's ergonomic tracing (context manager + decorator).
+
+Run from sdks/python:  python -m pytest test_sdk_ergonomics.py
+No live API needed — TraceroClient.log_trace is stubbed to capture the payload.
+"""
+
+from uuid import uuid4
+
+from traceroai import TraceroClient
+
+
+def _client_capturing(captured: list[dict]) -> TraceroClient:
+    client = TraceroClient(base_url="http://test")
+
+    def fake_log_trace(**payload):
+        captured.append(payload)
+        return uuid4()
+
+    client.log_trace = fake_log_trace  # type: ignore[method-assign]
+    return client
+
+
+def test_context_manager_builds_and_sends_trace() -> None:
+    captured: list[dict] = []
+    client = _client_capturing(captured)
+
+    with client.trace("How long is a refund?") as t:
+        t.log_retrieval([{"rank": 1, "chunk_id": "c1", "text": "5 to 7 days"}], strategy="lexical")
+        t.log_generation("5 to 7 business days", model="gpt-4o-mini")
+
+    assert t.trace_id is not None
+    assert len(captured) == 1
+    payload = captured[0]
+    assert payload["query"]["original"] == "How long is a refund?"
+    assert payload["retrieval"]["strategy"] == "lexical"
+    assert payload["generation"]["answered"] is True
+    assert payload["generation"]["model"] == "gpt-4o-mini"
+    assert "total_ms" in payload["latency"]
+
+
+def test_exception_marks_trace_unanswered_and_still_sends() -> None:
+    captured: list[dict] = []
+    client = _client_capturing(captured)
+
+    try:
+        with client.trace("boom") as t:
+            t.log_generation("partial", model="gpt-4o-mini")
+            raise ValueError("pipeline failed")
+    except ValueError:
+        pass
+
+    # Trace still sent (for debugging the failure), marked unanswered.
+    assert len(captured) == 1
+    assert captured[0]["generation"]["answered"] is False
+
+
+def test_decorator_traces_and_returns_answer() -> None:
+    captured: list[dict] = []
+    client = _client_capturing(captured)
+
+    @client.traced(model="gpt-4o-mini", strategy="lexical_top_k")
+    def answer(query: str):
+        return "an answer", [{"rank": 1, "chunk_id": "c1", "text": "ctx"}]
+
+    result = answer("a question")
+
+    assert result == "an answer"  # caller gets the answer transparently
+    assert len(captured) == 1
+    assert captured[0]["query"]["original"] == "a question"
+    assert captured[0]["retrieval"]["strategy"] == "lexical_top_k"

traceroai-0.1.0/test_sdk_send.py

@@ -0,0 +1,52 @@
+from traceroai import TraceroClient
+
+client = TraceroClient(base_url="http://127.0.0.1:8000")
+
+trace_id = client.log_trace(
+    query={
+        "original": "Can admins change the workspace region themselves?",
+        "rewritten": "Can admins change the workspace region themselves?",
+        "rewrite_changed": False,
+        "rewrite_method": "rule_based_v1",
+    },
+    retrieval={
+        "strategy": "hybrid_rrf_rerank",
+        "config": {
+            "lexical_top_k": 5,
+            "dense_top_k": 5,
+            "final_top_k": 3,
+            "fusion": "rrf",
+            "reranker": "rule_based_v1",
+        },
+        "chunks": [
+            {
+                "rank": 1,
+                "chunk_id": "product_faq_2",
+                "document_id": "product_faq",
+                "document_title": "Product FAQ",
+                "section": "Can I change my workspace region?",
+                "source": "product_faq.md",
+                "final_score": 1.08,
+                "text": "Customers cannot directly change a workspace region after the workspace is created. To request a region change, customers must contact support.",
+            }
+        ],
+    },
+    generation={
+        "provider": "openai",
+        "model": "gpt-4o-mini",
+        "temperature": 0,
+        "answer": "No, admins cannot change the workspace region themselves. They must contact support [1].",
+        "answered": True,
+    },
+    latency={
+        "retrieval_ms": 17,
+        "generation_ms": 1154,
+        "total_ms": 1171,
+    },
+    diagnosis={
+        "label": "healthy_answer",
+        "reason": "The retriever found useful context and the model answered the question.",
+    },
+)
+
+print(f"Sent trace: {trace_id}")

traceroai-0.1.0/traceroai/__init__.py

@@ -0,0 +1,4 @@
+from traceroai.client import TraceroClient
+from traceroai.trace import TraceContext
+
+__all__ = ["TraceroClient", "TraceContext"]

traceroai-0.1.0/traceroai/client.py

@@ -0,0 +1,89 @@
+from typing import Any
+from uuid import UUID
+
+import httpx
+
+from traceroai.trace import TraceContext
+
+
+class TraceroClient:
+    def __init__(self, base_url: str, api_key: str| None = None, timeout_seconds: float = 10.0,)-> None:
+        self.base_url = base_url
+        self.api_key = api_key
+        self.timeout_seconds = timeout_seconds
+
+    def trace(
+        self,
+        query: str,
+        *,
+        rewritten: str | None = None,
+        project: dict[str, Any] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> TraceContext:
+        """Open a trace context manager that auto-times and auto-sends.
+
+        with client.trace("How long is a refund?") as t:
+            t.log_retrieval(chunks)
+            t.log_generation(answer, model="gpt-4o-mini")
+        """
+        return TraceContext(
+            self, query, rewritten=rewritten, project=project, metadata=metadata
+        )
+
+    def traced(self, *, model: str, strategy: str = "vector"):
+        """Decorator for a RAG function that returns (answer, chunks).
+
+        The first positional arg of the wrapped function is treated as the query.
+
+            @client.traced(model="gpt-4o-mini")
+            def answer(query: str) -> tuple[str, list[dict]]:
+                chunks = retrieve(query)
+                return generate(query, chunks), chunks
+        """
+
+        def decorator(func):
+            def wrapper(query: str, *args: Any, **kwargs: Any):
+                with self.trace(query) as t:
+                    answer, chunks = func(query, *args, **kwargs)
+                    t.log_retrieval(chunks, strategy=strategy)
+                    t.log_generation(answer, model=model)
+                    return answer
+
+            return wrapper
+
+        return decorator
+
+    def log_trace(self , *,query:dict[str, Any],
+                  retrieval: dict[str, Any],
+                  generation: dict[str, Any],
+                  prompt: dict[str, Any] | None = None,
+                  latency: dict[str, Any] | None = None,
+                  evaluation: dict[str, Any] | None = None,
+                  diagnosis: dict[str, Any] | None = None,
+                  project: dict[str, Any]| None = None,
+                  metadata: dict[str, Any] | None = None,
+            )-> UUID:
+                payload = {
+                "query": query,
+                "retrieval": retrieval,
+                "generation": generation,    
+                "prompt": prompt or {},
+                "latency": latency or {},
+                "evaluation": evaluation or {},
+                "diagnosis": diagnosis or {},
+                "project": project or {},
+                "metadata": metadata or {},
+                }
+
+                headers= {}
+                if self.api_key:
+                    headers["Authorization"] = f"Bearer {self.api_key}"
+                
+                response = httpx.post(f"{self.base_url}/v1/traces", json=payload, headers=headers, timeout=self.timeout_seconds,)
+
+                response.raise_for_status()
+
+                data = response.json()
+                return UUID(data["trace_id"])
+                
+                

traceroai-0.1.0/traceroai/trace.py

@@ -0,0 +1,107 @@
+"""Ergonomic tracing: a context manager that builds and sends a trace for you.
+
+Instead of hand-assembling the log_trace(...) payload, wrap the RAG work:
+
+    with client.trace(query="How long is a refund?") as t:
+        t.log_retrieval(chunks, strategy="hybrid", config={"final_top_k": 3})
+        t.log_prompt(prompt_text, version="grounded_v1")
+        t.log_generation(answer, model="gpt-4o-mini")
+
+The context manager times the block automatically, fills latency.total_ms, marks
+the trace unanswered if an exception escapes, and sends it on exit.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import TYPE_CHECKING, Any
+from uuid import UUID
+
+if TYPE_CHECKING:
+    from traceroai.client import TraceroClient
+
+
+class TraceContext:
+    def __init__(
+        self,
+        client: "TraceroClient",
+        query: str,
+        *,
+        rewritten: str | None = None,
+        project: dict[str, Any] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        self._client = client
+        self._query: dict[str, Any] = {"original": query}
+        if rewritten is not None and rewritten != query:
+            self._query.update({"rewritten": rewritten, "rewrite_changed": True})
+
+        self._retrieval: dict[str, Any] = {"chunks": []}
+        self._prompt: dict[str, Any] | None = None
+        self._generation: dict[str, Any] = {"answer": "", "answered": False}
+        self._project = project
+        self._metadata = metadata
+
+        self._start: float = 0.0
+        self.trace_id: UUID | None = None
+
+    # --- piece loggers (call these inside the `with` block) ---
+
+    def log_retrieval(
+        self,
+        chunks: list[dict[str, Any]],
+        *,
+        strategy: str = "vector",
+        config: dict[str, Any] | None = None,
+    ) -> None:
+        self._retrieval = {"strategy": strategy, "chunks": chunks}
+        if config is not None:
+            self._retrieval["config"] = config
+
+    def log_prompt(
+        self, content: str, *, version: str | None = None, template_name: str | None = None
+    ) -> None:
+        self._prompt = {"content": content, "version": version, "template_name": template_name}
+
+    def log_generation(
+        self,
+        answer: str,
+        *,
+        model: str,
+        provider: str | None = None,
+        temperature: float | None = None,
+    ) -> None:
+        self._generation = {
+            "answer": answer,
+            "answered": True,
+            "model": model,
+            "provider": provider,
+            "temperature": temperature,
+        }
+
+    # --- context manager protocol ---
+
+    def __enter__(self) -> "TraceContext":
+        self._start = time.perf_counter()
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        total_ms = int((time.perf_counter() - self._start) * 1000)
+
+        # A generation is only "answered" if one was logged and nothing blew up.
+        if exc_type is not None:
+            self._generation["answered"] = False
+
+        # The schema requires a model; default it so a half-finished trace still sends.
+        self._generation.setdefault("model", "unknown")
+
+        self.trace_id = self._client.log_trace(
+            query=self._query,
+            retrieval=self._retrieval,
+            generation=self._generation,
+            prompt=self._prompt,
+            latency={"total_ms": total_ms},
+            project=self._project,
+            metadata=self._metadata,
+        )
+        return False  # never suppress exceptions