pixie-qa 0.1.11__tar.gz → 0.2.0__tar.gz

{pixie_qa-0.1.11 → pixie_qa-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pixie-qa
-Version: 0.1.11
+Version: 0.2.0
 Summary: Automated quality assurance for AI applications
 Project-URL: Homepage, https://github.com/yiouli/pixie-qa
 Project-URL: Repository, https://github.com/yiouli/pixie-qa
@@ -40,6 +40,7 @@ Classifier: Topic :: Software Development :: Testing
 Requires-Python: >=3.11
 Requires-Dist: autoevals>=0.1.0
 Requires-Dist: jsonpickle>=4.0.0
+Requires-Dist: openai>=2.29.0
 Requires-Dist: openinference-instrumentation>=0.1.44
 Requires-Dist: opentelemetry-api>=1.27.0
 Requires-Dist: opentelemetry-sdk>=1.27.0

pixie_qa-0.2.0/changelogs/observe-sensitive-field-stripping.md

@@ -0,0 +1,22 @@
+# @observe: Strip `self` and `cls` from captured input
+
+## What changed
+
+The `@observe` decorator now automatically removes `self` and `cls` from the
+captured function arguments before serialization. Previously, decorating a
+method with `@observe` would serialize the entire instance (including API keys,
+client objects, and other sensitive state) into `eval_input` via jsonpickle.
+
+## Files affected
+
+- `pixie/instrumentation/observation.py` — `sync_wrapper` and `async_wrapper`
+  now pop `self`/`cls` from `bound.arguments` before calling `_serialize()`.
+- `tests/pixie/instrumentation/test_observation.py` — Two new tests:
+  `test_self_excluded_from_input` and `test_cls_excluded_from_input`.
+
+## Migration notes
+
+No API changes. This is a backward-compatible fix. Existing `@observe` usage on
+functions (not methods) is unaffected. Methods that previously leaked `self` into
+`eval_input` will now produce cleaner, smaller input data containing only the
+semantic arguments.

{pixie_qa-0.1.11 → pixie_qa-0.2.0}/pixie/__init__.py

@@ -17,6 +17,7 @@ from pixie.evals.eval_utils import (
     run_and_evaluate,
 )
 from pixie.evals.evaluation import Evaluation, Evaluator, evaluate
+from pixie.evals.llm_evaluator import create_llm_evaluator
 from pixie.evals.scorers import (
     AnswerCorrectnessEval,
     AnswerRelevancyEval,
@@ -96,6 +97,7 @@ __all__ = [
     "assert_dataset_pass",
     "assert_pass",
     "capture_traces",
+    "create_llm_evaluator",
     "evaluate",
     "last_llm_call",
     "root",

{pixie_qa-0.1.11 → pixie_qa-0.2.0}/pixie/cli/main.py

@@ -30,6 +30,7 @@ from pixie.cli.dataset_command import (
     dataset_save,
     format_dataset_table,
 )
+from pixie.cli.trace_command import trace_last, trace_list, trace_show
 from pixie.config import get_config
 from pixie.dataset.store import DatasetStore
 from pixie.storage.evaluable import UNSET, _Unset
@@ -88,6 +89,58 @@ def _build_parser() -> argparse.ArgumentParser:
         help="Optional notes to attach to the evaluable metadata",
     )
 
+    # -- pixie trace ---------------------------------------------------------
+    trace_parser = subparsers.add_parser("trace", help="Inspect captured traces")
+    trace_sub = trace_parser.add_subparsers(dest="trace_action", help="Trace actions")
+
+    # pixie trace list [--limit N] [--errors]
+    trace_list_parser = trace_sub.add_parser("list", help="List recent traces")
+    trace_list_parser.add_argument(
+        "--limit",
+        type=int,
+        default=10,
+        help="Maximum number of traces to show (default: 10)",
+    )
+    trace_list_parser.add_argument(
+        "--errors",
+        action="store_true",
+        default=False,
+        help="Show only traces with errors",
+    )
+
+    # pixie trace show <trace_id> [-v] [--json]
+    trace_show_parser = trace_sub.add_parser("show", help="Show span tree for a trace")
+    trace_show_parser.add_argument(
+        "trace_id",
+        help="Trace ID (or prefix, minimum 8 characters)",
+    )
+    trace_show_parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="Show full input/output data for each span",
+    )
+    trace_show_parser.add_argument(
+        "--json",
+        dest="as_json",
+        action="store_true",
+        default=False,
+        help="Output as JSON",
+    )
+
+    # pixie trace last [--json]
+    trace_last_parser = trace_sub.add_parser(
+        "last", help="Show the most recent trace (verbose)"
+    )
+    trace_last_parser.add_argument(
+        "--json",
+        dest="as_json",
+        action="store_true",
+        default=False,
+        help="Output as JSON",
+    )
+
     # -- pixie test ----------------------------------------------------------
     test_parser = subparsers.add_parser("test", help="Run pixie eval tests")
     test_parser.add_argument(
@@ -213,6 +266,29 @@ def main(argv: list[str] | None = None) -> int:
             print(f"Error: {exc}", file=sys.stderr)  # noqa: T201
             return 1
 
+    elif args.command == "trace":
+        if args.trace_action is None:
+            parser.parse_args(["trace", "--help"])
+            return 1
+
+        try:
+            if args.trace_action == "list":
+                return trace_list(
+                    limit=args.limit,
+                    errors_only=args.errors,
+                )
+            elif args.trace_action == "show":
+                return trace_show(
+                    trace_id=args.trace_id,
+                    verbose=args.verbose,
+                    as_json=args.as_json,
+                )
+            elif args.trace_action == "last":
+                return trace_last(as_json=args.as_json)
+        except Exception as exc:
+            print(f"Error: {exc}", file=sys.stderr)  # noqa: T201
+            return 1
+
     elif args.command == "test":
         from pixie.cli.test_command import main as test_main
 

pixie_qa-0.2.0/pixie/cli/trace_command.py

@@ -0,0 +1,186 @@
+"""``pixie trace`` CLI subcommands — list, show, and last.
+
+Provides read-only inspection of captured traces via the
+:class:`~pixie.storage.store.ObservationStore`.
+
+Commands::
+
+    pixie trace list [--limit N] [--errors]
+    pixie trace show <trace_id> [-v | --verbose] [--json]
+    pixie trace last [--json]
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from datetime import datetime
+from typing import Any
+
+from piccolo.engine.sqlite import SQLiteEngine
+
+from pixie.config import get_config
+from pixie.storage.store import ObservationStore
+from pixie.storage.tree import ObservationNode
+
+
+def _make_store() -> ObservationStore:
+    """Create an ObservationStore from config."""
+    config = get_config()
+    engine = SQLiteEngine(path=config.db_path)
+    return ObservationStore(engine=engine)
+
+
+def _format_datetime(value: Any) -> str:
+    """Format a datetime value to a human-readable string."""
+    if isinstance(value, datetime):
+        return value.strftime("%Y-%m-%d %H:%M")
+    if isinstance(value, str):
+        try:
+            dt = datetime.fromisoformat(value)
+            return dt.strftime("%Y-%m-%d %H:%M")
+        except (ValueError, TypeError):
+            return str(value)
+    return str(value) if value is not None else ""
+
+
+async def _trace_list(limit: int, errors_only: bool) -> list[dict[str, Any]]:
+    """Fetch trace summaries from the store."""
+    store = _make_store()
+    traces = await store.list_traces(limit=limit)
+    if errors_only:
+        traces = [t for t in traces if t.get("has_error")]
+    return traces
+
+
+async def _trace_show(
+    trace_id: str,
+    verbose: bool,
+    as_json: bool,
+) -> str:
+    """Fetch and render a single trace."""
+    store = _make_store()
+    # Support prefix matching
+    traces = await store.list_traces(limit=500)
+    matched = [t for t in traces if t["trace_id"].startswith(trace_id)]
+    if not matched:
+        return f"Error: No trace found matching '{trace_id}'"
+    if len(matched) > 1:
+        ids = "\n  ".join(t["trace_id"] for t in matched[:10])
+        return f"Error: Multiple traces match '{trace_id}'. Be more specific:\n  {ids}"
+    full_id = matched[0]["trace_id"]
+
+    tree = await store.get_trace(full_id)
+    if not tree:
+        return f"Error: No spans found for trace '{full_id}'"
+
+    if as_json:
+        spans_data = []
+        for node in tree:
+            spans_data.extend(_collect_serialized(node))
+        return json.dumps(spans_data, indent=2, default=str)
+
+    # Text rendering — to_text already handles both compact and verbose
+    # For compact mode, we use a stripped-down version
+    if verbose:
+        lines = [f"[trace_id: {full_id}]\n"]
+        for root_node in tree:
+            lines.append(root_node.to_text(indent=0))
+        return "\n".join(lines)
+
+    # Compact mode: just names and timing
+    lines = [f"[trace_id: {full_id}]\n"]
+    for root_node in tree:
+        lines.append(_compact_text(root_node, indent=0))
+    return "\n".join(lines)
+
+
+async def _trace_last(as_json: bool) -> str:
+    """Show the most recent trace in verbose mode."""
+    store = _make_store()
+    traces = await store.list_traces(limit=1)
+    if not traces:
+        return "No traces found."
+    trace_id = traces[0]["trace_id"]
+    return await _trace_show(trace_id, verbose=True, as_json=as_json)
+
+
+def _compact_text(node: ObservationNode, indent: int = 0) -> str:
+    """Render a compact text view (names and timing only)."""
+    from pixie.instrumentation.spans import LLMSpan
+
+    prefix = "  " * indent
+    lines: list[str] = []
+    if isinstance(node.span, LLMSpan):
+        span = node.span
+        header = (
+            f"{prefix}{span.request_model} [{span.provider}, {span.duration_ms:.0f}ms]"
+        )
+        lines.append(header)
+        token_parts: list[str] = []
+        if span.input_tokens > 0 or span.output_tokens > 0:
+            token_parts.append(f"{span.input_tokens} in / {span.output_tokens} out")
+            lines.append(f"{prefix}  tokens: {' '.join(token_parts)}")
+    else:
+        name = node.span.name or "(unnamed)"
+        lines.append(f"{prefix}{name} [{node.span.duration_ms:.0f}ms]")
+
+    for child in node.children:
+        lines.append(_compact_text(child, indent + 1))
+    return "\n".join(lines)
+
+
+def _collect_serialized(node: ObservationNode) -> list[dict[str, Any]]:
+    """Recursively collect serialized spans from a tree."""
+    from pixie.storage.serialization import serialize_span
+
+    result: list[dict[str, Any]] = [serialize_span(node.span)]
+    for child in node.children:
+        result.extend(_collect_serialized(child))
+    return result
+
+
+def trace_list(limit: int = 10, errors_only: bool = False) -> int:
+    """Entry point for ``pixie trace list``."""
+    traces = asyncio.run(_trace_list(limit, errors_only))
+    if not traces:
+        print("No traces found.")  # noqa: T201
+        return 0
+
+    # Table header
+    header = (
+        f"{'TRACE_ID':<34}"
+        f"{'ROOT SPAN':<25}"
+        f"{'STARTED':<20}"
+        f"{'SPANS':>6}"
+        f"{'ERRORS':>7}"
+    )
+    print(header)  # noqa: T201
+    for t in traces:
+        row = (
+            f"{t['trace_id']:<34}"
+            f"{(t.get('root_name') or '(unknown)'):<25}"
+            f"{_format_datetime(t.get('started_at')):<20}"
+            f"{t.get('observation_count', 0):>6}"
+            f"{('yes' if t.get('has_error') else ''):>7}"
+        )
+        print(row)  # noqa: T201
+    return 0
+
+
+def trace_show(
+    trace_id: str,
+    verbose: bool = False,
+    as_json: bool = False,
+) -> int:
+    """Entry point for ``pixie trace show``."""
+    output = asyncio.run(_trace_show(trace_id, verbose, as_json))
+    print(output)  # noqa: T201
+    return 1 if output.startswith("Error:") else 0
+
+
+def trace_last(as_json: bool = False) -> int:
+    """Entry point for ``pixie trace last``."""
+    output = asyncio.run(_trace_last(as_json))
+    print(output)  # noqa: T201
+    return 0

pixie_qa-0.2.0/pixie/evals/llm_evaluator.py

@@ -0,0 +1,207 @@
+"""Factory for custom LLM-as-judge evaluators from prompt templates.
+
+Usage::
+
+    from pixie import create_llm_evaluator
+
+    concise_voice_style = create_llm_evaluator(
+        name="ConciseVoiceStyle",
+        prompt_template=\"\"\"
+        You are evaluating whether a voice agent response is concise and
+        phone-friendly.
+
+        User said: {eval_input}
+        Agent responded: {eval_output}
+        Expected behavior: {expected_output}
+
+        Score 1.0 if the response is concise (under 3 sentences), directly
+        addresses the question, and uses conversational language suitable for
+        a phone call. Score 0.0 if it's verbose, off-topic, or uses
+        written-style formatting.
+        \"\"\",
+    )
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+from openai import OpenAI
+
+from pixie.evals.evaluation import Evaluation
+from pixie.storage.evaluable import Evaluable, _Unset
+from pixie.storage.tree import ObservationNode
+
+logger = logging.getLogger(__name__)
+
+# Default model for LLM-as-judge calls
+_DEFAULT_MODEL = "gpt-4o-mini"
+
+# Regex to detect nested field access like {eval_input[key]} in templates
+_NESTED_ACCESS_RE = re.compile(
+    r"\{(eval_input|eval_output|expected_output)\["
+)
+
+
+def _value_to_str(value: Any) -> str:
+    """Convert an eval value to a string for template substitution."""
+    if value is None:
+        return ""
+    if isinstance(value, _Unset):
+        return "(not provided)"
+    if isinstance(value, (dict, list)):
+        return json.dumps(value, default=str)
+    return str(value)
+
+
+def _parse_score(text: str) -> tuple[float, str]:
+    """Extract a 0-1 score and reasoning from LLM response text.
+
+    Looks for patterns like "Score: 0.8", "score: 1.0", "0.7/1.0", or
+    a bare float on its own line. Returns (score, reasoning).
+    """
+    # Try "Score: X" pattern first
+    match = re.search(r"[Ss]core\s*[:=]\s*([01](?:\.\d+)?)", text)
+    if match:
+        score = float(match.group(1))
+        return min(max(score, 0.0), 1.0), text.strip()
+
+    # Try "X/1" or "X/1.0" pattern
+    match = re.search(r"([01](?:\.\d+)?)\s*/\s*1(?:\.0)?", text)
+    if match:
+        score = float(match.group(1))
+        return min(max(score, 0.0), 1.0), text.strip()
+
+    # Try bare float on a line
+    match = re.search(r"^([01](?:\.\d+)?)\s*$", text, re.MULTILINE)
+    if match:
+        score = float(match.group(1))
+        return min(max(score, 0.0), 1.0), text.strip()
+
+    # Fallback: couldn't parse score
+    logger.warning("Could not parse score from LLM response: %s", text[:200])
+    return 0.0, f"Failed to parse score. Raw response: {text.strip()}"
+
+
+class _LLMEvaluator:
+    """Evaluator that uses an LLM to judge quality via a prompt template."""
+
+    def __init__(
+        self,
+        name: str,
+        prompt_template: str,
+        model: str,
+        client: Any | None,
+    ) -> None:
+        self._name = name
+        self._prompt_template = prompt_template
+        self._model = model
+        self._client = client
+
+    @property
+    def name(self) -> str:
+        """Return the evaluator's display name."""
+        return self._name
+
+    def _get_client(self) -> Any:
+        """Get or create the OpenAI client."""
+        if self._client is not None:
+            return self._client
+
+        return OpenAI()
+
+    def _render_prompt(self, evaluable: Evaluable) -> str:
+        """Fill in the template with evaluable fields."""
+        expected = evaluable.expected_output
+        rendered = self._prompt_template.format(
+            eval_input=_value_to_str(evaluable.eval_input),
+            eval_output=_value_to_str(evaluable.eval_output),
+            expected_output=_value_to_str(expected),
+        )
+        return rendered + "\n\nRespond with 'Score: X.X' followed by reasoning."
+
+    async def __call__(
+        self,
+        evaluable: Evaluable,
+        *,
+        trace: list[ObservationNode] | None = None,
+    ) -> Evaluation:
+        """Run the LLM judge and parse the score."""
+        import asyncio
+
+        prompt = self._render_prompt(evaluable)
+        client = self._get_client()
+
+        response = await asyncio.to_thread(
+            client.chat.completions.create,
+            model=self._model,
+            messages=[
+                {
+                    "role": "system",
+                    "content": (
+                        "You are an evaluation judge. Score the following on "
+                        "a scale of 0.0 to 1.0. Always include 'Score: X.X' "
+                        "in your response, followed by your reasoning."
+                    ),
+                },
+                {"role": "user", "content": prompt},
+            ],
+            temperature=0.0,
+        )
+
+        text = response.choices[0].message.content or ""
+        score, reasoning = _parse_score(text)
+
+        return Evaluation(
+            score=score,
+            reasoning=reasoning,
+            details={"evaluator": self._name, "model": self._model},
+        )
+
+
+def create_llm_evaluator(
+    name: str,
+    prompt_template: str,
+    *,
+    model: str = _DEFAULT_MODEL,
+    client: Any | None = None,
+) -> _LLMEvaluator:
+    """Create a custom LLM-as-judge evaluator from a prompt template.
+
+    The template may reference these variables (populated from the
+    :class:`~pixie.storage.evaluable.Evaluable` fields):
+
+    - ``{eval_input}`` — the evaluable's input
+    - ``{eval_output}`` — the evaluable's output
+    - ``{expected_output}`` — the evaluable's expected output
+
+    Args:
+        name: Display name for the evaluator (shown in scorecard).
+        prompt_template: A string template with ``{eval_input}``,
+            ``{eval_output}``, and/or ``{expected_output}`` placeholders.
+        model: OpenAI model name (default: ``gpt-4o-mini``).
+        client: Optional pre-configured OpenAI client instance.
+
+    Returns:
+        An evaluator callable satisfying the ``Evaluator`` protocol.
+
+    Raises:
+        ValueError: If the template uses nested field access like
+            ``{eval_input[key]}`` (only top-level placeholders are supported).
+    """
+    match = _NESTED_ACCESS_RE.search(prompt_template)
+    if match:
+        raise ValueError(
+            f"Nested field access like '{{{match.group(1)}[...]}}' is not "
+            f"supported in prompt templates. Use '{{{match.group(1)}}}' "
+            f"instead — dict values are serialized to JSON automatically."
+        )
+    return _LLMEvaluator(
+        name=name,
+        prompt_template=prompt_template,
+        model=model,
+        client=client,
+    )

{pixie_qa-0.1.11 → pixie_qa-0.2.0}/pixie/instrumentation/observation.py

@@ -58,7 +58,10 @@ def observe(
                 sig = inspect.signature(fn)
                 bound = sig.bind(*args, **kwargs)
                 bound.apply_defaults()
-                serialized_input = _serialize(dict(bound.arguments))
+                arguments = dict(bound.arguments)
+                arguments.pop("self", None)
+                arguments.pop("cls", None)
+                serialized_input = _serialize(arguments)
 
                 with start_observation(
                     input=serialized_input, name=span_name
@@ -76,7 +79,10 @@ def observe(
                 sig = inspect.signature(fn)
                 bound = sig.bind(*args, **kwargs)
                 bound.apply_defaults()
-                serialized_input = _serialize(dict(bound.arguments))
+                arguments = dict(bound.arguments)
+                arguments.pop("self", None)
+                arguments.pop("cls", None)
+                serialized_input = _serialize(arguments)
 
                 with start_observation(
                     input=serialized_input, name=span_name

{pixie_qa-0.1.11 → pixie_qa-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pixie-qa"
-version = "0.1.11"
+version = "0.2.0"
 description = "Automated quality assurance for AI applications"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -25,6 +25,7 @@ dependencies = [
     "pydantic>=2.0",
     "jsonpickle>=4.0.0",
     "python-dotenv>=1.2.2",
+    "openai>=2.29.0",
 ]
 
 [project.urls]