pixie-qa 0.1.10__tar.gz → 0.1.12__tar.gz

{pixie_qa-0.1.10 → pixie_qa-0.1.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pixie-qa
-Version: 0.1.10
+Version: 0.1.12
 Summary: Automated quality assurance for AI applications
 Project-URL: Homepage, https://github.com/yiouli/pixie-qa
 Project-URL: Repository, https://github.com/yiouli/pixie-qa
@@ -66,11 +66,11 @@ Description-Content-Type: text/markdown
 
 # pixie-qa
 
-An agent skill for **eval-driven development** of LLM-powered applications.
+An agent skill that make coding agent the QA engineer for LLM applications.
 
 ## What the Skill Does
 
-The `eval-driven-dev` skill guides your coding agent through the full QA loop for LLM applications:
+The `qa-eval` skill guides your coding agent through the full eval-based QA loop for LLM applications:
 
 1. **Understand the code** — read the codebase, trace the data flow, learn what the code is supposed to do
 2. **Instrument it** — add `enable_storage()` and `@observe` so every run is captured to a local SQLite database

{pixie_qa-0.1.10 → pixie_qa-0.1.12}/README.md

@@ -1,10 +1,10 @@
 # pixie-qa
 
-An agent skill for **eval-driven development** of LLM-powered applications.
+An agent skill that make coding agent the QA engineer for LLM applications.
 
 ## What the Skill Does
 
-The `eval-driven-dev` skill guides your coding agent through the full QA loop for LLM applications:
+The `qa-eval` skill guides your coding agent through the full eval-based QA loop for LLM applications:
 
 1. **Understand the code** — read the codebase, trace the data flow, learn what the code is supposed to do
 2. **Instrument it** — add `enable_storage()` and `@observe` so every run is captured to a local SQLite database

pixie_qa-0.1.12/changelogs/observe-sensitive-field-stripping.md

@@ -0,0 +1,22 @@
+# @observe: Strip `self` and `cls` from captured input
+
+## What changed
+
+The `@observe` decorator now automatically removes `self` and `cls` from the
+captured function arguments before serialization. Previously, decorating a
+method with `@observe` would serialize the entire instance (including API keys,
+client objects, and other sensitive state) into `eval_input` via jsonpickle.
+
+## Files affected
+
+- `pixie/instrumentation/observation.py` — `sync_wrapper` and `async_wrapper`
+  now pop `self`/`cls` from `bound.arguments` before calling `_serialize()`.
+- `tests/pixie/instrumentation/test_observation.py` — Two new tests:
+  `test_self_excluded_from_input` and `test_cls_excluded_from_input`.
+
+## Migration notes
+
+No API changes. This is a backward-compatible fix. Existing `@observe` usage on
+functions (not methods) is unaffected. Methods that previously leaked `self` into
+`eval_input` will now produce cleaner, smaller input data containing only the
+semantic arguments.

{pixie_qa-0.1.10 → pixie_qa-0.1.12}/pixie/instrumentation/observation.py

@@ -58,7 +58,10 @@ def observe(
                 sig = inspect.signature(fn)
                 bound = sig.bind(*args, **kwargs)
                 bound.apply_defaults()
-                serialized_input = _serialize(dict(bound.arguments))
+                arguments = dict(bound.arguments)
+                arguments.pop("self", None)
+                arguments.pop("cls", None)
+                serialized_input = _serialize(arguments)
 
                 with start_observation(
                     input=serialized_input, name=span_name
@@ -76,7 +79,10 @@ def observe(
                 sig = inspect.signature(fn)
                 bound = sig.bind(*args, **kwargs)
                 bound.apply_defaults()
-                serialized_input = _serialize(dict(bound.arguments))
+                arguments = dict(bound.arguments)
+                arguments.pop("self", None)
+                arguments.pop("cls", None)
+                serialized_input = _serialize(arguments)
 
                 with start_observation(
                     input=serialized_input, name=span_name

{pixie_qa-0.1.10 → pixie_qa-0.1.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pixie-qa"
-version = "0.1.10"
+version = "0.1.12"
 description = "Automated quality assurance for AI applications"
 readme = "README.md"
 requires-python = ">=3.11"

{pixie_qa-0.1.10 → pixie_qa-0.1.12}/skills/eval-driven-dev/SKILL.md

@@ -1,23 +1,23 @@
 ---
 name: eval-driven-dev
-description: Instrument Python LLM apps, build golden datasets, write eval-based tests, run them, and root-cause failures — covering the full eval-driven development cycle. Make sure to use this skill whenever a user is developing, testing, QA-ing, evaluating, or benchmarking a Python project that calls an LLM, even if they don't say "evals" explicitly. Use for making sure an AI app works correctly, catching regressions after prompt changes, debugging why an agent started behaving differently, or validating output quality before shipping.
+description: Add instrumentation, build golden datasets, write eval-based tests, run them, root-cause failures, and iterate — Ensure your Python LLM application works correctly. Make sure to use this skill whenever a user is developing, testing, QA-ing, evaluating, or benchmarking a Python project that calls an LLM. Use for making sure an LLM application works correctly, catching regressions after prompt changes, fixing unexpected behavior, or validating output quality before shipping.
 license: MIT
 compatibility: Python 3.11+
 metadata:
-  version: 0.1.10
+  version: 0.1.11
 ---
 
-# Eval-Driven Development with pixie
+# Evaluation-Driven Development for Python LLM Applications
 
 This skill is about doing the work, not describing it. When a user asks you to set up evals for their app, you should be reading their code, editing their files, running commands, and producing a working test pipeline — not writing a plan for them to follow later.
 
 ## Startup checks (always first)
 
-Before doing anything else, perform these two steps in order:
+This skill uses the python `pixie-qa` package. Before doing anything else, perform these two steps:
 
 ### 1. Upgrade pixie-qa
 
-Always attempt to upgrade the `pixie-qa` package in the user's environment regardless of whether the skill itself is outdated. Detect the package manager from the project (check for `uv.lock`, `poetry.lock`, `requirements.txt`, or a plain `pip` environment) and run the appropriate upgrade command:
+Attempt to upgrade the `pixie-qa` package in the user's environment. Detect the package manager from the project (check for `uv.lock`, `poetry.lock`, `requirements.txt`, or a plain `pip` environment) and run the appropriate upgrade command:
 
 - **uv**: `uv add pixie-qa --upgrade` (or `uv sync --upgrade-package pixie-qa`)
 - **poetry**: `poetry add pixie-qa@latest`
@@ -27,7 +27,7 @@ If the upgrade fails (e.g., no network, version conflict), log the error and con
 
 ### 2. Check skill version
 
-After upgrading pixie-qa, check whether a newer version of this skill itself is available by running the script `check_version.py`.
+Check whether a newer version of this skill itself is available by running the script `check_version.py`.
 
 If there is a newer version, reinstall the skill with `npx skills add` before proceeding.
 

{pixie_qa-0.1.10 → pixie_qa-0.1.12}/skills/eval-driven-dev/resources/check_version.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Check whether the eval-driven-dev skill is outdated and update it if needed."""
+"""Check whether the qa-eval skill is outdated and update it if needed."""
 
 from __future__ import annotations
 
@@ -9,9 +9,7 @@ from urllib.error import URLError
 from urllib.request import urlopen
 
 SKILL_REPO = "/yiouli/pixie-qa/"
-SKILL_URL = (
-    f"https://raw.githubusercontent.com{SKILL_REPO}main/skills/eval-driven-dev/SKILL.md"
-)
+SKILL_URL = f"https://raw.githubusercontent.com{SKILL_REPO}main/skills/qa-eval/SKILL.md"
 
 _RE_FRONTMATTER = re.compile(r"^---\s*\n(.*?)\n---", re.DOTALL)
 _RE_NAME = re.compile(r"^name:\s*(.+)$", re.MULTILINE)
@@ -57,7 +55,7 @@ def _normalise_version(version: str) -> tuple[int, ...]:
 
 def main() -> int:
     resource_dir = Path(__file__).resolve().parent
-    skill_dir = resource_dir.parent  # skills/eval-driven-dev/
+    skill_dir = resource_dir.parent  # skills/ai-qa/
 
     local_data = _load_local_version(skill_dir)
     local_version = local_data.get("version", "0.0.0")

{pixie_qa-0.1.10 → pixie_qa-0.1.12}/tests/pixie/instrumentation/test_observation.py

@@ -148,3 +148,49 @@ class TestObserveDecorator:
         # Both positional args should appear in the serialized input
         assert "a" in obs.input
         assert "b" in obs.input
+
+    def test_self_excluded_from_input(
+        self, recording_handler: RecordingHandler
+    ) -> None:
+        """@observe on a method strips 'self' from captured input."""
+        px.init()
+        px.add_handler(recording_handler)
+
+        class Agent:
+            secret_key = "sk-SHOULD-NOT-APPEAR"
+
+            @observe()
+            def respond(self, message: str) -> str:
+                return f"Got: {message}"
+
+        agent = Agent()
+        result = agent.respond("hello")
+        px.flush()
+
+        assert result == "Got: hello"
+        obs = recording_handler.observe_spans[0]
+        assert "hello" in obs.input
+        assert "self" not in obs.input
+        assert "sk-SHOULD-NOT-APPEAR" not in obs.input
+
+    def test_cls_excluded_from_input(self, recording_handler: RecordingHandler) -> None:
+        """@observe on a classmethod strips 'cls' from captured input."""
+        px.init()
+        px.add_handler(recording_handler)
+
+        class Service:
+            api_key = "sk-SECRET"
+
+            @classmethod
+            @observe()
+            def handle(cls, query: str) -> str:
+                return f"Handled: {query}"
+
+        result = Service.handle("test")
+        px.flush()
+
+        assert result == "Handled: test"
+        obs = recording_handler.observe_spans[0]
+        assert "test" in obs.input
+        assert "cls" not in obs.input
+        assert "sk-SECRET" not in obs.input