pixie-qa 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/PKG-INFO +1 -1
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/test_command.py +14 -2
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pyproject.toml +1 -1
- pixie_qa-0.2.2/tests/pixie/cli/conftest.py +15 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/.github/copilot-instructions.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/.github/workflows/publish.yml +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/.gitignore +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/LICENSE +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/README.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/async-handler-processing.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/autoevals-adapters.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/cli-dataset-commands.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/dataset-management.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/deep-research-demo.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/eval-harness.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/expected-output-in-evals.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/instrumentation-module-implementation.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/loud-failure-mode.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/manual-instrumentation-usability.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/observation-store-implementation.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/observe-sensitive-field-stripping.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/pixie-directory-and-skill-improvements.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/pixie-test-e2e-suite.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/root-package-exports-and-trace-id.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/scorecard-branding-and-skill-version-check.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/scorecard-eval-detail-dialog.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/skill-v2-and-rootdir-discovery.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/test-scorecard.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/usability-utils.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/docs/package.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/dataset_command.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/main.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/trace_command.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/config.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/dataset/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/dataset/models.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/dataset/store.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/criteria.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/eval_utils.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/evaluation.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/llm_evaluator.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/runner.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/scorecard.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/scorers.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/trace_capture.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/trace_helpers.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/favicon.png +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/context.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/handler.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/handlers.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/instrumentors.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/observation.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/processor.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/queue.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/spans.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/evaluable.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/piccolo_conf.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/piccolo_migrations/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/serialization.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/store.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/tables.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/tree.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/SKILL.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/dataset-generation.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/eval-tests.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/instrumentation.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/investigation.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/pixie-api.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/run-harness-patterns.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/understanding-app.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/agent-skill-1.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/agent-skill.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/autoevals-adapters.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/dataset-management.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/evals-harness.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/expected-output-in-evals.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/instrumentation.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/manual-instrumentation-usability.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/storage.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/usability-utils.md +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_cases.json +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_fixtures/conftest.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_fixtures/datasets/customer-faq.json +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_fixtures/mock_evaluators.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_fixtures/test_customer_faq.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/test_dataset_command.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/test_e2e_pixie_test.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/test_main.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/test_trace_command.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/dataset/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/dataset/test_models.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/dataset/test_store.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_criteria.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_eval_utils.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_evaluation.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_llm_evaluator.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_runner.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_scorecard.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_scorers.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_trace_capture.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_trace_helpers.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/conftest.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_context.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_handler.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_integration.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_observation.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_processor.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_queue.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_spans.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_storage_handler.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/__init__.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/conftest.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/test_evaluable.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/test_serialization.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/test_store.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/test_tree.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/test_config.py +0 -0
- {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/test_init.py +0 -0
|
@@ -2,18 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
Usage::
|
|
4
4
|
|
|
5
|
-
pixie test [path] [--filter PATTERN] [--verbose]
|
|
5
|
+
pixie test [path] [--filter PATTERN] [--verbose] [--no-open]
|
|
6
6
|
|
|
7
7
|
Discovers and runs eval test functions, reporting pass/fail results.
|
|
8
8
|
Generates an HTML scorecard report saved to
|
|
9
|
-
``{config.root}/scorecards/<timestamp>.html
|
|
9
|
+
``{config.root}/scorecards/<timestamp>.html`` and opens it in the default
|
|
10
|
+
browser (pass ``--no-open`` to suppress).
|
|
10
11
|
"""
|
|
11
12
|
|
|
12
13
|
from __future__ import annotations
|
|
13
14
|
|
|
14
15
|
import argparse
|
|
15
16
|
import sys
|
|
17
|
+
import webbrowser
|
|
16
18
|
from collections.abc import Sequence
|
|
19
|
+
from pathlib import Path
|
|
17
20
|
|
|
18
21
|
import pixie.instrumentation as px
|
|
19
22
|
from pixie.evals.runner import discover_tests, format_results, run_tests
|
|
@@ -82,6 +85,12 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
82
85
|
default=False,
|
|
83
86
|
help="Show detailed evaluation results",
|
|
84
87
|
)
|
|
88
|
+
parser.add_argument(
|
|
89
|
+
"--no-open",
|
|
90
|
+
action="store_true",
|
|
91
|
+
default=False,
|
|
92
|
+
help="Do not automatically open the scorecard HTML in a browser",
|
|
93
|
+
)
|
|
85
94
|
|
|
86
95
|
args = parser.parse_args(argv)
|
|
87
96
|
|
|
@@ -100,6 +109,9 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
100
109
|
scorecard_path = save_scorecard(report)
|
|
101
110
|
print(f"\nSee {scorecard_path} for test details") # noqa: T201
|
|
102
111
|
|
|
112
|
+
if not args.no_open:
|
|
113
|
+
webbrowser.open(Path(scorecard_path).as_uri())
|
|
114
|
+
|
|
103
115
|
all_passed = all(r.status == "passed" for r in results)
|
|
104
116
|
return 0 if all_passed else 1
|
|
105
117
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Shared pytest fixtures for pixie CLI tests."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Generator
|
|
6
|
+
from unittest.mock import MagicMock, patch
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture(autouse=True)
|
|
12
|
+
def no_browser_open() -> Generator[None, None, None]:
|
|
13
|
+
"""Prevent ``webbrowser.open`` from launching a real browser in tests."""
|
|
14
|
+
with patch("webbrowser.open", MagicMock()):
|
|
15
|
+
yield
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|