pixie-qa 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/PKG-INFO +1 -1
  2. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/test_command.py +14 -2
  3. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pyproject.toml +1 -1
  4. pixie_qa-0.2.2/tests/pixie/cli/conftest.py +15 -0
  5. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/.github/copilot-instructions.md +0 -0
  6. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/.github/workflows/publish.yml +0 -0
  7. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/.gitignore +0 -0
  8. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/LICENSE +0 -0
  9. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/README.md +0 -0
  10. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/async-handler-processing.md +0 -0
  11. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/autoevals-adapters.md +0 -0
  12. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/cli-dataset-commands.md +0 -0
  13. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/dataset-management.md +0 -0
  14. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/deep-research-demo.md +0 -0
  15. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/eval-harness.md +0 -0
  16. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/expected-output-in-evals.md +0 -0
  17. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/instrumentation-module-implementation.md +0 -0
  18. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/loud-failure-mode.md +0 -0
  19. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/manual-instrumentation-usability.md +0 -0
  20. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/observation-store-implementation.md +0 -0
  21. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/observe-sensitive-field-stripping.md +0 -0
  22. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/pixie-directory-and-skill-improvements.md +0 -0
  23. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/pixie-test-e2e-suite.md +0 -0
  24. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/root-package-exports-and-trace-id.md +0 -0
  25. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/scorecard-branding-and-skill-version-check.md +0 -0
  26. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/scorecard-eval-detail-dialog.md +0 -0
  27. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/skill-v2-and-rootdir-discovery.md +0 -0
  28. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/test-scorecard.md +0 -0
  29. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/changelogs/usability-utils.md +0 -0
  30. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/docs/package.md +0 -0
  31. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/__init__.py +0 -0
  32. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/__init__.py +0 -0
  33. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/dataset_command.py +0 -0
  34. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/main.py +0 -0
  35. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/cli/trace_command.py +0 -0
  36. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/config.py +0 -0
  37. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/dataset/__init__.py +0 -0
  38. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/dataset/models.py +0 -0
  39. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/dataset/store.py +0 -0
  40. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/__init__.py +0 -0
  41. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/criteria.py +0 -0
  42. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/eval_utils.py +0 -0
  43. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/evaluation.py +0 -0
  44. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/llm_evaluator.py +0 -0
  45. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/runner.py +0 -0
  46. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/scorecard.py +0 -0
  47. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/scorers.py +0 -0
  48. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/trace_capture.py +0 -0
  49. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/evals/trace_helpers.py +0 -0
  50. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/favicon.png +0 -0
  51. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/__init__.py +0 -0
  52. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/context.py +0 -0
  53. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/handler.py +0 -0
  54. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/handlers.py +0 -0
  55. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/instrumentors.py +0 -0
  56. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/observation.py +0 -0
  57. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/processor.py +0 -0
  58. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/queue.py +0 -0
  59. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/instrumentation/spans.py +0 -0
  60. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/__init__.py +0 -0
  61. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/evaluable.py +0 -0
  62. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/piccolo_conf.py +0 -0
  63. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/piccolo_migrations/__init__.py +0 -0
  64. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/serialization.py +0 -0
  65. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/store.py +0 -0
  66. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/tables.py +0 -0
  67. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/pixie/storage/tree.py +0 -0
  68. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/SKILL.md +0 -0
  69. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/dataset-generation.md +0 -0
  70. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/eval-tests.md +0 -0
  71. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/instrumentation.md +0 -0
  72. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/investigation.md +0 -0
  73. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/pixie-api.md +0 -0
  74. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/run-harness-patterns.md +0 -0
  75. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/skills/eval-driven-dev/references/understanding-app.md +0 -0
  76. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/agent-skill-1.md +0 -0
  77. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/agent-skill.md +0 -0
  78. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/autoevals-adapters.md +0 -0
  79. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/dataset-management.md +0 -0
  80. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/evals-harness.md +0 -0
  81. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/expected-output-in-evals.md +0 -0
  82. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/instrumentation.md +0 -0
  83. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/manual-instrumentation-usability.md +0 -0
  84. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/storage.md +0 -0
  85. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/specs/usability-utils.md +0 -0
  86. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/__init__.py +0 -0
  87. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/__init__.py +0 -0
  88. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/__init__.py +0 -0
  89. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_cases.json +0 -0
  90. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_fixtures/conftest.py +0 -0
  91. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_fixtures/datasets/customer-faq.json +0 -0
  92. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_fixtures/mock_evaluators.py +0 -0
  93. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/e2e_fixtures/test_customer_faq.py +0 -0
  94. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/test_dataset_command.py +0 -0
  95. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/test_e2e_pixie_test.py +0 -0
  96. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/test_main.py +0 -0
  97. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/cli/test_trace_command.py +0 -0
  98. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/dataset/__init__.py +0 -0
  99. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/dataset/test_models.py +0 -0
  100. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/dataset/test_store.py +0 -0
  101. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/__init__.py +0 -0
  102. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_criteria.py +0 -0
  103. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_eval_utils.py +0 -0
  104. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_evaluation.py +0 -0
  105. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_llm_evaluator.py +0 -0
  106. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_runner.py +0 -0
  107. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_scorecard.py +0 -0
  108. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_scorers.py +0 -0
  109. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_trace_capture.py +0 -0
  110. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/evals/test_trace_helpers.py +0 -0
  111. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/__init__.py +0 -0
  112. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/conftest.py +0 -0
  113. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_context.py +0 -0
  114. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_handler.py +0 -0
  115. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_integration.py +0 -0
  116. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_observation.py +0 -0
  117. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_processor.py +0 -0
  118. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_queue.py +0 -0
  119. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_spans.py +0 -0
  120. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/instrumentation/test_storage_handler.py +0 -0
  121. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/__init__.py +0 -0
  122. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/conftest.py +0 -0
  123. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/test_evaluable.py +0 -0
  124. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/test_serialization.py +0 -0
  125. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/test_store.py +0 -0
  126. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/observation_store/test_tree.py +0 -0
  127. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/test_config.py +0 -0
  128. {pixie_qa-0.2.1 → pixie_qa-0.2.2}/tests/pixie/test_init.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pixie-qa
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Automated quality assurance for AI applications
5
5
  Project-URL: Homepage, https://github.com/yiouli/pixie-qa
6
6
  Project-URL: Repository, https://github.com/yiouli/pixie-qa
@@ -2,18 +2,21 @@
2
2
 
3
3
  Usage::
4
4
 
5
- pixie test [path] [--filter PATTERN] [--verbose]
5
+ pixie test [path] [--filter PATTERN] [--verbose] [--no-open]
6
6
 
7
7
  Discovers and runs eval test functions, reporting pass/fail results.
8
8
  Generates an HTML scorecard report saved to
9
- ``{config.root}/scorecards/<timestamp>.html``.
9
+ ``{config.root}/scorecards/<timestamp>.html`` and opens it in the default
10
+ browser (pass ``--no-open`` to suppress).
10
11
  """
11
12
 
12
13
  from __future__ import annotations
13
14
 
14
15
  import argparse
15
16
  import sys
17
+ import webbrowser
16
18
  from collections.abc import Sequence
19
+ from pathlib import Path
17
20
 
18
21
  import pixie.instrumentation as px
19
22
  from pixie.evals.runner import discover_tests, format_results, run_tests
@@ -82,6 +85,12 @@ def main(argv: list[str] | None = None) -> int:
82
85
  default=False,
83
86
  help="Show detailed evaluation results",
84
87
  )
88
+ parser.add_argument(
89
+ "--no-open",
90
+ action="store_true",
91
+ default=False,
92
+ help="Do not automatically open the scorecard HTML in a browser",
93
+ )
85
94
 
86
95
  args = parser.parse_args(argv)
87
96
 
@@ -100,6 +109,9 @@ def main(argv: list[str] | None = None) -> int:
100
109
  scorecard_path = save_scorecard(report)
101
110
  print(f"\nSee {scorecard_path} for test details") # noqa: T201
102
111
 
112
+ if not args.no_open:
113
+ webbrowser.open(Path(scorecard_path).as_uri())
114
+
103
115
  all_passed = all(r.status == "passed" for r in results)
104
116
  return 0 if all_passed else 1
105
117
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "pixie-qa"
3
- version = "0.2.1"
3
+ version = "0.2.2"
4
4
  description = "Automated quality assurance for AI applications"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.11"
@@ -0,0 +1,15 @@
1
+ """Shared pytest fixtures for pixie CLI tests."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Generator
6
+ from unittest.mock import MagicMock, patch
7
+
8
+ import pytest
9
+
10
+
11
+ @pytest.fixture(autouse=True)
12
+ def no_browser_open() -> Generator[None, None, None]:
13
+ """Prevent ``webbrowser.open`` from launching a real browser in tests."""
14
+ with patch("webbrowser.open", MagicMock()):
15
+ yield
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes