alignmenter 0.3.5__tar.gz → 0.3.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. {alignmenter-0.3.5/src/alignmenter.egg-info → alignmenter-0.3.6}/PKG-INFO +1 -1
  2. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/_version.py +1 -1
  3. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/durable_judge.py +19 -3
  4. {alignmenter-0.3.5 → alignmenter-0.3.6/src/alignmenter.egg-info}/PKG-INFO +1 -1
  5. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/SOURCES.txt +1 -0
  6. alignmenter-0.3.6/tests/test_judge_schema.py +68 -0
  7. {alignmenter-0.3.5 → alignmenter-0.3.6}/LICENSE +0 -0
  8. {alignmenter-0.3.5 → alignmenter-0.3.6}/MANIFEST.in +0 -0
  9. {alignmenter-0.3.5 → alignmenter-0.3.6}/README.md +0 -0
  10. {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/demo_config.yaml +0 -0
  11. {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/judges/safety_prompt.txt +0 -0
  12. {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/persona/default.yaml +0 -0
  13. {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/run-grounded.yaml +0 -0
  14. {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/run.yaml +0 -0
  15. {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/safety_keywords.yaml +0 -0
  16. {alignmenter-0.3.5 → alignmenter-0.3.6}/datasets/README.md +0 -0
  17. {alignmenter-0.3.5 → alignmenter-0.3.6}/datasets/demo_conversations.jsonl +0 -0
  18. {alignmenter-0.3.5 → alignmenter-0.3.6}/datasets/grounded_demo.jsonl +0 -0
  19. {alignmenter-0.3.5 → alignmenter-0.3.6}/datasets/wendys_twitter.jsonl +0 -0
  20. {alignmenter-0.3.5 → alignmenter-0.3.6}/pyproject.toml +0 -0
  21. {alignmenter-0.3.5 → alignmenter-0.3.6}/setup.cfg +0 -0
  22. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/__init__.py +0 -0
  23. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/__init__.py +0 -0
  24. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/analyze.py +0 -0
  25. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/bounds.py +0 -0
  26. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/diagnose.py +0 -0
  27. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/generate.py +0 -0
  28. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/label.py +0 -0
  29. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/optimize.py +0 -0
  30. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/sampling.py +0 -0
  31. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/validate.py +0 -0
  32. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/cli.py +0 -0
  33. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/config.py +0 -0
  34. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/demo_config.yaml +0 -0
  35. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/judges/safety_prompt.txt +0 -0
  36. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/persona/default.yaml +0 -0
  37. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/run-grounded.yaml +0 -0
  38. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/run.yaml +0 -0
  39. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/safety_keywords.yaml +0 -0
  40. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/datasets/demo_conversations.jsonl +0 -0
  41. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/datasets/grounded_demo.jsonl +0 -0
  42. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/dataset_cli.py +0 -0
  43. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/__init__.py +0 -0
  44. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/custom.py +0 -0
  45. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/evidence.py +0 -0
  46. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/faithfulness.py +0 -0
  47. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/grounding.py +0 -0
  48. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/metrics.py +0 -0
  49. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/examples/__init__.py +0 -0
  50. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/examples/resource_task.py +0 -0
  51. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/__init__.py +0 -0
  52. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/archive.py +0 -0
  53. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/artifacts.py +0 -0
  54. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/comparison.py +0 -0
  55. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/evaluation.py +0 -0
  56. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/gates.py +0 -0
  57. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/leases.py +0 -0
  58. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/legacy.py +0 -0
  59. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/recovery.py +0 -0
  60. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/review.py +0 -0
  61. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/suite.py +0 -0
  62. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/importers/__init__.py +0 -0
  63. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/importers/healthbench.py +0 -0
  64. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/judges/__init__.py +0 -0
  65. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/judges/authenticity_judge.py +0 -0
  66. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/judges/prompts.py +0 -0
  67. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/__init__.py +0 -0
  68. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/anthropic.py +0 -0
  69. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/base.py +0 -0
  70. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/callable.py +0 -0
  71. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/classifiers.py +0 -0
  72. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/embeddings.py +0 -0
  73. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/judges.py +0 -0
  74. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/local.py +0 -0
  75. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/openai.py +0 -0
  76. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/release_cli.py +0 -0
  77. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/__init__.py +0 -0
  78. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/durable.py +0 -0
  79. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/github_comment.py +0 -0
  80. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/html.py +0 -0
  81. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/json_out.py +0 -0
  82. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/rubric_grade.py +0 -0
  83. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/rubric_grade_cli.py +0 -0
  84. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/run_config.py +0 -0
  85. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/runner.py +0 -0
  86. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/__init__.py +0 -0
  87. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/dataset.py +0 -0
  88. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/evaluation.py +0 -0
  89. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/execution.py +0 -0
  90. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/gates.py +0 -0
  91. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/metrics.py +0 -0
  92. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/review.py +0 -0
  93. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/scoring.py +0 -0
  94. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/suite.py +0 -0
  95. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/__init__.py +0 -0
  96. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/authenticity.py +0 -0
  97. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/faithfulness.py +0 -0
  98. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/grounding.py +0 -0
  99. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/safety.py +0 -0
  100. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/stability.py +0 -0
  101. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/__init__.py +0 -0
  102. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/bootstrap_dataset.py +0 -0
  103. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/calibrate_persona.py +0 -0
  104. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/run_openai_demo.py +0 -0
  105. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/sanitize_dataset.py +0 -0
  106. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/sdk.py +0 -0
  107. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/storage/__init__.py +0 -0
  108. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/storage/evaluations.py +0 -0
  109. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/storage/reviews.py +0 -0
  110. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/storage/runs.py +0 -0
  111. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/__init__.py +0 -0
  112. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/io.py +0 -0
  113. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/optional.py +0 -0
  114. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/tokens.py +0 -0
  115. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/yaml.py +0 -0
  116. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/dependency_links.txt +0 -0
  117. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/entry_points.txt +0 -0
  118. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/requires.txt +0 -0
  119. {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/top_level.txt +0 -0
  120. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/__init__.py +0 -0
  121. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/conftest.py +0 -0
  122. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_evaluation_judge.py +0 -0
  123. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_evaluation_worker.py +0 -0
  124. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_recovery_target.py +0 -0
  125. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_recovery_worker.py +0 -0
  126. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_run_worker.py +0 -0
  127. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/mini_cli_dataset.jsonl +0 -0
  128. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_authenticity_judge.py +0 -0
  129. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_builtin_evaluations.py +0 -0
  130. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_calibrate_persona.py +0 -0
  131. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_capture_recovery.py +0 -0
  132. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_errors.py +0 -0
  133. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_grounded.py +0 -0
  134. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_helpers.py +0 -0
  135. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_import.py +0 -0
  136. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_init.py +0 -0
  137. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_run_config.py +0 -0
  138. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_config.py +0 -0
  139. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_dataset_import.py +0 -0
  140. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_dataset_management.py +0 -0
  141. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_dataset_sample.py +0 -0
  142. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_durable_evaluations.py +0 -0
  143. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_durable_execution.py +0 -0
  144. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_faithfulness.py +0 -0
  145. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_github_comment.py +0 -0
  146. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_grounding.py +0 -0
  147. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_html_report.py +0 -0
  148. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_judge_providers.py +0 -0
  149. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_offline_safety.py +0 -0
  150. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_persona_gpt.py +0 -0
  151. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_provider_local.py +0 -0
  152. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_provider_openai.py +0 -0
  153. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_providers.py +0 -0
  154. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_release_workflow.py +0 -0
  155. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_review_workflow.py +0 -0
  156. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_rubric_grade.py +0 -0
  157. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_run_config_grounded.py +0 -0
  158. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_run_config_loader.py +0 -0
  159. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_run_openai_demo.py +0 -0
  160. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_runner.py +0 -0
  161. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_sampling.py +0 -0
  162. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_scorers.py +0 -0
  163. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_smoke.py +0 -0
  164. {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_suite_archive.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: alignmenter
3
- Version: 0.3.5
3
+ Version: 0.3.6
4
4
  Summary: Durable application alignment evaluations, evidence review, saved comparisons, and CI gates.
5
5
  Author: Alignmenter
6
6
  License-Expression: Apache-2.0
@@ -1,3 +1,3 @@
1
1
  """Single source for distribution and runtime version metadata."""
2
2
 
3
- __version__ = "0.3.5"
3
+ __version__ = "0.3.6"
@@ -3,6 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  from collections.abc import Callable
6
+ from copy import deepcopy
6
7
  from dataclasses import dataclass
7
8
  from importlib.metadata import version
8
9
  from typing import Protocol
@@ -37,7 +38,8 @@ class ChatCompletionJudge:
37
38
  """
38
39
 
39
40
  def __init__(self, *, client, model: str, revision: str, max_completion_tokens: int = 2048,
40
- timeout: float = 60.0, json_mode: bool = True) -> None:
41
+ timeout: float = 60.0, json_mode: bool = True,
42
+ response_schema: dict | None = None) -> None:
41
43
  if type(max_completion_tokens) is not int or max_completion_tokens < 1:
42
44
  raise ValueError("max_completion_tokens must be a positive integer")
43
45
  import math
@@ -46,9 +48,16 @@ class ChatCompletionJudge:
46
48
  raise ValueError("Judge timeout must be positive and finite")
47
49
  if not revision.strip():
48
50
  raise ValueError("Judge deployment revision is required")
51
+ if response_schema is not None:
52
+ if not json_mode:
53
+ raise ValueError("response_schema requires json_mode=True")
54
+ if not isinstance(response_schema, dict) or response_schema.get("type") != "object":
55
+ raise ValueError("response_schema must describe a JSON object")
56
+ content_digest(response_schema) # Reject non-JSON/non-finite configuration before dispatch.
49
57
  self._client = client.with_options(max_retries=0, timeout=timeout)
50
58
  self._model, self._tokens, self._json_mode = model, max_completion_tokens, json_mode
51
- self.contract = JudgeContract(model=model, configuration_digest=content_digest({
59
+ self._schema = deepcopy(response_schema)
60
+ configuration = {
52
61
  "adapter": "chat-completion-v1", "model": model, "deployment_revision": revision,
53
62
  "sdk_version": version("openai"),
54
63
  "organization": getattr(self._client, "organization", None),
@@ -56,10 +65,17 @@ class ChatCompletionJudge:
56
65
  # Persist only the digest; endpoint URLs may contain private routing details.
57
66
  "endpoint": str(self._client.base_url), "max_completion_tokens": max_completion_tokens,
58
67
  "timeout": timeout, "json_mode": json_mode, "max_retries": 0,
59
- }))
68
+ }
69
+ if self._schema is not None:
70
+ configuration.update(adapter="chat-completion-schema-v1", response_schema=self._schema)
71
+ self.contract = JudgeContract(model=model, configuration_digest=content_digest(configuration))
60
72
 
61
73
  def evaluate(self, request: JudgeRequest) -> JudgeReply:
62
74
  options = {"response_format": {"type": "json_object"}} if self._json_mode else {}
75
+ if self._schema is not None:
76
+ options = {"response_format": {"type": "json_schema", "json_schema": {
77
+ "name": "alignmenter_verdict", "strict": True, "schema": deepcopy(self._schema),
78
+ }}}
63
79
  response = self._client.chat.completions.create(
64
80
  model=self._model, extra_body={"max_completion_tokens": self._tokens},
65
81
  messages=[{"role": "system", "content": request.system},
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: alignmenter
3
- Version: 0.3.5
3
+ Version: 0.3.6
4
4
  Summary: Durable application alignment evaluations, evidence review, saved comparisons, and CI gates.
5
5
  Author: Alignmenter
6
6
  License-Expression: Apache-2.0
@@ -137,6 +137,7 @@ tests/test_github_comment.py
137
137
  tests/test_grounding.py
138
138
  tests/test_html_report.py
139
139
  tests/test_judge_providers.py
140
+ tests/test_judge_schema.py
140
141
  tests/test_offline_safety.py
141
142
  tests/test_persona_gpt.py
142
143
  tests/test_provider_local.py
@@ -0,0 +1,68 @@
1
+ """Optional strict provider schemas preserve raw replies and single-dispatch accounting."""
2
+ import json
3
+
4
+ import httpx
5
+ import pytest
6
+ from openai import APIStatusError, OpenAI
7
+
8
+ from alignmenter.providers.durable_judge import ChatCompletionJudge
9
+ from alignmenter.schemas.evaluation import JudgeRequest
10
+
11
+
12
+ def schema():
13
+ return {"type": "object", "properties": {"reason": {"type": ["string", "null"]}},
14
+ "required": ["reason"], "additionalProperties": False}
15
+
16
+
17
+ @pytest.mark.parametrize("mode", ["schema", "json", "text"])
18
+ def test_explicit_schema_and_legacy_modes_preserve_raw_response(mode):
19
+ calls = []
20
+ def handler(request):
21
+ calls.append(json.loads(request.content))
22
+ return httpx.Response(200, json={"id": "test", "object": "chat.completion", "created": 0,
23
+ "model": "fixture", "choices": [{"index": 0, "finish_reason": "stop",
24
+ "message": {"role": "assistant", "content": '{"reason":null}'}}],
25
+ "usage": {"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5}})
26
+ with httpx.Client(transport=httpx.MockTransport(handler)) as http_client:
27
+ client = OpenAI(api_key="fixture-unused", base_url="http://fixture.invalid/v1", http_client=http_client)
28
+ original = schema()
29
+ judge = ChatCompletionJudge(client=client, model="fixture", revision="v1", json_mode=mode != "text",
30
+ response_schema=original if mode == "schema" else None)
31
+ digest = judge.contract.configuration_digest
32
+ original["required"].clear() # Caller mutation must not change a frozen judge's wire contract.
33
+ reply = judge.evaluate(JudgeRequest(system="Grade.", prompt="Fixture."))
34
+ assert len(calls) == 1 and reply.text == '{"reason":null}'
35
+ assert reply.usage["total_tokens"] == 5 and judge.contract.configuration_digest == digest
36
+ if mode == "schema":
37
+ form = calls[0]["response_format"]
38
+ assert form["type"] == "json_schema" and form["json_schema"]["strict"] is True
39
+ assert form["json_schema"]["schema"]["required"] == ["reason"]
40
+ elif mode == "json":
41
+ assert calls[0]["response_format"] == {"type": "json_object"}
42
+ else:
43
+ assert "response_format" not in calls[0]
44
+
45
+
46
+ def test_schema_is_in_cache_identity_and_retries_stay_disabled():
47
+ calls = []
48
+ def handler(request):
49
+ calls.append(request)
50
+ return httpx.Response(429, json={"error": {"message": "Fixture rate limit"}})
51
+ with httpx.Client(transport=httpx.MockTransport(handler)) as http_client:
52
+ client = OpenAI(api_key="fixture-unused", base_url="http://fixture.invalid/v1", http_client=http_client, max_retries=7)
53
+ legacy = ChatCompletionJudge(client=client, model="fixture", revision="v1")
54
+ strict = ChatCompletionJudge(client=client, model="fixture", revision="v1", response_schema=schema())
55
+ assert legacy.contract.configuration_digest != strict.contract.configuration_digest
56
+ with pytest.raises(APIStatusError):
57
+ strict.evaluate(JudgeRequest(system="Grade.", prompt="Fixture."))
58
+ assert len(calls) == 1 and strict.contract.max_dispatches_per_request == 1
59
+
60
+
61
+ @pytest.mark.parametrize("value", [[], {"type": "array"}, {"type": "object", "bad": float("nan")}])
62
+ def test_invalid_schema_rejected_before_dispatch(value):
63
+ with httpx.Client(transport=httpx.MockTransport(lambda _: pytest.fail("No dispatch expected"))) as http_client:
64
+ client = OpenAI(api_key="fixture-unused", base_url="http://fixture.invalid/v1", http_client=http_client)
65
+ with pytest.raises((ValueError, TypeError)):
66
+ ChatCompletionJudge(client=client, model="fixture", revision="v1", response_schema=value)
67
+ with pytest.raises(ValueError, match="json_mode"):
68
+ ChatCompletionJudge(client=client, model="fixture", revision="v1", json_mode=False, response_schema=schema())
File without changes
File without changes
File without changes
File without changes
File without changes