alignmenter 0.3.5__tar.gz → 0.3.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {alignmenter-0.3.5/src/alignmenter.egg-info → alignmenter-0.3.6}/PKG-INFO +1 -1
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/_version.py +1 -1
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/durable_judge.py +19 -3
- {alignmenter-0.3.5 → alignmenter-0.3.6/src/alignmenter.egg-info}/PKG-INFO +1 -1
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/SOURCES.txt +1 -0
- alignmenter-0.3.6/tests/test_judge_schema.py +68 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/LICENSE +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/MANIFEST.in +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/README.md +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/demo_config.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/judges/safety_prompt.txt +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/persona/default.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/run-grounded.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/run.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/configs/safety_keywords.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/datasets/README.md +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/datasets/demo_conversations.jsonl +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/datasets/grounded_demo.jsonl +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/datasets/wendys_twitter.jsonl +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/pyproject.toml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/setup.cfg +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/analyze.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/bounds.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/diagnose.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/generate.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/label.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/optimize.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/sampling.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/calibration/validate.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/cli.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/config.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/demo_config.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/judges/safety_prompt.txt +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/persona/default.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/run-grounded.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/run.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/safety_keywords.yaml +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/datasets/demo_conversations.jsonl +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/datasets/grounded_demo.jsonl +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/dataset_cli.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/custom.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/evidence.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/faithfulness.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/grounding.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/evaluators/metrics.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/examples/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/examples/resource_task.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/archive.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/artifacts.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/comparison.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/evaluation.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/gates.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/leases.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/legacy.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/recovery.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/review.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/execution/suite.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/importers/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/importers/healthbench.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/judges/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/judges/authenticity_judge.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/judges/prompts.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/anthropic.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/base.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/callable.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/classifiers.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/embeddings.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/judges.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/local.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/providers/openai.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/release_cli.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/durable.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/github_comment.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/html.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/reporting/json_out.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/rubric_grade.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/rubric_grade_cli.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/run_config.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/runner.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/dataset.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/evaluation.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/execution.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/gates.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/metrics.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/review.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/scoring.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/schemas/suite.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/authenticity.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/faithfulness.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/grounding.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/safety.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scorers/stability.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/bootstrap_dataset.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/calibrate_persona.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/run_openai_demo.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/scripts/sanitize_dataset.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/sdk.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/storage/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/storage/evaluations.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/storage/reviews.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/storage/runs.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/io.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/optional.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/tokens.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/utils/yaml.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/dependency_links.txt +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/entry_points.txt +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/requires.txt +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter.egg-info/top_level.txt +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/__init__.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/conftest.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_evaluation_judge.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_evaluation_worker.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_recovery_target.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_recovery_worker.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/durable_run_worker.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/data/mini_cli_dataset.jsonl +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_authenticity_judge.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_builtin_evaluations.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_calibrate_persona.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_capture_recovery.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_errors.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_grounded.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_helpers.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_import.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_init.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_cli_run_config.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_config.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_dataset_import.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_dataset_management.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_dataset_sample.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_durable_evaluations.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_durable_execution.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_faithfulness.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_github_comment.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_grounding.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_html_report.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_judge_providers.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_offline_safety.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_persona_gpt.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_provider_local.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_provider_openai.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_providers.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_release_workflow.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_review_workflow.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_rubric_grade.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_run_config_grounded.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_run_config_loader.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_run_openai_demo.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_runner.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_sampling.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_scorers.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_smoke.py +0 -0
- {alignmenter-0.3.5 → alignmenter-0.3.6}/tests/test_suite_archive.py +0 -0
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from collections.abc import Callable
|
|
6
|
+
from copy import deepcopy
|
|
6
7
|
from dataclasses import dataclass
|
|
7
8
|
from importlib.metadata import version
|
|
8
9
|
from typing import Protocol
|
|
@@ -37,7 +38,8 @@ class ChatCompletionJudge:
|
|
|
37
38
|
"""
|
|
38
39
|
|
|
39
40
|
def __init__(self, *, client, model: str, revision: str, max_completion_tokens: int = 2048,
|
|
40
|
-
timeout: float = 60.0, json_mode: bool = True
|
|
41
|
+
timeout: float = 60.0, json_mode: bool = True,
|
|
42
|
+
response_schema: dict | None = None) -> None:
|
|
41
43
|
if type(max_completion_tokens) is not int or max_completion_tokens < 1:
|
|
42
44
|
raise ValueError("max_completion_tokens must be a positive integer")
|
|
43
45
|
import math
|
|
@@ -46,9 +48,16 @@ class ChatCompletionJudge:
|
|
|
46
48
|
raise ValueError("Judge timeout must be positive and finite")
|
|
47
49
|
if not revision.strip():
|
|
48
50
|
raise ValueError("Judge deployment revision is required")
|
|
51
|
+
if response_schema is not None:
|
|
52
|
+
if not json_mode:
|
|
53
|
+
raise ValueError("response_schema requires json_mode=True")
|
|
54
|
+
if not isinstance(response_schema, dict) or response_schema.get("type") != "object":
|
|
55
|
+
raise ValueError("response_schema must describe a JSON object")
|
|
56
|
+
content_digest(response_schema) # Reject non-JSON/non-finite configuration before dispatch.
|
|
49
57
|
self._client = client.with_options(max_retries=0, timeout=timeout)
|
|
50
58
|
self._model, self._tokens, self._json_mode = model, max_completion_tokens, json_mode
|
|
51
|
-
self.
|
|
59
|
+
self._schema = deepcopy(response_schema)
|
|
60
|
+
configuration = {
|
|
52
61
|
"adapter": "chat-completion-v1", "model": model, "deployment_revision": revision,
|
|
53
62
|
"sdk_version": version("openai"),
|
|
54
63
|
"organization": getattr(self._client, "organization", None),
|
|
@@ -56,10 +65,17 @@ class ChatCompletionJudge:
|
|
|
56
65
|
# Persist only the digest; endpoint URLs may contain private routing details.
|
|
57
66
|
"endpoint": str(self._client.base_url), "max_completion_tokens": max_completion_tokens,
|
|
58
67
|
"timeout": timeout, "json_mode": json_mode, "max_retries": 0,
|
|
59
|
-
}
|
|
68
|
+
}
|
|
69
|
+
if self._schema is not None:
|
|
70
|
+
configuration.update(adapter="chat-completion-schema-v1", response_schema=self._schema)
|
|
71
|
+
self.contract = JudgeContract(model=model, configuration_digest=content_digest(configuration))
|
|
60
72
|
|
|
61
73
|
def evaluate(self, request: JudgeRequest) -> JudgeReply:
|
|
62
74
|
options = {"response_format": {"type": "json_object"}} if self._json_mode else {}
|
|
75
|
+
if self._schema is not None:
|
|
76
|
+
options = {"response_format": {"type": "json_schema", "json_schema": {
|
|
77
|
+
"name": "alignmenter_verdict", "strict": True, "schema": deepcopy(self._schema),
|
|
78
|
+
}}}
|
|
63
79
|
response = self._client.chat.completions.create(
|
|
64
80
|
model=self._model, extra_body={"max_completion_tokens": self._tokens},
|
|
65
81
|
messages=[{"role": "system", "content": request.system},
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Optional strict provider schemas preserve raw replies and single-dispatch accounting."""
|
|
2
|
+
import json
|
|
3
|
+
|
|
4
|
+
import httpx
|
|
5
|
+
import pytest
|
|
6
|
+
from openai import APIStatusError, OpenAI
|
|
7
|
+
|
|
8
|
+
from alignmenter.providers.durable_judge import ChatCompletionJudge
|
|
9
|
+
from alignmenter.schemas.evaluation import JudgeRequest
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def schema():
|
|
13
|
+
return {"type": "object", "properties": {"reason": {"type": ["string", "null"]}},
|
|
14
|
+
"required": ["reason"], "additionalProperties": False}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.mark.parametrize("mode", ["schema", "json", "text"])
|
|
18
|
+
def test_explicit_schema_and_legacy_modes_preserve_raw_response(mode):
|
|
19
|
+
calls = []
|
|
20
|
+
def handler(request):
|
|
21
|
+
calls.append(json.loads(request.content))
|
|
22
|
+
return httpx.Response(200, json={"id": "test", "object": "chat.completion", "created": 0,
|
|
23
|
+
"model": "fixture", "choices": [{"index": 0, "finish_reason": "stop",
|
|
24
|
+
"message": {"role": "assistant", "content": '{"reason":null}'}}],
|
|
25
|
+
"usage": {"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5}})
|
|
26
|
+
with httpx.Client(transport=httpx.MockTransport(handler)) as http_client:
|
|
27
|
+
client = OpenAI(api_key="fixture-unused", base_url="http://fixture.invalid/v1", http_client=http_client)
|
|
28
|
+
original = schema()
|
|
29
|
+
judge = ChatCompletionJudge(client=client, model="fixture", revision="v1", json_mode=mode != "text",
|
|
30
|
+
response_schema=original if mode == "schema" else None)
|
|
31
|
+
digest = judge.contract.configuration_digest
|
|
32
|
+
original["required"].clear() # Caller mutation must not change a frozen judge's wire contract.
|
|
33
|
+
reply = judge.evaluate(JudgeRequest(system="Grade.", prompt="Fixture."))
|
|
34
|
+
assert len(calls) == 1 and reply.text == '{"reason":null}'
|
|
35
|
+
assert reply.usage["total_tokens"] == 5 and judge.contract.configuration_digest == digest
|
|
36
|
+
if mode == "schema":
|
|
37
|
+
form = calls[0]["response_format"]
|
|
38
|
+
assert form["type"] == "json_schema" and form["json_schema"]["strict"] is True
|
|
39
|
+
assert form["json_schema"]["schema"]["required"] == ["reason"]
|
|
40
|
+
elif mode == "json":
|
|
41
|
+
assert calls[0]["response_format"] == {"type": "json_object"}
|
|
42
|
+
else:
|
|
43
|
+
assert "response_format" not in calls[0]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_schema_is_in_cache_identity_and_retries_stay_disabled():
|
|
47
|
+
calls = []
|
|
48
|
+
def handler(request):
|
|
49
|
+
calls.append(request)
|
|
50
|
+
return httpx.Response(429, json={"error": {"message": "Fixture rate limit"}})
|
|
51
|
+
with httpx.Client(transport=httpx.MockTransport(handler)) as http_client:
|
|
52
|
+
client = OpenAI(api_key="fixture-unused", base_url="http://fixture.invalid/v1", http_client=http_client, max_retries=7)
|
|
53
|
+
legacy = ChatCompletionJudge(client=client, model="fixture", revision="v1")
|
|
54
|
+
strict = ChatCompletionJudge(client=client, model="fixture", revision="v1", response_schema=schema())
|
|
55
|
+
assert legacy.contract.configuration_digest != strict.contract.configuration_digest
|
|
56
|
+
with pytest.raises(APIStatusError):
|
|
57
|
+
strict.evaluate(JudgeRequest(system="Grade.", prompt="Fixture."))
|
|
58
|
+
assert len(calls) == 1 and strict.contract.max_dispatches_per_request == 1
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@pytest.mark.parametrize("value", [[], {"type": "array"}, {"type": "object", "bad": float("nan")}])
|
|
62
|
+
def test_invalid_schema_rejected_before_dispatch(value):
|
|
63
|
+
with httpx.Client(transport=httpx.MockTransport(lambda _: pytest.fail("No dispatch expected"))) as http_client:
|
|
64
|
+
client = OpenAI(api_key="fixture-unused", base_url="http://fixture.invalid/v1", http_client=http_client)
|
|
65
|
+
with pytest.raises((ValueError, TypeError)):
|
|
66
|
+
ChatCompletionJudge(client=client, model="fixture", revision="v1", response_schema=value)
|
|
67
|
+
with pytest.raises(ValueError, match="json_mode"):
|
|
68
|
+
ChatCompletionJudge(client=client, model="fixture", revision="v1", json_mode=False, response_schema=schema())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/configs/judges/safety_prompt.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{alignmenter-0.3.5 → alignmenter-0.3.6}/src/alignmenter/data/datasets/demo_conversations.jsonl
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|