jev-mcp-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jev_mcp/__init__.py +1 -0
- jev_mcp/__main__.py +3 -0
- jev_mcp/domain/__init__.py +32 -0
- jev_mcp/domain/answers.py +25 -0
- jev_mcp/domain/json.py +49 -0
- jev_mcp/domain/questions.py +75 -0
- jev_mcp/domain/usage.py +16 -0
- jev_mcp/errors.py +59 -0
- jev_mcp/extract/__init__.py +1 -0
- jev_mcp/extract/candidates.py +75 -0
- jev_mcp/extract/dialect.py +400 -0
- jev_mcp/extract/executor.py +118 -0
- jev_mcp/extract/worker.py +198 -0
- jev_mcp/ids.py +49 -0
- jev_mcp/limits.py +218 -0
- jev_mcp/policy/__init__.py +98 -0
- jev_mcp/policy/actions.py +41 -0
- jev_mcp/policy/claims.py +103 -0
- jev_mcp/policy/extract.py +73 -0
- jev_mcp/policy/ranking.py +41 -0
- jev_mcp/policy/review.py +73 -0
- jev_mcp/policy/screen.py +48 -0
- jev_mcp/policy/thresholds.py +74 -0
- jev_mcp/providers/__init__.py +26 -0
- jev_mcp/providers/base.py +236 -0
- jev_mcp/providers/cloudflare.py +59 -0
- jev_mcp/providers/compatible.py +43 -0
- jev_mcp/providers/openrouter.py +47 -0
- jev_mcp/providers/resolver.py +106 -0
- jev_mcp/providers/typesafe.py +127 -0
- jev_mcp/py.typed +0 -0
- jev_mcp/serialize.py +199 -0
- jev_mcp/server.py +176 -0
- jev_mcp/settings.py +73 -0
- jev_mcp/stdio.py +99 -0
- jev_mcp/telemetry.py +223 -0
- jev_mcp/text.py +42 -0
- jev_mcp/tools/__init__.py +20 -0
- jev_mcp/tools/arguments.py +447 -0
- jev_mcp/tools/base.py +153 -0
- jev_mcp/tools/classify.py +187 -0
- jev_mcp/tools/common.py +96 -0
- jev_mcp/tools/compare.py +143 -0
- jev_mcp/tools/decide.py +206 -0
- jev_mcp/tools/extract.py +262 -0
- jev_mcp/tools/find.py +113 -0
- jev_mcp/tools/gate.py +236 -0
- jev_mcp/tools/observed.py +69 -0
- jev_mcp/tools/rerank.py +139 -0
- jev_mcp/tools/review.py +236 -0
- jev_mcp/tools/screen.py +126 -0
- jev_mcp/tools/toolset.py +92 -0
- jev_mcp/tools/verify.py +141 -0
- jev_mcp/validation/__init__.py +25 -0
- jev_mcp/validation/caps.py +93 -0
- jev_mcp/validation/choice.py +65 -0
- jev_mcp/validation/extract.py +48 -0
- jev_mcp/validation/noul.py +15 -0
- jev_mcp/validation/numbers.py +21 -0
- jev_mcp/validation/score.py +20 -0
- jev_mcp_python-0.1.0.dist-info/METADATA +18 -0
- jev_mcp_python-0.1.0.dist-info/RECORD +65 -0
- jev_mcp_python-0.1.0.dist-info/WHEEL +4 -0
- jev_mcp_python-0.1.0.dist-info/entry_points.txt +2 -0
- jev_mcp_python-0.1.0.dist-info/licenses/LICENSE +21 -0
jev_mcp/policy/claims.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Claim policy: per-claim actions, gate reason codes, and decide's requirement contradictions.
|
|
2
|
+
|
|
3
|
+
Sources: `claimAction` (`lib.ts:344-353`), the gate's reason-code assembly (`index.ts:1471-1482`),
|
|
4
|
+
and `contradictsRecommendation` (`lib.ts:153-160`).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from collections.abc import Iterable, Sequence
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
from jev_mcp.policy.actions import Action
|
|
12
|
+
from jev_mcp.policy.thresholds import PolicyThresholds
|
|
13
|
+
|
|
14
|
+
type ClaimVerdict = Literal["verified", "contradicted", "unsupported"]
|
|
15
|
+
|
|
16
|
+
GATE_REASON_CODES = (
|
|
17
|
+
"incomplete_context",
|
|
18
|
+
"invalid_response",
|
|
19
|
+
"review_escalated",
|
|
20
|
+
"review_required",
|
|
21
|
+
"claims_contradicted",
|
|
22
|
+
"claims_unsupported",
|
|
23
|
+
"claim_confidence_low",
|
|
24
|
+
"claim_confidence_below_auto_accept",
|
|
25
|
+
"accepted",
|
|
26
|
+
)
|
|
27
|
+
"""Every gate reason code, in the frozen order they are emitted (`parity-manifest.json` `policy`)."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def claim_action(verdict: ClaimVerdict, confidence: float | None, auto_accept: float, review_at: float) -> Action:
|
|
31
|
+
"""`claimAction`: unknown or low confidence and confident contradictions escalate.
|
|
32
|
+
|
|
33
|
+
Only confident verification is auto. Unknown confidence never satisfies a threshold, even a zero one.
|
|
34
|
+
"""
|
|
35
|
+
if confidence is None or confidence < review_at:
|
|
36
|
+
return "escalate"
|
|
37
|
+
if verdict == "contradicted" and confidence >= auto_accept:
|
|
38
|
+
return "escalate"
|
|
39
|
+
return "auto" if verdict == "verified" and confidence >= auto_accept else "review"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True, slots=True)
|
|
43
|
+
class ClaimJudgment:
|
|
44
|
+
"""A gate claim whose Choice answer validated."""
|
|
45
|
+
|
|
46
|
+
verdict: ClaimVerdict
|
|
47
|
+
confidence: float | None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def gate_reason_codes(
|
|
51
|
+
*,
|
|
52
|
+
truncated: bool,
|
|
53
|
+
review_action: Action,
|
|
54
|
+
review_invalid: bool,
|
|
55
|
+
claims: Sequence[ClaimJudgment | None],
|
|
56
|
+
action: Action,
|
|
57
|
+
thresholds: PolicyThresholds,
|
|
58
|
+
) -> list[str]:
|
|
59
|
+
"""The gate's reason codes, in `GATE_REASON_CODES` order (`index.ts:1472-1482`).
|
|
60
|
+
|
|
61
|
+
`claims` holds one entry per claim, `None` for a claim whose answer failed validation. `action` is
|
|
62
|
+
the gate's overall action. A valid claim with unknown confidence counts as confidence -1, so it
|
|
63
|
+
reports `claim_confidence_low` rather than nothing.
|
|
64
|
+
"""
|
|
65
|
+
valid = [claim for claim in claims if claim is not None]
|
|
66
|
+
codes: list[str] = []
|
|
67
|
+
if truncated:
|
|
68
|
+
codes.append("incomplete_context")
|
|
69
|
+
if review_invalid or len(valid) < len(claims):
|
|
70
|
+
codes.append("invalid_response")
|
|
71
|
+
if review_action == "escalate":
|
|
72
|
+
codes.append("review_escalated")
|
|
73
|
+
if review_action == "review":
|
|
74
|
+
codes.append("review_required")
|
|
75
|
+
if any(claim.verdict == "contradicted" for claim in valid):
|
|
76
|
+
codes.append("claims_contradicted")
|
|
77
|
+
if any(claim.verdict == "unsupported" for claim in valid):
|
|
78
|
+
codes.append("claims_unsupported")
|
|
79
|
+
confidences = [claim.confidence if claim.confidence is not None else -1 for claim in valid]
|
|
80
|
+
if any(c < thresholds.review_at for c in confidences):
|
|
81
|
+
codes.append("claim_confidence_low")
|
|
82
|
+
if any(thresholds.review_at <= c < thresholds.auto_accept for c in confidences):
|
|
83
|
+
codes.append("claim_confidence_below_auto_accept")
|
|
84
|
+
if action == "auto":
|
|
85
|
+
codes.append("accepted")
|
|
86
|
+
return codes
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass(frozen=True, slots=True)
|
|
90
|
+
class RequirementCheck:
|
|
91
|
+
"""One validated jev_decide requirement check: a candidate id, a requirement index, and the answer."""
|
|
92
|
+
|
|
93
|
+
candidate: str
|
|
94
|
+
requirement: int
|
|
95
|
+
answer: str
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def contradicts_recommendation(checks: Iterable[RequirementCheck], recommended: str) -> list[int]:
|
|
99
|
+
"""`contradictsRecommendation`: requirement indexes whose check contradicts the recommended candidate.
|
|
100
|
+
|
|
101
|
+
Independent questions may disagree with the recommendation; surface it, do not average it away.
|
|
102
|
+
"""
|
|
103
|
+
return [c.requirement for c in checks if c.candidate == recommended and c.answer == "contradicted"]
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""jev_extract's per-field decision (`index.ts:1062-1110`): auto/review/not_found and its Reason Code.
|
|
2
|
+
|
|
3
|
+
Validation has already accepted the answer (or the field had no candidates to ask about); pattern
|
|
4
|
+
failures and invalid answers never reach here. The tool projects the decision into its payload.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Literal
|
|
9
|
+
|
|
10
|
+
from jev_mcp.policy.actions import classification_decision
|
|
11
|
+
|
|
12
|
+
type ExtractStatus = Literal["auto", "review", "not_found"]
|
|
13
|
+
|
|
14
|
+
type ExtractReasonCode = Literal[
|
|
15
|
+
"no_regex_matches", "matches_too_long", "candidate_limit", "none_matched", "none_matched_ambiguous"
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
EXTRACT_REASON_CODES: tuple[ExtractReasonCode, ...] = (
|
|
19
|
+
"no_regex_matches",
|
|
20
|
+
"matches_too_long",
|
|
21
|
+
"candidate_limit",
|
|
22
|
+
"none_matched",
|
|
23
|
+
"none_matched_ambiguous",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True, slots=True)
|
|
28
|
+
class ExtractJudgment:
|
|
29
|
+
"""Jev's validated pick for a field: a candidate, or none of them, with its top probability and gap."""
|
|
30
|
+
|
|
31
|
+
none_matched: bool
|
|
32
|
+
top_probability: float
|
|
33
|
+
gap: float
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True, slots=True)
|
|
37
|
+
class ExtractFieldEvidence:
|
|
38
|
+
"""One field's candidate universe and, when it had candidates, the judgment over them.
|
|
39
|
+
|
|
40
|
+
The universe is incomplete when candidates were capped (`truncated`) or matches were skipped as
|
|
41
|
+
too long (`too_long > 0`): the right value may be among the matches never sent. `truncated` implies
|
|
42
|
+
candidates, so a field without any is judged on `too_long` alone, as the reference does.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
too_long: int
|
|
46
|
+
truncated: bool
|
|
47
|
+
judgment: ExtractJudgment | None
|
|
48
|
+
"""`None` iff the field had no candidates, so nothing was asked."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True, slots=True)
|
|
52
|
+
class ExtractFieldDecision:
|
|
53
|
+
status: ExtractStatus
|
|
54
|
+
reason: ExtractReasonCode | None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def decide_extract_field(evidence: ExtractFieldEvidence, *, threshold: float, margin: float) -> ExtractFieldDecision:
|
|
58
|
+
"""Gate the pick on top probability >= threshold and gap >= margin; an incomplete universe is never
|
|
59
|
+
`auto` and never a definite `not_found`, and a negative answer is gated like a positive one."""
|
|
60
|
+
judgment = evidence.judgment
|
|
61
|
+
if judgment is None:
|
|
62
|
+
if evidence.too_long > 0:
|
|
63
|
+
return ExtractFieldDecision("review", "matches_too_long")
|
|
64
|
+
return ExtractFieldDecision("not_found", "no_regex_matches")
|
|
65
|
+
incomplete = evidence.truncated or evidence.too_long > 0
|
|
66
|
+
if incomplete:
|
|
67
|
+
return ExtractFieldDecision("review", "candidate_limit")
|
|
68
|
+
decision = classification_decision(judgment.top_probability, judgment.gap, threshold, margin)
|
|
69
|
+
if judgment.none_matched:
|
|
70
|
+
if decision == "auto":
|
|
71
|
+
return ExtractFieldDecision("not_found", "none_matched")
|
|
72
|
+
return ExtractFieldDecision("review", "none_matched_ambiguous")
|
|
73
|
+
return ExtractFieldDecision(decision, None)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Ranking policy for jev_find and jev_rerank (`lib.ts:90-104`, `lib.ts:210-214`)."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping, Sequence
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
from jev_mcp.policy.thresholds import EXISTS_ABSENT_BELOW, EXISTS_FOUND_AT
|
|
7
|
+
|
|
8
|
+
type ExistsVerdict = Literal["answered", "partial", "absent"]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def exists_verdict(exists: float, found: float = EXISTS_FOUND_AT, absent: float = EXISTS_ABSENT_BELOW) -> ExistsVerdict:
|
|
12
|
+
"""`existsVerdict`: answered at or above `found`, absent below `absent`, else partial."""
|
|
13
|
+
if exists >= found:
|
|
14
|
+
return "answered"
|
|
15
|
+
return "absent" if exists < absent else "partial"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def rank_candidates(
|
|
19
|
+
candidates: Sequence[Mapping[str, object]], probabilities: Mapping[str, float]
|
|
20
|
+
) -> list[dict[str, object]]:
|
|
21
|
+
"""`rankCandidates`: each candidate plus its `probability`, descending; ties keep caller order.
|
|
22
|
+
|
|
23
|
+
A candidate id missing from `probabilities` ranks at 0. `sorted` is stable, as the reference's
|
|
24
|
+
index tie-break is.
|
|
25
|
+
"""
|
|
26
|
+
scored = [(probabilities.get(str(candidate["id"]), 0.0), candidate) for candidate in candidates]
|
|
27
|
+
return [{**candidate, "probability": p} for p, candidate in _descending(scored)]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def rerank_by_score(candidates: Sequence[Mapping[str, object]], scores: Sequence[float]) -> list[dict[str, object]]:
|
|
31
|
+
"""`rerankByScore`: each candidate plus its index-aligned `relevance`, descending; ties keep caller order.
|
|
32
|
+
|
|
33
|
+
Scores are validated before this runs. The 0 for a missing score only guards a wiring mistake
|
|
34
|
+
(fewer scores than candidates), never a model answer.
|
|
35
|
+
"""
|
|
36
|
+
scored = [(scores[i] if i < len(scores) else 0.0, candidate) for i, candidate in enumerate(candidates)]
|
|
37
|
+
return [{**candidate, "relevance": r} for r, candidate in _descending(scored)]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _descending[T](scored: list[tuple[float, T]]) -> list[tuple[float, T]]:
|
|
41
|
+
return sorted(scored, key=lambda pair: pair[0], reverse=True)
|
jev_mcp/policy/review.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Patch-review policy: the weighted composite and the review action (`lib.ts:245-337`, `index.ts:1254-1257`)."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from collections.abc import Iterable
|
|
5
|
+
|
|
6
|
+
from jev_mcp.policy.actions import Action
|
|
7
|
+
|
|
8
|
+
REVIEW_WEIGHTS: dict[str, float] = {
|
|
9
|
+
"correctness": 0.4,
|
|
10
|
+
"spec_match": 0.3,
|
|
11
|
+
"test_gap": 0.15,
|
|
12
|
+
"blast_radius": 0.15,
|
|
13
|
+
}
|
|
14
|
+
"""`REVIEW_WEIGHTS` in its reference key order, which is also the serialized order of `weights`."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _clamp(value: float, low: float, high: float) -> float:
|
|
18
|
+
"""`Math.min(high, Math.max(low, value))`: NaN stays NaN, where Python's `min`/`max` would drop it."""
|
|
19
|
+
if math.isnan(value):
|
|
20
|
+
return value
|
|
21
|
+
return min(high, max(low, value))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def review_composite(correctness: float, spec_match: float, test_gap: float, blast_radius: float) -> float:
|
|
25
|
+
"""`reviewComposite`: weighted 0..1 composite of 0..2 rubric scores, test gap and blast radius inverted.
|
|
26
|
+
|
|
27
|
+
Inputs are clamped to [0, 2] first. The sum is written out left to right so each float64 add
|
|
28
|
+
happens in the reference's order.
|
|
29
|
+
"""
|
|
30
|
+
c = _clamp(_clamp(correctness, 0.0, 2.0) / 2, 0.0, 1.0)
|
|
31
|
+
s = _clamp(_clamp(spec_match, 0.0, 2.0) / 2, 0.0, 1.0)
|
|
32
|
+
t = _clamp(1 - _clamp(test_gap, 0.0, 2.0) / 2, 0.0, 1.0)
|
|
33
|
+
b = _clamp(1 - _clamp(blast_radius, 0.0, 2.0) / 2, 0.0, 1.0)
|
|
34
|
+
return (
|
|
35
|
+
REVIEW_WEIGHTS["correctness"] * c
|
|
36
|
+
+ REVIEW_WEIGHTS["spec_match"] * s
|
|
37
|
+
+ REVIEW_WEIGHTS["test_gap"] * t
|
|
38
|
+
+ REVIEW_WEIGHTS["blast_radius"] * b
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def min_confidence(confidences: Iterable[float | None]) -> float | None:
|
|
43
|
+
"""The lowest rubric confidence, or `None` when any is unknown (`index.ts:1256-1257`).
|
|
44
|
+
|
|
45
|
+
Unknown on one rubric is unknown overall: it must not become a number that could satisfy a
|
|
46
|
+
threshold. No confidences at all is `inf`, as `Math.min()` is.
|
|
47
|
+
"""
|
|
48
|
+
values = list(confidences)
|
|
49
|
+
if any(value is None for value in values):
|
|
50
|
+
return None
|
|
51
|
+
return min((value for value in values if value is not None), default=math.inf)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def review_action(
|
|
55
|
+
*,
|
|
56
|
+
composite: float,
|
|
57
|
+
safe_to_apply: float,
|
|
58
|
+
min_confidence: float | None,
|
|
59
|
+
auto_accept: float,
|
|
60
|
+
review_at: float,
|
|
61
|
+
composite_floor: float,
|
|
62
|
+
) -> Action:
|
|
63
|
+
"""`reviewAction` (`lib.ts:319-337`).
|
|
64
|
+
|
|
65
|
+
Escalate when min_confidence is unknown or below review_at, or safe_to_apply is below review_at.
|
|
66
|
+
Auto when safe_to_apply and min_confidence reach auto_accept and composite reaches
|
|
67
|
+
composite_floor. Otherwise review. Truncation is applied after, by `require_complete_context`.
|
|
68
|
+
"""
|
|
69
|
+
if min_confidence is None or min_confidence < review_at or safe_to_apply < review_at:
|
|
70
|
+
return "escalate"
|
|
71
|
+
if safe_to_apply >= auto_accept and composite >= composite_floor and min_confidence >= auto_accept:
|
|
72
|
+
return "auto"
|
|
73
|
+
return "review"
|
jev_mcp/policy/screen.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""jev_screen recommendation (`screenRecommendation`, `lib.ts:70-87`) and its fail-closed fallback."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Literal
|
|
5
|
+
|
|
6
|
+
from jev_mcp.policy.thresholds import SCREEN_RELEVANCE_SKIP_BELOW, SCREEN_SUBSTANCE_SKIP_BELOW
|
|
7
|
+
from jev_mcp.serialize import number_to_string, to_fixed
|
|
8
|
+
|
|
9
|
+
type ScreenAction = Literal["pass", "review", "block", "skip"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class ScreenRecommendation:
|
|
14
|
+
action: ScreenAction
|
|
15
|
+
reason: str
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def screen_recommendation(
|
|
19
|
+
*,
|
|
20
|
+
injection: float,
|
|
21
|
+
block_at: float,
|
|
22
|
+
review_at: float,
|
|
23
|
+
relevance: float | None = None,
|
|
24
|
+
substance: float | None = None,
|
|
25
|
+
) -> ScreenRecommendation:
|
|
26
|
+
"""Block, then review, on injection; then skip on low substance, then low relevance; else pass.
|
|
27
|
+
|
|
28
|
+
`None` means the answer was not asked for. Probabilities print with `toFixed(2)` and thresholds
|
|
29
|
+
with `Number::toString`, as the reference's template literals do.
|
|
30
|
+
"""
|
|
31
|
+
if injection >= block_at:
|
|
32
|
+
return ScreenRecommendation(
|
|
33
|
+
"block", f"injection probability {to_fixed(injection)} >= block threshold {number_to_string(block_at)}"
|
|
34
|
+
)
|
|
35
|
+
if injection >= review_at:
|
|
36
|
+
return ScreenRecommendation(
|
|
37
|
+
"review", f"injection probability {to_fixed(injection)} >= review threshold {number_to_string(review_at)}"
|
|
38
|
+
)
|
|
39
|
+
if substance is not None and substance < SCREEN_SUBSTANCE_SKIP_BELOW:
|
|
40
|
+
return ScreenRecommendation("skip", f"little substantive content (substance {to_fixed(substance)})")
|
|
41
|
+
if relevance is not None and relevance < SCREEN_RELEVANCE_SKIP_BELOW:
|
|
42
|
+
return ScreenRecommendation("skip", f"not relevant to the stated purpose (relevance {to_fixed(relevance)})")
|
|
43
|
+
return ScreenRecommendation("pass", "no signals above thresholds")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def screen_fail_closed() -> ScreenRecommendation:
|
|
47
|
+
"""A missing or malformed answer is not a clean bill of health: review, never pass."""
|
|
48
|
+
return ScreenRecommendation("review", "missing or malformed answers; cannot screen safely")
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Policy thresholds: the reference defaults (`parity-manifest.json` `defaults`) and the review/gate pair.
|
|
2
|
+
|
|
3
|
+
Every threshold a tool compares against lives here (ADR-0002); a tool that hardcodes one is a bug.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import math
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
DEFAULT_AUTO_ACCEPT = 0.8
|
|
10
|
+
"""jev_verify, jev_review, jev_gate (`index.ts:1304`, `index.ts:1378`)."""
|
|
11
|
+
|
|
12
|
+
DEFAULT_REVIEW_AT_CAP = 0.5
|
|
13
|
+
"""An omitted review_at becomes `min(0.5, auto_accept)` (`lib.ts:282`)."""
|
|
14
|
+
|
|
15
|
+
DEFAULT_COMPOSITE_FLOOR = 0.7
|
|
16
|
+
"""`lib.ts:238`."""
|
|
17
|
+
|
|
18
|
+
DEFAULT_CLASSIFY_AUTO_ACCEPT = 0.85
|
|
19
|
+
"""jev_classify, jev_compare, jev_extract."""
|
|
20
|
+
|
|
21
|
+
DEFAULT_MINIMUM_MARGIN = 0.5
|
|
22
|
+
"""jev_classify, jev_compare, jev_extract."""
|
|
23
|
+
|
|
24
|
+
DEFAULT_SCREEN_BLOCK_AT = 0.75
|
|
25
|
+
DEFAULT_SCREEN_REVIEW_AT = 0.25
|
|
26
|
+
|
|
27
|
+
SCREEN_SUBSTANCE_SKIP_BELOW = 0.3
|
|
28
|
+
"""Hardcoded in `screenRecommendation` (`lib.ts:82`), not a parameter."""
|
|
29
|
+
|
|
30
|
+
SCREEN_RELEVANCE_SKIP_BELOW = 0.3
|
|
31
|
+
"""Hardcoded in `screenRecommendation` (`lib.ts:84`), not a parameter."""
|
|
32
|
+
|
|
33
|
+
EXISTS_FOUND_AT = 0.7
|
|
34
|
+
"""`existsVerdict` default (`lib.ts:90`)."""
|
|
35
|
+
|
|
36
|
+
EXISTS_ABSENT_BELOW = 0.35
|
|
37
|
+
"""`existsVerdict` default (`lib.ts:90`)."""
|
|
38
|
+
|
|
39
|
+
THRESHOLD_INVARIANT_MESSAGE = "Thresholds must satisfy 0 <= review_at <= auto_accept <= 1."
|
|
40
|
+
"""The reference's thrown text (`lib.ts:273`); the tool reports it verbatim."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True, slots=True)
|
|
44
|
+
class PolicyThresholds:
|
|
45
|
+
"""A validated review/gate threshold pair: 0 <= review_at <= auto_accept <= 1."""
|
|
46
|
+
|
|
47
|
+
auto_accept: float
|
|
48
|
+
review_at: float
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def validate_policy_thresholds(auto_accept: float, review_at: float) -> None:
|
|
52
|
+
"""Raise `ValueError` unless both are finite and 0 <= review_at <= auto_accept <= 1 (`lib.ts:263-275`)."""
|
|
53
|
+
if (
|
|
54
|
+
not math.isfinite(auto_accept)
|
|
55
|
+
or not math.isfinite(review_at)
|
|
56
|
+
or auto_accept < 0
|
|
57
|
+
or auto_accept > 1
|
|
58
|
+
or review_at < 0
|
|
59
|
+
or review_at > 1
|
|
60
|
+
or review_at > auto_accept
|
|
61
|
+
):
|
|
62
|
+
raise ValueError(THRESHOLD_INVARIANT_MESSAGE)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def resolve_policy_thresholds(
|
|
66
|
+
auto_accept: float = DEFAULT_AUTO_ACCEPT, review_at: float | None = None
|
|
67
|
+
) -> PolicyThresholds:
|
|
68
|
+
"""Fill an omitted review_at so a lone low auto_accept cannot invert the pair, then validate (`lib.ts:278-285`).
|
|
69
|
+
|
|
70
|
+
Only `None` means omitted, as JS `??`: an explicit `review_at=0` stands.
|
|
71
|
+
"""
|
|
72
|
+
resolved = review_at if review_at is not None else min(DEFAULT_REVIEW_AT_CAP, auto_accept)
|
|
73
|
+
validate_policy_thresholds(auto_accept, resolved)
|
|
74
|
+
return PolicyThresholds(auto_accept=auto_accept, review_at=resolved)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Provider transports; they validate only the envelope (ADR-0003).
|
|
2
|
+
|
|
3
|
+
`typesafe-sdk` is imported only when the TypeSafe provider sends its first request. A provider owns
|
|
4
|
+
an HTTP client: whoever resolves one closes it with `aclose()`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from jev_mcp.providers.base import (
|
|
8
|
+
Evaluation,
|
|
9
|
+
JevProvider,
|
|
10
|
+
ProviderConfigError,
|
|
11
|
+
ProviderError,
|
|
12
|
+
ProviderName,
|
|
13
|
+
ProviderTimeoutError,
|
|
14
|
+
)
|
|
15
|
+
from jev_mcp.providers.resolver import resolve_model, resolve_provider
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"Evaluation",
|
|
19
|
+
"JevProvider",
|
|
20
|
+
"ProviderConfigError",
|
|
21
|
+
"ProviderError",
|
|
22
|
+
"ProviderName",
|
|
23
|
+
"ProviderTimeoutError",
|
|
24
|
+
"resolve_model",
|
|
25
|
+
"resolve_provider",
|
|
26
|
+
]
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""The provider contract, uniform envelope validation (ADR-0003), and the shared error path.
|
|
2
|
+
|
|
3
|
+
Every provider sends `{state, questions}` under a model name and returns an `Evaluation`. Adapters
|
|
4
|
+
differ only in URL, auth, model slug, the request/response envelope, and usage. Everything else is
|
|
5
|
+
here: the envelope rules the reference applies to `compatible` alone (`provider.ts:175-194`) apply to
|
|
6
|
+
every provider, every error message passes through secret redaction (ADR-0008), and a timeout or an
|
|
7
|
+
MCP cancellation aborts the in-flight request (ADR-0011).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import math
|
|
12
|
+
from abc import ABC, abstractmethod
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from typing import ClassVar, Literal, override
|
|
16
|
+
from urllib.parse import urlsplit
|
|
17
|
+
|
|
18
|
+
import anyio
|
|
19
|
+
import httpx
|
|
20
|
+
|
|
21
|
+
from jev_mcp.domain import JsonValue, Question, RawAnswer, Usage, as_number, decode_json, questions_to_wire
|
|
22
|
+
from jev_mcp.errors import Redactor
|
|
23
|
+
from jev_mcp.serialize import stringify_compact
|
|
24
|
+
from jev_mcp.text import head
|
|
25
|
+
|
|
26
|
+
type ProviderName = Literal["typesafe", "openrouter", "cloudflare", "compatible"]
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger("jev_mcp.providers")
|
|
29
|
+
|
|
30
|
+
ERROR_BODY_UNITS = 200
|
|
31
|
+
"""Error bodies are cut to `.slice(0, 200)` UTF-16 units (`provider.ts:146,172,247`)."""
|
|
32
|
+
|
|
33
|
+
_DEFAULT_PORTS = {"http": 80, "https": 443}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _origin_of(url: str | httpx.URL) -> tuple[str, str, int]:
|
|
37
|
+
"""The scheme/host/effective-port triple an origin comparison needs (ADR-0023)."""
|
|
38
|
+
parsed = url if isinstance(url, httpx.URL) else httpx.URL(url)
|
|
39
|
+
return (parsed.scheme, parsed.host or "", parsed.port or _DEFAULT_PORTS.get(parsed.scheme, 0))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class ProviderError(Exception):
|
|
43
|
+
"""A provider failure. Its text is MCP-visible, so `evaluate` redacts it before raising."""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class ProviderConfigError(ProviderError):
|
|
47
|
+
"""Provider resolution failed before any request (`provider.ts:35-77`)."""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ProviderTimeoutError(ProviderError):
|
|
51
|
+
"""The request did not complete within the caller's timeout."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True, slots=True)
|
|
55
|
+
class Evaluation:
|
|
56
|
+
"""One provider reply: raw answers for per-tool validation, plus usage and the model to report."""
|
|
57
|
+
|
|
58
|
+
answers: dict[str, RawAnswer]
|
|
59
|
+
usage: Usage
|
|
60
|
+
provider: ProviderName
|
|
61
|
+
model: str
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True, slots=True)
|
|
65
|
+
class Envelope:
|
|
66
|
+
"""A reply envelope that passed `parse_envelope`. `model` is `None` when the body carried none."""
|
|
67
|
+
|
|
68
|
+
answers: dict[str, RawAnswer]
|
|
69
|
+
usage: Usage
|
|
70
|
+
model: str | None
|
|
71
|
+
|
|
72
|
+
def model_or(self, default: str) -> str:
|
|
73
|
+
return default if self.model is None else self.model
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def parse_envelope(body: object, label: str) -> Envelope:
|
|
77
|
+
"""Validate the envelope only, with the reference's `compatible` rules and messages (`provider.ts:175-194`).
|
|
78
|
+
|
|
79
|
+
`usage` absent or null reports zeros; otherwise both counts must be finite non-negative numbers.
|
|
80
|
+
`model` must be absent, null, or a string. Per-answer validity is each tool's job.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def invalid(why: str) -> ProviderError:
|
|
84
|
+
return ProviderError(f"{label} returned an invalid response: {why}")
|
|
85
|
+
|
|
86
|
+
if not isinstance(body, dict):
|
|
87
|
+
raise invalid("expected a JSON object.")
|
|
88
|
+
envelope: dict[str, object] = body # pyright: ignore[reportUnknownVariableType]
|
|
89
|
+
answers = envelope.get("answers")
|
|
90
|
+
if not isinstance(answers, dict):
|
|
91
|
+
raise invalid("expected an answers object.")
|
|
92
|
+
usage = Usage()
|
|
93
|
+
raw_usage = envelope.get("usage")
|
|
94
|
+
if raw_usage is not None:
|
|
95
|
+
counts = _usage_counts(raw_usage)
|
|
96
|
+
if counts is None:
|
|
97
|
+
raise invalid("usage must report finite non-negative input_tokens and output_tokens.")
|
|
98
|
+
usage = Usage(*counts)
|
|
99
|
+
model = envelope.get("model")
|
|
100
|
+
if model is not None and not isinstance(model, str):
|
|
101
|
+
raise invalid("model must be absent or a string.")
|
|
102
|
+
return Envelope(answers=answers, usage=usage, model=model) # pyright: ignore[reportUnknownArgumentType]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _usage_counts(usage: object) -> tuple[int | float, int | float] | None:
|
|
106
|
+
if not isinstance(usage, dict):
|
|
107
|
+
return None
|
|
108
|
+
record: dict[str, object] = usage # pyright: ignore[reportUnknownVariableType]
|
|
109
|
+
counts: list[int | float] = []
|
|
110
|
+
for key in ("input_tokens", "output_tokens"):
|
|
111
|
+
value = record.get(key)
|
|
112
|
+
number = as_number(value)
|
|
113
|
+
if number is None or not math.isfinite(number) or number < 0:
|
|
114
|
+
return None
|
|
115
|
+
counts.append(value if isinstance(value, int) else number)
|
|
116
|
+
return counts[0], counts[1]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def decode_text(content: bytes) -> str:
|
|
120
|
+
"""`await response.text()`: UTF-8 with replacement characters, a leading BOM dropped."""
|
|
121
|
+
return content.decode("utf-8", errors="replace").removeprefix("")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def decode_body(content: bytes) -> object | None:
|
|
125
|
+
"""`await response.json().catch(() => null)`: `None` for a body `JSON.parse` would reject."""
|
|
126
|
+
try:
|
|
127
|
+
return decode_json(decode_text(content))
|
|
128
|
+
except ValueError:
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def refuse_credentials_in_url(url: str, label: str) -> None:
|
|
133
|
+
"""Node `fetch` refuses a URL with userinfo, so the reference fails there; failing is also safer here.
|
|
134
|
+
|
|
135
|
+
`httpx` would turn the userinfo into Basic auth and overwrite the Bearer header. The text is
|
|
136
|
+
V8's minus the URL, which redaction would blank anyway (ADR-0008).
|
|
137
|
+
"""
|
|
138
|
+
if urlsplit(url).netloc.rpartition("@")[1]:
|
|
139
|
+
raise ProviderError(
|
|
140
|
+
f"{label} request failed: Request cannot be constructed from a URL that includes credentials"
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def encode_json(value: object) -> bytes:
|
|
145
|
+
"""`JSON.stringify(value)` as the request body."""
|
|
146
|
+
return stringify_compact(value).encode("utf-8")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class JevProvider(ABC):
|
|
150
|
+
"""A transport to Jev. `evaluate` is the only entry point; adapters implement `_send`."""
|
|
151
|
+
|
|
152
|
+
name: ClassVar[ProviderName]
|
|
153
|
+
label: ClassVar[str]
|
|
154
|
+
"""Prefix of this provider's error messages."""
|
|
155
|
+
|
|
156
|
+
def __init__(self, redact: Redactor) -> None:
|
|
157
|
+
self._redact = redact
|
|
158
|
+
|
|
159
|
+
async def evaluate(
|
|
160
|
+
self, state: JsonValue, questions: Mapping[str, Question], model: str, timeout: float | None
|
|
161
|
+
) -> Evaluation:
|
|
162
|
+
"""Ask Jev `questions` about `state`. `timeout` bounds the whole call, in seconds; `None` waits.
|
|
163
|
+
|
|
164
|
+
Raises `ProviderError` with redacted text. Cancellation of the calling task propagates and
|
|
165
|
+
aborts the request (ADR-0011).
|
|
166
|
+
"""
|
|
167
|
+
wire = questions_to_wire(questions)
|
|
168
|
+
try:
|
|
169
|
+
with anyio.fail_after(timeout):
|
|
170
|
+
return await self._send(state, wire, model, timeout)
|
|
171
|
+
except ProviderError as error:
|
|
172
|
+
raise type(error)(self._redact(str(error))) from None
|
|
173
|
+
except (TimeoutError, httpx.TimeoutException):
|
|
174
|
+
after = "" if timeout is None else f" after {timeout:g} s"
|
|
175
|
+
raise ProviderTimeoutError(f"{self.label} request timed out{after}.") from None
|
|
176
|
+
except Exception as error:
|
|
177
|
+
# The original may carry a secret-bearing URL or header; only the redacted text leaves.
|
|
178
|
+
# Some carry no message at all (a reset is a bare `RemoteProtocolError`): name the type.
|
|
179
|
+
cause = str(error) or type(error).__name__
|
|
180
|
+
raise ProviderError(self._redact(f"{self.label} request failed: {cause}")) from None
|
|
181
|
+
|
|
182
|
+
@abstractmethod
|
|
183
|
+
async def _send(
|
|
184
|
+
self, state: JsonValue, questions: dict[str, JsonValue], model: str, timeout: float | None
|
|
185
|
+
) -> Evaluation: ...
|
|
186
|
+
|
|
187
|
+
def _status_error(self, status: int | str, body: str) -> ProviderError:
|
|
188
|
+
"""`{label} {status}: {body}`, the body redacted and then cut to 200 units (`provider.ts:172`).
|
|
189
|
+
|
|
190
|
+
Redacting first means a secret straddling the cut cannot leak its head.
|
|
191
|
+
"""
|
|
192
|
+
return ProviderError(f"{self.label} {status}: {head(self._redact(body), ERROR_BODY_UNITS)}")
|
|
193
|
+
|
|
194
|
+
@abstractmethod
|
|
195
|
+
async def aclose(self) -> None:
|
|
196
|
+
"""Release network resources."""
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class HttpProvider(JevProvider):
|
|
200
|
+
"""A provider that POSTs JSON with `httpx`, as the reference does with `fetch` — except that a
|
|
201
|
+
redirect may never leave the origin the first request used (ADR-0023): the reference's `fetch`
|
|
202
|
+
would re-send the caller's state to a cross-origin 307/308 target, and the state is more
|
|
203
|
+
sensitive than the credential (which httpx strips there anyway). httpx still builds every
|
|
204
|
+
redirect request itself — method semantics and the redirect cap are httpx's, not ours; the
|
|
205
|
+
hook below only refuses the hop before it is sent.
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
def __init__(self, redact: Redactor, client: httpx.AsyncClient | None = None) -> None:
|
|
209
|
+
super().__init__(redact)
|
|
210
|
+
# The allowed origin is fixed per instance: it comes from process config (ADR-0008), so the
|
|
211
|
+
# lazy first-request assignment races only with itself, writing the same value.
|
|
212
|
+
self._origin: tuple[str, str, int] | None = None
|
|
213
|
+
# Like `fetch`: no timeout of its own (`evaluate` owns the deadline), and redirects are followed.
|
|
214
|
+
self._client = client or httpx.AsyncClient(
|
|
215
|
+
timeout=None, # noqa: S113 - `evaluate` owns the deadline
|
|
216
|
+
follow_redirects=True,
|
|
217
|
+
event_hooks={"request": [self._reject_cross_origin]},
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
async def _reject_cross_origin(self, request: httpx.Request) -> None:
|
|
221
|
+
"""Async: the client awaits every request hook, redirects included."""
|
|
222
|
+
origin = _origin_of(request.url)
|
|
223
|
+
if self._origin is None:
|
|
224
|
+
self._origin = origin
|
|
225
|
+
elif origin != self._origin:
|
|
226
|
+
logger.warning("blocked a cross-origin redirect to %s://%s", request.url.scheme, request.url.host)
|
|
227
|
+
raise ProviderError(f"{self.label} request failed: a redirect left the configured origin and was blocked")
|
|
228
|
+
|
|
229
|
+
async def _post(self, url: str, headers: Mapping[str, str], body: object) -> httpx.Response:
|
|
230
|
+
return await self._client.post(
|
|
231
|
+
url, content=encode_json(body), headers={**headers, "Content-Type": "application/json"}
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
@override
|
|
235
|
+
async def aclose(self) -> None:
|
|
236
|
+
await self._client.aclose()
|