jev-mcp-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jev_mcp/__init__.py +1 -0
- jev_mcp/__main__.py +3 -0
- jev_mcp/domain/__init__.py +32 -0
- jev_mcp/domain/answers.py +25 -0
- jev_mcp/domain/json.py +49 -0
- jev_mcp/domain/questions.py +75 -0
- jev_mcp/domain/usage.py +16 -0
- jev_mcp/errors.py +59 -0
- jev_mcp/extract/__init__.py +1 -0
- jev_mcp/extract/candidates.py +75 -0
- jev_mcp/extract/dialect.py +400 -0
- jev_mcp/extract/executor.py +118 -0
- jev_mcp/extract/worker.py +198 -0
- jev_mcp/ids.py +49 -0
- jev_mcp/limits.py +218 -0
- jev_mcp/policy/__init__.py +98 -0
- jev_mcp/policy/actions.py +41 -0
- jev_mcp/policy/claims.py +103 -0
- jev_mcp/policy/extract.py +73 -0
- jev_mcp/policy/ranking.py +41 -0
- jev_mcp/policy/review.py +73 -0
- jev_mcp/policy/screen.py +48 -0
- jev_mcp/policy/thresholds.py +74 -0
- jev_mcp/providers/__init__.py +26 -0
- jev_mcp/providers/base.py +236 -0
- jev_mcp/providers/cloudflare.py +59 -0
- jev_mcp/providers/compatible.py +43 -0
- jev_mcp/providers/openrouter.py +47 -0
- jev_mcp/providers/resolver.py +106 -0
- jev_mcp/providers/typesafe.py +127 -0
- jev_mcp/py.typed +0 -0
- jev_mcp/serialize.py +199 -0
- jev_mcp/server.py +176 -0
- jev_mcp/settings.py +73 -0
- jev_mcp/stdio.py +99 -0
- jev_mcp/telemetry.py +223 -0
- jev_mcp/text.py +42 -0
- jev_mcp/tools/__init__.py +20 -0
- jev_mcp/tools/arguments.py +447 -0
- jev_mcp/tools/base.py +153 -0
- jev_mcp/tools/classify.py +187 -0
- jev_mcp/tools/common.py +96 -0
- jev_mcp/tools/compare.py +143 -0
- jev_mcp/tools/decide.py +206 -0
- jev_mcp/tools/extract.py +262 -0
- jev_mcp/tools/find.py +113 -0
- jev_mcp/tools/gate.py +236 -0
- jev_mcp/tools/observed.py +69 -0
- jev_mcp/tools/rerank.py +139 -0
- jev_mcp/tools/review.py +236 -0
- jev_mcp/tools/screen.py +126 -0
- jev_mcp/tools/toolset.py +92 -0
- jev_mcp/tools/verify.py +141 -0
- jev_mcp/validation/__init__.py +25 -0
- jev_mcp/validation/caps.py +93 -0
- jev_mcp/validation/choice.py +65 -0
- jev_mcp/validation/extract.py +48 -0
- jev_mcp/validation/noul.py +15 -0
- jev_mcp/validation/numbers.py +21 -0
- jev_mcp/validation/score.py +20 -0
- jev_mcp_python-0.1.0.dist-info/METADATA +18 -0
- jev_mcp_python-0.1.0.dist-info/RECORD +65 -0
- jev_mcp_python-0.1.0.dist-info/WHEEL +4 -0
- jev_mcp_python-0.1.0.dist-info/entry_points.txt +2 -0
- jev_mcp_python-0.1.0.dist-info/licenses/LICENSE +21 -0
jev_mcp/tools/extract.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""jev_extract: regex finds candidates, Jev picks one, the value is returned verbatim (`index.ts:884-1128`).
|
|
2
|
+
|
|
3
|
+
Hard invariants: a value is one of its field's candidates or null; a call with no candidates at all
|
|
4
|
+
makes no provider request (and so never resolves the provider); an incomplete candidate universe
|
|
5
|
+
(capped, or matches skipped as too long) is never `auto` and never a definite `not_found`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from jev_mcp.domain import ChoiceQuestion, Question
|
|
12
|
+
from jev_mcp.extract.candidates import Refused, find_candidates
|
|
13
|
+
from jev_mcp.extract.dialect import to_units
|
|
14
|
+
from jev_mcp.limits import EXTRACT
|
|
15
|
+
from jev_mcp.policy import DEFAULT_CLASSIFY_AUTO_ACCEPT, DEFAULT_MINIMUM_MARGIN, ExtractFieldEvidence, ExtractJudgment
|
|
16
|
+
from jev_mcp.serialize import quote
|
|
17
|
+
from jev_mcp.text import length
|
|
18
|
+
from jev_mcp.tools.base import JevTool, Runtime, ToolError, ToolResult, caller_actions, define, frame, headline
|
|
19
|
+
from jev_mcp.tools.observed import decide_extract_field, validate_extract_choice
|
|
20
|
+
from jev_mcp.validation import margin, top_probability
|
|
21
|
+
from jev_mcp.validation.caps import CapLedger, candidate_budget_error, exceeds
|
|
22
|
+
|
|
23
|
+
NONE_OF_THEM = "none_of_them"
|
|
24
|
+
|
|
25
|
+
DEFINITION = define(
|
|
26
|
+
"jev_extract",
|
|
27
|
+
"Extract fields by regex, Jev picks the right match",
|
|
28
|
+
"Extract structured fields from a document with TypeSafe Jev as the picker, not the generator: your regex finds "
|
|
29
|
+
"candidate substrings in code, Jev chooses which candidate is the field's true value, and the result is "
|
|
30
|
+
"returned verbatim — never model-generated text. Fields with zero regex matches never reach the model "
|
|
31
|
+
"(not_found); if no field has matches, no API call is made. Ambiguous picks are flagged for review. Use for "
|
|
32
|
+
"prices, dates, version numbers, IDs, and anything with a recognizable shape; keep documents bounded.",
|
|
33
|
+
{
|
|
34
|
+
"type": "object",
|
|
35
|
+
"properties": {
|
|
36
|
+
"document": {
|
|
37
|
+
"type": "string",
|
|
38
|
+
"minLength": EXTRACT.document_min,
|
|
39
|
+
"maxLength": EXTRACT.document_max,
|
|
40
|
+
"description": f"The document to extract from. Rejected above {EXTRACT.document_max:,} characters.",
|
|
41
|
+
},
|
|
42
|
+
"fields": {
|
|
43
|
+
"type": "array",
|
|
44
|
+
"items": {
|
|
45
|
+
"type": "object",
|
|
46
|
+
"properties": {
|
|
47
|
+
"id": {
|
|
48
|
+
"type": "string",
|
|
49
|
+
"pattern": "^[a-z][a-z0-9_-]*$",
|
|
50
|
+
"maxLength": EXTRACT.field_id_max,
|
|
51
|
+
"description": "Field name, e.g. 'price' or 'version'.",
|
|
52
|
+
},
|
|
53
|
+
"pattern": {
|
|
54
|
+
"type": "string",
|
|
55
|
+
"minLength": EXTRACT.pattern_min,
|
|
56
|
+
"maxLength": EXTRACT.pattern_max,
|
|
57
|
+
"description": "JavaScript regex source (without delimiters) that matches candidate "
|
|
58
|
+
"values. Runs in a sandboxed worker with a hard timeout.",
|
|
59
|
+
},
|
|
60
|
+
"flags": {
|
|
61
|
+
"type": "string",
|
|
62
|
+
"maxLength": EXTRACT.flags_max,
|
|
63
|
+
"description": "Regex flags (e.g. 'i'). 'g' is always added; non-letters are dropped.",
|
|
64
|
+
},
|
|
65
|
+
"description": {
|
|
66
|
+
"type": "string",
|
|
67
|
+
"minLength": EXTRACT.description_min,
|
|
68
|
+
"maxLength": EXTRACT.description_max,
|
|
69
|
+
"description": "What the field is, so Jev can pick the right candidate among regex "
|
|
70
|
+
"matches.",
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
"required": ["id", "pattern", "description"],
|
|
74
|
+
"additionalProperties": False,
|
|
75
|
+
},
|
|
76
|
+
"minItems": EXTRACT.fields_min,
|
|
77
|
+
"maxItems": EXTRACT.fields_max,
|
|
78
|
+
"description": f"Fields to extract. Up to {EXTRACT.fields_max} per call, all judged in one request.",
|
|
79
|
+
},
|
|
80
|
+
"purpose": {"type": "string", "description": "What the extraction is for; shared across fields."},
|
|
81
|
+
"auto_accept": {
|
|
82
|
+
"type": "number",
|
|
83
|
+
"minimum": 0,
|
|
84
|
+
"maximum": 1,
|
|
85
|
+
"description": "Minimum top probability for auto. Default 0.85.",
|
|
86
|
+
},
|
|
87
|
+
"minimum_margin": {
|
|
88
|
+
"type": "number",
|
|
89
|
+
"minimum": 0,
|
|
90
|
+
"maximum": 1,
|
|
91
|
+
"description": "Minimum winner-to-runner-up gap for auto. Default 0.5.",
|
|
92
|
+
},
|
|
93
|
+
},
|
|
94
|
+
"required": ["document", "fields"],
|
|
95
|
+
"additionalProperties": False,
|
|
96
|
+
},
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True, slots=True)
|
|
101
|
+
class _Field:
|
|
102
|
+
id: str
|
|
103
|
+
key: str
|
|
104
|
+
pattern: str
|
|
105
|
+
description: str
|
|
106
|
+
candidates: list[str]
|
|
107
|
+
too_long: int
|
|
108
|
+
truncated: bool
|
|
109
|
+
error: str | None
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def flags(self) -> dict[str, object]:
|
|
113
|
+
return {"candidates_truncated": self.truncated, "matches_skipped_too_long": self.too_long}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
async def _match(runtime: Runtime, index: int, raw: dict[str, str], units: str) -> _Field:
|
|
117
|
+
telemetry = runtime.telemetry
|
|
118
|
+
with telemetry.span("regex.extract") as span:
|
|
119
|
+
telemetry.payload(span, "pattern", lambda: raw["pattern"])
|
|
120
|
+
found = await find_candidates(runtime.regex_executor, raw["pattern"], raw.get("flags", ""), units)
|
|
121
|
+
if isinstance(found, Refused):
|
|
122
|
+
span.attributes["outcome"] = found.outcome
|
|
123
|
+
candidates, too_long, truncated, error = [], 0, False, found.reason
|
|
124
|
+
else:
|
|
125
|
+
candidates, too_long, truncated, error = found.candidates, found.too_long, found.truncated, None
|
|
126
|
+
span.attributes.update(
|
|
127
|
+
outcome="ok", candidates=len(candidates), too_long=too_long, candidates_truncated=truncated
|
|
128
|
+
)
|
|
129
|
+
return _Field(raw["id"], f"f{index}", raw["pattern"], raw["description"], candidates, too_long, truncated, error)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _result(field: _Field, answer: object, auto_accept: float, minimum_margin: float) -> dict[str, object]:
|
|
133
|
+
if field.error is not None:
|
|
134
|
+
return {
|
|
135
|
+
"id": field.id,
|
|
136
|
+
"value": None,
|
|
137
|
+
"status": "invalid_pattern",
|
|
138
|
+
"reason": field.error,
|
|
139
|
+
"candidates_considered": 0,
|
|
140
|
+
**field.flags,
|
|
141
|
+
}
|
|
142
|
+
if not field.candidates:
|
|
143
|
+
decision = decide_extract_field(
|
|
144
|
+
ExtractFieldEvidence(field.too_long, field.truncated, None), threshold=auto_accept, margin=minimum_margin
|
|
145
|
+
)
|
|
146
|
+
return {
|
|
147
|
+
"id": field.id,
|
|
148
|
+
"value": None,
|
|
149
|
+
"status": decision.status,
|
|
150
|
+
"reason": decision.reason,
|
|
151
|
+
"candidates_considered": 0,
|
|
152
|
+
**field.flags,
|
|
153
|
+
}
|
|
154
|
+
keys = [f"c{j}" for j in range(len(field.candidates))] + [NONE_OF_THEM]
|
|
155
|
+
validated = validate_extract_choice(answer, keys)
|
|
156
|
+
considered = len(field.candidates)
|
|
157
|
+
if validated is None:
|
|
158
|
+
return {
|
|
159
|
+
"id": field.id,
|
|
160
|
+
"value": None,
|
|
161
|
+
"status": "invalid_response",
|
|
162
|
+
"reason": None,
|
|
163
|
+
"candidates_considered": considered,
|
|
164
|
+
**field.flags,
|
|
165
|
+
}
|
|
166
|
+
gap = margin(validated.probabilities)
|
|
167
|
+
top = top_probability(validated)
|
|
168
|
+
none_matched = validated.choice == NONE_OF_THEM
|
|
169
|
+
decision = decide_extract_field(
|
|
170
|
+
ExtractFieldEvidence(field.too_long, field.truncated, ExtractJudgment(none_matched, top, gap)),
|
|
171
|
+
threshold=auto_accept,
|
|
172
|
+
margin=minimum_margin,
|
|
173
|
+
)
|
|
174
|
+
return {
|
|
175
|
+
"id": field.id,
|
|
176
|
+
"value": None if none_matched else field.candidates[int(validated.choice[1:])],
|
|
177
|
+
"status": decision.status,
|
|
178
|
+
"reason": decision.reason,
|
|
179
|
+
"confidence": validated.confidence,
|
|
180
|
+
"top_probability": top,
|
|
181
|
+
"margin": gap,
|
|
182
|
+
"candidates_considered": considered,
|
|
183
|
+
**field.flags,
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
async def handle(args: dict[str, Any], runtime: Runtime) -> ToolResult:
|
|
188
|
+
auto_accept: float = args.get("auto_accept", DEFAULT_CLASSIFY_AUTO_ACCEPT)
|
|
189
|
+
minimum_margin: float = args.get("minimum_margin", DEFAULT_MINIMUM_MARGIN)
|
|
190
|
+
# The schema rejects a document over its cap first, so only the candidate universe records a cut.
|
|
191
|
+
ledger = CapLedger()
|
|
192
|
+
document = ledger.text(args["document"], EXTRACT.document_max, "context")
|
|
193
|
+
raw_fields: list[dict[str, str]] = args["fields"]
|
|
194
|
+
|
|
195
|
+
seen: set[str] = set()
|
|
196
|
+
for raw in raw_fields:
|
|
197
|
+
if raw["id"] in seen:
|
|
198
|
+
raise ToolError(f"Duplicate field id: {raw['id']}")
|
|
199
|
+
seen.add(raw["id"])
|
|
200
|
+
|
|
201
|
+
# Fields run one after another, in caller order (ADR-0012 D5).
|
|
202
|
+
units = to_units(document)
|
|
203
|
+
fields = [await _match(runtime, index, raw, units) for index, raw in enumerate(raw_fields)]
|
|
204
|
+
|
|
205
|
+
total = sum(length(candidate) for field in fields for candidate in field.candidates)
|
|
206
|
+
if exceeds(total, EXTRACT.aggregate_candidate_units):
|
|
207
|
+
raise ToolError(
|
|
208
|
+
candidate_budget_error(total, EXTRACT.aggregate_candidate_units, "Tighten the patterns or split the call.")
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
# One Choice per field with candidates; the document is sent once, candidates only in their criteria.
|
|
212
|
+
questions: dict[str, Question] = {}
|
|
213
|
+
state_fields: list[dict[str, object]] = []
|
|
214
|
+
for field in fields:
|
|
215
|
+
if field.error is not None or not field.candidates:
|
|
216
|
+
continue
|
|
217
|
+
criteria: dict[str, str] = {f"c{j}": f"Candidate value: {quote(c)}" for j, c in enumerate(field.candidates)}
|
|
218
|
+
criteria[NONE_OF_THEM] = "None of the candidates is the value this field asks for"
|
|
219
|
+
questions[field.key] = ChoiceQuestion(
|
|
220
|
+
f'Which candidate is the correct value of the field "{field.id}" ({field.description}) in the document '
|
|
221
|
+
"in the state? Pick the exact substring the document presents as this field's value.",
|
|
222
|
+
criteria,
|
|
223
|
+
)
|
|
224
|
+
state_fields.append({"id": field.key, "description": field.description, "pattern": field.pattern})
|
|
225
|
+
|
|
226
|
+
# No field with candidates: nothing is asked, and the frame reports no provider (`frame`).
|
|
227
|
+
evaluation = (
|
|
228
|
+
await runtime.ask({"purpose": args.get("purpose"), "document": document, "fields": state_fields}, questions)
|
|
229
|
+
if state_fields
|
|
230
|
+
else None
|
|
231
|
+
)
|
|
232
|
+
answers = evaluation.answers if evaluation is not None else {}
|
|
233
|
+
|
|
234
|
+
results = [_result(field, answers.get(field.key), auto_accept, minimum_margin) for field in fields]
|
|
235
|
+
if any(field.truncated or field.too_long > 0 for field in fields):
|
|
236
|
+
ledger.note("context") # an incomplete candidate universe: Policy never auto-accepts over it
|
|
237
|
+
item_actions = caller_actions(r["status"] for r in results)
|
|
238
|
+
return ToolResult(
|
|
239
|
+
frame(
|
|
240
|
+
"jev_extract",
|
|
241
|
+
evaluation,
|
|
242
|
+
{
|
|
243
|
+
"summary": {
|
|
244
|
+
"fields": len(results),
|
|
245
|
+
"extracted": sum(1 for r in results if r["value"] is not None),
|
|
246
|
+
"auto": sum(1 for r in results if r["status"] == "auto"),
|
|
247
|
+
"review": sum(1 for r in results if r["status"] == "review"),
|
|
248
|
+
"not_found": sum(1 for r in results if r["status"] == "not_found"),
|
|
249
|
+
"invalid": sum(1 for r in results if r["status"] in ("invalid_pattern", "invalid_response")),
|
|
250
|
+
},
|
|
251
|
+
"thresholds": {"auto_accept": auto_accept, "minimum_margin": minimum_margin},
|
|
252
|
+
"results": results,
|
|
253
|
+
},
|
|
254
|
+
model=runtime.model,
|
|
255
|
+
),
|
|
256
|
+
action=headline(item_actions),
|
|
257
|
+
item_actions=item_actions,
|
|
258
|
+
truncated=ledger.scopes,
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
TOOL = JevTool(DEFINITION, handle)
|
jev_mcp/tools/find.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""jev_find: semantic search over candidates (`index.ts:325-392`)."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from jev_mcp.domain import ChoiceQuestion, NoulCriteria, NoulQuestion
|
|
6
|
+
from jev_mcp.ids import ensure_unique_ids
|
|
7
|
+
from jev_mcp.limits import CANDIDATES, FIND
|
|
8
|
+
from jev_mcp.serialize import to_fixed
|
|
9
|
+
from jev_mcp.tools.base import JevTool, Runtime, ToolResult, define, frame
|
|
10
|
+
from jev_mcp.tools.common import candidates_schema
|
|
11
|
+
from jev_mcp.tools.observed import exists_verdict, rank_candidates, validate_choice, validate_noul
|
|
12
|
+
from jev_mcp.validation.caps import CapLedger
|
|
13
|
+
|
|
14
|
+
EXISTS_CRITERIA = NoulCriteria(
|
|
15
|
+
"At least one candidate states or directly implies the answer", "No candidate addresses this"
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
DEFINITION = define(
|
|
19
|
+
"jev_find",
|
|
20
|
+
"Semantic search over candidates",
|
|
21
|
+
"Rank candidates against a plain-language query with TypeSafe Jev — no embeddings needed. One Choice scores "
|
|
22
|
+
"every candidate id by how well it answers the query, plus a Noul checks whether any candidate addresses the "
|
|
23
|
+
"query at all (so a confident 'top hit' cannot masquerade as an answer). Pattern: "
|
|
24
|
+
f"docs.typesafe.ai/cookbooks/semantic_find. Use for 'which file/note/line covers X' across up to "
|
|
25
|
+
f"{CANDIDATES.max_items} candidates.",
|
|
26
|
+
{
|
|
27
|
+
"type": "object",
|
|
28
|
+
"properties": {
|
|
29
|
+
"query": {
|
|
30
|
+
"type": "string",
|
|
31
|
+
"minLength": 1,
|
|
32
|
+
"description": "What you are looking for, in natural language.",
|
|
33
|
+
},
|
|
34
|
+
"candidates": candidates_schema(
|
|
35
|
+
f"Candidates to search. Up to {CANDIDATES.max_items} in one call; texts are truncated at "
|
|
36
|
+
f"{CANDIDATES.text_units} chars."
|
|
37
|
+
),
|
|
38
|
+
"top_k": {
|
|
39
|
+
"type": "integer",
|
|
40
|
+
"minimum": FIND.top_k_min,
|
|
41
|
+
"maximum": FIND.top_k_max,
|
|
42
|
+
"description": f"How many ranked candidates to return. Default {FIND.top_k_default}.",
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
"required": ["query", "candidates"],
|
|
46
|
+
"additionalProperties": False,
|
|
47
|
+
},
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
async def handle(args: dict[str, Any], runtime: Runtime) -> ToolResult:
|
|
52
|
+
query: str = args["query"]
|
|
53
|
+
top_k = int(args.get("top_k", FIND.top_k_default))
|
|
54
|
+
ledger = CapLedger()
|
|
55
|
+
candidates = ensure_unique_ids(
|
|
56
|
+
[
|
|
57
|
+
{"id": c.get("id", ""), "text": ledger.text(c["text"], CANDIDATES.text_units, "item")}
|
|
58
|
+
for c in args["candidates"]
|
|
59
|
+
],
|
|
60
|
+
"candidate",
|
|
61
|
+
).items
|
|
62
|
+
ids = [str(c["id"]) for c in candidates]
|
|
63
|
+
|
|
64
|
+
questions = {
|
|
65
|
+
"best": ChoiceQuestion(f'Which candidate contains the best answer to: "{query}"?', dict.fromkeys(ids)),
|
|
66
|
+
"exists": NoulQuestion(f'Does any candidate address or answer: "{query}"?', EXISTS_CRITERIA),
|
|
67
|
+
}
|
|
68
|
+
evaluation = await runtime.ask({"query": query, "candidates": candidates}, questions)
|
|
69
|
+
exists = validate_noul(evaluation.answers.get("exists"))
|
|
70
|
+
best = validate_choice(evaluation.answers.get("best"), ids)
|
|
71
|
+
|
|
72
|
+
if exists is None or best is None:
|
|
73
|
+
# A missing answer is neither "no match" nor "no ranking": report the protocol failure.
|
|
74
|
+
return ToolResult(
|
|
75
|
+
frame(
|
|
76
|
+
"jev_find",
|
|
77
|
+
evaluation,
|
|
78
|
+
{
|
|
79
|
+
"query": query,
|
|
80
|
+
"status": "invalid_response",
|
|
81
|
+
"exists": exists,
|
|
82
|
+
"exists_verdict": None,
|
|
83
|
+
"top": [],
|
|
84
|
+
"reason": "missing or malformed best or exists answer; cannot rank safely",
|
|
85
|
+
},
|
|
86
|
+
),
|
|
87
|
+
truncated=ledger.scopes,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
ranked = rank_candidates(candidates, best.probabilities)[:top_k]
|
|
91
|
+
return ToolResult(
|
|
92
|
+
frame(
|
|
93
|
+
"jev_find",
|
|
94
|
+
evaluation,
|
|
95
|
+
{
|
|
96
|
+
"query": query,
|
|
97
|
+
"exists": exists,
|
|
98
|
+
"exists_verdict": exists_verdict(exists),
|
|
99
|
+
"top": [
|
|
100
|
+
{
|
|
101
|
+
"id": c["id"],
|
|
102
|
+
"probability": float(to_fixed(best.probabilities.get(str(c["id"]), 0.0), 4)),
|
|
103
|
+
"text": c["text"],
|
|
104
|
+
}
|
|
105
|
+
for c in ranked
|
|
106
|
+
],
|
|
107
|
+
},
|
|
108
|
+
),
|
|
109
|
+
truncated=ledger.scopes,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
TOOL = JevTool(DEFINITION, handle)
|
jev_mcp/tools/gate.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""jev_gate: review a patch and verify completion claims in one call (`index.ts:1342-1495`)."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from jev_mcp.domain import ChoiceQuestion
|
|
6
|
+
from jev_mcp.limits import GATE
|
|
7
|
+
from jev_mcp.policy import Action, ClaimJudgment, ClaimVerdict
|
|
8
|
+
from jev_mcp.serialize import js_number_to_locale_string_en_us
|
|
9
|
+
from jev_mcp.text import length
|
|
10
|
+
from jev_mcp.tools.arguments import Refinement
|
|
11
|
+
from jev_mcp.tools.base import JevTool, Runtime, ToolResult, define, frame
|
|
12
|
+
from jev_mcp.tools.common import EVIDENCE_SCHEMA, has_non_empty_evidence, normalize_evidence
|
|
13
|
+
from jev_mcp.tools.observed import (
|
|
14
|
+
claim_action,
|
|
15
|
+
gate_reason_codes,
|
|
16
|
+
require_complete_context,
|
|
17
|
+
validate_choice,
|
|
18
|
+
worst_action,
|
|
19
|
+
)
|
|
20
|
+
from jev_mcp.tools.review import (
|
|
21
|
+
ANTI_INJECTION,
|
|
22
|
+
project_review,
|
|
23
|
+
review_docs,
|
|
24
|
+
review_questions,
|
|
25
|
+
review_settings,
|
|
26
|
+
)
|
|
27
|
+
from jev_mcp.validation.caps import CapLedger, exceeds, gate_evidence_aggregate_error, gate_evidence_items_error
|
|
28
|
+
|
|
29
|
+
CLAIM_CRITERIA = {
|
|
30
|
+
"verified": "The evidence clearly supports the claim",
|
|
31
|
+
"contradicted": "The evidence contradicts the claim",
|
|
32
|
+
"unsupported": "The evidence neither supports nor contradicts the claim",
|
|
33
|
+
}
|
|
34
|
+
"""`VERIFY_CLAIM_CRITERIA` (`lib.ts:250-254`)."""
|
|
35
|
+
CLAIM_VERDICTS: dict[str, ClaimVerdict] = {
|
|
36
|
+
"verified": "verified",
|
|
37
|
+
"contradicted": "contradicted",
|
|
38
|
+
"unsupported": "unsupported",
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
REVIEW_FRAMING = " Claims are assertions to check, not evidence that the patch is correct or tested."
|
|
42
|
+
|
|
43
|
+
DEFINITION = define(
|
|
44
|
+
"jev_gate",
|
|
45
|
+
"Gate completion: review a patch and verify claims",
|
|
46
|
+
"Review a proposed patch and verify completion claims against supplied evidence in one TypeSafe Jev call. Auto "
|
|
47
|
+
"only when the patch review is accepted and every claim is verified at or above auto_accept. Unsupported claims "
|
|
48
|
+
"require review; confident contradictions, unknown confidence, or low confidence escalate. The request and "
|
|
49
|
+
"claims are assertions to check, never proof; put supporting diff excerpts and test logs in evidence. Evidence "
|
|
50
|
+
f"is capped at {GATE.evidence_items} items and {js_number_to_locale_string_en_us(GATE.aggregate_evidence_units)} "
|
|
51
|
+
"characters in aggregate. Does not run tests or apply changes. Use jev_review "
|
|
52
|
+
"for a patch without claims, jev_verify for claims without a patch review.",
|
|
53
|
+
{
|
|
54
|
+
"type": "object",
|
|
55
|
+
"properties": {
|
|
56
|
+
"request": {
|
|
57
|
+
"type": "string",
|
|
58
|
+
"minLength": 1,
|
|
59
|
+
"description": "What the user asked for; this is not evidence of completion.",
|
|
60
|
+
},
|
|
61
|
+
"diff": {
|
|
62
|
+
"type": "string",
|
|
63
|
+
"minLength": 1,
|
|
64
|
+
"description": f"Proposed patch, file excerpt, or change summary. Truncated at {GATE.doc_units} chars.",
|
|
65
|
+
},
|
|
66
|
+
"claims": {
|
|
67
|
+
"type": "array",
|
|
68
|
+
"items": {"type": "string", "minLength": 1},
|
|
69
|
+
"minItems": GATE.claims_min,
|
|
70
|
+
"maxItems": GATE.claims_max,
|
|
71
|
+
"description": "Completion claims to check against evidence, each truncated at "
|
|
72
|
+
f"{GATE.claim_units} chars. Up to {GATE.claims_max} per call.",
|
|
73
|
+
},
|
|
74
|
+
"evidence": EVIDENCE_SCHEMA,
|
|
75
|
+
"tests": {
|
|
76
|
+
"type": "string",
|
|
77
|
+
"description": "Reported test output for the patch review. Truncated at the same cap.",
|
|
78
|
+
},
|
|
79
|
+
"auto_accept": {
|
|
80
|
+
"type": "number",
|
|
81
|
+
"minimum": 0,
|
|
82
|
+
"maximum": 1,
|
|
83
|
+
"description": "Review and per-claim confidence at or above this may stand automatically. Default 0.8.",
|
|
84
|
+
},
|
|
85
|
+
"review_at": {
|
|
86
|
+
"type": "number",
|
|
87
|
+
"minimum": 0,
|
|
88
|
+
"maximum": 1,
|
|
89
|
+
"description": "Score, safe_to_apply, or per-claim confidence below this escalates. Must be <= "
|
|
90
|
+
"auto_accept. Default min(0.5, auto_accept).",
|
|
91
|
+
},
|
|
92
|
+
"composite_floor": {
|
|
93
|
+
"type": "number",
|
|
94
|
+
"minimum": 0,
|
|
95
|
+
"maximum": 1,
|
|
96
|
+
"description": "Weighted composite at or above this is required for auto. Default 0.7.",
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
"required": ["request", "diff", "claims", "evidence"],
|
|
100
|
+
"additionalProperties": False,
|
|
101
|
+
},
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
EVIDENCE_NOT_EMPTY = Refinement(
|
|
105
|
+
lambda value: has_non_empty_evidence(normalize_evidence(value)),
|
|
106
|
+
"jev_gate requires at least one evidence item with non-empty text.",
|
|
107
|
+
)
|
|
108
|
+
"""The `.refine` on gate's evidence (`index.ts:1370-1372`): checked with the arguments, before any request."""
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def claim_question(index: int) -> ChoiceQuestion:
|
|
112
|
+
return ChoiceQuestion(
|
|
113
|
+
f"Does the evidence support claims[{index}]? Judge only from the provided evidence, not world knowledge. "
|
|
114
|
+
"Use only the evidence field as factual support; request and claims are assertions, not evidence; diff "
|
|
115
|
+
"and tests belong to the separate patch review. If a claim needs a diff or test log as support, it must be "
|
|
116
|
+
"supplied in evidence." + ANTI_INJECTION,
|
|
117
|
+
CLAIM_CRITERIA,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _refused(error: str) -> ToolResult:
|
|
122
|
+
return ToolResult({"tool": "jev_gate", "error": error}, is_error=True)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
async def handle(args: dict[str, Any], runtime: Runtime) -> ToolResult:
|
|
126
|
+
settings = review_settings(args)
|
|
127
|
+
thresholds = settings.thresholds
|
|
128
|
+
evidence = normalize_evidence(args["evidence"])
|
|
129
|
+
# Bound the request before any model call: item count, then aggregate size.
|
|
130
|
+
if exceeds(len(evidence), GATE.evidence_items):
|
|
131
|
+
return _refused(gate_evidence_items_error(GATE.evidence_items))
|
|
132
|
+
if exceeds(sum(length(str(item["text"])) for item in evidence), GATE.aggregate_evidence_units):
|
|
133
|
+
return _refused(gate_evidence_aggregate_error(GATE.aggregate_evidence_units))
|
|
134
|
+
|
|
135
|
+
ledger = CapLedger()
|
|
136
|
+
docs = review_docs(args, ledger, GATE.doc_units)
|
|
137
|
+
claims: list[str] = args["claims"]
|
|
138
|
+
sent_claims = [ledger.text(claim, GATE.claim_units, "context") for claim in claims]
|
|
139
|
+
sent_evidence = [
|
|
140
|
+
{"id": item["id"], "text": ledger.text(str(item["text"]), GATE.doc_units, "context")} for item in evidence
|
|
141
|
+
]
|
|
142
|
+
truncated = ledger.context_cut
|
|
143
|
+
|
|
144
|
+
state = {
|
|
145
|
+
"purpose": "Review the proposed diff against the request, then check each completion claim against the "
|
|
146
|
+
"evidence only.",
|
|
147
|
+
"request": docs.request,
|
|
148
|
+
"diff": docs.diff,
|
|
149
|
+
"tests": docs.tests,
|
|
150
|
+
"claims": sent_claims,
|
|
151
|
+
"evidence": sent_evidence,
|
|
152
|
+
}
|
|
153
|
+
# Review questions carry extra framing so claims cannot read as proof; claims use evidence only.
|
|
154
|
+
questions = review_questions(REVIEW_FRAMING)
|
|
155
|
+
for index in range(len(claims)):
|
|
156
|
+
questions[f"claim_{index}"] = claim_question(index)
|
|
157
|
+
evaluation = await runtime.ask(state, questions)
|
|
158
|
+
answers = evaluation.answers
|
|
159
|
+
|
|
160
|
+
review = project_review(answers, settings, truncated)
|
|
161
|
+
|
|
162
|
+
results: list[dict[str, object]] = []
|
|
163
|
+
judgments: list[ClaimJudgment | None] = []
|
|
164
|
+
claim_actions: list[Action] = []
|
|
165
|
+
for index, claim in enumerate(claims):
|
|
166
|
+
answer = validate_choice(answers.get(f"claim_{index}"), CLAIM_CRITERIA)
|
|
167
|
+
if answer is None:
|
|
168
|
+
judgments.append(None)
|
|
169
|
+
claim_actions.append("escalate")
|
|
170
|
+
results.append(
|
|
171
|
+
{
|
|
172
|
+
"claim": claim,
|
|
173
|
+
"verdict": None,
|
|
174
|
+
"confidence": None,
|
|
175
|
+
"probabilities": None,
|
|
176
|
+
"action": "escalate",
|
|
177
|
+
"status": "invalid_response",
|
|
178
|
+
}
|
|
179
|
+
)
|
|
180
|
+
continue
|
|
181
|
+
verdict = CLAIM_VERDICTS[answer.choice]
|
|
182
|
+
judgments.append(ClaimJudgment(verdict, answer.confidence))
|
|
183
|
+
action = require_complete_context(
|
|
184
|
+
claim_action(verdict, answer.confidence, thresholds.auto_accept, thresholds.review_at), truncated
|
|
185
|
+
)
|
|
186
|
+
claim_actions.append(action)
|
|
187
|
+
results.append(
|
|
188
|
+
{
|
|
189
|
+
"claim": claim,
|
|
190
|
+
"verdict": verdict,
|
|
191
|
+
"confidence": answer.confidence,
|
|
192
|
+
"probabilities": answer.probabilities,
|
|
193
|
+
"action": action,
|
|
194
|
+
}
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
verification_action = worst_action(claim_actions)
|
|
198
|
+
verification = {
|
|
199
|
+
"action": verification_action,
|
|
200
|
+
"summary": {
|
|
201
|
+
"verified": sum(1 for r in results if r["verdict"] == "verified"),
|
|
202
|
+
"contradicted": sum(1 for r in results if r["verdict"] == "contradicted"),
|
|
203
|
+
"unsupported": sum(1 for r in results if r["verdict"] == "unsupported"),
|
|
204
|
+
"needs_review": sum(1 for r in results if r["action"] != "auto"),
|
|
205
|
+
"invalid_response": sum(1 for r in results if r.get("status") == "invalid_response"),
|
|
206
|
+
},
|
|
207
|
+
"thresholds": {"auto_accept": thresholds.auto_accept, "review_at": thresholds.review_at},
|
|
208
|
+
"results": results,
|
|
209
|
+
}
|
|
210
|
+
action = worst_action([review.action, verification_action])
|
|
211
|
+
reason_codes = gate_reason_codes(
|
|
212
|
+
truncated=truncated,
|
|
213
|
+
review_action=review.action,
|
|
214
|
+
review_invalid=review.invalid,
|
|
215
|
+
claims=judgments,
|
|
216
|
+
action=action,
|
|
217
|
+
thresholds=thresholds,
|
|
218
|
+
)
|
|
219
|
+
return ToolResult(
|
|
220
|
+
frame(
|
|
221
|
+
"jev_gate",
|
|
222
|
+
evaluation,
|
|
223
|
+
{
|
|
224
|
+
"truncated": truncated,
|
|
225
|
+
"action": action,
|
|
226
|
+
"reason_codes": reason_codes,
|
|
227
|
+
"review": review.payload,
|
|
228
|
+
"verification": verification,
|
|
229
|
+
},
|
|
230
|
+
),
|
|
231
|
+
action=action,
|
|
232
|
+
truncated=ledger.scopes,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
TOOL = JevTool(DEFINITION, handle, {"evidence": EVIDENCE_NOT_EMPTY})
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Answer validators and policy functions as tools call them: each call is a span (ROADMAP P9).
|
|
2
|
+
|
|
3
|
+
`jev_mcp.validation` and `jev_mcp.policy` stay pure (ADR-0002); the spans are opened here, in the
|
|
4
|
+
tools layer. A validator call is a `jev.validate` span labelled with its kind and whether the answer
|
|
5
|
+
was valid — a rejected answer is a fail-closed. A policy call is a `jev.policy` span labelled with
|
|
6
|
+
the function and, when it returns one, the action. Tools import these callables from here, never
|
|
7
|
+
from the pure packages (`tests/unit/test_telemetry.py` checks).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from functools import wraps
|
|
12
|
+
|
|
13
|
+
from jev_mcp import policy, validation
|
|
14
|
+
from jev_mcp.telemetry import ACTIONS, span
|
|
15
|
+
|
|
16
|
+
TRACED: set[str] = set()
|
|
17
|
+
"""Every traced name; tools must not import these from `jev_mcp.policy` or `jev_mcp.validation`."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _validator[**P, R](kind: str, validate: Callable[P, R | None]) -> Callable[P, R | None]:
|
|
21
|
+
TRACED.add(validate.__name__)
|
|
22
|
+
|
|
23
|
+
@wraps(validate)
|
|
24
|
+
def traced(*args: P.args, **kwargs: P.kwargs) -> R | None:
|
|
25
|
+
with span("jev.validate", kind=kind) as current:
|
|
26
|
+
answer = validate(*args, **kwargs)
|
|
27
|
+
current.attributes["valid"] = answer is not None
|
|
28
|
+
return answer
|
|
29
|
+
|
|
30
|
+
return traced
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _policy[**P, R](decide: Callable[P, R]) -> Callable[P, R]:
|
|
34
|
+
TRACED.add(decide.__name__)
|
|
35
|
+
|
|
36
|
+
@wraps(decide)
|
|
37
|
+
def traced(*args: P.args, **kwargs: P.kwargs) -> R:
|
|
38
|
+
with span("jev.policy", decision=decide.__name__) as current:
|
|
39
|
+
result = decide(*args, **kwargs)
|
|
40
|
+
if isinstance(result, str) and result in ACTIONS:
|
|
41
|
+
current.attributes["action"] = result
|
|
42
|
+
return result
|
|
43
|
+
|
|
44
|
+
return traced
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
validate_choice = _validator("choice", validation.validate_choice)
|
|
48
|
+
validate_extract_choice = _validator("extract_choice", validation.validate_extract_choice)
|
|
49
|
+
validate_noul = _validator("noul", validation.validate_noul)
|
|
50
|
+
validate_score = _validator("score", validation.validate_score)
|
|
51
|
+
|
|
52
|
+
claim_action = _policy(policy.claim_action)
|
|
53
|
+
classification_decision = _policy(policy.classification_decision)
|
|
54
|
+
contradicts_recommendation = _policy(policy.contradicts_recommendation)
|
|
55
|
+
decide_extract_field = _policy(policy.decide_extract_field)
|
|
56
|
+
exists_verdict = _policy(policy.exists_verdict)
|
|
57
|
+
gate_reason_codes = _policy(policy.gate_reason_codes)
|
|
58
|
+
min_confidence = _policy(policy.min_confidence)
|
|
59
|
+
rank_candidates = _policy(policy.rank_candidates)
|
|
60
|
+
require_complete_context = _policy(policy.require_complete_context)
|
|
61
|
+
rerank_by_score = _policy(policy.rerank_by_score)
|
|
62
|
+
resolve_policy_thresholds = _policy(policy.resolve_policy_thresholds)
|
|
63
|
+
review_action = _policy(policy.review_action)
|
|
64
|
+
review_composite = _policy(policy.review_composite)
|
|
65
|
+
screen_fail_closed = _policy(policy.screen_fail_closed)
|
|
66
|
+
screen_recommendation = _policy(policy.screen_recommendation)
|
|
67
|
+
validate_policy_thresholds = _policy(policy.validate_policy_thresholds)
|
|
68
|
+
verify_action = _policy(policy.verify_action)
|
|
69
|
+
worst_action = _policy(policy.worst_action)
|