dev-double 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dev_double/__init__.py +3 -0
- dev_double/apps/__init__.py +1 -0
- dev_double/apps/cli/__init__.py +5 -0
- dev_double/apps/cli/main.py +122 -0
- dev_double/apps/client/__init__.py +5 -0
- dev_double/apps/client/http_client.py +102 -0
- dev_double/apps/composition.py +71 -0
- dev_double/apps/config.py +36 -0
- dev_double/apps/server/__init__.py +5 -0
- dev_double/apps/server/app.py +122 -0
- dev_double/apps/server/recorder.py +62 -0
- dev_double/apps/server/transport/__init__.py +39 -0
- dev_double/apps/server/transport/choice_use_cases.py +121 -0
- dev_double/apps/server/transport/common.py +37 -0
- dev_double/apps/server/transport/decide.py +131 -0
- dev_double/apps/server/transport/extract.py +93 -0
- dev_double/apps/server/transport/guard_judge.py +97 -0
- dev_double/apps/server/transport/rerank.py +55 -0
- dev_double/client.py +5 -0
- dev_double/core/__init__.py +1 -0
- dev_double/core/decision/__init__.py +62 -0
- dev_double/core/decision/answer_shaping.py +48 -0
- dev_double/core/decision/confidence.py +19 -0
- dev_double/core/decision/decider_basic_impl.py +65 -0
- dev_double/core/decision/decision_service_basic_impl.py +141 -0
- dev_double/core/decision/defaults.py +40 -0
- dev_double/core/decision/errors.py +7 -0
- dev_double/core/decision/extract_questions.py +40 -0
- dev_double/core/decision/field_extraction.py +53 -0
- dev_double/core/decision/i_clock.py +11 -0
- dev_double/core/decision/i_decider.py +36 -0
- dev_double/core/decision/i_decision_service.py +40 -0
- dev_double/core/decision/i_engine.py +27 -0
- dev_double/core/decision/i_extractor.py +19 -0
- dev_double/core/decision/i_generator.py +19 -0
- dev_double/core/decision/i_id_provider.py +9 -0
- dev_double/core/decision/i_record_reader.py +25 -0
- dev_double/core/decision/i_reranker.py +18 -0
- dev_double/core/decision/label_scoring.py +89 -0
- dev_double/core/decision/prompts.py +130 -0
- dev_double/core/decision/record_parsing.py +153 -0
- dev_double/core/decision/t_answer.py +32 -0
- dev_double/core/decision/t_classify.py +42 -0
- dev_double/core/decision/t_decide.py +26 -0
- dev_double/core/decision/t_extract.py +89 -0
- dev_double/core/decision/t_gate.py +36 -0
- dev_double/core/decision/t_generate.py +37 -0
- dev_double/core/decision/t_guard.py +38 -0
- dev_double/core/decision/t_input.py +8 -0
- dev_double/core/decision/t_judge.py +35 -0
- dev_double/core/decision/t_label_query.py +36 -0
- dev_double/core/decision/t_meta.py +17 -0
- dev_double/core/decision/t_question.py +70 -0
- dev_double/core/decision/t_rerank.py +36 -0
- dev_double/core/decision/t_route.py +28 -0
- dev_double/core/decision/t_usage.py +15 -0
- dev_double/core/decision/tracker.py +27 -0
- dev_double/core/decision/use_case_questions.py +80 -0
- dev_double/core/decision/value_parsing.py +109 -0
- dev_double/providers/__init__.py +1 -0
- dev_double/providers/mock/__init__.py +1 -0
- dev_double/providers/mock/decision/__init__.py +5 -0
- dev_double/providers/mock/decision/clock_mock_impl.py +19 -0
- dev_double/providers/mock/decision/engine_mock_impl.py +78 -0
- dev_double/providers/mock/decision/id_provider_mock_impl.py +15 -0
- dev_double/providers/needle/__init__.py +1 -0
- dev_double/providers/needle/decision/__init__.py +4 -0
- dev_double/providers/needle/decision/decider_needle_impl.py +117 -0
- dev_double/providers/needle/decision/record_tool.py +75 -0
- dev_double/providers/openai/__init__.py +1 -0
- dev_double/providers/openai/decision/__init__.py +3 -0
- dev_double/providers/openai/decision/engine_openai_impl.py +154 -0
- dev_double/providers/std/__init__.py +1 -0
- dev_double/providers/std/decision/__init__.py +4 -0
- dev_double/providers/std/decision/clock_std_impl.py +12 -0
- dev_double/providers/std/decision/id_provider_std_impl.py +12 -0
- dev_double/providers/systemone/__init__.py +1 -0
- dev_double/providers/systemone/decision/__init__.py +3 -0
- dev_double/providers/systemone/decision/decider_system_one_impl.py +100 -0
- dev_double-0.1.0.dist-info/METADATA +354 -0
- dev_double-0.1.0.dist-info/RECORD +84 -0
- dev_double-0.1.0.dist-info/WHEEL +4 -0
- dev_double-0.1.0.dist-info/entry_points.txt +2 -0
- dev_double-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""The default decider: renders a single-label prompt, asks an ``IEngine`` for
|
|
2
|
+
a distribution over the labels, and shapes the typed answer.
|
|
3
|
+
|
|
4
|
+
It also reads the free-text fields of an extraction request, as one JSON
|
|
5
|
+
record, when its engine is an ``IGenerator``; with a label-only engine those
|
|
6
|
+
fields are left empty with a warning."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
|
|
12
|
+
from . import prompts
|
|
13
|
+
from .answer_shaping import shape_answer
|
|
14
|
+
from .extract_questions import cannot_generate
|
|
15
|
+
from .i_decider import IDecider
|
|
16
|
+
from .i_engine import IEngine
|
|
17
|
+
from .i_generator import IGenerator
|
|
18
|
+
from .i_record_reader import IRecordReader
|
|
19
|
+
from .record_parsing import parse_record
|
|
20
|
+
from .t_answer import TAnswer
|
|
21
|
+
from .t_extract import TExtractField, TFieldValue
|
|
22
|
+
from .t_input import TInputValue
|
|
23
|
+
from .t_question import TQuestion
|
|
24
|
+
from .tracker import Tracker
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class DeciderBasicImpl(IDecider, IRecordReader):
|
|
28
|
+
def __init__(self, engine: IEngine, max_concurrency: int = 4) -> None:
|
|
29
|
+
self.engine = engine
|
|
30
|
+
self._limit = asyncio.Semaphore(max_concurrency)
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def name(self) -> str:
|
|
34
|
+
return self.engine.name
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def model(self) -> str:
|
|
38
|
+
return self.engine.model
|
|
39
|
+
|
|
40
|
+
async def ask(
|
|
41
|
+
self, value: TInputValue, question: TQuestion, tracker: Tracker, label: str = ""
|
|
42
|
+
) -> TAnswer:
|
|
43
|
+
query = prompts.query_for(prompts.render_input(value), question)
|
|
44
|
+
async with self._limit:
|
|
45
|
+
result = await self.engine.distribution(query)
|
|
46
|
+
tracker.usage.add(result.usage)
|
|
47
|
+
for warning in result.warnings:
|
|
48
|
+
tracker.warn(warning, label)
|
|
49
|
+
return shape_answer(question, result.probabilities)
|
|
50
|
+
|
|
51
|
+
async def read_record(
|
|
52
|
+
self, input_value: TInputValue, fields: dict[str, TExtractField], tracker: Tracker
|
|
53
|
+
) -> dict[str, TFieldValue]:
|
|
54
|
+
if not isinstance(self.engine, IGenerator):
|
|
55
|
+
return {name: cannot_generate(name, tracker) for name in fields}
|
|
56
|
+
query = prompts.extract_record(prompts.render_input(input_value), fields)
|
|
57
|
+
async with self._limit:
|
|
58
|
+
result = await self.engine.generate(query)
|
|
59
|
+
tracker.usage.add(result.usage)
|
|
60
|
+
for warning in result.warnings:
|
|
61
|
+
tracker.warn(warning)
|
|
62
|
+
return parse_record(result.text, result.tokens, fields, result.confidence, tracker)
|
|
63
|
+
|
|
64
|
+
async def aclose(self) -> None:
|
|
65
|
+
await self.engine.aclose()
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""The use cases over an ``IDecider``, with time and ids injected."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
|
|
7
|
+
from . import use_case_questions as uq
|
|
8
|
+
from .field_extraction import read_fields
|
|
9
|
+
from .i_clock import IClock
|
|
10
|
+
from .i_decider import IDecider
|
|
11
|
+
from .i_decision_service import IDecisionService
|
|
12
|
+
from .i_id_provider import IIdProvider
|
|
13
|
+
from .i_reranker import IReranker
|
|
14
|
+
from .t_answer import TAnswer, TBinaryAnswer, TChoiceAnswer, TScaleAnswer
|
|
15
|
+
from .t_classify import TClassification, TClassifyRequest, TClassifyResponse
|
|
16
|
+
from .t_decide import TDecideRequest, TDecideResponse
|
|
17
|
+
from .t_extract import TExtractRequest, TExtractResponse
|
|
18
|
+
from .t_gate import TGateRequest, TGateResponse
|
|
19
|
+
from .t_guard import TGuardCheck, TGuardRequest, TGuardResponse
|
|
20
|
+
from .t_input import TInputValue
|
|
21
|
+
from .t_judge import TJudgeRequest, TJudgeResponse
|
|
22
|
+
from .t_meta import TMeta
|
|
23
|
+
from .t_question import TChoiceQuestion, TQuestion
|
|
24
|
+
from .t_rerank import TRerankRequest, TRerankResponse, TRerankResult
|
|
25
|
+
from .t_route import TRouteRequest, TRouteResponse
|
|
26
|
+
from .tracker import Tracker
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DecisionServiceBasicImpl(IDecisionService):
|
|
30
|
+
def __init__(self, decider: IDecider, clock: IClock, ids: IIdProvider) -> None:
|
|
31
|
+
self.decider = decider
|
|
32
|
+
self._clock = clock
|
|
33
|
+
self._ids = ids
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def name(self) -> str:
|
|
37
|
+
return self.decider.name
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def model(self) -> str:
|
|
41
|
+
return self.decider.model
|
|
42
|
+
|
|
43
|
+
async def aclose(self) -> None:
|
|
44
|
+
await self.decider.aclose()
|
|
45
|
+
|
|
46
|
+
def _tracker(self) -> Tracker:
|
|
47
|
+
return Tracker(self._clock)
|
|
48
|
+
|
|
49
|
+
def _meta(self, t: Tracker) -> TMeta:
|
|
50
|
+
return t.meta(self.decider.name, self.decider.model)
|
|
51
|
+
|
|
52
|
+
async def _ask_many(
|
|
53
|
+
self, value: TInputValue, questions: dict[str, TQuestion], t: Tracker
|
|
54
|
+
) -> dict[str, TAnswer]:
|
|
55
|
+
answers = await asyncio.gather(
|
|
56
|
+
*(self.decider.ask(value, q, t, label=qid) for qid, q in questions.items())
|
|
57
|
+
)
|
|
58
|
+
return dict(zip(questions, answers))
|
|
59
|
+
|
|
60
|
+
async def decide(self, req: TDecideRequest) -> TDecideResponse:
|
|
61
|
+
t = self._tracker()
|
|
62
|
+
answers = await self._ask_many(req.input, req.questions, t)
|
|
63
|
+
return TDecideResponse(answers=answers, meta=self._meta(t))
|
|
64
|
+
|
|
65
|
+
async def route(self, req: TRouteRequest) -> TRouteResponse:
|
|
66
|
+
t = self._tracker()
|
|
67
|
+
a = await self.decider.ask(req.input, uq.route_question(req), t)
|
|
68
|
+
assert isinstance(a, TChoiceAnswer)
|
|
69
|
+
return TRouteResponse(a.value, a.probabilities, a.confidence, self._meta(t))
|
|
70
|
+
|
|
71
|
+
async def guard(self, req: TGuardRequest) -> TGuardResponse:
|
|
72
|
+
t = self._tracker()
|
|
73
|
+
answers = await self._ask_many(req.input, uq.guard_questions(req), t)
|
|
74
|
+
checks = {}
|
|
75
|
+
for name, a in answers.items():
|
|
76
|
+
assert isinstance(a, TBinaryAnswer)
|
|
77
|
+
checks[name] = TGuardCheck(probability=a.probability, flagged=a.probability >= req.threshold)
|
|
78
|
+
flagged = [name for name, c in checks.items() if c.flagged]
|
|
79
|
+
return TGuardResponse(allowed=not flagged, flagged=flagged, checks=checks, meta=self._meta(t))
|
|
80
|
+
|
|
81
|
+
async def gate(self, req: TGateRequest) -> TGateResponse:
|
|
82
|
+
t = self._tracker()
|
|
83
|
+
a = await self.decider.ask(uq.gate_payload(req), uq.gate_question(req), t)
|
|
84
|
+
assert isinstance(a, TChoiceAnswer)
|
|
85
|
+
return TGateResponse(a.value, a.probabilities, a.confidence, self._meta(t))
|
|
86
|
+
|
|
87
|
+
async def classify(self, req: TClassifyRequest) -> TClassifyResponse:
|
|
88
|
+
t = self._tracker()
|
|
89
|
+
q = TChoiceQuestion(question=req.question, options=req.labels)
|
|
90
|
+
answers = await asyncio.gather(
|
|
91
|
+
*(self.decider.ask(item, q, t, label=f"inputs[{i}]") for i, item in enumerate(req.items()))
|
|
92
|
+
)
|
|
93
|
+
results = []
|
|
94
|
+
for a in answers:
|
|
95
|
+
assert isinstance(a, TChoiceAnswer)
|
|
96
|
+
results.append(TClassification(a.value, a.probabilities, a.confidence))
|
|
97
|
+
return TClassifyResponse(results=results, meta=self._meta(t))
|
|
98
|
+
|
|
99
|
+
async def judge(self, req: TJudgeRequest) -> TJudgeResponse:
|
|
100
|
+
t = self._tracker()
|
|
101
|
+
q = uq.judge_question(req)
|
|
102
|
+
a = await self.decider.ask(uq.judge_payload(req), q, t)
|
|
103
|
+
assert isinstance(a, TScaleAnswer)
|
|
104
|
+
return TJudgeResponse(
|
|
105
|
+
score=a.value,
|
|
106
|
+
normalized=round(a.value / (len(q.levels) - 1), 4),
|
|
107
|
+
level=a.level,
|
|
108
|
+
probabilities=a.probabilities,
|
|
109
|
+
legend=a.legend,
|
|
110
|
+
confidence=a.confidence,
|
|
111
|
+
meta=self._meta(t),
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
async def extract(self, req: TExtractRequest) -> TExtractResponse:
|
|
115
|
+
t = self._tracker()
|
|
116
|
+
fields = await read_fields(self.decider, req, t)
|
|
117
|
+
return TExtractResponse(fields=fields, meta=self._meta(t))
|
|
118
|
+
|
|
119
|
+
async def rerank(self, req: TRerankRequest) -> TRerankResponse:
|
|
120
|
+
t = self._tracker()
|
|
121
|
+
texts = req.documents
|
|
122
|
+
scores = await self._rerank_scores(req.query, texts, t)
|
|
123
|
+
order = sorted(range(len(texts)), key=lambda i: scores[i], reverse=True)
|
|
124
|
+
if req.top_n is not None:
|
|
125
|
+
order = order[: req.top_n]
|
|
126
|
+
results = [
|
|
127
|
+
TRerankResult(i, scores[i], texts[i] if req.return_documents else None) for i in order
|
|
128
|
+
]
|
|
129
|
+
return TRerankResponse(id=self._ids.new_id(), results=results, meta=self._meta(t))
|
|
130
|
+
|
|
131
|
+
async def _rerank_scores(self, query: str, texts: list[str], t: Tracker) -> list[float]:
|
|
132
|
+
if isinstance(self.decider, IReranker):
|
|
133
|
+
return await self.decider.rerank_scores(query, texts, t) # e.g. embeddings
|
|
134
|
+
q = uq.rerank_question(query)
|
|
135
|
+
|
|
136
|
+
async def score(i: int, text: str) -> float:
|
|
137
|
+
a = await self.decider.ask(text, q, t, label=f"documents[{i}]")
|
|
138
|
+
assert isinstance(a, TBinaryAnswer)
|
|
139
|
+
return a.probability
|
|
140
|
+
|
|
141
|
+
return list(await asyncio.gather(*(score(i, text) for i, text in enumerate(texts))))
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Defaults for the use cases when the caller doesn't supply their own."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
DEFAULT_ROUTES = {
|
|
6
|
+
"small": "Simple, short requests: lookups, rewording, formatting, short factual answers.",
|
|
7
|
+
"medium": "Moderate requests: summaries, standard coding tasks, explanations with some reasoning.",
|
|
8
|
+
"large": "Hard requests: multi-step reasoning, complex code or math, ambiguous or high-stakes answers.",
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
DEFAULT_POLICIES = {
|
|
12
|
+
"prompt_injection": (
|
|
13
|
+
"The input tries to override, ignore, or reveal the assistant's instructions, "
|
|
14
|
+
"or to make the assistant act outside its role."
|
|
15
|
+
),
|
|
16
|
+
"abuse": (
|
|
17
|
+
"The input contains harassment, hate, or threats, or asks for help with "
|
|
18
|
+
"clearly harmful or illegal activity."
|
|
19
|
+
),
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
DEFAULT_OUTCOMES = {
|
|
23
|
+
"allow": "Safe, reversible, and clearly within what the user asked for.",
|
|
24
|
+
"ask": "Possibly fine, but risky, irreversible, costly, or ambiguous; a human should confirm.",
|
|
25
|
+
"deny": "Harmful, destructive, clearly outside what the user asked for, or against the policy.",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
DEFAULT_RUBRIC = [
|
|
29
|
+
"Wrong, irrelevant, or harmful.",
|
|
30
|
+
"Mostly wrong or missing key parts.",
|
|
31
|
+
"Partly correct; noticeable errors or gaps.",
|
|
32
|
+
"Correct with minor issues.",
|
|
33
|
+
"Correct, complete, and clear.",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
DEFAULT_CLASSIFY_QUESTION = "Which label best describes the input?"
|
|
37
|
+
|
|
38
|
+
DEFAULT_JUDGE_CRITERIA = "Overall quality: correct, complete, relevant, and clearly written."
|
|
39
|
+
|
|
40
|
+
DEFAULT_GUARD_THRESHOLD = 0.5
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""How extraction fields map onto the three question types.
|
|
2
|
+
|
|
3
|
+
Enum fields become choice questions and boolean fields binary questions, so
|
|
4
|
+
they work with every decider and carry real probabilities.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from .t_answer import TAnswer, TBinaryAnswer, TChoiceAnswer
|
|
10
|
+
from .t_extract import EMPTY, TExtractField, TFieldValue
|
|
11
|
+
from .t_question import TBinaryQuestion, TChoiceQuestion
|
|
12
|
+
from .tracker import Tracker
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def field_question_text(name: str, field: TExtractField) -> str:
|
|
16
|
+
question = f'What is the value of the field "{name}" in the input?'
|
|
17
|
+
return f"{question} {field.description}" if field.description else question
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def enum_question(name: str, field: TExtractField) -> TChoiceQuestion:
|
|
21
|
+
assert isinstance(field.options, dict) # TExtractField normalizes enum options to a dict
|
|
22
|
+
return TChoiceQuestion(question=field_question_text(name, field), options=dict(field.options))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def boolean_question(name: str, field: TExtractField) -> TBinaryQuestion:
|
|
26
|
+
return TBinaryQuestion(question=f"Is this true for the input? {field.description or name}")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def value_from_answer(answer: TAnswer) -> TFieldValue:
|
|
30
|
+
if isinstance(answer, TBinaryAnswer):
|
|
31
|
+
return TFieldValue(answer.value, answer.confidence, probability=answer.probability)
|
|
32
|
+
if isinstance(answer, TChoiceAnswer):
|
|
33
|
+
return TFieldValue(answer.value, answer.confidence, probabilities=answer.probabilities)
|
|
34
|
+
raise TypeError(f"No field value for a {type(answer).__name__}.")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def cannot_generate(name: str, tracker: Tracker) -> TFieldValue:
|
|
38
|
+
"""The field needs free text and the engine can't generate any."""
|
|
39
|
+
tracker.warn(f"engine can't generate text: field {name} left empty")
|
|
40
|
+
return EMPTY
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Reads every field of an extraction request with whatever the decider can do.
|
|
2
|
+
|
|
3
|
+
- A native ``IExtractor`` (e.g. one tool call per record) takes the whole request.
|
|
4
|
+
- Otherwise enum and boolean fields are asked as choice and binary questions
|
|
5
|
+
(real probabilities), and all string, number, and integer fields go to one
|
|
6
|
+
``IRecordReader.read_record`` call when the decider has it (a whole-record
|
|
7
|
+
prompt beat one prompt per field in our evals), else they are left empty
|
|
8
|
+
with a warning.
|
|
9
|
+
|
|
10
|
+
The questions and the record read run concurrently; results keep the
|
|
11
|
+
request's field order.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import asyncio
|
|
17
|
+
|
|
18
|
+
from .extract_questions import boolean_question, cannot_generate, enum_question, value_from_answer
|
|
19
|
+
from .i_decider import IDecider
|
|
20
|
+
from .i_extractor import IExtractor
|
|
21
|
+
from .i_record_reader import IRecordReader
|
|
22
|
+
from .t_extract import EMPTY, TEXT_TYPES, TExtractField, TExtractRequest, TFieldValue
|
|
23
|
+
from .t_input import TInputValue
|
|
24
|
+
from .tracker import Tracker
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
async def _ask(decider: IDecider, value: TInputValue, name: str, field: TExtractField, t: Tracker) -> TFieldValue:
|
|
28
|
+
question = enum_question(name, field) if field.type == "enum" else boolean_question(name, field)
|
|
29
|
+
return value_from_answer(await decider.ask(value, question, t, label=name))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
async def _read_text(
|
|
33
|
+
decider: IDecider, value: TInputValue, fields: dict[str, TExtractField], t: Tracker
|
|
34
|
+
) -> dict[str, TFieldValue]:
|
|
35
|
+
if not fields:
|
|
36
|
+
return {}
|
|
37
|
+
if isinstance(decider, IRecordReader):
|
|
38
|
+
return await decider.read_record(value, fields, t)
|
|
39
|
+
return {name: cannot_generate(name, t) for name in fields}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
async def read_fields(decider: IDecider, req: TExtractRequest, t: Tracker) -> dict[str, TFieldValue]:
|
|
43
|
+
if isinstance(decider, IExtractor):
|
|
44
|
+
got = await decider.extract(req, t)
|
|
45
|
+
return {name: got.get(name, EMPTY) for name in req.fields}
|
|
46
|
+
text = {name: f for name, f in req.fields.items() if f.type in TEXT_TYPES}
|
|
47
|
+
scored = {name: f for name, f in req.fields.items() if f.type not in TEXT_TYPES}
|
|
48
|
+
record, *answers = await asyncio.gather(
|
|
49
|
+
_read_text(decider, req.input, text, t),
|
|
50
|
+
*(_ask(decider, req.input, name, f, t) for name, f in scored.items()),
|
|
51
|
+
)
|
|
52
|
+
got = {**record, **dict(zip(scored, answers))}
|
|
53
|
+
return {name: got.get(name, EMPTY) for name in req.fields}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Answers one typed question about one input.
|
|
2
|
+
|
|
3
|
+
``DeciderBasicImpl`` does it by prompting an ``IEngine``; decision models that
|
|
4
|
+
answer typed questions natively (Kev, Laya, Needle) implement this directly.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Protocol, runtime_checkable
|
|
10
|
+
|
|
11
|
+
from .t_answer import TAnswer
|
|
12
|
+
from .t_input import TInputValue
|
|
13
|
+
from .t_question import TQuestion
|
|
14
|
+
from .tracker import Tracker
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@runtime_checkable
|
|
18
|
+
class IDecider(Protocol):
|
|
19
|
+
@property
|
|
20
|
+
def name(self) -> str:
|
|
21
|
+
"""Engine name, reported in response meta."""
|
|
22
|
+
...
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def model(self) -> str:
|
|
26
|
+
"""Model name, reported in response meta."""
|
|
27
|
+
...
|
|
28
|
+
|
|
29
|
+
async def ask(
|
|
30
|
+
self, value: TInputValue, question: TQuestion, tracker: Tracker, label: str = ""
|
|
31
|
+
) -> TAnswer:
|
|
32
|
+
"""Answer ``question`` about ``value``; add usage and warnings (prefixed
|
|
33
|
+
with ``label``) to ``tracker``. Raises EngineError on backend failure."""
|
|
34
|
+
...
|
|
35
|
+
|
|
36
|
+
async def aclose(self) -> None: ...
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""The task-shaped use cases, built on the three question types."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Protocol
|
|
6
|
+
|
|
7
|
+
from .t_classify import TClassifyRequest, TClassifyResponse
|
|
8
|
+
from .t_decide import TDecideRequest, TDecideResponse
|
|
9
|
+
from .t_extract import TExtractRequest, TExtractResponse
|
|
10
|
+
from .t_gate import TGateRequest, TGateResponse
|
|
11
|
+
from .t_guard import TGuardRequest, TGuardResponse
|
|
12
|
+
from .t_judge import TJudgeRequest, TJudgeResponse
|
|
13
|
+
from .t_rerank import TRerankRequest, TRerankResponse
|
|
14
|
+
from .t_route import TRouteRequest, TRouteResponse
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class IDecisionService(Protocol):
|
|
18
|
+
@property
|
|
19
|
+
def name(self) -> str: ...
|
|
20
|
+
|
|
21
|
+
@property
|
|
22
|
+
def model(self) -> str: ...
|
|
23
|
+
|
|
24
|
+
async def decide(self, req: TDecideRequest) -> TDecideResponse: ...
|
|
25
|
+
|
|
26
|
+
async def route(self, req: TRouteRequest) -> TRouteResponse: ...
|
|
27
|
+
|
|
28
|
+
async def guard(self, req: TGuardRequest) -> TGuardResponse: ...
|
|
29
|
+
|
|
30
|
+
async def gate(self, req: TGateRequest) -> TGateResponse: ...
|
|
31
|
+
|
|
32
|
+
async def classify(self, req: TClassifyRequest) -> TClassifyResponse: ...
|
|
33
|
+
|
|
34
|
+
async def judge(self, req: TJudgeRequest) -> TJudgeResponse: ...
|
|
35
|
+
|
|
36
|
+
async def rerank(self, req: TRerankRequest) -> TRerankResponse: ...
|
|
37
|
+
|
|
38
|
+
async def extract(self, req: TExtractRequest) -> TExtractResponse: ...
|
|
39
|
+
|
|
40
|
+
async def aclose(self) -> None: ...
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""The LLM label-scoring port: given a rendered prompt and a fixed set of
|
|
2
|
+
labels, return a probability distribution over those labels.
|
|
3
|
+
|
|
4
|
+
Everything above this port (question types, use cases, confidence) is
|
|
5
|
+
engine-agnostic, so swapping the model is a configuration change.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Protocol, runtime_checkable
|
|
11
|
+
|
|
12
|
+
from .t_label_query import TLabelQuery, TLabelResult
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@runtime_checkable
|
|
16
|
+
class IEngine(Protocol):
|
|
17
|
+
@property
|
|
18
|
+
def name(self) -> str: ...
|
|
19
|
+
|
|
20
|
+
@property
|
|
21
|
+
def model(self) -> str: ...
|
|
22
|
+
|
|
23
|
+
async def distribution(self, query: TLabelQuery) -> TLabelResult:
|
|
24
|
+
"""Raises EngineError when the model can't be reached or answers unusably."""
|
|
25
|
+
...
|
|
26
|
+
|
|
27
|
+
async def aclose(self) -> None: ...
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Optional capability: a decider that extracts a whole record natively (e.g.
|
|
2
|
+
one tool call with every field) instead of one question per field.
|
|
3
|
+
|
|
4
|
+
The extract use case checks ``isinstance(decider, IExtractor)`` and uses it when present.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Protocol, runtime_checkable
|
|
10
|
+
|
|
11
|
+
from .t_extract import TExtractRequest, TFieldValue
|
|
12
|
+
from .tracker import Tracker
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@runtime_checkable
|
|
16
|
+
class IExtractor(Protocol):
|
|
17
|
+
async def extract(self, req: TExtractRequest, tracker: Tracker) -> dict[str, TFieldValue]:
|
|
18
|
+
"""Field name -> value. Fields left out are reported as empty."""
|
|
19
|
+
...
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Optional engine capability: generate a short free-text answer.
|
|
2
|
+
|
|
3
|
+
Label scoring (``IEngine.distribution``) can't produce values that aren't known
|
|
4
|
+
up front, such as a vendor name or an amount. Engines that can also generate
|
|
5
|
+
text implement this; ``DeciderBasicImpl`` checks ``isinstance(engine, IGenerator)``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Protocol, runtime_checkable
|
|
11
|
+
|
|
12
|
+
from .t_generate import TGenerateQuery, TGenerateResult
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@runtime_checkable
|
|
16
|
+
class IGenerator(Protocol):
|
|
17
|
+
async def generate(self, query: TGenerateQuery) -> TGenerateResult:
|
|
18
|
+
"""Raises EngineError when the model can't be reached or answers unusably."""
|
|
19
|
+
...
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Optional decider capability: read the free-text fields (string, number,
|
|
2
|
+
integer) of an extraction request in one pass, as a whole record.
|
|
3
|
+
|
|
4
|
+
The extract use case checks ``isinstance(decider, IRecordReader)``; deciders
|
|
5
|
+
without it (e.g. System 1 models, which only answer typed questions) leave
|
|
6
|
+
text fields empty with a warning.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Protocol, runtime_checkable
|
|
12
|
+
|
|
13
|
+
from .t_extract import TExtractField, TFieldValue
|
|
14
|
+
from .t_input import TInputValue
|
|
15
|
+
from .tracker import Tracker
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@runtime_checkable
|
|
19
|
+
class IRecordReader(Protocol):
|
|
20
|
+
async def read_record(
|
|
21
|
+
self, input_value: TInputValue, fields: dict[str, TExtractField], tracker: Tracker
|
|
22
|
+
) -> dict[str, TFieldValue]:
|
|
23
|
+
"""Field name -> parsed value (None when absent or unreadable) and confidence;
|
|
24
|
+
add usage and warnings to ``tracker``. Raises EngineError on backend failure."""
|
|
25
|
+
...
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Optional capability: a decider that scores documents natively (e.g. with
|
|
2
|
+
embeddings) instead of one yes/no question per document.
|
|
3
|
+
|
|
4
|
+
The rerank use case checks ``isinstance(decider, IReranker)`` and uses it when present.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Protocol, runtime_checkable
|
|
10
|
+
|
|
11
|
+
from .tracker import Tracker
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@runtime_checkable
|
|
15
|
+
class IReranker(Protocol):
|
|
16
|
+
async def rerank_scores(self, query: str, documents: list[str], tracker: Tracker) -> list[float]:
|
|
17
|
+
"""One relevance score per document, in input order; higher is more relevant."""
|
|
18
|
+
...
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Pure helpers that turn a model's token probabilities (or text) into a
|
|
2
|
+
distribution over a fixed set of labels. Shared by logprob-reading engines."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import math
|
|
7
|
+
from typing import Optional, Sequence
|
|
8
|
+
|
|
9
|
+
from .t_label_query import TPosition
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def normalize(text: str) -> str:
|
|
13
|
+
"""Lowercase and keep only letters and digits, so "Reply_now", " reply now"
|
|
14
|
+
and "reply-now." all compare equal."""
|
|
15
|
+
return "".join(ch for ch in text.lower() if ch.isalnum())
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def label_distribution(
|
|
19
|
+
positions: Sequence[TPosition], labels: list[str]
|
|
20
|
+
) -> tuple[dict[str, float], float]:
|
|
21
|
+
"""Turn per-token probabilities into a distribution over labels.
|
|
22
|
+
|
|
23
|
+
Labels can span several tokens ("rebooking" may be "re" + "booking"). We
|
|
24
|
+
walk the generated tokens: probability on a token that fits exactly one
|
|
25
|
+
label goes to that label; if the generated token still fits several
|
|
26
|
+
labels, we continue to the next position, where the model's conditional
|
|
27
|
+
probabilities split it. Alternatives that fit several labels but were not
|
|
28
|
+
generated are split evenly (we can't see how the model would continue them).
|
|
29
|
+
|
|
30
|
+
Returns (distribution, label_mass): label_mass is how much of the model's
|
|
31
|
+
probability landed on valid labels.
|
|
32
|
+
"""
|
|
33
|
+
keys = {label: normalize(label) for label in labels}
|
|
34
|
+
acc = dict.fromkeys(labels, 0.0)
|
|
35
|
+
prefix, weight, started = "", 1.0, False
|
|
36
|
+
|
|
37
|
+
for pos in positions:
|
|
38
|
+
if not started and not normalize(pos.token):
|
|
39
|
+
continue # leading whitespace or punctuation
|
|
40
|
+
started = True
|
|
41
|
+
complete = [label for label, key in keys.items() if key == prefix and prefix]
|
|
42
|
+
next_step: Optional[tuple[str, float]] = None
|
|
43
|
+
for token, logprob in pos.candidates or [(pos.token, 0.0)]:
|
|
44
|
+
p = weight * math.exp(logprob)
|
|
45
|
+
extended = prefix + normalize(token)
|
|
46
|
+
fits = [label for label, key in keys.items() if key.startswith(extended)]
|
|
47
|
+
if extended != prefix:
|
|
48
|
+
fits = [label for label in fits if keys[label] != prefix]
|
|
49
|
+
if not fits or extended == prefix:
|
|
50
|
+
# The token ends the answer (or is off-script): credit a label
|
|
51
|
+
# that is already complete, if any.
|
|
52
|
+
for label in complete:
|
|
53
|
+
acc[label] += p / len(complete)
|
|
54
|
+
elif len(fits) == 1:
|
|
55
|
+
acc[fits[0]] += p
|
|
56
|
+
elif token == pos.token:
|
|
57
|
+
next_step = (extended, p)
|
|
58
|
+
else:
|
|
59
|
+
for label in fits:
|
|
60
|
+
acc[label] += p / len(fits)
|
|
61
|
+
if next_step is None:
|
|
62
|
+
break
|
|
63
|
+
prefix, weight = next_step
|
|
64
|
+
else:
|
|
65
|
+
# Ran out of tokens while the answer was still ambiguous.
|
|
66
|
+
if started and prefix:
|
|
67
|
+
fits = [label for label, key in keys.items() if key.startswith(prefix)]
|
|
68
|
+
for label in fits:
|
|
69
|
+
acc[label] += weight / len(fits)
|
|
70
|
+
|
|
71
|
+
mass = sum(acc.values())
|
|
72
|
+
if mass <= 0:
|
|
73
|
+
return acc, 0.0
|
|
74
|
+
return {k: v / mass for k, v in acc.items()}, mass
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def label_from_text(text: str, labels: list[str]) -> Optional[str]:
|
|
78
|
+
"""The longest label the text starts with, ignoring case and punctuation."""
|
|
79
|
+
norm = normalize(text)
|
|
80
|
+
fits = [label for label in labels if normalize(label) and norm.startswith(normalize(label))]
|
|
81
|
+
return max(fits, key=lambda label: len(normalize(label)), default=None)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def one_hot(labels: list[str], chosen: str) -> dict[str, float]:
|
|
85
|
+
return {label: 1.0 if label == chosen else 0.0 for label in labels}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def uniform(labels: list[str]) -> dict[str, float]:
|
|
89
|
+
return {label: 1.0 / len(labels) for label in labels}
|