dev-double 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. dev_double/__init__.py +3 -0
  2. dev_double/apps/__init__.py +1 -0
  3. dev_double/apps/cli/__init__.py +5 -0
  4. dev_double/apps/cli/main.py +122 -0
  5. dev_double/apps/client/__init__.py +5 -0
  6. dev_double/apps/client/http_client.py +102 -0
  7. dev_double/apps/composition.py +71 -0
  8. dev_double/apps/config.py +36 -0
  9. dev_double/apps/server/__init__.py +5 -0
  10. dev_double/apps/server/app.py +122 -0
  11. dev_double/apps/server/recorder.py +62 -0
  12. dev_double/apps/server/transport/__init__.py +39 -0
  13. dev_double/apps/server/transport/choice_use_cases.py +121 -0
  14. dev_double/apps/server/transport/common.py +37 -0
  15. dev_double/apps/server/transport/decide.py +131 -0
  16. dev_double/apps/server/transport/extract.py +93 -0
  17. dev_double/apps/server/transport/guard_judge.py +97 -0
  18. dev_double/apps/server/transport/rerank.py +55 -0
  19. dev_double/client.py +5 -0
  20. dev_double/core/__init__.py +1 -0
  21. dev_double/core/decision/__init__.py +62 -0
  22. dev_double/core/decision/answer_shaping.py +48 -0
  23. dev_double/core/decision/confidence.py +19 -0
  24. dev_double/core/decision/decider_basic_impl.py +65 -0
  25. dev_double/core/decision/decision_service_basic_impl.py +141 -0
  26. dev_double/core/decision/defaults.py +40 -0
  27. dev_double/core/decision/errors.py +7 -0
  28. dev_double/core/decision/extract_questions.py +40 -0
  29. dev_double/core/decision/field_extraction.py +53 -0
  30. dev_double/core/decision/i_clock.py +11 -0
  31. dev_double/core/decision/i_decider.py +36 -0
  32. dev_double/core/decision/i_decision_service.py +40 -0
  33. dev_double/core/decision/i_engine.py +27 -0
  34. dev_double/core/decision/i_extractor.py +19 -0
  35. dev_double/core/decision/i_generator.py +19 -0
  36. dev_double/core/decision/i_id_provider.py +9 -0
  37. dev_double/core/decision/i_record_reader.py +25 -0
  38. dev_double/core/decision/i_reranker.py +18 -0
  39. dev_double/core/decision/label_scoring.py +89 -0
  40. dev_double/core/decision/prompts.py +130 -0
  41. dev_double/core/decision/record_parsing.py +153 -0
  42. dev_double/core/decision/t_answer.py +32 -0
  43. dev_double/core/decision/t_classify.py +42 -0
  44. dev_double/core/decision/t_decide.py +26 -0
  45. dev_double/core/decision/t_extract.py +89 -0
  46. dev_double/core/decision/t_gate.py +36 -0
  47. dev_double/core/decision/t_generate.py +37 -0
  48. dev_double/core/decision/t_guard.py +38 -0
  49. dev_double/core/decision/t_input.py +8 -0
  50. dev_double/core/decision/t_judge.py +35 -0
  51. dev_double/core/decision/t_label_query.py +36 -0
  52. dev_double/core/decision/t_meta.py +17 -0
  53. dev_double/core/decision/t_question.py +70 -0
  54. dev_double/core/decision/t_rerank.py +36 -0
  55. dev_double/core/decision/t_route.py +28 -0
  56. dev_double/core/decision/t_usage.py +15 -0
  57. dev_double/core/decision/tracker.py +27 -0
  58. dev_double/core/decision/use_case_questions.py +80 -0
  59. dev_double/core/decision/value_parsing.py +109 -0
  60. dev_double/providers/__init__.py +1 -0
  61. dev_double/providers/mock/__init__.py +1 -0
  62. dev_double/providers/mock/decision/__init__.py +5 -0
  63. dev_double/providers/mock/decision/clock_mock_impl.py +19 -0
  64. dev_double/providers/mock/decision/engine_mock_impl.py +78 -0
  65. dev_double/providers/mock/decision/id_provider_mock_impl.py +15 -0
  66. dev_double/providers/needle/__init__.py +1 -0
  67. dev_double/providers/needle/decision/__init__.py +4 -0
  68. dev_double/providers/needle/decision/decider_needle_impl.py +117 -0
  69. dev_double/providers/needle/decision/record_tool.py +75 -0
  70. dev_double/providers/openai/__init__.py +1 -0
  71. dev_double/providers/openai/decision/__init__.py +3 -0
  72. dev_double/providers/openai/decision/engine_openai_impl.py +154 -0
  73. dev_double/providers/std/__init__.py +1 -0
  74. dev_double/providers/std/decision/__init__.py +4 -0
  75. dev_double/providers/std/decision/clock_std_impl.py +12 -0
  76. dev_double/providers/std/decision/id_provider_std_impl.py +12 -0
  77. dev_double/providers/systemone/__init__.py +1 -0
  78. dev_double/providers/systemone/decision/__init__.py +3 -0
  79. dev_double/providers/systemone/decision/decider_system_one_impl.py +100 -0
  80. dev_double-0.1.0.dist-info/METADATA +354 -0
  81. dev_double-0.1.0.dist-info/RECORD +84 -0
  82. dev_double-0.1.0.dist-info/WHEEL +4 -0
  83. dev_double-0.1.0.dist-info/entry_points.txt +2 -0
  84. dev_double-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,65 @@
1
+ """The default decider: renders a single-label prompt, asks an ``IEngine`` for
2
+ a distribution over the labels, and shapes the typed answer.
3
+
4
+ It also reads the free-text fields of an extraction request, as one JSON
5
+ record, when its engine is an ``IGenerator``; with a label-only engine those
6
+ fields are left empty with a warning."""
7
+
8
+ from __future__ import annotations
9
+
10
+ import asyncio
11
+
12
+ from . import prompts
13
+ from .answer_shaping import shape_answer
14
+ from .extract_questions import cannot_generate
15
+ from .i_decider import IDecider
16
+ from .i_engine import IEngine
17
+ from .i_generator import IGenerator
18
+ from .i_record_reader import IRecordReader
19
+ from .record_parsing import parse_record
20
+ from .t_answer import TAnswer
21
+ from .t_extract import TExtractField, TFieldValue
22
+ from .t_input import TInputValue
23
+ from .t_question import TQuestion
24
+ from .tracker import Tracker
25
+
26
+
27
+ class DeciderBasicImpl(IDecider, IRecordReader):
28
+ def __init__(self, engine: IEngine, max_concurrency: int = 4) -> None:
29
+ self.engine = engine
30
+ self._limit = asyncio.Semaphore(max_concurrency)
31
+
32
+ @property
33
+ def name(self) -> str:
34
+ return self.engine.name
35
+
36
+ @property
37
+ def model(self) -> str:
38
+ return self.engine.model
39
+
40
+ async def ask(
41
+ self, value: TInputValue, question: TQuestion, tracker: Tracker, label: str = ""
42
+ ) -> TAnswer:
43
+ query = prompts.query_for(prompts.render_input(value), question)
44
+ async with self._limit:
45
+ result = await self.engine.distribution(query)
46
+ tracker.usage.add(result.usage)
47
+ for warning in result.warnings:
48
+ tracker.warn(warning, label)
49
+ return shape_answer(question, result.probabilities)
50
+
51
+ async def read_record(
52
+ self, input_value: TInputValue, fields: dict[str, TExtractField], tracker: Tracker
53
+ ) -> dict[str, TFieldValue]:
54
+ if not isinstance(self.engine, IGenerator):
55
+ return {name: cannot_generate(name, tracker) for name in fields}
56
+ query = prompts.extract_record(prompts.render_input(input_value), fields)
57
+ async with self._limit:
58
+ result = await self.engine.generate(query)
59
+ tracker.usage.add(result.usage)
60
+ for warning in result.warnings:
61
+ tracker.warn(warning)
62
+ return parse_record(result.text, result.tokens, fields, result.confidence, tracker)
63
+
64
+ async def aclose(self) -> None:
65
+ await self.engine.aclose()
@@ -0,0 +1,141 @@
1
+ """The use cases over an ``IDecider``, with time and ids injected."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+
7
+ from . import use_case_questions as uq
8
+ from .field_extraction import read_fields
9
+ from .i_clock import IClock
10
+ from .i_decider import IDecider
11
+ from .i_decision_service import IDecisionService
12
+ from .i_id_provider import IIdProvider
13
+ from .i_reranker import IReranker
14
+ from .t_answer import TAnswer, TBinaryAnswer, TChoiceAnswer, TScaleAnswer
15
+ from .t_classify import TClassification, TClassifyRequest, TClassifyResponse
16
+ from .t_decide import TDecideRequest, TDecideResponse
17
+ from .t_extract import TExtractRequest, TExtractResponse
18
+ from .t_gate import TGateRequest, TGateResponse
19
+ from .t_guard import TGuardCheck, TGuardRequest, TGuardResponse
20
+ from .t_input import TInputValue
21
+ from .t_judge import TJudgeRequest, TJudgeResponse
22
+ from .t_meta import TMeta
23
+ from .t_question import TChoiceQuestion, TQuestion
24
+ from .t_rerank import TRerankRequest, TRerankResponse, TRerankResult
25
+ from .t_route import TRouteRequest, TRouteResponse
26
+ from .tracker import Tracker
27
+
28
+
29
+ class DecisionServiceBasicImpl(IDecisionService):
30
+ def __init__(self, decider: IDecider, clock: IClock, ids: IIdProvider) -> None:
31
+ self.decider = decider
32
+ self._clock = clock
33
+ self._ids = ids
34
+
35
+ @property
36
+ def name(self) -> str:
37
+ return self.decider.name
38
+
39
+ @property
40
+ def model(self) -> str:
41
+ return self.decider.model
42
+
43
+ async def aclose(self) -> None:
44
+ await self.decider.aclose()
45
+
46
+ def _tracker(self) -> Tracker:
47
+ return Tracker(self._clock)
48
+
49
+ def _meta(self, t: Tracker) -> TMeta:
50
+ return t.meta(self.decider.name, self.decider.model)
51
+
52
+ async def _ask_many(
53
+ self, value: TInputValue, questions: dict[str, TQuestion], t: Tracker
54
+ ) -> dict[str, TAnswer]:
55
+ answers = await asyncio.gather(
56
+ *(self.decider.ask(value, q, t, label=qid) for qid, q in questions.items())
57
+ )
58
+ return dict(zip(questions, answers))
59
+
60
+ async def decide(self, req: TDecideRequest) -> TDecideResponse:
61
+ t = self._tracker()
62
+ answers = await self._ask_many(req.input, req.questions, t)
63
+ return TDecideResponse(answers=answers, meta=self._meta(t))
64
+
65
+ async def route(self, req: TRouteRequest) -> TRouteResponse:
66
+ t = self._tracker()
67
+ a = await self.decider.ask(req.input, uq.route_question(req), t)
68
+ assert isinstance(a, TChoiceAnswer)
69
+ return TRouteResponse(a.value, a.probabilities, a.confidence, self._meta(t))
70
+
71
+ async def guard(self, req: TGuardRequest) -> TGuardResponse:
72
+ t = self._tracker()
73
+ answers = await self._ask_many(req.input, uq.guard_questions(req), t)
74
+ checks = {}
75
+ for name, a in answers.items():
76
+ assert isinstance(a, TBinaryAnswer)
77
+ checks[name] = TGuardCheck(probability=a.probability, flagged=a.probability >= req.threshold)
78
+ flagged = [name for name, c in checks.items() if c.flagged]
79
+ return TGuardResponse(allowed=not flagged, flagged=flagged, checks=checks, meta=self._meta(t))
80
+
81
+ async def gate(self, req: TGateRequest) -> TGateResponse:
82
+ t = self._tracker()
83
+ a = await self.decider.ask(uq.gate_payload(req), uq.gate_question(req), t)
84
+ assert isinstance(a, TChoiceAnswer)
85
+ return TGateResponse(a.value, a.probabilities, a.confidence, self._meta(t))
86
+
87
+ async def classify(self, req: TClassifyRequest) -> TClassifyResponse:
88
+ t = self._tracker()
89
+ q = TChoiceQuestion(question=req.question, options=req.labels)
90
+ answers = await asyncio.gather(
91
+ *(self.decider.ask(item, q, t, label=f"inputs[{i}]") for i, item in enumerate(req.items()))
92
+ )
93
+ results = []
94
+ for a in answers:
95
+ assert isinstance(a, TChoiceAnswer)
96
+ results.append(TClassification(a.value, a.probabilities, a.confidence))
97
+ return TClassifyResponse(results=results, meta=self._meta(t))
98
+
99
+ async def judge(self, req: TJudgeRequest) -> TJudgeResponse:
100
+ t = self._tracker()
101
+ q = uq.judge_question(req)
102
+ a = await self.decider.ask(uq.judge_payload(req), q, t)
103
+ assert isinstance(a, TScaleAnswer)
104
+ return TJudgeResponse(
105
+ score=a.value,
106
+ normalized=round(a.value / (len(q.levels) - 1), 4),
107
+ level=a.level,
108
+ probabilities=a.probabilities,
109
+ legend=a.legend,
110
+ confidence=a.confidence,
111
+ meta=self._meta(t),
112
+ )
113
+
114
+ async def extract(self, req: TExtractRequest) -> TExtractResponse:
115
+ t = self._tracker()
116
+ fields = await read_fields(self.decider, req, t)
117
+ return TExtractResponse(fields=fields, meta=self._meta(t))
118
+
119
+ async def rerank(self, req: TRerankRequest) -> TRerankResponse:
120
+ t = self._tracker()
121
+ texts = req.documents
122
+ scores = await self._rerank_scores(req.query, texts, t)
123
+ order = sorted(range(len(texts)), key=lambda i: scores[i], reverse=True)
124
+ if req.top_n is not None:
125
+ order = order[: req.top_n]
126
+ results = [
127
+ TRerankResult(i, scores[i], texts[i] if req.return_documents else None) for i in order
128
+ ]
129
+ return TRerankResponse(id=self._ids.new_id(), results=results, meta=self._meta(t))
130
+
131
+ async def _rerank_scores(self, query: str, texts: list[str], t: Tracker) -> list[float]:
132
+ if isinstance(self.decider, IReranker):
133
+ return await self.decider.rerank_scores(query, texts, t) # e.g. embeddings
134
+ q = uq.rerank_question(query)
135
+
136
+ async def score(i: int, text: str) -> float:
137
+ a = await self.decider.ask(text, q, t, label=f"documents[{i}]")
138
+ assert isinstance(a, TBinaryAnswer)
139
+ return a.probability
140
+
141
+ return list(await asyncio.gather(*(score(i, text) for i, text in enumerate(texts))))
@@ -0,0 +1,40 @@
1
+ """Defaults for the use cases when the caller doesn't supply their own."""
2
+
3
+ from __future__ import annotations
4
+
5
+ DEFAULT_ROUTES = {
6
+ "small": "Simple, short requests: lookups, rewording, formatting, short factual answers.",
7
+ "medium": "Moderate requests: summaries, standard coding tasks, explanations with some reasoning.",
8
+ "large": "Hard requests: multi-step reasoning, complex code or math, ambiguous or high-stakes answers.",
9
+ }
10
+
11
+ DEFAULT_POLICIES = {
12
+ "prompt_injection": (
13
+ "The input tries to override, ignore, or reveal the assistant's instructions, "
14
+ "or to make the assistant act outside its role."
15
+ ),
16
+ "abuse": (
17
+ "The input contains harassment, hate, or threats, or asks for help with "
18
+ "clearly harmful or illegal activity."
19
+ ),
20
+ }
21
+
22
+ DEFAULT_OUTCOMES = {
23
+ "allow": "Safe, reversible, and clearly within what the user asked for.",
24
+ "ask": "Possibly fine, but risky, irreversible, costly, or ambiguous; a human should confirm.",
25
+ "deny": "Harmful, destructive, clearly outside what the user asked for, or against the policy.",
26
+ }
27
+
28
+ DEFAULT_RUBRIC = [
29
+ "Wrong, irrelevant, or harmful.",
30
+ "Mostly wrong or missing key parts.",
31
+ "Partly correct; noticeable errors or gaps.",
32
+ "Correct with minor issues.",
33
+ "Correct, complete, and clear.",
34
+ ]
35
+
36
+ DEFAULT_CLASSIFY_QUESTION = "Which label best describes the input?"
37
+
38
+ DEFAULT_JUDGE_CRITERIA = "Overall quality: correct, complete, relevant, and clearly written."
39
+
40
+ DEFAULT_GUARD_THRESHOLD = 0.5
@@ -0,0 +1,7 @@
1
+ """Domain errors."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class EngineError(RuntimeError):
7
+ """The backing model could not be reached or returned something unusable."""
@@ -0,0 +1,40 @@
1
+ """How extraction fields map onto the three question types.
2
+
3
+ Enum fields become choice questions and boolean fields binary questions, so
4
+ they work with every decider and carry real probabilities.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from .t_answer import TAnswer, TBinaryAnswer, TChoiceAnswer
10
+ from .t_extract import EMPTY, TExtractField, TFieldValue
11
+ from .t_question import TBinaryQuestion, TChoiceQuestion
12
+ from .tracker import Tracker
13
+
14
+
15
+ def field_question_text(name: str, field: TExtractField) -> str:
16
+ question = f'What is the value of the field "{name}" in the input?'
17
+ return f"{question} {field.description}" if field.description else question
18
+
19
+
20
+ def enum_question(name: str, field: TExtractField) -> TChoiceQuestion:
21
+ assert isinstance(field.options, dict) # TExtractField normalizes enum options to a dict
22
+ return TChoiceQuestion(question=field_question_text(name, field), options=dict(field.options))
23
+
24
+
25
+ def boolean_question(name: str, field: TExtractField) -> TBinaryQuestion:
26
+ return TBinaryQuestion(question=f"Is this true for the input? {field.description or name}")
27
+
28
+
29
+ def value_from_answer(answer: TAnswer) -> TFieldValue:
30
+ if isinstance(answer, TBinaryAnswer):
31
+ return TFieldValue(answer.value, answer.confidence, probability=answer.probability)
32
+ if isinstance(answer, TChoiceAnswer):
33
+ return TFieldValue(answer.value, answer.confidence, probabilities=answer.probabilities)
34
+ raise TypeError(f"No field value for a {type(answer).__name__}.")
35
+
36
+
37
+ def cannot_generate(name: str, tracker: Tracker) -> TFieldValue:
38
+ """The field needs free text and the engine can't generate any."""
39
+ tracker.warn(f"engine can't generate text: field {name} left empty")
40
+ return EMPTY
@@ -0,0 +1,53 @@
1
+ """Reads every field of an extraction request with whatever the decider can do.
2
+
3
+ - A native ``IExtractor`` (e.g. one tool call per record) takes the whole request.
4
+ - Otherwise enum and boolean fields are asked as choice and binary questions
5
+ (real probabilities), and all string, number, and integer fields go to one
6
+ ``IRecordReader.read_record`` call when the decider has it (a whole-record
7
+ prompt beat one prompt per field in our evals), else they are left empty
8
+ with a warning.
9
+
10
+ The questions and the record read run concurrently; results keep the
11
+ request's field order.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import asyncio
17
+
18
+ from .extract_questions import boolean_question, cannot_generate, enum_question, value_from_answer
19
+ from .i_decider import IDecider
20
+ from .i_extractor import IExtractor
21
+ from .i_record_reader import IRecordReader
22
+ from .t_extract import EMPTY, TEXT_TYPES, TExtractField, TExtractRequest, TFieldValue
23
+ from .t_input import TInputValue
24
+ from .tracker import Tracker
25
+
26
+
27
+ async def _ask(decider: IDecider, value: TInputValue, name: str, field: TExtractField, t: Tracker) -> TFieldValue:
28
+ question = enum_question(name, field) if field.type == "enum" else boolean_question(name, field)
29
+ return value_from_answer(await decider.ask(value, question, t, label=name))
30
+
31
+
32
+ async def _read_text(
33
+ decider: IDecider, value: TInputValue, fields: dict[str, TExtractField], t: Tracker
34
+ ) -> dict[str, TFieldValue]:
35
+ if not fields:
36
+ return {}
37
+ if isinstance(decider, IRecordReader):
38
+ return await decider.read_record(value, fields, t)
39
+ return {name: cannot_generate(name, t) for name in fields}
40
+
41
+
42
+ async def read_fields(decider: IDecider, req: TExtractRequest, t: Tracker) -> dict[str, TFieldValue]:
43
+ if isinstance(decider, IExtractor):
44
+ got = await decider.extract(req, t)
45
+ return {name: got.get(name, EMPTY) for name in req.fields}
46
+ text = {name: f for name, f in req.fields.items() if f.type in TEXT_TYPES}
47
+ scored = {name: f for name, f in req.fields.items() if f.type not in TEXT_TYPES}
48
+ record, *answers = await asyncio.gather(
49
+ _read_text(decider, req.input, text, t),
50
+ *(_ask(decider, req.input, name, f, t) for name, f in scored.items()),
51
+ )
52
+ got = {**record, **dict(zip(scored, answers))}
53
+ return {name: got.get(name, EMPTY) for name in req.fields}
@@ -0,0 +1,11 @@
1
+ """Time port."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Protocol
6
+
7
+
8
+ class IClock(Protocol):
9
+ def now(self) -> float:
10
+ """Monotonic seconds; only differences are meaningful."""
11
+ ...
@@ -0,0 +1,36 @@
1
+ """Answers one typed question about one input.
2
+
3
+ ``DeciderBasicImpl`` does it by prompting an ``IEngine``; decision models that
4
+ answer typed questions natively (Kev, Laya, Needle) implement this directly.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Protocol, runtime_checkable
10
+
11
+ from .t_answer import TAnswer
12
+ from .t_input import TInputValue
13
+ from .t_question import TQuestion
14
+ from .tracker import Tracker
15
+
16
+
17
+ @runtime_checkable
18
+ class IDecider(Protocol):
19
+ @property
20
+ def name(self) -> str:
21
+ """Engine name, reported in response meta."""
22
+ ...
23
+
24
+ @property
25
+ def model(self) -> str:
26
+ """Model name, reported in response meta."""
27
+ ...
28
+
29
+ async def ask(
30
+ self, value: TInputValue, question: TQuestion, tracker: Tracker, label: str = ""
31
+ ) -> TAnswer:
32
+ """Answer ``question`` about ``value``; add usage and warnings (prefixed
33
+ with ``label``) to ``tracker``. Raises EngineError on backend failure."""
34
+ ...
35
+
36
+ async def aclose(self) -> None: ...
@@ -0,0 +1,40 @@
1
+ """The task-shaped use cases, built on the three question types."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Protocol
6
+
7
+ from .t_classify import TClassifyRequest, TClassifyResponse
8
+ from .t_decide import TDecideRequest, TDecideResponse
9
+ from .t_extract import TExtractRequest, TExtractResponse
10
+ from .t_gate import TGateRequest, TGateResponse
11
+ from .t_guard import TGuardRequest, TGuardResponse
12
+ from .t_judge import TJudgeRequest, TJudgeResponse
13
+ from .t_rerank import TRerankRequest, TRerankResponse
14
+ from .t_route import TRouteRequest, TRouteResponse
15
+
16
+
17
+ class IDecisionService(Protocol):
18
+ @property
19
+ def name(self) -> str: ...
20
+
21
+ @property
22
+ def model(self) -> str: ...
23
+
24
+ async def decide(self, req: TDecideRequest) -> TDecideResponse: ...
25
+
26
+ async def route(self, req: TRouteRequest) -> TRouteResponse: ...
27
+
28
+ async def guard(self, req: TGuardRequest) -> TGuardResponse: ...
29
+
30
+ async def gate(self, req: TGateRequest) -> TGateResponse: ...
31
+
32
+ async def classify(self, req: TClassifyRequest) -> TClassifyResponse: ...
33
+
34
+ async def judge(self, req: TJudgeRequest) -> TJudgeResponse: ...
35
+
36
+ async def rerank(self, req: TRerankRequest) -> TRerankResponse: ...
37
+
38
+ async def extract(self, req: TExtractRequest) -> TExtractResponse: ...
39
+
40
+ async def aclose(self) -> None: ...
@@ -0,0 +1,27 @@
1
+ """The LLM label-scoring port: given a rendered prompt and a fixed set of
2
+ labels, return a probability distribution over those labels.
3
+
4
+ Everything above this port (question types, use cases, confidence) is
5
+ engine-agnostic, so swapping the model is a configuration change.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Protocol, runtime_checkable
11
+
12
+ from .t_label_query import TLabelQuery, TLabelResult
13
+
14
+
15
+ @runtime_checkable
16
+ class IEngine(Protocol):
17
+ @property
18
+ def name(self) -> str: ...
19
+
20
+ @property
21
+ def model(self) -> str: ...
22
+
23
+ async def distribution(self, query: TLabelQuery) -> TLabelResult:
24
+ """Raises EngineError when the model can't be reached or answers unusably."""
25
+ ...
26
+
27
+ async def aclose(self) -> None: ...
@@ -0,0 +1,19 @@
1
+ """Optional capability: a decider that extracts a whole record natively (e.g.
2
+ one tool call with every field) instead of one question per field.
3
+
4
+ The extract use case checks ``isinstance(decider, IExtractor)`` and uses it when present.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Protocol, runtime_checkable
10
+
11
+ from .t_extract import TExtractRequest, TFieldValue
12
+ from .tracker import Tracker
13
+
14
+
15
+ @runtime_checkable
16
+ class IExtractor(Protocol):
17
+ async def extract(self, req: TExtractRequest, tracker: Tracker) -> dict[str, TFieldValue]:
18
+ """Field name -> value. Fields left out are reported as empty."""
19
+ ...
@@ -0,0 +1,19 @@
1
+ """Optional engine capability: generate a short free-text answer.
2
+
3
+ Label scoring (``IEngine.distribution``) can't produce values that aren't known
4
+ up front, such as a vendor name or an amount. Engines that can also generate
5
+ text implement this; ``DeciderBasicImpl`` checks ``isinstance(engine, IGenerator)``.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Protocol, runtime_checkable
11
+
12
+ from .t_generate import TGenerateQuery, TGenerateResult
13
+
14
+
15
+ @runtime_checkable
16
+ class IGenerator(Protocol):
17
+ async def generate(self, query: TGenerateQuery) -> TGenerateResult:
18
+ """Raises EngineError when the model can't be reached or answers unusably."""
19
+ ...
@@ -0,0 +1,9 @@
1
+ """Randomness port: mints response ids."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Protocol
6
+
7
+
8
+ class IIdProvider(Protocol):
9
+ def new_id(self) -> str: ...
@@ -0,0 +1,25 @@
1
+ """Optional decider capability: read the free-text fields (string, number,
2
+ integer) of an extraction request in one pass, as a whole record.
3
+
4
+ The extract use case checks ``isinstance(decider, IRecordReader)``; deciders
5
+ without it (e.g. System 1 models, which only answer typed questions) leave
6
+ text fields empty with a warning.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import Protocol, runtime_checkable
12
+
13
+ from .t_extract import TExtractField, TFieldValue
14
+ from .t_input import TInputValue
15
+ from .tracker import Tracker
16
+
17
+
18
+ @runtime_checkable
19
+ class IRecordReader(Protocol):
20
+ async def read_record(
21
+ self, input_value: TInputValue, fields: dict[str, TExtractField], tracker: Tracker
22
+ ) -> dict[str, TFieldValue]:
23
+ """Field name -> parsed value (None when absent or unreadable) and confidence;
24
+ add usage and warnings to ``tracker``. Raises EngineError on backend failure."""
25
+ ...
@@ -0,0 +1,18 @@
1
+ """Optional capability: a decider that scores documents natively (e.g. with
2
+ embeddings) instead of one yes/no question per document.
3
+
4
+ The rerank use case checks ``isinstance(decider, IReranker)`` and uses it when present.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Protocol, runtime_checkable
10
+
11
+ from .tracker import Tracker
12
+
13
+
14
+ @runtime_checkable
15
+ class IReranker(Protocol):
16
+ async def rerank_scores(self, query: str, documents: list[str], tracker: Tracker) -> list[float]:
17
+ """One relevance score per document, in input order; higher is more relevant."""
18
+ ...
@@ -0,0 +1,89 @@
1
+ """Pure helpers that turn a model's token probabilities (or text) into a
2
+ distribution over a fixed set of labels. Shared by logprob-reading engines."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import math
7
+ from typing import Optional, Sequence
8
+
9
+ from .t_label_query import TPosition
10
+
11
+
12
+ def normalize(text: str) -> str:
13
+ """Lowercase and keep only letters and digits, so "Reply_now", " reply now"
14
+ and "reply-now." all compare equal."""
15
+ return "".join(ch for ch in text.lower() if ch.isalnum())
16
+
17
+
18
+ def label_distribution(
19
+ positions: Sequence[TPosition], labels: list[str]
20
+ ) -> tuple[dict[str, float], float]:
21
+ """Turn per-token probabilities into a distribution over labels.
22
+
23
+ Labels can span several tokens ("rebooking" may be "re" + "booking"). We
24
+ walk the generated tokens: probability on a token that fits exactly one
25
+ label goes to that label; if the generated token still fits several
26
+ labels, we continue to the next position, where the model's conditional
27
+ probabilities split it. Alternatives that fit several labels but were not
28
+ generated are split evenly (we can't see how the model would continue them).
29
+
30
+ Returns (distribution, label_mass): label_mass is how much of the model's
31
+ probability landed on valid labels.
32
+ """
33
+ keys = {label: normalize(label) for label in labels}
34
+ acc = dict.fromkeys(labels, 0.0)
35
+ prefix, weight, started = "", 1.0, False
36
+
37
+ for pos in positions:
38
+ if not started and not normalize(pos.token):
39
+ continue # leading whitespace or punctuation
40
+ started = True
41
+ complete = [label for label, key in keys.items() if key == prefix and prefix]
42
+ next_step: Optional[tuple[str, float]] = None
43
+ for token, logprob in pos.candidates or [(pos.token, 0.0)]:
44
+ p = weight * math.exp(logprob)
45
+ extended = prefix + normalize(token)
46
+ fits = [label for label, key in keys.items() if key.startswith(extended)]
47
+ if extended != prefix:
48
+ fits = [label for label in fits if keys[label] != prefix]
49
+ if not fits or extended == prefix:
50
+ # The token ends the answer (or is off-script): credit a label
51
+ # that is already complete, if any.
52
+ for label in complete:
53
+ acc[label] += p / len(complete)
54
+ elif len(fits) == 1:
55
+ acc[fits[0]] += p
56
+ elif token == pos.token:
57
+ next_step = (extended, p)
58
+ else:
59
+ for label in fits:
60
+ acc[label] += p / len(fits)
61
+ if next_step is None:
62
+ break
63
+ prefix, weight = next_step
64
+ else:
65
+ # Ran out of tokens while the answer was still ambiguous.
66
+ if started and prefix:
67
+ fits = [label for label, key in keys.items() if key.startswith(prefix)]
68
+ for label in fits:
69
+ acc[label] += weight / len(fits)
70
+
71
+ mass = sum(acc.values())
72
+ if mass <= 0:
73
+ return acc, 0.0
74
+ return {k: v / mass for k, v in acc.items()}, mass
75
+
76
+
77
+ def label_from_text(text: str, labels: list[str]) -> Optional[str]:
78
+ """The longest label the text starts with, ignoring case and punctuation."""
79
+ norm = normalize(text)
80
+ fits = [label for label in labels if normalize(label) and norm.startswith(normalize(label))]
81
+ return max(fits, key=lambda label: len(normalize(label)), default=None)
82
+
83
+
84
+ def one_hot(labels: list[str], chosen: str) -> dict[str, float]:
85
+ return {label: 1.0 if label == chosen else 0.0 for label in labels}
86
+
87
+
88
+ def uniform(labels: list[str]) -> dict[str, float]:
89
+ return {label: 1.0 / len(labels) for label in labels}