semantic-operators 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- semantic_operators/__init__.py +6 -0
- semantic_operators/bench.py +172 -0
- semantic_operators/provider.py +22 -0
- semantic_operators/providers/__init__.py +0 -0
- semantic_operators/providers/jev.py +67 -0
- semantic_operators/providers/laya.py +69 -0
- semantic_operators/py.typed +0 -0
- semantic_operators/types.py +57 -0
- semantic_operators-0.1.0.dist-info/METADATA +195 -0
- semantic_operators-0.1.0.dist-info/RECORD +12 -0
- semantic_operators-0.1.0.dist-info/WHEEL +4 -0
- semantic_operators-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Semantic Operators: one small interface for System One models."""
|
|
2
|
+
|
|
3
|
+
from .provider import AsyncProvider, Provider
|
|
4
|
+
from .types import Answer, Boolean, Choice, Question, Score, State
|
|
5
|
+
|
|
6
|
+
__all__ = ["Answer", "AsyncProvider", "Boolean", "Choice", "Provider", "Question", "Score", "State"]
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""A tiny benchmark: run labeled cases through a provider and score the answers.
|
|
2
|
+
|
|
3
|
+
Higher layer: built only on the base layer (types + Provider).
|
|
4
|
+
|
|
5
|
+
For each question we report:
|
|
6
|
+
- accuracy: how often the provider's decision matches the label.
|
|
7
|
+
- p(correct): the average probability the provider gave the labeled answer.
|
|
8
|
+
Two providers can be equally accurate while one is far more sure
|
|
9
|
+
of itself when it's right (and, worse, when it's wrong).
|
|
10
|
+
|
|
11
|
+
``run_async`` is ``run`` for an ``AsyncProvider``, with up to ``concurrency``
|
|
12
|
+
calls in flight at once.
|
|
13
|
+
|
|
14
|
+
``stability`` compares runs of differently worded versions of the same
|
|
15
|
+
questions: how often does the decision stay the same when only the wording changes?
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
import statistics
|
|
20
|
+
import time
|
|
21
|
+
from collections.abc import Mapping, Sequence
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
|
|
24
|
+
from .provider import AsyncProvider, Provider
|
|
25
|
+
from .types import Answer, Boolean, Choice, Question, Score, State
|
|
26
|
+
|
|
27
|
+
# A decision is an answer reduced to one discrete label:
|
|
28
|
+
# a bool for Boolean, an option name for Choice, a level index (0, 1, ...) for Score.
|
|
29
|
+
Decision = bool | str | int
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class Case:
|
|
34
|
+
"""One labeled input. ``expected`` maps question name -> the correct decision.
|
|
35
|
+
|
|
36
|
+
Score labels are level indexes, not level text, so the same labels still apply
|
|
37
|
+
when the rubric is reworded.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
state: State
|
|
41
|
+
expected: dict[str, Decision]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class QuestionStats:
|
|
46
|
+
correct: int = 0
|
|
47
|
+
total: int = 0
|
|
48
|
+
p_correct: list[float] = field(default_factory=list)
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def accuracy(self) -> float:
|
|
52
|
+
return self.correct / self.total if self.total else 0.0
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def mean_p_correct(self) -> float:
|
|
56
|
+
return statistics.fmean(self.p_correct) if self.p_correct else 0.0
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class Miss:
|
|
61
|
+
case: int
|
|
62
|
+
question: str
|
|
63
|
+
expected: Decision
|
|
64
|
+
got: Answer
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class Report:
|
|
69
|
+
questions: dict[str, QuestionStats]
|
|
70
|
+
latencies_ms: list[float] # per call
|
|
71
|
+
total_ms: float # wall-clock time for all cases (excludes warm-up)
|
|
72
|
+
misses: list[Miss]
|
|
73
|
+
decisions: list[dict[str, Decision]] # per case, per question
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def decide(question: Question, answer: Answer) -> Decision:
|
|
77
|
+
"""Reduce an answer to a discrete decision (Score: round half up to a level index)."""
|
|
78
|
+
match question:
|
|
79
|
+
case Boolean() | Choice():
|
|
80
|
+
return answer.value
|
|
81
|
+
case Score():
|
|
82
|
+
return int(float(answer.value) + 0.5)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def run(provider: Provider, questions: Mapping[str, Question], cases: Sequence[Case],
|
|
86
|
+
warmup: int = 1) -> Report:
|
|
87
|
+
"""Ask every case's state all ``questions`` in one call per case.
|
|
88
|
+
|
|
89
|
+
The first ``warmup`` cases are also run once beforehand, untimed, so one-time
|
|
90
|
+
setup (loading, connecting) doesn't count as latency.
|
|
91
|
+
"""
|
|
92
|
+
for case in cases[:warmup]:
|
|
93
|
+
provider.ask(case.state, questions)
|
|
94
|
+
|
|
95
|
+
answers: list[dict[str, Answer]] = []
|
|
96
|
+
latencies: list[float] = []
|
|
97
|
+
begin = time.perf_counter()
|
|
98
|
+
for case in cases:
|
|
99
|
+
start = time.perf_counter()
|
|
100
|
+
answers.append(provider.ask(case.state, questions))
|
|
101
|
+
latencies.append((time.perf_counter() - start) * 1000)
|
|
102
|
+
total = (time.perf_counter() - begin) * 1000
|
|
103
|
+
return score(questions, cases, answers, latencies, total)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
async def run_async(provider: AsyncProvider, questions: Mapping[str, Question],
|
|
107
|
+
cases: Sequence[Case], warmup: int = 1, concurrency: int = 4) -> Report:
|
|
108
|
+
"""Like ``run``, but with up to ``concurrency`` calls in flight at once.
|
|
109
|
+
|
|
110
|
+
Per-call latency is measured from when a call gets a concurrency slot. It still
|
|
111
|
+
includes any queueing inside the provider (``AsyncLaya`` runs one call at a time),
|
|
112
|
+
so under concurrency it measures what a caller waits, not model speed.
|
|
113
|
+
``total_ms`` shows the throughput gain, if any.
|
|
114
|
+
"""
|
|
115
|
+
for case in cases[:warmup]:
|
|
116
|
+
await provider.ask(case.state, questions)
|
|
117
|
+
|
|
118
|
+
slots = asyncio.Semaphore(concurrency)
|
|
119
|
+
|
|
120
|
+
async def one(case: Case) -> tuple[dict[str, Answer], float]:
|
|
121
|
+
async with slots:
|
|
122
|
+
start = time.perf_counter()
|
|
123
|
+
answers = await provider.ask(case.state, questions)
|
|
124
|
+
return answers, (time.perf_counter() - start) * 1000
|
|
125
|
+
|
|
126
|
+
begin = time.perf_counter()
|
|
127
|
+
results = await asyncio.gather(*(one(case) for case in cases)) # keeps case order
|
|
128
|
+
total = (time.perf_counter() - begin) * 1000
|
|
129
|
+
return score(questions, cases, [a for a, _ in results], [ms for _, ms in results], total)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def score(questions: Mapping[str, Question], cases: Sequence[Case],
|
|
133
|
+
answers: Sequence[Mapping[str, Answer]], latencies_ms: list[float],
|
|
134
|
+
total_ms: float) -> Report:
|
|
135
|
+
"""Score already-collected answers (one dict of answers per case) against the labels."""
|
|
136
|
+
stats = {name: QuestionStats() for name in questions}
|
|
137
|
+
misses: list[Miss] = []
|
|
138
|
+
decisions: list[dict[str, Decision]] = []
|
|
139
|
+
for i, (case, case_answers) in enumerate(zip(cases, answers, strict=True)):
|
|
140
|
+
decisions.append({name: decide(q, case_answers[name]) for name, q in questions.items()})
|
|
141
|
+
for name, label in case.expected.items():
|
|
142
|
+
answer, s = case_answers[name], stats[name]
|
|
143
|
+
ok = decisions[-1][name] == label
|
|
144
|
+
s.total += 1
|
|
145
|
+
s.correct += ok
|
|
146
|
+
s.p_correct.append(answer.probabilities[_key(questions[name], label)])
|
|
147
|
+
if not ok:
|
|
148
|
+
misses.append(Miss(i, name, label, answer))
|
|
149
|
+
return Report(stats, latencies_ms, total_ms, misses, decisions)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def stability(reports: Sequence[Report]) -> dict[str, float]:
|
|
153
|
+
"""Per question: the fraction of cases whose decision is identical in every report.
|
|
154
|
+
|
|
155
|
+
Pass reports from differently worded versions of the same questions. Labels play
|
|
156
|
+
no part: a model can be perfectly stable and consistently wrong.
|
|
157
|
+
"""
|
|
158
|
+
names = reports[0].decisions[0].keys()
|
|
159
|
+
cases = range(len(reports[0].decisions))
|
|
160
|
+
return {name: sum(len({r.decisions[i][name] for r in reports}) == 1 for i in cases) / len(cases)
|
|
161
|
+
for name in names}
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _key(question: Question, label: Decision) -> str:
|
|
165
|
+
# Answer.probabilities keys: "true"/"false", option names, or level text.
|
|
166
|
+
match question:
|
|
167
|
+
case Boolean():
|
|
168
|
+
return str(label).lower()
|
|
169
|
+
case Choice():
|
|
170
|
+
return str(label)
|
|
171
|
+
case Score():
|
|
172
|
+
return question.levels[int(label)]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""The provider interfaces: anything with this ``ask`` method is a provider.
|
|
2
|
+
|
|
3
|
+
They're Protocols, so providers don't inherit from anything or register anywhere.
|
|
4
|
+
``AsyncProvider`` is the same contract with ``ask`` awaited.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
from typing import Protocol
|
|
9
|
+
|
|
10
|
+
from .types import Answer, Question, State
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Provider(Protocol):
|
|
14
|
+
def ask(self, state: State, questions: Mapping[str, Question]) -> dict[str, Answer]:
|
|
15
|
+
"""Answer every named question about ``state``. Returns answers by the same names."""
|
|
16
|
+
...
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class AsyncProvider(Protocol):
|
|
20
|
+
async def ask(self, state: State, questions: Mapping[str, Question]) -> dict[str, Answer]:
|
|
21
|
+
"""Answer every named question about ``state``. Returns answers by the same names."""
|
|
22
|
+
...
|
|
File without changes
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Jev, via the TypeSafe SDK (``pip install semantic-operators[jev]``).
|
|
2
|
+
|
|
3
|
+
Translation only: our questions -> SDK questions, one ``system_one`` call,
|
|
4
|
+
SDK answers -> our ``Answer``. You create and own the SDK client:
|
|
5
|
+
``TypeSafeClient`` for ``Jev``, ``AsyncTypeSafeClient`` for ``AsyncJev``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
|
|
10
|
+
import typesafe_sdk as ts
|
|
11
|
+
|
|
12
|
+
from ..types import Answer, Boolean, Choice, Question, Score, State
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Jev:
|
|
16
|
+
def __init__(self, client: ts.TypeSafeClient, model: str = "jev-latest") -> None:
|
|
17
|
+
self.client = client
|
|
18
|
+
self.model = model
|
|
19
|
+
|
|
20
|
+
def ask(self, state: State, questions: Mapping[str, Question]) -> dict[str, Answer]:
|
|
21
|
+
response = self.client.system_one(
|
|
22
|
+
state=state,
|
|
23
|
+
questions={name: _to_jev(q) for name, q in questions.items()},
|
|
24
|
+
model=self.model,
|
|
25
|
+
)
|
|
26
|
+
return {name: _from_jev(q, response.answers[name]) for name, q in questions.items()}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class AsyncJev:
|
|
30
|
+
def __init__(self, client: ts.AsyncTypeSafeClient, model: str = "jev-latest") -> None:
|
|
31
|
+
self.client = client
|
|
32
|
+
self.model = model
|
|
33
|
+
|
|
34
|
+
async def ask(self, state: State, questions: Mapping[str, Question]) -> dict[str, Answer]:
|
|
35
|
+
response = await self.client.system_one(
|
|
36
|
+
state=state,
|
|
37
|
+
questions={name: _to_jev(q) for name, q in questions.items()},
|
|
38
|
+
model=self.model,
|
|
39
|
+
)
|
|
40
|
+
return {name: _from_jev(q, response.answers[name]) for name, q in questions.items()}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _to_jev(question: Question) -> ts.Noul | ts.Choice | ts.Score:
|
|
44
|
+
match question:
|
|
45
|
+
case Boolean():
|
|
46
|
+
described = question.true is not None or question.false is not None
|
|
47
|
+
criteria = {"true": question.true, "false": question.false} if described else None
|
|
48
|
+
return ts.Noul(instructions=question.instructions, criteria=criteria)
|
|
49
|
+
case Choice():
|
|
50
|
+
return ts.Choice(instructions=question.instructions, criteria=dict(question.options))
|
|
51
|
+
case Score():
|
|
52
|
+
return ts.Score(instructions=question.instructions, criteria=list(question.levels))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _from_jev(question: Question, answer: ts.Answer) -> Answer:
|
|
56
|
+
match question:
|
|
57
|
+
case Boolean():
|
|
58
|
+
# Jev returns one number: the probability the answer is "true".
|
|
59
|
+
p = answer.noul
|
|
60
|
+
return Answer(value=p > 0.5, probabilities={"true": p, "false": 1 - p}, raw=answer)
|
|
61
|
+
case Choice():
|
|
62
|
+
probabilities = {option: answer.probabilities[option] for option in question.options}
|
|
63
|
+
return Answer(value=answer.choice, probabilities=probabilities, raw=answer)
|
|
64
|
+
case Score():
|
|
65
|
+
# Jev keys probabilities by level index (0, 1, 2...); we key them by level text.
|
|
66
|
+
probabilities = {level: answer.probabilities[i] for i, level in enumerate(question.levels)}
|
|
67
|
+
return Answer(value=answer.score, probabilities=probabilities, raw=answer)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Laya, the open-weight Jev-compatible model (``pip install semantic-operators[laya]``).
|
|
2
|
+
|
|
3
|
+
Runs locally. You load the model with the ``laya`` package and pass it in:
|
|
4
|
+
``laya.load("convaiinnovations/laya")`` for one checkpoint, or ``laya.Router()`` to
|
|
5
|
+
pick a checkpoint by language. Both have the ``predict`` method used here.
|
|
6
|
+
|
|
7
|
+
``AsyncLaya`` runs predictions in a worker thread so they don't block the event
|
|
8
|
+
loop, one at a time (the model is a single local compute resource; Laya's own
|
|
9
|
+
HTTP server serializes calls the same way).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from ..types import Answer, Boolean, Choice, Question, Score, State
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Laya:
|
|
20
|
+
def __init__(self, model: Any) -> None:
|
|
21
|
+
self.model = model
|
|
22
|
+
|
|
23
|
+
def ask(self, state: State, questions: Mapping[str, Question]) -> dict[str, Answer]:
|
|
24
|
+
result = self.model.predict(state, {name: _to_laya(q) for name, q in questions.items()})
|
|
25
|
+
answers = result["answers"]
|
|
26
|
+
return {name: _from_laya(q, answers[name]) for name, q in questions.items()}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class AsyncLaya:
|
|
30
|
+
def __init__(self, model: Any) -> None:
|
|
31
|
+
self._laya = Laya(model)
|
|
32
|
+
self._lock = asyncio.Lock()
|
|
33
|
+
|
|
34
|
+
async def ask(self, state: State, questions: Mapping[str, Question]) -> dict[str, Answer]:
|
|
35
|
+
async with self._lock:
|
|
36
|
+
return await asyncio.to_thread(self._laya.ask, state, questions)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# Laya takes and returns plain dicts in Jev's request/response shape.
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _to_laya(question: Question) -> dict[str, Any]:
|
|
43
|
+
match question:
|
|
44
|
+
case Boolean():
|
|
45
|
+
q: dict[str, Any] = {"type": "noul", "instructions": question.instructions}
|
|
46
|
+
if question.true is not None or question.false is not None:
|
|
47
|
+
q["criteria"] = {"true": question.true, "false": question.false}
|
|
48
|
+
return q
|
|
49
|
+
case Choice():
|
|
50
|
+
return {"type": "choice", "instructions": question.instructions,
|
|
51
|
+
"criteria": dict(question.options)}
|
|
52
|
+
case Score():
|
|
53
|
+
return {"type": "score", "instructions": question.instructions,
|
|
54
|
+
"criteria": list(question.levels)}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _from_laya(question: Question, answer: dict[str, Any]) -> Answer:
|
|
58
|
+
match question:
|
|
59
|
+
case Boolean():
|
|
60
|
+
p = answer["noul"]
|
|
61
|
+
return Answer(value=p > 0.5, probabilities={"true": p, "false": 1 - p}, raw=answer)
|
|
62
|
+
case Choice():
|
|
63
|
+
probabilities = {option: answer["probabilities"][option] for option in question.options}
|
|
64
|
+
return Answer(value=answer["choice"], probabilities=probabilities, raw=answer)
|
|
65
|
+
case Score():
|
|
66
|
+
# Laya keys probabilities by level index as a string ("0", "1", ...).
|
|
67
|
+
probabilities = {level: answer["probabilities"][str(i)]
|
|
68
|
+
for i, level in enumerate(question.levels)}
|
|
69
|
+
return Answer(value=answer["score"], probabilities=probabilities, raw=answer)
|
|
File without changes
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""The three question types and the one answer type.
|
|
2
|
+
|
|
3
|
+
These are our words, not any provider's. A provider translates them into
|
|
4
|
+
its own API and translates its answers back into an ``Answer``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
# What a question is asked about: text, or JSON-shaped data.
|
|
11
|
+
State = str | dict[str, Any] | list[Any]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class Boolean:
|
|
16
|
+
"""A yes/no question. ``true``/``false`` optionally describe each outcome."""
|
|
17
|
+
|
|
18
|
+
instructions: str
|
|
19
|
+
true: str | None = None
|
|
20
|
+
false: str | None = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Choice:
|
|
25
|
+
"""Pick one of several named options. Maps option name -> description."""
|
|
26
|
+
|
|
27
|
+
instructions: str
|
|
28
|
+
options: dict[str, str | None]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class Score:
|
|
33
|
+
"""Rate on an ordered rubric. ``levels[0]`` is score 0, ``levels[1]`` is 1, ..."""
|
|
34
|
+
|
|
35
|
+
instructions: str
|
|
36
|
+
levels: list[str]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
Question = Boolean | Choice | Score
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class Answer:
|
|
44
|
+
"""The answer to one question.
|
|
45
|
+
|
|
46
|
+
- Boolean: ``value`` is a bool; ``probabilities`` has keys "true" and "false".
|
|
47
|
+
- Choice: ``value`` is the chosen option name; ``probabilities`` is keyed by option,
|
|
48
|
+
in the same order as the question's options.
|
|
49
|
+
- Score: ``value`` is the expected score (a float, may fall between levels);
|
|
50
|
+
``probabilities`` is keyed by level text, in level order.
|
|
51
|
+
|
|
52
|
+
``raw`` is the provider's own answer object, for when you need more.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
value: bool | str | float
|
|
56
|
+
probabilities: dict[str, float]
|
|
57
|
+
raw: Any = field(default=None, repr=False)
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: semantic-operators
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: One small, provider-neutral interface for System One models (Jev, Laya, and compatibles).
|
|
5
|
+
Project-URL: Homepage, https://github.com/jasonduncan/semantic-operators
|
|
6
|
+
Project-URL: Source, https://github.com/jasonduncan/semantic-operators
|
|
7
|
+
Project-URL: Issues, https://github.com/jasonduncan/semantic-operators/issues
|
|
8
|
+
Author: Jason Duncan
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: benchmark,classification,jev,laya,llm,system-one,typesafe
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Provides-Extra: jev
|
|
22
|
+
Requires-Dist: typesafe-sdk>=0.7.1; extra == 'jev'
|
|
23
|
+
Provides-Extra: laya
|
|
24
|
+
Requires-Dist: laya>=0.3.20; extra == 'laya'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# Semantic Operators
|
|
28
|
+
|
|
29
|
+
One small interface for **System One models**: fast models that answer structured
|
|
30
|
+
questions about text or data with probabilities, not prose. [Jev](https://typesafe.ai)
|
|
31
|
+
was the first; [Laya](https://huggingface.co/convaiinnovations/laya) is an open-weight,
|
|
32
|
+
Jev-compatible alternative you can run locally. More are coming. This library lets you
|
|
33
|
+
write your code once and swap the model underneath.
|
|
34
|
+
|
|
35
|
+
| Provider | Where it runs | Install |
|
|
36
|
+
|----------|---------------|---------|
|
|
37
|
+
| `providers.jev.Jev` | TypeSafe's hosted API (needs `TYPESAFE_API_KEY`) | `[jev]` |
|
|
38
|
+
| `providers.laya.Laya` | on your machine (~800 MB download on first use) | `[laya]` |
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
pip install "semantic-operators[jev]" # Jev (hosted)
|
|
44
|
+
pip install "semantic-operators[laya]" # Laya (local; pulls in torch)
|
|
45
|
+
pip install "semantic-operators[jev,laya]" # both
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The core alone (`pip install semantic-operators`) has no dependencies.
|
|
49
|
+
|
|
50
|
+
## The whole idea
|
|
51
|
+
|
|
52
|
+
A System One model is asked **named questions about a piece of state** and returns an
|
|
53
|
+
answer with probabilities for each. There are three kinds of question:
|
|
54
|
+
|
|
55
|
+
| Question | You give it | `answer.value` |
|
|
56
|
+
|-----------|-----------------------------------------------|--------------------------------------|
|
|
57
|
+
| `Boolean` | instructions (+ optional true/false meanings) | `True` / `False` |
|
|
58
|
+
| `Choice` | instructions + named options | the chosen option name |
|
|
59
|
+
| `Score` | instructions + ordered rubric levels | expected level as a float, e.g. `1.7` |
|
|
60
|
+
|
|
61
|
+
Every `Answer` also carries `probabilities` (a dict, in the question's option/level
|
|
62
|
+
order) and `raw` (the provider's own answer object).
|
|
63
|
+
|
|
64
|
+
A **provider** is anything with one method:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
def ask(self, state, questions: dict[str, Question]) -> dict[str, Answer]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
That's the entire abstraction.
|
|
71
|
+
|
|
72
|
+
## Quick start
|
|
73
|
+
|
|
74
|
+
```sh
|
|
75
|
+
echo "TYPESAFE_API_KEY=..." > .env
|
|
76
|
+
uv run --env-file .env --extra jev python examples/hello.py
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from typesafe_sdk import TypeSafeClient
|
|
81
|
+
from semantic_operators import Boolean, Choice, Score
|
|
82
|
+
from semantic_operators.providers.jev import Jev
|
|
83
|
+
|
|
84
|
+
with TypeSafeClient() as client: # you create and own the SDK client
|
|
85
|
+
jev = Jev(client) # model defaults to "jev-latest"
|
|
86
|
+
answers = jev.ask(
|
|
87
|
+
"I was charged twice and I'm furious.",
|
|
88
|
+
{
|
|
89
|
+
"is_complaint": Boolean("Is the customer complaining?"),
|
|
90
|
+
"department": Choice("Which team should handle this?",
|
|
91
|
+
{"billing": "Payments, refunds", "other": "Anything else"}),
|
|
92
|
+
"urgency": Score("How urgent is this?", ["low", "medium", "high"]),
|
|
93
|
+
},
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
answers["department"].value # "billing"
|
|
97
|
+
answers["department"].probabilities # {"billing": 0.97, "other": 0.03}
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Swapping to Laya changes only how the provider is built:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
import laya
|
|
104
|
+
from semantic_operators.providers.laya import Laya
|
|
105
|
+
|
|
106
|
+
provider = Laya(laya.load("convaiinnovations/laya")) # or Laya(laya.Router())
|
|
107
|
+
answers = provider.ask(state, questions) # same questions, same Answer type
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Compare both side by side:
|
|
111
|
+
|
|
112
|
+
```sh
|
|
113
|
+
uv run --env-file .env --extra jev --extra laya python examples/compare.py
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Benchmark
|
|
117
|
+
|
|
118
|
+
`bench.run(provider, questions, cases)` asks each labeled case all questions in one call
|
|
119
|
+
and reports, per question, **accuracy** (Score values are rounded to the nearest level)
|
|
120
|
+
and **p(correct)**, the average probability the provider gave the right answer, plus
|
|
121
|
+
latency and every miss.
|
|
122
|
+
|
|
123
|
+
```sh
|
|
124
|
+
uv run --env-file .env --extra jev --extra laya python benchmarks/run.py
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
`benchmarks/support_tickets.py` holds 20 hand-written, hand-labeled support messages
|
|
128
|
+
and the same 3 questions in three wordings. `bench.stability(reports)` reports how often
|
|
129
|
+
a provider's decision stays the same when only the wording changes (labels play no part). It's a smoke test, not a verdict: small, authored, one person's labels.
|
|
130
|
+
|
|
131
|
+
## Async
|
|
132
|
+
|
|
133
|
+
Every provider has an async twin with the same contract, `await provider.ask(...)`:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from typesafe_sdk import AsyncTypeSafeClient
|
|
137
|
+
from semantic_operators.providers.jev import AsyncJev
|
|
138
|
+
from semantic_operators.providers.laya import AsyncLaya
|
|
139
|
+
|
|
140
|
+
async with AsyncTypeSafeClient() as client:
|
|
141
|
+
answers = await AsyncJev(client).ask(state, questions)
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
`AsyncLaya` runs the local model in a worker thread, one call at a time. Concurrency
|
|
145
|
+
speeds up a hosted API (many requests in flight), not a single local model.
|
|
146
|
+
`bench.run_async(provider, questions, cases, concurrency=8)` benchmarks async providers:
|
|
147
|
+
|
|
148
|
+
```sh
|
|
149
|
+
uv run --env-file .env --extra jev --extra laya python benchmarks/run_async.py
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Layout
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
src/semantic_operators/
|
|
156
|
+
types.py Boolean, Choice, Score, Answer: our vocabulary
|
|
157
|
+
provider.py Provider and AsyncProvider (one method each)
|
|
158
|
+
providers/jev.py translates to/from the TypeSafe SDK
|
|
159
|
+
providers/laya.py translates to/from the laya package
|
|
160
|
+
bench.py (higher layer) run labeled cases through a provider, score them
|
|
161
|
+
examples/
|
|
162
|
+
hello.py one real call to Jev
|
|
163
|
+
compare.py the same questions through Jev and Laya
|
|
164
|
+
benchmarks/
|
|
165
|
+
support_tickets.py 20 labeled messages + the questions
|
|
166
|
+
run.py runs the suite through Jev and Laya
|
|
167
|
+
run_async.py concurrency, and both providers at once
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
## Layers
|
|
171
|
+
|
|
172
|
+
Semantic Operators is built in layers inside one package:
|
|
173
|
+
|
|
174
|
+
1. **Base layer:** a clean, provider-neutral abstraction over System One
|
|
175
|
+
models: `types.py`, `provider.py`, `providers/`.
|
|
176
|
+
2. **Higher layers:** built only on the base layer. So far: `bench.py`. Later: reusable
|
|
177
|
+
named operators and composition.
|
|
178
|
+
|
|
179
|
+
The base layer never imports from a higher layer, so it could later be split out as its
|
|
180
|
+
own package without changing how it's used.
|
|
181
|
+
|
|
182
|
+
## Rules
|
|
183
|
+
|
|
184
|
+
- The library never reads API keys or environment variables. You build the client.
|
|
185
|
+
- The core has no dependencies. Each provider's SDK is an optional extra (`[jev]`, `[laya]`).
|
|
186
|
+
- Our names, not the provider's: `Boolean`, not `noul`.
|
|
187
|
+
|
|
188
|
+
## Not here yet (on purpose)
|
|
189
|
+
|
|
190
|
+
Reusable named operators, error types, and
|
|
191
|
+
"don't know" answers. Each will be added as its own small step.
|
|
192
|
+
|
|
193
|
+
## License
|
|
194
|
+
|
|
195
|
+
MIT
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
semantic_operators/__init__.py,sha256=qr16WoMff280u5FNF1tVlSIY2Nd1fJT5sye6PAP9tBg,285
|
|
2
|
+
semantic_operators/bench.py,sha256=fV5yB5UA496OHfoBntbPYs4cQBJpbvnzq39XP9XfuOw,6439
|
|
3
|
+
semantic_operators/provider.py,sha256=qoq0Zedo4NVb6g5o5iYPqL_gJgUNvc25Fo9EBRJ3c2Y,791
|
|
4
|
+
semantic_operators/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
semantic_operators/types.py,sha256=4aYTGjWYx1jx8ars31yPCY64rBa-vN4BUN8rUmpu8tc,1612
|
|
6
|
+
semantic_operators/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
semantic_operators/providers/jev.py,sha256=Da1sOjMjJyAQnbo-DpLC7T-QeqZyBOTz_r0eAmtiLmc,2923
|
|
8
|
+
semantic_operators/providers/laya.py,sha256=i7rH8D565GEXxeH49HPc7Y8nx3Rh3OruInkDJneF9ks,2939
|
|
9
|
+
semantic_operators-0.1.0.dist-info/METADATA,sha256=XE8irLsHuHedFsOWtxTh3fbjYfy7yHw3uQRv2FhXZoo,7397
|
|
10
|
+
semantic_operators-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
11
|
+
semantic_operators-0.1.0.dist-info/licenses/LICENSE,sha256=Xndu7f16mot0SJ4RVZ45Ho_2qqezkHHStDHIaUkSJrQ,1069
|
|
12
|
+
semantic_operators-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jason Duncan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|