python-fastllm 0.0.70__py3-none-any.whl → 0.0.71__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastllm/__init__.py +1 -1
- fastllm/_modidx.py +10 -2
- fastllm/acomplete.py +8 -3
- fastllm/chat.py +5 -2
- fastllm/gemini.py +108 -18
- fastllm/model_prices.json +21214 -1291
- fastllm/types.py +3 -3
- {python_fastllm-0.0.70.dist-info → python_fastllm-0.0.71.dist-info}/METADATA +1 -1
- python_fastllm-0.0.71.dist-info/RECORD +18 -0
- python_fastllm-0.0.70.dist-info/RECORD +0 -18
- {python_fastllm-0.0.70.dist-info → python_fastllm-0.0.71.dist-info}/WHEEL +0 -0
- {python_fastllm-0.0.70.dist-info → python_fastllm-0.0.71.dist-info}/entry_points.txt +0 -0
- {python_fastllm-0.0.70.dist-info → python_fastllm-0.0.71.dist-info}/top_level.txt +0 -0
fastllm/__init__.py
CHANGED
fastllm/_modidx.py
CHANGED
|
@@ -145,10 +145,17 @@ d = { 'settings': { 'branch': 'main',
|
|
|
145
145
|
'fastllm.codex.main': ('codex.html#main', 'fastllm/codex.py'),
|
|
146
146
|
'fastllm.codex.mk_jwt': ('codex.html#mk_jwt', 'fastllm/codex.py'),
|
|
147
147
|
'fastllm.codex.token_expired': ('codex.html#token_expired', 'fastllm/codex.py')},
|
|
148
|
-
'fastllm.gemini': { 'fastllm.gemini.
|
|
148
|
+
'fastllm.gemini': { 'fastllm.gemini._GeminiStream': ('gemini.html#_geministream', 'fastllm/gemini.py'),
|
|
149
|
+
'fastllm.gemini._GeminiStream.__aiter__': ('gemini.html#_geministream.__aiter__', 'fastllm/gemini.py'),
|
|
150
|
+
'fastllm.gemini._GeminiStream.__init__': ('gemini.html#_geministream.__init__', 'fastllm/gemini.py'),
|
|
151
|
+
'fastllm.gemini._continuation_token': ('gemini.html#_continuation_token', 'fastllm/gemini.py'),
|
|
152
|
+
'fastllm.gemini._fn_call': ('gemini.html#_fn_call', 'fastllm/gemini.py'),
|
|
149
153
|
'fastllm.gemini._fn_calls': ('gemini.html#_fn_calls', 'fastllm/gemini.py'),
|
|
150
154
|
'fastllm.gemini._gem_filter_sch': ('gemini.html#_gem_filter_sch', 'fastllm/gemini.py'),
|
|
151
155
|
'fastllm.gemini._gem_part_type': ('gemini.html#_gem_part_type', 'fastllm/gemini.py'),
|
|
156
|
+
'fastllm.gemini._merge_metadata': ('gemini.html#_merge_metadata', 'fastllm/gemini.py'),
|
|
157
|
+
'fastllm.gemini._merge_response': ('gemini.html#_merge_response', 'fastllm/gemini.py'),
|
|
158
|
+
'fastllm.gemini._total_usage': ('gemini.html#_total_usage', 'fastllm/gemini.py'),
|
|
152
159
|
'fastllm.gemini._url_cite': ('gemini.html#_url_cite', 'fastllm/gemini.py'),
|
|
153
160
|
'fastllm.gemini.acollect_stream': ('gemini.html#acollect_stream', 'fastllm/gemini.py'),
|
|
154
161
|
'fastllm.gemini.collate_raw': ('gemini.html#collate_raw', 'fastllm/gemini.py'),
|
|
@@ -178,7 +185,8 @@ d = { 'settings': { 'branch': 'main',
|
|
|
178
185
|
'fastllm.gemini.norm_sse_event': ('gemini.html#norm_sse_event', 'fastllm/gemini.py'),
|
|
179
186
|
'fastllm.gemini.norm_tool_calls': ('gemini.html#norm_tool_calls', 'fastllm/gemini.py'),
|
|
180
187
|
'fastllm.gemini.norm_usage': ('gemini.html#norm_usage', 'fastllm/gemini.py'),
|
|
181
|
-
'fastllm.gemini.raw_msg': ('gemini.html#raw_msg', 'fastllm/gemini.py')
|
|
188
|
+
'fastllm.gemini.raw_msg': ('gemini.html#raw_msg', 'fastllm/gemini.py'),
|
|
189
|
+
'fastllm.gemini.request': ('gemini.html#request', 'fastllm/gemini.py')},
|
|
182
190
|
'fastllm.openai_chat': { 'fastllm.openai_chat.acollect_stream': ('oai_chat.html#acollect_stream', 'fastllm/openai_chat.py'),
|
|
183
191
|
'fastllm.openai_chat.collate_raw': ('oai_chat.html#collate_raw', 'fastllm/openai_chat.py'),
|
|
184
192
|
'fastllm.openai_chat.cost': ('oai_chat.html#cost', 'fastllm/openai_chat.py'),
|
fastllm/acomplete.py
CHANGED
|
@@ -194,14 +194,19 @@ async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None,
|
|
|
194
194
|
ep = endpoint or api.endpoint
|
|
195
195
|
path,params = ep(model, stream) if callable(ep) else (ep, None)
|
|
196
196
|
if defaults.debug_mode: _debug_print(model, api_name, vendor_name, body, None if own else cli._url(path))
|
|
197
|
-
async def _call(
|
|
197
|
+
async def _call(body=body):
|
|
198
|
+
async def send(): return await provider_req(cli, path, body, params=params, stream=stream, headers=headers)
|
|
199
|
+
return await send() if stream else await _retry_call(send, retries, retry_delay)
|
|
200
|
+
async def _request():
|
|
201
|
+
if req := getattr(api, 'request', None): return await req(_call, body, stream=stream)
|
|
202
|
+
return await _call()
|
|
198
203
|
async def _mk_gen():
|
|
199
|
-
resp = payload if own else await
|
|
204
|
+
resp = payload if own else await _request() # an own-transport api's collector makes its request from the payload
|
|
200
205
|
async for o in api.acollect_stream(resp, model=model, vendor_name=vendor_name, stop_callables=stop_callables): yield o
|
|
201
206
|
if stream or own:
|
|
202
207
|
result = _retry_stream(_mk_gen, retries, retry_delay)
|
|
203
208
|
if stream: return result
|
|
204
209
|
async for completion in result: pass # own-transport apis are streaming-only: drain for the completion
|
|
205
210
|
return completion
|
|
206
|
-
resp = await
|
|
211
|
+
resp = await _request()
|
|
207
212
|
return mk_completion(resp, model=model, api_name=api_name, vendor_name=vendor_name)
|
fastllm/chat.py
CHANGED
|
@@ -9,7 +9,7 @@ __all__ = ['effort', 'contents', 'stop_reason', 'cite_footnote', 'postproc', 'Re
|
|
|
9
9
|
'StopSequencesCallback', 'StreamAccum', 'adisplay_stream']
|
|
10
10
|
|
|
11
11
|
# %% ../nbs/09_chat.ipynb #d5a3bc1f
|
|
12
|
-
import secrets
|
|
12
|
+
import secrets, inspect
|
|
13
13
|
from typing import Optional,Callable
|
|
14
14
|
from fastcore.funccall import mk_ns, call_func, call_func_async, get_schema
|
|
15
15
|
from fastcore.utils import *
|
|
@@ -94,7 +94,10 @@ async def structured(
|
|
|
94
94
|
if not forced: sp = f"{part_txt(sp)}\n\nAnswer by calling the `{name}` tool.".strip()
|
|
95
95
|
r = await acomplete(msgs, m, system=sp, tools=[t], tool_choice=name if forced else 'auto', **kwargs)
|
|
96
96
|
if not r.tool_calls: raise ValueError(f"{m} answered without calling `{name}`")
|
|
97
|
-
|
|
97
|
+
args = r.tool_calls[0].arguments
|
|
98
|
+
try: inspect.signature(tool).bind(**args)
|
|
99
|
+
except TypeError as e: raise TypeError(f"{m} called `{name}` with arguments that don't fit it ({e}): {args}") from None
|
|
100
|
+
return tool(**args)
|
|
98
101
|
|
|
99
102
|
# %% ../nbs/09_chat.ipynb #1fe8a9bc
|
|
100
103
|
def _has_search(info): return bool(info.get('search_context_cost_per_query') or info.get('supports_web_search'))
|
fastllm/gemini.py
CHANGED
|
@@ -4,17 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
# %% auto #0
|
|
6
6
|
__all__ = ['api_ns', 'endpoint', 'grounding', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'raw_msg',
|
|
7
|
-
'norm_sse_event', 'delta_index_fn', 'collate_raw', '
|
|
8
|
-
'denorm_tool', 'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice',
|
|
9
|
-
'denorm_web_search', 'denorm_system', 'denorm_user', 'denorm_image', 'denorm_audio',
|
|
10
|
-
'denorm_file', 'denorm_tool_result', 'mk_payload', 'get_hdrs', 'cost']
|
|
7
|
+
'norm_sse_event', 'delta_index_fn', 'collate_raw', 'request', 'acollect_stream', 'denorm_tool_use',
|
|
8
|
+
'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice',
|
|
9
|
+
'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user', 'denorm_image', 'denorm_audio',
|
|
10
|
+
'denorm_video', 'denorm_file', 'denorm_tool_result', 'mk_payload', 'get_hdrs', 'cost']
|
|
11
11
|
|
|
12
12
|
# %% ../nbs/07_gemini.ipynb #02afd3d7
|
|
13
13
|
import html
|
|
14
|
+
from copy import deepcopy
|
|
14
15
|
from collections import Counter
|
|
15
16
|
from fastcore.utils import *
|
|
16
17
|
from fastcore.meta import *
|
|
17
|
-
from fasttransport.errors import api_error_from_event
|
|
18
|
+
from fasttransport.errors import APIError, api_error_from_event
|
|
18
19
|
|
|
19
20
|
from .types import *
|
|
20
21
|
from aidialog.msg_parts import (MdStr, Part, PartType, Msg, Completion, Text, Thinking, ToolUse, ToolResult, tool_text,
|
|
@@ -67,9 +68,9 @@ def norm_usage(resp):
|
|
|
67
68
|
reasoning = int(usg.get("thoughtsTokenCount", 0) or 0)
|
|
68
69
|
parts = nested_idx(resp, 'candidates', 0, 'content', 'parts') or []
|
|
69
70
|
cand = nested_idx(resp, 'candidates', 0) or {}
|
|
70
|
-
stu = {}
|
|
71
|
-
if any("executableCode" in p for p in parts): stu
|
|
72
|
-
if call := grounding(cand)[1]: stu
|
|
71
|
+
stu = dict(usg.get('server_tool_use') or {})
|
|
72
|
+
if any("executableCode" in p for p in parts): stu.setdefault('code_execution', sum(1 for p in parts if "executableCode" in p))
|
|
73
|
+
if call := grounding(cand)[1]: stu.setdefault('google_search', len(call.arguments['queries'] or [1]))
|
|
73
74
|
if stu: usg['server_tool_use'] = stu
|
|
74
75
|
return Usage(prompt_tokens=pt, completion_tokens=ct, total_tokens=tt,
|
|
75
76
|
cached_tokens=cached, reasoning_tokens=reasoning, search_queries=stu.get('google_search', 0), raw=usg)
|
|
@@ -79,7 +80,8 @@ def norm_finish(resp, tcs=None):
|
|
|
79
80
|
"Canonicalize finish_reason to OpenAI Chat values: stop, tool_calls, length, content_filter."
|
|
80
81
|
reason = nested_idx(resp, 'candidates', 0, 'finishReason')
|
|
81
82
|
if reason is not None: reason = reason.lower()
|
|
82
|
-
mp = dict(stop=FinishReason.stop, max_tokens=FinishReason.length,
|
|
83
|
+
mp = dict(stop=FinishReason.stop, max_tokens=FinishReason.length, continuation=FinishReason.length,
|
|
84
|
+
safety=FinishReason.content_filter, blocklist=FinishReason.content_filter)
|
|
83
85
|
r = mp.get(reason, reason)
|
|
84
86
|
return FinishReason.tool_calls if r==FinishReason.stop and any(~L(tcs).attrgot('server')) else r
|
|
85
87
|
|
|
@@ -134,21 +136,108 @@ def delta_index_fn(d, typ, last_typ, last_idx):
|
|
|
134
136
|
# %% ../nbs/07_gemini.ipynb #33506d56
|
|
135
137
|
def collate_raw(deltas):
|
|
136
138
|
"The model content rebuilt from a stream's chunks"
|
|
137
|
-
parts = []
|
|
139
|
+
parts,start = [],0
|
|
138
140
|
for ev in L(deltas).attrgot('raw'):
|
|
141
|
+
if ev.get('_continuation_start'): start = len(parts)
|
|
139
142
|
for p in nested_idx(ev, 'candidates', 0, 'content', 'parts') or []:
|
|
140
|
-
last = parts[-1] if parts else None
|
|
143
|
+
last = parts[-1] if len(parts) > start else None
|
|
141
144
|
if 'text' in p and last and 'text' in last and last.get('thought') == p.get('thought') and 'thoughtSignature' not in last:
|
|
142
145
|
last['text'] += p['text']
|
|
143
146
|
if 'thoughtSignature' in p: last['thoughtSignature'] = p['thoughtSignature']
|
|
144
147
|
else: parts.append(dict(p))
|
|
145
148
|
return dict(role='model', parts=parts)
|
|
146
149
|
|
|
150
|
+
# %% ../nbs/07_gemini.ipynb #e9f0cac6
|
|
151
|
+
def _merge_metadata(prev, cur):
|
|
152
|
+
res = prev.copy()
|
|
153
|
+
for k,v in cur.items():
|
|
154
|
+
old = res.get(k)
|
|
155
|
+
if isinstance(old, dict) and isinstance(v, dict): v = _merge_metadata(old, v)
|
|
156
|
+
elif isinstance(old, list) and isinstance(v, list): v = old + v
|
|
157
|
+
elif isinstance(old, (int, float)) and isinstance(v, (int, float)): v = old + v
|
|
158
|
+
res[k] = v
|
|
159
|
+
return res
|
|
160
|
+
|
|
161
|
+
def _total_usage(hops):
|
|
162
|
+
res = {}
|
|
163
|
+
for u in hops: res = _merge_metadata(res, u)
|
|
164
|
+
if len(hops) > 1: res['continuation_usage'] = hops.copy()
|
|
165
|
+
return res
|
|
166
|
+
|
|
167
|
+
def _continuation_token(resp, previous=None):
|
|
168
|
+
cand = nested_idx(resp, 'candidates', 0) or {}
|
|
169
|
+
token = cand.get('continuationToken')
|
|
170
|
+
if cand.get('finishReason') == 'CONTINUATION' and token != previous and not resp.get('promptFeedback'): return token
|
|
171
|
+
|
|
172
|
+
def _merge_response(prev, cur):
|
|
173
|
+
res = prev | cur
|
|
174
|
+
if not (prev.get('candidates') or cur.get('candidates')): return res
|
|
175
|
+
a = nested_idx(prev, 'candidates', 0) or {}
|
|
176
|
+
b = deepcopy(nested_idx(cur, 'candidates', 0) or {})
|
|
177
|
+
ap,bp = nested_idx(a, 'content', 'parts') or [], nested_idx(b, 'content', 'parts') or []
|
|
178
|
+
ag,bg = a.get('groundingMetadata') or {}, b.get('groundingMetadata') or {}
|
|
179
|
+
for support in bg.get('groundingSupports', []):
|
|
180
|
+
if 'groundingChunkIndices' in support:
|
|
181
|
+
support['groundingChunkIndices'] = [i + len(ag.get('groundingChunks', [])) for i in support['groundingChunkIndices']]
|
|
182
|
+
if (seg := support.get('segment')) is not None: seg['partIndex'] = seg.get('partIndex', 0) + len(ap)
|
|
183
|
+
cand = a | b
|
|
184
|
+
if a.get('content') or b.get('content'):
|
|
185
|
+
cand['content'] = (a.get('content') or {}) | (b.get('content') or {}) | dict(parts=ap + bp)
|
|
186
|
+
if ag or bg: cand['groundingMetadata'] = _merge_metadata(ag, bg)
|
|
187
|
+
if b.get('finishReason'):
|
|
188
|
+
for k in ('continuationToken', 'finishMessage'):
|
|
189
|
+
if k not in b: cand.pop(k, None)
|
|
190
|
+
res['candidates'] = [cand]
|
|
191
|
+
return res
|
|
192
|
+
|
|
193
|
+
# %% ../nbs/07_gemini.ipynb #594ca72a
|
|
194
|
+
class _GeminiStream:
|
|
195
|
+
def __init__(self, call, body):
|
|
196
|
+
self.call,self.body = call,body
|
|
197
|
+
self.stopped = self.continued = False
|
|
198
|
+
|
|
199
|
+
async def __aiter__(self):
|
|
200
|
+
hops,token = [],self.body.get('continuationToken')
|
|
201
|
+
while True:
|
|
202
|
+
hop,usg = {},None
|
|
203
|
+
resp = await self.call(self.body | dict(continuationToken=token) if token else self.body)
|
|
204
|
+
async for ev in resp:
|
|
205
|
+
ev,first = obj2dict(ev),not hop
|
|
206
|
+
hop = _merge_response(hop, ev)
|
|
207
|
+
usg = norm_usage(hop | dict(usageMetadata=dict(hop.get('usageMetadata') or {})))
|
|
208
|
+
out = ev | dict(usageMetadata=_total_usage(hops + [usg.raw if usg else {}]), _continuation_start=first)
|
|
209
|
+
if not self.stopped and _continuation_token(hop, token):
|
|
210
|
+
out = out | dict(candidates=[dict(c, finishReason=None) for c in out.get('candidates', [])])
|
|
211
|
+
yield out
|
|
212
|
+
if usg: hops.append(usg.raw)
|
|
213
|
+
token = None if self.stopped else _continuation_token(hop, token)
|
|
214
|
+
if not token: return
|
|
215
|
+
self.continued = True
|
|
216
|
+
|
|
217
|
+
async def request(call, body, stream=False):
|
|
218
|
+
"Send a Gemini generation, resuming only at explicit continuation boundaries."
|
|
219
|
+
if stream: return _GeminiStream(call, body)
|
|
220
|
+
res,hops,token = {},[],body.get('continuationToken')
|
|
221
|
+
while True:
|
|
222
|
+
resp = obj2dict(await call(body | dict(continuationToken=token) if token else body))
|
|
223
|
+
if usg := norm_usage(resp): hops.append(usg.raw)
|
|
224
|
+
res = _merge_response(res, resp)
|
|
225
|
+
token = _continuation_token(resp, token)
|
|
226
|
+
if not token:
|
|
227
|
+
if hops: res['usageMetadata'] = _total_usage(hops)
|
|
228
|
+
return res
|
|
229
|
+
|
|
147
230
|
# %% ../nbs/07_gemini.ipynb #328af4d5
|
|
148
231
|
@delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
|
|
149
232
|
async def acollect_stream(resp, **kwargs):
|
|
150
233
|
res = mk_acollect_stream(norm_and_yield(resp, norm_sse_event), index_fn=delta_index_fn, api_name='gemini', **kwargs)
|
|
151
|
-
|
|
234
|
+
try:
|
|
235
|
+
async for o in res:
|
|
236
|
+
if isinstance(resp, _GeminiStream) and isinstance(o, Status): resp.stopped = True
|
|
237
|
+
yield o
|
|
238
|
+
except APIError as e:
|
|
239
|
+
if isinstance(resp, _GeminiStream) and resp.continued: e.retryable = False # never replay a completed segment
|
|
240
|
+
raise
|
|
152
241
|
|
|
153
242
|
# %% ../nbs/07_gemini.ipynb #58d0cb74
|
|
154
243
|
def denorm_tool_use(p:ToolUse):
|
|
@@ -311,6 +400,7 @@ def get_hdrs(api_key=None, oauth_token=None):
|
|
|
311
400
|
# %% ../nbs/07_gemini.ipynb #4ee3891f
|
|
312
401
|
def cost(usage, m):
|
|
313
402
|
raw = usage.raw
|
|
403
|
+
if hops := raw.get('continuation_usage'): return sum(cost(Usage(raw=u), m) for u in hops)
|
|
314
404
|
prompt_tot = raw.get('promptTokenCount', 0)
|
|
315
405
|
tier = price_tier(m, prompt_tot)
|
|
316
406
|
in_rate = tier_rate(m, 'input_cost_per_token', tier)
|
|
@@ -324,14 +414,14 @@ def cost(usage, m):
|
|
|
324
414
|
thoughts = raw.get('thoughtsTokenCount', 0) or 0
|
|
325
415
|
cands = raw.get('candidatesTokenCount', 0) or 0
|
|
326
416
|
reason_rate = m.get('output_cost_per_reasoning_token') or out_rate
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
return
|
|
417
|
+
res = in_txt * in_rate + cands * out_rate
|
|
418
|
+
res += cached * cache_rate
|
|
419
|
+
res += audio * (audio_rate or 0)
|
|
420
|
+
res += thoughts * reason_rate
|
|
421
|
+
return res
|
|
332
422
|
|
|
333
423
|
# %% ../nbs/07_gemini.ipynb #f7c0b989
|
|
334
424
|
api_ns = dict(norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
|
|
335
425
|
acollect_stream=acollect_stream, raw_msg=raw_msg, collate_raw=collate_raw, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs,
|
|
336
|
-
endpoint=endpoint)
|
|
426
|
+
endpoint=endpoint, request=request)
|
|
337
427
|
api_registry.register('gemini', **api_ns)
|