python-fastllm 0.0.70__tar.gz → 0.0.71__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/PKG-INFO +1 -1
  2. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/__init__.py +1 -1
  3. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/_modidx.py +10 -2
  4. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/acomplete.py +8 -3
  5. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/chat.py +5 -2
  6. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/gemini.py +108 -18
  7. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/model_prices.json +21214 -1291
  8. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/types.py +3 -3
  9. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/python_fastllm.egg-info/PKG-INFO +1 -1
  10. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/README.md +0 -0
  11. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/anthropic.py +0 -0
  12. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/codex.py +0 -0
  13. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/openai_chat.py +0 -0
  14. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/openai_responses.py +0 -0
  15. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/responses.py +0 -0
  16. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/fastllm/streaming.py +0 -0
  17. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/pyproject.toml +0 -0
  18. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/python_fastllm.egg-info/SOURCES.txt +0 -0
  19. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/python_fastllm.egg-info/dependency_links.txt +0 -0
  20. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/python_fastllm.egg-info/entry_points.txt +0 -0
  21. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/python_fastllm.egg-info/requires.txt +0 -0
  22. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/python_fastllm.egg-info/top_level.txt +0 -0
  23. {python_fastllm-0.0.70 → python_fastllm-0.0.71}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: python-fastllm
3
- Version: 0.0.70
3
+ Version: 0.0.71
4
4
  Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
5
5
  License-Expression: Apache-2.0
6
6
  Project-URL: Repository, https://github.com/AnswerDotAI/fastllm
@@ -2,4 +2,4 @@
2
2
 
3
3
  - `fastllm.responses`: Provider-independent Responses resources and streaming over FastLLM"""
4
4
 
5
- __version__ = "0.0.70"
5
+ __version__ = "0.0.71"
@@ -145,10 +145,17 @@ d = { 'settings': { 'branch': 'main',
145
145
  'fastllm.codex.main': ('codex.html#main', 'fastllm/codex.py'),
146
146
  'fastllm.codex.mk_jwt': ('codex.html#mk_jwt', 'fastllm/codex.py'),
147
147
  'fastllm.codex.token_expired': ('codex.html#token_expired', 'fastllm/codex.py')},
148
- 'fastllm.gemini': { 'fastllm.gemini._fn_call': ('gemini.html#_fn_call', 'fastllm/gemini.py'),
148
+ 'fastllm.gemini': { 'fastllm.gemini._GeminiStream': ('gemini.html#_geministream', 'fastllm/gemini.py'),
149
+ 'fastllm.gemini._GeminiStream.__aiter__': ('gemini.html#_geministream.__aiter__', 'fastllm/gemini.py'),
150
+ 'fastllm.gemini._GeminiStream.__init__': ('gemini.html#_geministream.__init__', 'fastllm/gemini.py'),
151
+ 'fastllm.gemini._continuation_token': ('gemini.html#_continuation_token', 'fastllm/gemini.py'),
152
+ 'fastllm.gemini._fn_call': ('gemini.html#_fn_call', 'fastllm/gemini.py'),
149
153
  'fastllm.gemini._fn_calls': ('gemini.html#_fn_calls', 'fastllm/gemini.py'),
150
154
  'fastllm.gemini._gem_filter_sch': ('gemini.html#_gem_filter_sch', 'fastllm/gemini.py'),
151
155
  'fastllm.gemini._gem_part_type': ('gemini.html#_gem_part_type', 'fastllm/gemini.py'),
156
+ 'fastllm.gemini._merge_metadata': ('gemini.html#_merge_metadata', 'fastllm/gemini.py'),
157
+ 'fastllm.gemini._merge_response': ('gemini.html#_merge_response', 'fastllm/gemini.py'),
158
+ 'fastllm.gemini._total_usage': ('gemini.html#_total_usage', 'fastllm/gemini.py'),
152
159
  'fastllm.gemini._url_cite': ('gemini.html#_url_cite', 'fastllm/gemini.py'),
153
160
  'fastllm.gemini.acollect_stream': ('gemini.html#acollect_stream', 'fastllm/gemini.py'),
154
161
  'fastllm.gemini.collate_raw': ('gemini.html#collate_raw', 'fastllm/gemini.py'),
@@ -178,7 +185,8 @@ d = { 'settings': { 'branch': 'main',
178
185
  'fastllm.gemini.norm_sse_event': ('gemini.html#norm_sse_event', 'fastllm/gemini.py'),
179
186
  'fastllm.gemini.norm_tool_calls': ('gemini.html#norm_tool_calls', 'fastllm/gemini.py'),
180
187
  'fastllm.gemini.norm_usage': ('gemini.html#norm_usage', 'fastllm/gemini.py'),
181
- 'fastllm.gemini.raw_msg': ('gemini.html#raw_msg', 'fastllm/gemini.py')},
188
+ 'fastllm.gemini.raw_msg': ('gemini.html#raw_msg', 'fastllm/gemini.py'),
189
+ 'fastllm.gemini.request': ('gemini.html#request', 'fastllm/gemini.py')},
182
190
  'fastllm.openai_chat': { 'fastllm.openai_chat.acollect_stream': ('oai_chat.html#acollect_stream', 'fastllm/openai_chat.py'),
183
191
  'fastllm.openai_chat.collate_raw': ('oai_chat.html#collate_raw', 'fastllm/openai_chat.py'),
184
192
  'fastllm.openai_chat.cost': ('oai_chat.html#cost', 'fastllm/openai_chat.py'),
@@ -194,14 +194,19 @@ async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None,
194
194
  ep = endpoint or api.endpoint
195
195
  path,params = ep(model, stream) if callable(ep) else (ep, None)
196
196
  if defaults.debug_mode: _debug_print(model, api_name, vendor_name, body, None if own else cli._url(path))
197
- async def _call(): return await provider_req(cli, path, body, params=params, stream=stream, headers=headers)
197
+ async def _call(body=body):
198
+ async def send(): return await provider_req(cli, path, body, params=params, stream=stream, headers=headers)
199
+ return await send() if stream else await _retry_call(send, retries, retry_delay)
200
+ async def _request():
201
+ if req := getattr(api, 'request', None): return await req(_call, body, stream=stream)
202
+ return await _call()
198
203
  async def _mk_gen():
199
- resp = payload if own else await _call() # an own-transport api's `acollect_stream` makes the request itself, from the payload
204
+ resp = payload if own else await _request() # an own-transport api's collector makes its request from the payload
200
205
  async for o in api.acollect_stream(resp, model=model, vendor_name=vendor_name, stop_callables=stop_callables): yield o
201
206
  if stream or own:
202
207
  result = _retry_stream(_mk_gen, retries, retry_delay)
203
208
  if stream: return result
204
209
  async for completion in result: pass # own-transport apis are streaming-only: drain for the completion
205
210
  return completion
206
- resp = await _retry_call(_call, retries, retry_delay)
211
+ resp = await _request()
207
212
  return mk_completion(resp, model=model, api_name=api_name, vendor_name=vendor_name)
@@ -9,7 +9,7 @@ __all__ = ['effort', 'contents', 'stop_reason', 'cite_footnote', 'postproc', 'Re
9
9
  'StopSequencesCallback', 'StreamAccum', 'adisplay_stream']
10
10
 
11
11
  # %% ../nbs/09_chat.ipynb #d5a3bc1f
12
- import secrets
12
+ import secrets, inspect
13
13
  from typing import Optional,Callable
14
14
  from fastcore.funccall import mk_ns, call_func, call_func_async, get_schema
15
15
  from fastcore.utils import *
@@ -94,7 +94,10 @@ async def structured(
94
94
  if not forced: sp = f"{part_txt(sp)}\n\nAnswer by calling the `{name}` tool.".strip()
95
95
  r = await acomplete(msgs, m, system=sp, tools=[t], tool_choice=name if forced else 'auto', **kwargs)
96
96
  if not r.tool_calls: raise ValueError(f"{m} answered without calling `{name}`")
97
- return tool(**r.tool_calls[0].arguments)
97
+ args = r.tool_calls[0].arguments
98
+ try: inspect.signature(tool).bind(**args)
99
+ except TypeError as e: raise TypeError(f"{m} called `{name}` with arguments that don't fit it ({e}): {args}") from None
100
+ return tool(**args)
98
101
 
99
102
  # %% ../nbs/09_chat.ipynb #1fe8a9bc
100
103
  def _has_search(info): return bool(info.get('search_context_cost_per_query') or info.get('supports_web_search'))
@@ -4,17 +4,18 @@
4
4
 
5
5
  # %% auto #0
6
6
  __all__ = ['api_ns', 'endpoint', 'grounding', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'raw_msg',
7
- 'norm_sse_event', 'delta_index_fn', 'collate_raw', 'acollect_stream', 'denorm_tool_use', 'denorm_assistant',
8
- 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice', 'denorm_reasoning',
9
- 'denorm_web_search', 'denorm_system', 'denorm_user', 'denorm_image', 'denorm_audio', 'denorm_video',
10
- 'denorm_file', 'denorm_tool_result', 'mk_payload', 'get_hdrs', 'cost']
7
+ 'norm_sse_event', 'delta_index_fn', 'collate_raw', 'request', 'acollect_stream', 'denorm_tool_use',
8
+ 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice',
9
+ 'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user', 'denorm_image', 'denorm_audio',
10
+ 'denorm_video', 'denorm_file', 'denorm_tool_result', 'mk_payload', 'get_hdrs', 'cost']
11
11
 
12
12
  # %% ../nbs/07_gemini.ipynb #02afd3d7
13
13
  import html
14
+ from copy import deepcopy
14
15
  from collections import Counter
15
16
  from fastcore.utils import *
16
17
  from fastcore.meta import *
17
- from fasttransport.errors import api_error_from_event
18
+ from fasttransport.errors import APIError, api_error_from_event
18
19
 
19
20
  from .types import *
20
21
  from aidialog.msg_parts import (MdStr, Part, PartType, Msg, Completion, Text, Thinking, ToolUse, ToolResult, tool_text,
@@ -67,9 +68,9 @@ def norm_usage(resp):
67
68
  reasoning = int(usg.get("thoughtsTokenCount", 0) or 0)
68
69
  parts = nested_idx(resp, 'candidates', 0, 'content', 'parts') or []
69
70
  cand = nested_idx(resp, 'candidates', 0) or {}
70
- stu = {}
71
- if any("executableCode" in p for p in parts): stu["code_execution"] = sum(1 for p in parts if "executableCode" in p)
72
- if call := grounding(cand)[1]: stu['google_search'] = len(call.arguments['queries'] or [1])
71
+ stu = dict(usg.get('server_tool_use') or {})
72
+ if any("executableCode" in p for p in parts): stu.setdefault('code_execution', sum(1 for p in parts if "executableCode" in p))
73
+ if call := grounding(cand)[1]: stu.setdefault('google_search', len(call.arguments['queries'] or [1]))
73
74
  if stu: usg['server_tool_use'] = stu
74
75
  return Usage(prompt_tokens=pt, completion_tokens=ct, total_tokens=tt,
75
76
  cached_tokens=cached, reasoning_tokens=reasoning, search_queries=stu.get('google_search', 0), raw=usg)
@@ -79,7 +80,8 @@ def norm_finish(resp, tcs=None):
79
80
  "Canonicalize finish_reason to OpenAI Chat values: stop, tool_calls, length, content_filter."
80
81
  reason = nested_idx(resp, 'candidates', 0, 'finishReason')
81
82
  if reason is not None: reason = reason.lower()
82
- mp = dict(stop=FinishReason.stop, max_tokens=FinishReason.length, safety=FinishReason.content_filter, blocklist=FinishReason.content_filter)
83
+ mp = dict(stop=FinishReason.stop, max_tokens=FinishReason.length, continuation=FinishReason.length,
84
+ safety=FinishReason.content_filter, blocklist=FinishReason.content_filter)
83
85
  r = mp.get(reason, reason)
84
86
  return FinishReason.tool_calls if r==FinishReason.stop and any(~L(tcs).attrgot('server')) else r
85
87
 
@@ -134,21 +136,108 @@ def delta_index_fn(d, typ, last_typ, last_idx):
134
136
  # %% ../nbs/07_gemini.ipynb #33506d56
135
137
  def collate_raw(deltas):
136
138
  "The model content rebuilt from a stream's chunks"
137
- parts = []
139
+ parts,start = [],0
138
140
  for ev in L(deltas).attrgot('raw'):
141
+ if ev.get('_continuation_start'): start = len(parts)
139
142
  for p in nested_idx(ev, 'candidates', 0, 'content', 'parts') or []:
140
- last = parts[-1] if parts else None
143
+ last = parts[-1] if len(parts) > start else None
141
144
  if 'text' in p and last and 'text' in last and last.get('thought') == p.get('thought') and 'thoughtSignature' not in last:
142
145
  last['text'] += p['text']
143
146
  if 'thoughtSignature' in p: last['thoughtSignature'] = p['thoughtSignature']
144
147
  else: parts.append(dict(p))
145
148
  return dict(role='model', parts=parts)
146
149
 
150
+ # %% ../nbs/07_gemini.ipynb #e9f0cac6
151
+ def _merge_metadata(prev, cur):
152
+ res = prev.copy()
153
+ for k,v in cur.items():
154
+ old = res.get(k)
155
+ if isinstance(old, dict) and isinstance(v, dict): v = _merge_metadata(old, v)
156
+ elif isinstance(old, list) and isinstance(v, list): v = old + v
157
+ elif isinstance(old, (int, float)) and isinstance(v, (int, float)): v = old + v
158
+ res[k] = v
159
+ return res
160
+
161
+ def _total_usage(hops):
162
+ res = {}
163
+ for u in hops: res = _merge_metadata(res, u)
164
+ if len(hops) > 1: res['continuation_usage'] = hops.copy()
165
+ return res
166
+
167
+ def _continuation_token(resp, previous=None):
168
+ cand = nested_idx(resp, 'candidates', 0) or {}
169
+ token = cand.get('continuationToken')
170
+ if cand.get('finishReason') == 'CONTINUATION' and token != previous and not resp.get('promptFeedback'): return token
171
+
172
+ def _merge_response(prev, cur):
173
+ res = prev | cur
174
+ if not (prev.get('candidates') or cur.get('candidates')): return res
175
+ a = nested_idx(prev, 'candidates', 0) or {}
176
+ b = deepcopy(nested_idx(cur, 'candidates', 0) or {})
177
+ ap,bp = nested_idx(a, 'content', 'parts') or [], nested_idx(b, 'content', 'parts') or []
178
+ ag,bg = a.get('groundingMetadata') or {}, b.get('groundingMetadata') or {}
179
+ for support in bg.get('groundingSupports', []):
180
+ if 'groundingChunkIndices' in support:
181
+ support['groundingChunkIndices'] = [i + len(ag.get('groundingChunks', [])) for i in support['groundingChunkIndices']]
182
+ if (seg := support.get('segment')) is not None: seg['partIndex'] = seg.get('partIndex', 0) + len(ap)
183
+ cand = a | b
184
+ if a.get('content') or b.get('content'):
185
+ cand['content'] = (a.get('content') or {}) | (b.get('content') or {}) | dict(parts=ap + bp)
186
+ if ag or bg: cand['groundingMetadata'] = _merge_metadata(ag, bg)
187
+ if b.get('finishReason'):
188
+ for k in ('continuationToken', 'finishMessage'):
189
+ if k not in b: cand.pop(k, None)
190
+ res['candidates'] = [cand]
191
+ return res
192
+
193
+ # %% ../nbs/07_gemini.ipynb #594ca72a
194
+ class _GeminiStream:
195
+ def __init__(self, call, body):
196
+ self.call,self.body = call,body
197
+ self.stopped = self.continued = False
198
+
199
+ async def __aiter__(self):
200
+ hops,token = [],self.body.get('continuationToken')
201
+ while True:
202
+ hop,usg = {},None
203
+ resp = await self.call(self.body | dict(continuationToken=token) if token else self.body)
204
+ async for ev in resp:
205
+ ev,first = obj2dict(ev),not hop
206
+ hop = _merge_response(hop, ev)
207
+ usg = norm_usage(hop | dict(usageMetadata=dict(hop.get('usageMetadata') or {})))
208
+ out = ev | dict(usageMetadata=_total_usage(hops + [usg.raw if usg else {}]), _continuation_start=first)
209
+ if not self.stopped and _continuation_token(hop, token):
210
+ out = out | dict(candidates=[dict(c, finishReason=None) for c in out.get('candidates', [])])
211
+ yield out
212
+ if usg: hops.append(usg.raw)
213
+ token = None if self.stopped else _continuation_token(hop, token)
214
+ if not token: return
215
+ self.continued = True
216
+
217
+ async def request(call, body, stream=False):
218
+ "Send a Gemini generation, resuming only at explicit continuation boundaries."
219
+ if stream: return _GeminiStream(call, body)
220
+ res,hops,token = {},[],body.get('continuationToken')
221
+ while True:
222
+ resp = obj2dict(await call(body | dict(continuationToken=token) if token else body))
223
+ if usg := norm_usage(resp): hops.append(usg.raw)
224
+ res = _merge_response(res, resp)
225
+ token = _continuation_token(resp, token)
226
+ if not token:
227
+ if hops: res['usageMetadata'] = _total_usage(hops)
228
+ return res
229
+
147
230
  # %% ../nbs/07_gemini.ipynb #328af4d5
148
231
  @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
149
232
  async def acollect_stream(resp, **kwargs):
150
233
  res = mk_acollect_stream(norm_and_yield(resp, norm_sse_event), index_fn=delta_index_fn, api_name='gemini', **kwargs)
151
- async for o in res: yield o
234
+ try:
235
+ async for o in res:
236
+ if isinstance(resp, _GeminiStream) and isinstance(o, Status): resp.stopped = True
237
+ yield o
238
+ except APIError as e:
239
+ if isinstance(resp, _GeminiStream) and resp.continued: e.retryable = False # never replay a completed segment
240
+ raise
152
241
 
153
242
  # %% ../nbs/07_gemini.ipynb #58d0cb74
154
243
  def denorm_tool_use(p:ToolUse):
@@ -311,6 +400,7 @@ def get_hdrs(api_key=None, oauth_token=None):
311
400
  # %% ../nbs/07_gemini.ipynb #4ee3891f
312
401
  def cost(usage, m):
313
402
  raw = usage.raw
403
+ if hops := raw.get('continuation_usage'): return sum(cost(Usage(raw=u), m) for u in hops)
314
404
  prompt_tot = raw.get('promptTokenCount', 0)
315
405
  tier = price_tier(m, prompt_tot)
316
406
  in_rate = tier_rate(m, 'input_cost_per_token', tier)
@@ -324,14 +414,14 @@ def cost(usage, m):
324
414
  thoughts = raw.get('thoughtsTokenCount', 0) or 0
325
415
  cands = raw.get('candidatesTokenCount', 0) or 0
326
416
  reason_rate = m.get('output_cost_per_reasoning_token') or out_rate
327
- cost = in_txt * in_rate + cands * out_rate
328
- cost += cached * cache_rate
329
- cost += audio * (audio_rate or 0)
330
- cost += thoughts * reason_rate
331
- return cost
417
+ res = in_txt * in_rate + cands * out_rate
418
+ res += cached * cache_rate
419
+ res += audio * (audio_rate or 0)
420
+ res += thoughts * reason_rate
421
+ return res
332
422
 
333
423
  # %% ../nbs/07_gemini.ipynb #f7c0b989
334
424
  api_ns = dict(norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
335
425
  acollect_stream=acollect_stream, raw_msg=raw_msg, collate_raw=collate_raw, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs,
336
- endpoint=endpoint)
426
+ endpoint=endpoint, request=request)
337
427
  api_registry.register('gemini', **api_ns)