python-fastllm 0.0.64__tar.gz → 0.0.66__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/PKG-INFO +5 -3
  2. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/__init__.py +1 -1
  3. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/_modidx.py +17 -5
  4. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/acomplete.py +29 -2
  5. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/anthropic.py +24 -28
  6. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/chat.py +41 -32
  7. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/gemini.py +5 -5
  8. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/model_prices.json +29478 -31958
  9. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/openai_chat.py +33 -22
  10. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/openai_responses.py +7 -9
  11. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/responses.py +84 -33
  12. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/streaming.py +5 -4
  13. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/types.py +41 -22
  14. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/pyproject.toml +4 -1
  15. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/PKG-INFO +5 -3
  16. python_fastllm-0.0.66/python_fastllm.egg-info/requires.txt +7 -0
  17. python_fastllm-0.0.64/python_fastllm.egg-info/requires.txt +0 -4
  18. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/README.md +0 -0
  19. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/codex.py +0 -0
  20. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/SOURCES.txt +0 -0
  21. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/dependency_links.txt +0 -0
  22. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/entry_points.txt +0 -0
  23. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/top_level.txt +0 -0
  24. {python_fastllm-0.0.64 → python_fastllm-0.0.66}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: python-fastllm
3
- Version: 0.0.64
3
+ Version: 0.0.66
4
4
  Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
5
5
  License-Expression: Apache-2.0
6
6
  Project-URL: Repository, https://github.com/AnswerDotAI/fastllm
@@ -9,10 +9,12 @@ Classifier: Programming Language :: Python :: 3
9
9
  Classifier: Programming Language :: Python :: 3 :: Only
10
10
  Requires-Python: >=3.10
11
11
  Description-Content-Type: text/markdown
12
- Requires-Dist: fastcore>=2.1.18
13
- Requires-Dist: aidialog>=0.0.27
12
+ Requires-Dist: fastcore>=2.2.14
13
+ Requires-Dist: aidialog>=0.0.38
14
14
  Requires-Dist: fasttransport>=0.0.2
15
15
  Requires-Dist: pillow
16
+ Provides-Extra: dev
17
+ Requires-Dist: pycachy>=0.0.22; extra == "dev"
16
18
 
17
19
  # fastllm
18
20
 
@@ -2,4 +2,4 @@
2
2
 
3
3
  - `fastllm.responses`: Provider-independent Responses resources and streaming over FastLLM"""
4
4
 
5
- __version__ = "0.0.64"
5
+ __version__ = "0.0.66"
@@ -12,17 +12,18 @@ d = { 'settings': { 'branch': 'main',
12
12
  'fastllm/acomplete.py'),
13
13
  'fastllm.acomplete._debug_print': ('acomplete.html#_debug_print', 'fastllm/acomplete.py'),
14
14
  'fastllm.acomplete._is_ctx_exceeded': ('acomplete.html#_is_ctx_exceeded', 'fastllm/acomplete.py'),
15
+ 'fastllm.acomplete._limit_tool_text': ('acomplete.html#_limit_tool_text', 'fastllm/acomplete.py'),
15
16
  'fastllm.acomplete._raise_if_done': ('acomplete.html#_raise_if_done', 'fastllm/acomplete.py'),
16
17
  'fastllm.acomplete._retry_call': ('acomplete.html#_retry_call', 'fastllm/acomplete.py'),
17
18
  'fastllm.acomplete._retry_stream': ('acomplete.html#_retry_stream', 'fastllm/acomplete.py'),
19
+ 'fastllm.acomplete._tool_messages': ('acomplete.html#_tool_messages', 'fastllm/acomplete.py'),
18
20
  'fastllm.acomplete.acomplete': ('acomplete.html#acomplete', 'fastllm/acomplete.py'),
19
21
  'fastllm.acomplete.mk_client': ('acomplete.html#mk_client', 'fastllm/acomplete.py'),
20
22
  'fastllm.acomplete.split_vendor': ('acomplete.html#split_vendor', 'fastllm/acomplete.py')},
21
23
  'fastllm.anthropic': { 'fastllm.anthropic._add_cache_control': ('anthropic.html#_add_cache_control', 'fastllm/anthropic.py'),
22
24
  'fastllm.anthropic._ant_cc': ('anthropic.html#_ant_cc', 'fastllm/anthropic.py'),
23
25
  'fastllm.anthropic._ant_part_type': ('anthropic.html#_ant_part_type', 'fastllm/anthropic.py'),
24
- 'fastllm.anthropic._clear_cache_control': ( 'anthropic.html#_clear_cache_control',
25
- 'fastllm/anthropic.py'),
26
+ 'fastllm.anthropic._copy_msg': ('anthropic.html#_copy_msg', 'fastllm/anthropic.py'),
26
27
  'fastllm.anthropic._sanid': ('anthropic.html#_sanid', 'fastllm/anthropic.py'),
27
28
  'fastllm.anthropic._url_cite': ('anthropic.html#_url_cite', 'fastllm/anthropic.py'),
28
29
  'fastllm.anthropic.acollect_stream': ('anthropic.html#acollect_stream', 'fastllm/anthropic.py'),
@@ -65,8 +66,10 @@ d = { 'settings': { 'branch': 'main',
65
66
  'fastllm.chat.AsyncChat._track': ('chat.html#asyncchat._track', 'fastllm/chat.py'),
66
67
  'fastllm.chat.AsyncChat.add_cb': ('chat.html#asyncchat.add_cb', 'fastllm/chat.py'),
67
68
  'fastllm.chat.AsyncChat.add_cbs': ('chat.html#asyncchat.add_cbs', 'fastllm/chat.py'),
69
+ 'fastllm.chat.AsyncChat.dumps': ('chat.html#asyncchat.dumps', 'fastllm/chat.py'),
68
70
  'fastllm.chat.AsyncChat.full': ('chat.html#asyncchat.full', 'fastllm/chat.py'),
69
71
  'fastllm.chat.AsyncChat.print_hist': ('chat.html#asyncchat.print_hist', 'fastllm/chat.py'),
72
+ 'fastllm.chat.AsyncChat.reply_msgs': ('chat.html#asyncchat.reply_msgs', 'fastllm/chat.py'),
70
73
  'fastllm.chat.AsyncChat.tcdict': ('chat.html#asyncchat.tcdict', 'fastllm/chat.py'),
71
74
  'fastllm.chat.ChatCallback': ('chat.html#chatcallback', 'fastllm/chat.py'),
72
75
  'fastllm.chat.ChatCallback.__repr__': ('chat.html#chatcallback.__repr__', 'fastllm/chat.py'),
@@ -100,17 +103,18 @@ d = { 'settings': { 'branch': 'main',
100
103
  'fastllm.chat.UsageStats.__init__': ('chat.html#usagestats.__init__', 'fastllm/chat.py'),
101
104
  'fastllm.chat.UsageStats.__radd__': ('chat.html#usagestats.__radd__', 'fastllm/chat.py'),
102
105
  'fastllm.chat.UsageStats.__repr__': ('chat.html#usagestats.__repr__', 'fastllm/chat.py'),
103
- 'fastllm.chat.UsageStats.fmt': ('chat.html#usagestats.fmt', 'fastllm/chat.py'),
104
106
  'fastllm.chat.UsageStats.from_response': ('chat.html#usagestats.from_response', 'fastllm/chat.py'),
105
107
  'fastllm.chat._active_fence_langs': ('chat.html#_active_fence_langs', 'fastllm/chat.py'),
106
108
  'fastllm.chat._alite_call_func': ('chat.html#_alite_call_func', 'fastllm/chat.py'),
107
109
  'fastllm.chat._call_func': ('chat.html#_call_func', 'fastllm/chat.py'),
110
+ 'fastllm.chat._cut': ('chat.html#_cut', 'fastllm/chat.py'),
108
111
  'fastllm.chat._handle_stop_reason': ('chat.html#_handle_stop_reason', 'fastllm/chat.py'),
109
112
  'fastllm.chat._has_search': ('chat.html#_has_search', 'fastllm/chat.py'),
110
113
  'fastllm.chat._has_stop': ('chat.html#_has_stop', 'fastllm/chat.py'),
111
114
  'fastllm.chat._inject_tool_reminder': ('chat.html#_inject_tool_reminder', 'fastllm/chat.py'),
112
115
  'fastllm.chat._lite_call_func': ('chat.html#_lite_call_func', 'fastllm/chat.py'),
113
116
  'fastllm.chat._mk_tool_result': ('chat.html#_mk_tool_result', 'fastllm/chat.py'),
117
+ 'fastllm.chat._trim': ('chat.html#_trim', 'fastllm/chat.py'),
114
118
  'fastllm.chat._usrtools': ('chat.html#_usrtools', 'fastllm/chat.py'),
115
119
  'fastllm.chat.add_warning': ('chat.html#add_warning', 'fastllm/chat.py'),
116
120
  'fastllm.chat.adisplay_stream': ('chat.html#adisplay_stream', 'fastllm/chat.py'),
@@ -122,8 +126,7 @@ d = { 'settings': { 'branch': 'main',
122
126
  'fastllm.chat.run_fence_tool': ('chat.html#run_fence_tool', 'fastllm/chat.py'),
123
127
  'fastllm.chat.search_count': ('chat.html#search_count', 'fastllm/chat.py'),
124
128
  'fastllm.chat.stop_reason': ('chat.html#stop_reason', 'fastllm/chat.py'),
125
- 'fastllm.chat.structured': ('chat.html#structured', 'fastllm/chat.py'),
126
- 'fastllm.chat.think_sep': ('chat.html#think_sep', 'fastllm/chat.py')},
129
+ 'fastllm.chat.structured': ('chat.html#structured', 'fastllm/chat.py')},
127
130
  'fastllm.codex': { 'fastllm.codex._codex_hdrs': ('codex.html#_codex_hdrs', 'fastllm/codex.py'),
128
131
  'fastllm.codex._codex_vendor_auth': ('codex.html#_codex_vendor_auth', 'fastllm/codex.py'),
129
132
  'fastllm.codex._merge_tokens': ('codex.html#_merge_tokens', 'fastllm/codex.py'),
@@ -304,19 +307,27 @@ d = { 'settings': { 'branch': 'main',
304
307
  'fastllm.responses._ResponseStream': ('responses.html#_responsestream', 'fastllm/responses.py'),
305
308
  'fastllm.responses._ResponseStream.__init__': ( 'responses.html#_responsestream.__init__',
306
309
  'fastllm/responses.py'),
310
+ 'fastllm.responses._ResponseStream.close_thinking': ( 'responses.html#_responsestream.close_thinking',
311
+ 'fastllm/responses.py'),
307
312
  'fastllm.responses._ResponseStream.done_events': ( 'responses.html#_responsestream.done_events',
308
313
  'fastllm/responses.py'),
314
+ 'fastllm.responses._ResponseStream.end_step': ( 'responses.html#_responsestream.end_step',
315
+ 'fastllm/responses.py'),
309
316
  'fastllm.responses._ResponseStream.event': ( 'responses.html#_responsestream.event',
310
317
  'fastllm/responses.py'),
311
318
  'fastllm.responses._ResponseStream.part_events': ( 'responses.html#_responsestream.part_events',
312
319
  'fastllm/responses.py'),
313
320
  'fastllm.responses._ResponseStream.text_events': ( 'responses.html#_responsestream.text_events',
314
321
  'fastllm/responses.py'),
322
+ 'fastllm.responses._ResponseStream.thinking_events': ( 'responses.html#_responsestream.thinking_events',
323
+ 'fastllm/responses.py'),
315
324
  'fastllm.responses._ResponseStream.tool_events': ( 'responses.html#_responsestream.tool_events',
316
325
  'fastllm/responses.py'),
317
326
  'fastllm.responses._arguments': ('responses.html#_arguments', 'fastllm/responses.py'),
327
+ 'fastllm.responses._is_srv': ('responses.html#_is_srv', 'fastllm/responses.py'),
318
328
  'fastllm.responses._msg_parts': ('responses.html#_msg_parts', 'fastllm/responses.py'),
319
329
  'fastllm.responses._new_id': ('responses.html#_new_id', 'fastllm/responses.py'),
330
+ 'fastllm.responses._reasoning_item': ('responses.html#_reasoning_item', 'fastllm/responses.py'),
320
331
  'fastllm.responses._response_base': ('responses.html#_response_base', 'fastllm/responses.py'),
321
332
  'fastllm.responses._server_call': ('responses.html#_server_call', 'fastllm/responses.py'),
322
333
  'fastllm.responses._text_content': ('responses.html#_text_content', 'fastllm/responses.py'),
@@ -390,6 +401,7 @@ d = { 'settings': { 'branch': 'main',
390
401
  'fastllm.types.payload_kwargs': ('types.html#payload_kwargs', 'fastllm/types.py'),
391
402
  'fastllm.types.price_tier': ('types.html#price_tier', 'fastllm/types.py'),
392
403
  'fastllm.types.provider_req': ('types.html#provider_req', 'fastllm/types.py'),
404
+ 'fastllm.types.rebuild_parts': ('types.html#rebuild_parts', 'fastllm/types.py'),
393
405
  'fastllm.types.register_model_info': ('types.html#register_model_info', 'fastllm/types.py'),
394
406
  'fastllm.types.register_sub_model': ('types.html#register_sub_model', 'fastllm/types.py'),
395
407
  'fastllm.types.resize_b64': ('types.html#resize_b64', 'fastllm/types.py'),
@@ -3,7 +3,7 @@
3
3
  # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03_acomplete.ipynb.
4
4
 
5
5
  # %% auto #0
6
- __all__ = ['vendor_mapping', 'defaults', 'split_vendor', 'mk_client', 'ContextWindowExceededError', 'acomplete']
6
+ __all__ = ['vendor_mapping', 'vendor_doms', 'defaults', 'split_vendor', 'mk_client', 'ContextWindowExceededError', 'acomplete']
7
7
 
8
8
  # %% ../nbs/03_acomplete.ipynb #f2f57253
9
9
  import asyncio,json,httpx2
@@ -40,6 +40,7 @@ vendor_mapping = {
40
40
  "minimax": ('anthropic', "https://api.minimax.io/anthropic", "MINIMAX_API_KEY"),
41
41
  "meta": ('openai_chat', "https://api.meta.ai/v1", "META_API_KEY"),
42
42
  "zai": ('openai_chat', "https://api.z.ai/api/paas/v4", "ZAI_API_KEY")}
43
+ vendor_doms = tuple(dict.fromkeys(httpx2.URL(u).host for _,u,_ in vendor_mapping.values())) # hosts of every vendor's API, for cachy
43
44
 
44
45
  # %% ../nbs/03_acomplete.ipynb #e3ed40fb
45
46
  def split_vendor(model):
@@ -134,6 +135,32 @@ async def _retry_stream(mk_gen, retries=2, retry_delay=0.5):
134
135
  except APIError as e: await _raise_if_done(e, n, retries, retry_delay, yielded=yielded)
135
136
 
136
137
  # %% ../nbs/03_acomplete.ipynb #2379ec94
138
+ _TOOL_TEXT_MAX = 1 << 20
139
+
140
+ def _limit_tool_text(text):
141
+ scalar = isinstance(text, str)
142
+ parts = [Text(text)] if scalar else text
143
+ if not isinstance(parts, list): return text
144
+ size = sum(len(p.text.encode()) for p in parts if isinstance(p, Text) and isinstance(p.text, str))
145
+ if size <= _TOOL_TEXT_MAX: return text
146
+ cut,limited = size-_TOOL_TEXT_MAX,[]
147
+ for p in parts:
148
+ if isinstance(p, Text) and isinstance(p.text, str):
149
+ data = p.text.encode()
150
+ kept = data[min(cut, len(data)):].decode(errors='ignore')
151
+ cut = max(0, cut-len(data))
152
+ p = p.replace(text=type(p.text)(kept))
153
+ limited.append(p)
154
+ removed = size-sum(len(p.text.encode()) for p in limited if isinstance(p, Text) and isinstance(p.text, str))
155
+ at = next(i for i,p in enumerate(limited) if isinstance(p, Text) and p.text)
156
+ limited[at] = limited[at].replace(text=type(limited[at].text)(f'[{removed} bytes of output removed]\n'+limited[at].text))
157
+ return limited[0].text if scalar else limited
158
+
159
+ def _tool_messages(msgs):
160
+ if not msgs: return msgs
161
+ return [Msg(m.role, [p.replace(text=_limit_tool_text(p.text)) if isinstance(p, ToolResult) else p for p in m.content],
162
+ raw=m.raw, raw_api=m.raw_api) if any(isinstance(p, ToolResult) for p in m.content) else m for m in msgs]
163
+
137
164
  @delegates(payload_kwargs)
138
165
  async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None, oauth_token=None, base_url=None, endpoint=None, xtra_body=None, xtra_hdrs=None,
139
166
  stream=False, previous_response_id=None, stop_callables=None, retries=2, retry_delay=0.5, **kwargs):
@@ -158,7 +185,7 @@ async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None,
158
185
  if not getattr(api, 'supports_previous_response_id', False):
159
186
  raise ValueError('previous_response_id is not supported by this transport')
160
187
  kwargs['previous_response_id'] = previous_response_id
161
- payload = api.mk_payload(msgs, model, stream=stream, **kwargs)
188
+ payload = api.mk_payload(_tool_messages(msgs), model, stream=stream, **kwargs)
162
189
  if fix := getattr(api, 'fix_payload', None): fix(payload, model, vendor_name)
163
190
  body = payload | ifnone(xtra_body, {})
164
191
  if not own:
@@ -11,7 +11,6 @@ __all__ = ['ant_tc_types', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'f
11
11
 
12
12
  # %% ../nbs/06_anthropic.ipynb #02afd3d7
13
13
  import json, copy
14
- from collections import Counter
15
14
  from fastcore.utils import *
16
15
  from fastcore.meta import *
17
16
  from fasttransport.errors import api_error_from_event
@@ -187,12 +186,12 @@ def denorm_tool_use(p:ToolUse):
187
186
  return _ant_cc(dict(type='tool_use', id=_sanid(p.id or ''), name=p.name or '', input=p.arguments or {}), p)
188
187
 
189
188
  def denorm_assistant(m:Msg):
190
- "The message as the API sent it, else its text and tool calls; a cache mark on the last part is copied onto the last raw block"
191
- if m.raw:
189
+ "The message as the API sent it, else its text and client tool calls; a cache mark on the last part is copied onto the last raw block"
190
+ if m.raw and m.raw_api == 'anthropic':
192
191
  cc = [p.cache_control for p in m.content if p.cache_control]
193
192
  if not cc: return m.raw
194
193
  return m.raw | dict(content=m.raw['content'][:-1] + [m.raw['content'][-1] | dict(cache_control=cc[-1])])
195
- blocks = [_ant_cc(dict(type='text', text=p.text or ''), p) if isinstance(p, Text) else denorm_tool_use(p) for p in m.content if isinstance(p, (Text, ToolUse))]
194
+ blocks = [_ant_cc(dict(type='text', text=p.text or ''), p) if isinstance(p, Text) else denorm_tool_use(p) for p in rebuild_parts(m)]
196
195
  return dict(role='assistant', content=blocks)
197
196
 
198
197
  def denorm_tool(m:Msg):
@@ -305,11 +304,11 @@ def _add_cache_control(
305
304
  return msg
306
305
 
307
306
  # %% ../nbs/06_anthropic.ipynb #6bc52ef5
308
- def _clear_cache_control(obj):
309
- "Clear canonical cache marks from a message or part."
310
- if isinstance(obj, Msg):
311
- for p in obj.content: _clear_cache_control(p)
312
- elif isinstance(obj, Part): obj.cache_control = None
307
+ def _copy_msg(m):
308
+ "A copy of `m` with copied parts, to mark without changing `m`"
309
+ m = copy.copy(m)
310
+ m.content = [copy.copy(p) for p in m.content]
311
+ return m
313
312
 
314
313
  def apply_cache_idxs(
315
314
  msgs, # Canonical Msgs to mark
@@ -317,27 +316,28 @@ def apply_cache_idxs(
317
316
  sp=None, # System prompt: str, Msg, or Part
318
317
  ttl=None # Cache TTL: '5m' (default) or '1h'
319
318
  ):
320
- "Replace cache breakpoints in `msgs` and `sp`, then return the system prompt."
321
- for m in msgs: _clear_cache_control(m)
322
- _clear_cache_control(sp)
319
+ "`msgs` and `sp` with breakpoints added at `cache_idxs`, marking copies"
320
+ msgs = list(msgs)
323
321
  if sp:
324
322
  if 0 in cache_idxs:
325
323
  if isinstance(sp, str): sp = Text(sp)
326
- if isinstance(sp, Part): sp = Msg('', [sp])
327
- _add_cache_control(sp, ttl)
324
+ sp = _add_cache_control(_copy_msg(sp if isinstance(sp, Msg) else Msg('', [sp])), ttl)
328
325
  cache_idxs = [o-1 if o>0 else o for o in cache_idxs if o]
329
326
  ms = [j for j,m in enumerate(msgs) if m.role != 'tool']
330
327
  for i in cache_idxs:
331
- try: _add_cache_control(msgs[ms[i]], ttl)
328
+ try: j = ms[i]
332
329
  except IndexError: continue
333
- return sp
330
+ msgs[j] = _add_cache_control(_copy_msg(msgs[j]), ttl)
331
+ return msgs,sp
334
332
 
335
333
  # %% ../nbs/06_anthropic.ipynb #587d4fe5
336
334
  @delegates(payload_kwargs)
337
335
  def mk_payload(msgs, model, **kwargs):
338
336
  sp = kwargs.get('system')
339
- if (ci:=kwargs.get('cache_idxs')) is not None: sp = apply_cache_idxs(msgs, ci, sp, kwargs.get('ttl'))
340
- payload = dict(model=model, messages=denorm_msgs(msgs), max_tokens=kwargs.get('max_tokens') or 1024)
337
+ if ci := kwargs.get('cache_idxs'): msgs,sp = apply_cache_idxs(msgs, ci, sp, kwargs.get('ttl'))
338
+ max_tokens = kwargs.get('max_tokens')
339
+ if max_tokens is None: max_tokens = get_model_info(model, 'anthropic').get('max_output_tokens') or 1024
340
+ payload = dict(model=model, messages=denorm_msgs(msgs), max_tokens=max_tokens)
341
341
  if kwargs.get('stream'): payload['stream'] = True
342
342
  if sp: payload['system'] = denorm_system(sp)
343
343
  if tools:=kwargs.get('tools'): payload['tools'] = denorm_tool_schs(tools)
@@ -358,17 +358,13 @@ def get_hdrs(api_key=None, oauth_token=None):
358
358
  # %% ../nbs/06_anthropic.ipynb #0d03643a
359
359
  def cost(usage, m):
360
360
  raw = usage.raw
361
- in_tok = raw['input_tokens']
362
- cache_read = raw.get('cache_read_input_tokens', 0)
363
361
  cc = raw.get('cache_creation', {}) or {}
364
- cache_5m = cc.get('ephemeral_5m_input_tokens', raw.get('cache_creation_input_tokens', 0))
365
- cache_1h = cc.get('ephemeral_1h_input_tokens', 0)
366
- cost = in_tok * m.input_cost_per_token
367
- cost += raw['output_tokens'] * m.output_cost_per_token
368
- cost += cache_read * m.get('cache_read_input_token_cost', 0)
369
- cost += cache_5m * m.get('cache_creation_input_token_cost', 0)
370
- cost += cache_1h * m.get('cache_creation_input_token_cost_above_1hr', 0)
371
- return cost
362
+ prompt = dict(input_cost_per_token=raw['input_tokens'], cache_read_input_token_cost=raw.get('cache_read_input_tokens', 0),
363
+ cache_creation_input_token_cost=cc.get('ephemeral_5m_input_tokens', raw.get('cache_creation_input_tokens', 0)),
364
+ cache_creation_input_token_cost_above_1hr=cc.get('ephemeral_1h_input_tokens', 0))
365
+ tier = price_tier(m, sum(prompt.values()))
366
+ toks = prompt | dict(output_cost_per_token=raw['output_tokens'])
367
+ return sum(n * tier_rate(m, k, tier) for k,n in toks.items())
372
368
 
373
369
  # %% ../nbs/06_anthropic.ipynb #f7c0b989
374
370
  api_registry.register('anthropic', norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
@@ -3,7 +3,7 @@
3
3
  # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/09_chat.ipynb.
4
4
 
5
5
  # %% auto #0
6
- __all__ = ['effort', 'contents', 'stop_reason', 'cite_footnote', 'postproc', 'Refresh', 'think_sep', 'lite_mk_func', 'structured',
6
+ __all__ = ['effort', 'contents', 'stop_reason', 'cite_footnote', 'postproc', 'Refresh', 'lite_mk_func', 'structured',
7
7
  'search_count', 'UsageStats', 'AsyncChat', 'astream_with_complete', 'ChatCallback', 'DeepseekMsgsCallback',
8
8
  'add_warning', 'StopReasonCallback', 'run_fence_tool', 'FenceToolCallback', 'ToolReminderCallback',
9
9
  'StopSequencesCallback', 'StreamAccum', 'adisplay_stream']
@@ -16,8 +16,8 @@ from fastcore.utils import *
16
16
  from fastcore.meta import delegates
17
17
 
18
18
  from .types import *
19
- from aidialog.msg_parts import (Msg, Part, PartType, Text, Thinking, ToolUse, tool_text, mk_tool_res_msg, tool_info, usage_info,
20
- StopResponse, ToolResponse, display_list, fmt2hist, hist2fmt, mk_result_fence, split_fence_msgs, extract_fence_call,
19
+ from aidialog.msg_parts import (Msg, Part, Text, Thinking, ToolUse, ToolResult, tool_text, mk_tool_res_msg, msg2dict,
20
+ StopResponse, ToolResponse, display_list, hist2fmt, mk_result_fence, split_fence_msgs, extract_fence_call,
21
21
  trunc_str, Completion, mk_msg, mk_msgs, part_txt)
22
22
  from .acomplete import *
23
23
  from .streaming import FenceToolStop, stop_sequences
@@ -53,11 +53,6 @@ class Refresh:
53
53
  "Stream signal: state-holding renderers should re-render in full (e.g. via `AsyncChat.full`)"
54
54
  formatted = ''
55
55
 
56
- def think_sep(last, typ, sep='\n\n'):
57
- "Separator to insert between `formatted` items when crossing a 🧠↔text boundary"
58
- think = {PartType.thinking, 'status'}
59
- return sep if last and typ and (typ in think) != (last in think) else ''
60
-
61
56
  # %% ../nbs/09_chat.ipynb #0e7d3980
62
57
  def lite_mk_func(f):
63
58
  if isinstance(f, dict): return f
@@ -65,9 +60,9 @@ def lite_mk_func(f):
65
60
 
66
61
  # %% ../nbs/09_chat.ipynb #bba6fd58
67
62
  def _mk_tool_result(res):
68
- "Unwrap `ToolResponse`, and format tool result message"
63
+ "Preserve canonical results, unwrap `ToolResponse`, or format other values as text"
69
64
  if isinstance(res, ToolResponse): return res.content
70
- return tool_text(res)
65
+ return res if isinstance(res, ToolResult) else tool_text(res)
71
66
 
72
67
  # %% ../nbs/09_chat.ipynb #a0fcc96e
73
68
  def _call_func(tc:ToolUse, tool_schemas, ns, callf):
@@ -155,11 +150,6 @@ class UsageStats:
155
150
  if self.model: parts.append(self.model.split('/')[-1])
156
151
  return ' | '.join(parts)
157
152
 
158
- def fmt(self):
159
- if not self.total_tokens: return ''
160
- d = {k:v for k,v in self.__dict__.items() if v and not k.startswith('_')}
161
- return "\n\n" + fenced(dumps(d, ensure_ascii=False), usage_info) + "\n"
162
-
163
153
  # %% ../nbs/09_chat.ipynb #cb3d7e77
164
154
  class AsyncChat:
165
155
  def __init__(
@@ -280,7 +270,7 @@ async def astream_with_complete(self, agen, postproc=noop):
280
270
  # %% ../nbs/09_chat.ipynb #a049cf52
281
271
  @patch
282
272
  @delegates(acomplete)
283
- async def _call(self:AsyncChat, msg=None, step=1, search=None, tool_choice=None, initial_body=None, **kwargs):
273
+ async def _call(self:AsyncChat, msg=None, step=1, search=None, tool_choice=None, **kwargs):
284
274
  "One model turn plus its tool round, recursing for the next; turn-constant options come from `turn_opts`"
285
275
  t = AttrDict(self.turn_opts)
286
276
  if step>t.max_steps+1: return
@@ -291,12 +281,10 @@ async def _call(self:AsyncChat, msg=None, step=1, search=None, tool_choice=None,
291
281
 
292
282
  self.turn_kwargs, self.stream = kwargs, t.stream
293
283
  async for o in self._call_cbs('before_acomplete'): yield o
294
- kw = self.turn_kwargs
295
- if initial_body: kw = merge(kw, dict(xtra_body=merge(kw.get('xtra_body'), initial_body)))
296
284
  res = await acomplete(self.turn_msgs, self.model, system=self.turn_sysp, stream=t.stream, tools=self.tool_schemas,
297
285
  tool_choice=tool_choice, max_tokens=int(max_tokens), temperature=None if t.think else ifnone(t.temp,self.temp),
298
286
  previous_response_id=self.response_id if self.use_previous_response_id else None,
299
- cache_idxs=self.cache_idxs if self.cache else [], ttl=self.ttl, **kw)
287
+ cache_idxs=self.cache_idxs if self.cache else [], ttl=self.ttl, **self.turn_kwargs)
300
288
  if t.stream:
301
289
  res = astream_with_complete(res, postproc=postproc)
302
290
  async for chunk in res:
@@ -346,7 +334,6 @@ async def __call__(
346
334
  stream=False, # Stream results
347
335
  max_steps=2, # Maximum number of tool calls
348
336
  final_prompt=_final_prompt, # Final prompt when tool calls have ran out
349
- initial_body=None, # Extra body fields sent with the first request only; tool-loop continuations use the standard payload
350
337
  return_all=False, # Returns all intermediate ModelResponses if not streaming and has tool calls
351
338
  tool_choice=None, # Force ('required'/a tool name) or disable ('none') tool use
352
339
  max_tokens=None, # Response token limit (default: the model's maximum)
@@ -360,17 +347,38 @@ async def __call__(
360
347
  self._turn_start = len(self.hist)
361
348
  self.turn_opts = dict(temp=temp, think=think, stream=stream, max_steps=max_steps, final_prompt=final_prompt,
362
349
  max_tokens=max_tokens, n_workers=n_workers, pause=pause, tc_timeout=tc_timeout)
363
- result_gen = self._call(msg, 1, search, tool_choice, initial_body, **kwargs)
350
+ result_gen = self._call(msg, 1, search, tool_choice, **kwargs)
364
351
  if stream or return_all: return result_gen
365
352
  async for res in result_gen: pass
366
353
  return res # normal chat behavior only return last msg
367
354
 
368
355
  # %% ../nbs/09_chat.ipynb #5a6b6554
356
+ @patch
357
+ def reply_msgs(self:AsyncChat):
358
+ "Completed assistant and tool messages from the current top-level turn"
359
+ return [m for m in self.hist[self._turn_start:] if m.role in ('assistant','tool')]
360
+
369
361
  @patch
370
362
  def full(self:AsyncChat, mx=2000, showthink=None):
371
- "Formatted text of the current turn so far: completed tool calls as `{.tool}` blocks, pending ones as ⏳ rows"
372
- msgs = [m for m in self.hist[self._turn_start:] if m.role in ('assistant','tool')]
373
- return hist2fmt(msgs, mx=mx, showthink=ifnone(showthink, self.showthink))
363
+ "Formatted text of the current turn so far, with completed tool calls as `{.tool}` blocks"
364
+ return hist2fmt(self.reply_msgs(), mx=mx, showthink=ifnone(showthink, self.showthink))
365
+
366
+ # %% ../nbs/09_chat.ipynb #4f1fba78
367
+ def _cut(p, mx):
368
+ "`p` with its tool or server result cut to `mx`, losing `raw` if that changes it"
369
+ if not (p.text and (isinstance(p, ToolResult) or isinstance(p, ToolUse) and p.server)): return p
370
+ t = trunc_str(tool_text(p.text), mx=mx)
371
+ return p if t == p.text else p.replace(text=t, raw=None)
372
+
373
+ def _trim(m, mx):
374
+ "`m` without thinking, with tool and server results cut to `mx`"
375
+ parts = [_cut(p, mx) for p in m.content if not isinstance(p, Thinking)]
376
+ return m if parts == m.content else Msg(m.role, parts)
377
+
378
+ @patch
379
+ def dumps(self:AsyncChat, mx=None):
380
+ "`hist` as JSON, one `msg2dict` dict per message; `mx` drops thinking and cuts tool and server results to `mx` characters"
381
+ return dumps([msg2dict(m if mx is None else _trim(m, mx)) for m in self.hist], ensure_ascii=False)
374
382
 
375
383
  # %% ../nbs/09_chat.ipynb #a4bbd2ce
376
384
  class ChatCallback(GetAttr):
@@ -433,7 +441,9 @@ async def run_fence_tool(lang, code, ns):
433
441
  tname = _lang2tool[lang]
434
442
  arg = dict(code=code) if lang == 'py' else dict(cmd=code)
435
443
  res = _mk_tool_result(await call_func_async(tname, arg, ns=ns, raise_on_err=False))
436
- return mk_result_fence(trunc_str(tool_text(res)))
444
+ mx = None if isinstance(res, ToolResult) and res.full else 2000
445
+ if isinstance(res, ToolResult): res = res.text
446
+ return mk_result_fence(trunc_str(tool_text(res), mx=mx))
437
447
 
438
448
  # %% ../nbs/09_chat.ipynb #740ee3a4
439
449
  class FenceToolCallback(ChatCallback):
@@ -497,16 +507,13 @@ defaults.chat_callbacks = [FenceToolCallback, ToolReminderCallback, StopReasonCa
497
507
  # %% ../nbs/09_chat.ipynb #ead4b56d
498
508
  class StreamAccum:
499
509
  "Accumulate formatted parts; refresh from `chat.full()` on `Refresh` or `Completion`"
500
- def __init__(self, chat=None, mx=2000): self.chat,self.mx,self.txt,self.last = chat,mx,'',None
510
+ def __init__(self, chat=None, mx=2000): self.chat,self.mx,self.txt = chat,mx,''
501
511
  def __call__(self, o):
502
512
  "Feed one stream item; True if the display text changed"
503
513
  if isinstance(o, (Refresh, Completion)):
504
514
  if self.chat is None: return False
505
- self.txt,self.last = self.chat.full(mx=self.mx)+'\n\n',None
506
- elif f := getattr(o, 'formatted', ''):
507
- typ = getattr(o, 'type', None)
508
- self.txt += think_sep(self.last, typ) + f
509
- self.last = typ
515
+ self.txt = self.chat.full(mx=self.mx)+'\n\n'
516
+ elif f := getattr(o, 'formatted', ''): self.txt += f
510
517
  else: return False
511
518
  return True
512
519
 
@@ -516,6 +523,8 @@ async def adisplay_stream(rs, chat=None, mx=2000):
516
523
  try: from IPython.display import display, Markdown
517
524
  except ModuleNotFoundError: raise ModuleNotFoundError("This function requires ipython. Please run `pip install ipython` to use.")
518
525
  acc,h = StreamAccum(chat, mx=mx), display(Markdown(' '), display_id=True)
526
+ upd = Debounce(lambda: h.update(Markdown(acc.txt)), wait=0.05, max_wait=0.2)
519
527
  async for o in rs:
520
- if acc(o) and acc.txt: h.update(Markdown(acc.txt))
528
+ if acc(o) and acc.txt: upd()
529
+ await upd.flush()
521
530
  return acc.txt
@@ -152,15 +152,15 @@ async def acollect_stream(resp, **kwargs):
152
152
 
153
153
  # %% ../nbs/07_gemini.ipynb #58d0cb74
154
154
  def denorm_tool_use(p:ToolUse):
155
- "Convert canonical tool_use Part to Gemini functionCall part."
155
+ "Convert canonical tool_use Part to a Gemini functionCall part, signed with Gemini's validator bypass."
156
156
  fc = dict(name=p.name or '', args=p.arguments or {})
157
157
  if p.id: fc['id'] = p.id
158
- return dict(functionCall=fc)
158
+ return dict(functionCall=fc, thoughtSignature='skip_thought_signature_validator')
159
159
 
160
160
  def denorm_assistant(m:Msg):
161
- "The content as the API sent it, else its text and tool calls"
162
- if m.raw: return m.raw
163
- parts = [dict(text=p.text or '') if isinstance(p, Text) else denorm_tool_use(p) for p in m.content if isinstance(p, (Text, ToolUse))]
161
+ "The content as the API sent it, else its text and client tool calls"
162
+ if m.raw and m.raw_api == 'gemini': return m.raw
163
+ parts = [dict(text=p.text or '') if isinstance(p, Text) else denorm_tool_use(p) for p in rebuild_parts(m)]
164
164
  return dict(role='model', parts=parts)
165
165
 
166
166
  def denorm_tool(m:Msg):