python-fastllm 0.0.64__tar.gz → 0.0.66__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/PKG-INFO +5 -3
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/__init__.py +1 -1
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/_modidx.py +17 -5
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/acomplete.py +29 -2
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/anthropic.py +24 -28
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/chat.py +41 -32
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/gemini.py +5 -5
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/model_prices.json +29478 -31958
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/openai_chat.py +33 -22
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/openai_responses.py +7 -9
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/responses.py +84 -33
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/streaming.py +5 -4
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/types.py +41 -22
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/pyproject.toml +4 -1
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/PKG-INFO +5 -3
- python_fastllm-0.0.66/python_fastllm.egg-info/requires.txt +7 -0
- python_fastllm-0.0.64/python_fastllm.egg-info/requires.txt +0 -4
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/README.md +0 -0
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/fastllm/codex.py +0 -0
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/SOURCES.txt +0 -0
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/dependency_links.txt +0 -0
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/entry_points.txt +0 -0
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/python_fastllm.egg-info/top_level.txt +0 -0
- {python_fastllm-0.0.64 → python_fastllm-0.0.66}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: python-fastllm
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.66
|
|
4
4
|
Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
|
|
5
5
|
License-Expression: Apache-2.0
|
|
6
6
|
Project-URL: Repository, https://github.com/AnswerDotAI/fastllm
|
|
@@ -9,10 +9,12 @@ Classifier: Programming Language :: Python :: 3
|
|
|
9
9
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
10
10
|
Requires-Python: >=3.10
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
|
-
Requires-Dist: fastcore>=2.
|
|
13
|
-
Requires-Dist: aidialog>=0.0.
|
|
12
|
+
Requires-Dist: fastcore>=2.2.14
|
|
13
|
+
Requires-Dist: aidialog>=0.0.38
|
|
14
14
|
Requires-Dist: fasttransport>=0.0.2
|
|
15
15
|
Requires-Dist: pillow
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pycachy>=0.0.22; extra == "dev"
|
|
16
18
|
|
|
17
19
|
# fastllm
|
|
18
20
|
|
|
@@ -12,17 +12,18 @@ d = { 'settings': { 'branch': 'main',
|
|
|
12
12
|
'fastllm/acomplete.py'),
|
|
13
13
|
'fastllm.acomplete._debug_print': ('acomplete.html#_debug_print', 'fastllm/acomplete.py'),
|
|
14
14
|
'fastllm.acomplete._is_ctx_exceeded': ('acomplete.html#_is_ctx_exceeded', 'fastllm/acomplete.py'),
|
|
15
|
+
'fastllm.acomplete._limit_tool_text': ('acomplete.html#_limit_tool_text', 'fastllm/acomplete.py'),
|
|
15
16
|
'fastllm.acomplete._raise_if_done': ('acomplete.html#_raise_if_done', 'fastllm/acomplete.py'),
|
|
16
17
|
'fastllm.acomplete._retry_call': ('acomplete.html#_retry_call', 'fastllm/acomplete.py'),
|
|
17
18
|
'fastllm.acomplete._retry_stream': ('acomplete.html#_retry_stream', 'fastllm/acomplete.py'),
|
|
19
|
+
'fastllm.acomplete._tool_messages': ('acomplete.html#_tool_messages', 'fastllm/acomplete.py'),
|
|
18
20
|
'fastllm.acomplete.acomplete': ('acomplete.html#acomplete', 'fastllm/acomplete.py'),
|
|
19
21
|
'fastllm.acomplete.mk_client': ('acomplete.html#mk_client', 'fastllm/acomplete.py'),
|
|
20
22
|
'fastllm.acomplete.split_vendor': ('acomplete.html#split_vendor', 'fastllm/acomplete.py')},
|
|
21
23
|
'fastllm.anthropic': { 'fastllm.anthropic._add_cache_control': ('anthropic.html#_add_cache_control', 'fastllm/anthropic.py'),
|
|
22
24
|
'fastllm.anthropic._ant_cc': ('anthropic.html#_ant_cc', 'fastllm/anthropic.py'),
|
|
23
25
|
'fastllm.anthropic._ant_part_type': ('anthropic.html#_ant_part_type', 'fastllm/anthropic.py'),
|
|
24
|
-
'fastllm.anthropic.
|
|
25
|
-
'fastllm/anthropic.py'),
|
|
26
|
+
'fastllm.anthropic._copy_msg': ('anthropic.html#_copy_msg', 'fastllm/anthropic.py'),
|
|
26
27
|
'fastllm.anthropic._sanid': ('anthropic.html#_sanid', 'fastllm/anthropic.py'),
|
|
27
28
|
'fastllm.anthropic._url_cite': ('anthropic.html#_url_cite', 'fastllm/anthropic.py'),
|
|
28
29
|
'fastllm.anthropic.acollect_stream': ('anthropic.html#acollect_stream', 'fastllm/anthropic.py'),
|
|
@@ -65,8 +66,10 @@ d = { 'settings': { 'branch': 'main',
|
|
|
65
66
|
'fastllm.chat.AsyncChat._track': ('chat.html#asyncchat._track', 'fastllm/chat.py'),
|
|
66
67
|
'fastllm.chat.AsyncChat.add_cb': ('chat.html#asyncchat.add_cb', 'fastllm/chat.py'),
|
|
67
68
|
'fastllm.chat.AsyncChat.add_cbs': ('chat.html#asyncchat.add_cbs', 'fastllm/chat.py'),
|
|
69
|
+
'fastllm.chat.AsyncChat.dumps': ('chat.html#asyncchat.dumps', 'fastllm/chat.py'),
|
|
68
70
|
'fastllm.chat.AsyncChat.full': ('chat.html#asyncchat.full', 'fastllm/chat.py'),
|
|
69
71
|
'fastllm.chat.AsyncChat.print_hist': ('chat.html#asyncchat.print_hist', 'fastllm/chat.py'),
|
|
72
|
+
'fastllm.chat.AsyncChat.reply_msgs': ('chat.html#asyncchat.reply_msgs', 'fastllm/chat.py'),
|
|
70
73
|
'fastllm.chat.AsyncChat.tcdict': ('chat.html#asyncchat.tcdict', 'fastllm/chat.py'),
|
|
71
74
|
'fastllm.chat.ChatCallback': ('chat.html#chatcallback', 'fastllm/chat.py'),
|
|
72
75
|
'fastllm.chat.ChatCallback.__repr__': ('chat.html#chatcallback.__repr__', 'fastllm/chat.py'),
|
|
@@ -100,17 +103,18 @@ d = { 'settings': { 'branch': 'main',
|
|
|
100
103
|
'fastllm.chat.UsageStats.__init__': ('chat.html#usagestats.__init__', 'fastllm/chat.py'),
|
|
101
104
|
'fastllm.chat.UsageStats.__radd__': ('chat.html#usagestats.__radd__', 'fastllm/chat.py'),
|
|
102
105
|
'fastllm.chat.UsageStats.__repr__': ('chat.html#usagestats.__repr__', 'fastllm/chat.py'),
|
|
103
|
-
'fastllm.chat.UsageStats.fmt': ('chat.html#usagestats.fmt', 'fastllm/chat.py'),
|
|
104
106
|
'fastllm.chat.UsageStats.from_response': ('chat.html#usagestats.from_response', 'fastllm/chat.py'),
|
|
105
107
|
'fastllm.chat._active_fence_langs': ('chat.html#_active_fence_langs', 'fastllm/chat.py'),
|
|
106
108
|
'fastllm.chat._alite_call_func': ('chat.html#_alite_call_func', 'fastllm/chat.py'),
|
|
107
109
|
'fastllm.chat._call_func': ('chat.html#_call_func', 'fastllm/chat.py'),
|
|
110
|
+
'fastllm.chat._cut': ('chat.html#_cut', 'fastllm/chat.py'),
|
|
108
111
|
'fastllm.chat._handle_stop_reason': ('chat.html#_handle_stop_reason', 'fastllm/chat.py'),
|
|
109
112
|
'fastllm.chat._has_search': ('chat.html#_has_search', 'fastllm/chat.py'),
|
|
110
113
|
'fastllm.chat._has_stop': ('chat.html#_has_stop', 'fastllm/chat.py'),
|
|
111
114
|
'fastllm.chat._inject_tool_reminder': ('chat.html#_inject_tool_reminder', 'fastllm/chat.py'),
|
|
112
115
|
'fastllm.chat._lite_call_func': ('chat.html#_lite_call_func', 'fastllm/chat.py'),
|
|
113
116
|
'fastllm.chat._mk_tool_result': ('chat.html#_mk_tool_result', 'fastllm/chat.py'),
|
|
117
|
+
'fastllm.chat._trim': ('chat.html#_trim', 'fastllm/chat.py'),
|
|
114
118
|
'fastllm.chat._usrtools': ('chat.html#_usrtools', 'fastllm/chat.py'),
|
|
115
119
|
'fastllm.chat.add_warning': ('chat.html#add_warning', 'fastllm/chat.py'),
|
|
116
120
|
'fastllm.chat.adisplay_stream': ('chat.html#adisplay_stream', 'fastllm/chat.py'),
|
|
@@ -122,8 +126,7 @@ d = { 'settings': { 'branch': 'main',
|
|
|
122
126
|
'fastllm.chat.run_fence_tool': ('chat.html#run_fence_tool', 'fastllm/chat.py'),
|
|
123
127
|
'fastllm.chat.search_count': ('chat.html#search_count', 'fastllm/chat.py'),
|
|
124
128
|
'fastllm.chat.stop_reason': ('chat.html#stop_reason', 'fastllm/chat.py'),
|
|
125
|
-
'fastllm.chat.structured': ('chat.html#structured', 'fastllm/chat.py'),
|
|
126
|
-
'fastllm.chat.think_sep': ('chat.html#think_sep', 'fastllm/chat.py')},
|
|
129
|
+
'fastllm.chat.structured': ('chat.html#structured', 'fastllm/chat.py')},
|
|
127
130
|
'fastllm.codex': { 'fastllm.codex._codex_hdrs': ('codex.html#_codex_hdrs', 'fastllm/codex.py'),
|
|
128
131
|
'fastllm.codex._codex_vendor_auth': ('codex.html#_codex_vendor_auth', 'fastllm/codex.py'),
|
|
129
132
|
'fastllm.codex._merge_tokens': ('codex.html#_merge_tokens', 'fastllm/codex.py'),
|
|
@@ -304,19 +307,27 @@ d = { 'settings': { 'branch': 'main',
|
|
|
304
307
|
'fastllm.responses._ResponseStream': ('responses.html#_responsestream', 'fastllm/responses.py'),
|
|
305
308
|
'fastllm.responses._ResponseStream.__init__': ( 'responses.html#_responsestream.__init__',
|
|
306
309
|
'fastllm/responses.py'),
|
|
310
|
+
'fastllm.responses._ResponseStream.close_thinking': ( 'responses.html#_responsestream.close_thinking',
|
|
311
|
+
'fastllm/responses.py'),
|
|
307
312
|
'fastllm.responses._ResponseStream.done_events': ( 'responses.html#_responsestream.done_events',
|
|
308
313
|
'fastllm/responses.py'),
|
|
314
|
+
'fastllm.responses._ResponseStream.end_step': ( 'responses.html#_responsestream.end_step',
|
|
315
|
+
'fastllm/responses.py'),
|
|
309
316
|
'fastllm.responses._ResponseStream.event': ( 'responses.html#_responsestream.event',
|
|
310
317
|
'fastllm/responses.py'),
|
|
311
318
|
'fastllm.responses._ResponseStream.part_events': ( 'responses.html#_responsestream.part_events',
|
|
312
319
|
'fastllm/responses.py'),
|
|
313
320
|
'fastllm.responses._ResponseStream.text_events': ( 'responses.html#_responsestream.text_events',
|
|
314
321
|
'fastllm/responses.py'),
|
|
322
|
+
'fastllm.responses._ResponseStream.thinking_events': ( 'responses.html#_responsestream.thinking_events',
|
|
323
|
+
'fastllm/responses.py'),
|
|
315
324
|
'fastllm.responses._ResponseStream.tool_events': ( 'responses.html#_responsestream.tool_events',
|
|
316
325
|
'fastllm/responses.py'),
|
|
317
326
|
'fastllm.responses._arguments': ('responses.html#_arguments', 'fastllm/responses.py'),
|
|
327
|
+
'fastllm.responses._is_srv': ('responses.html#_is_srv', 'fastllm/responses.py'),
|
|
318
328
|
'fastllm.responses._msg_parts': ('responses.html#_msg_parts', 'fastllm/responses.py'),
|
|
319
329
|
'fastllm.responses._new_id': ('responses.html#_new_id', 'fastllm/responses.py'),
|
|
330
|
+
'fastllm.responses._reasoning_item': ('responses.html#_reasoning_item', 'fastllm/responses.py'),
|
|
320
331
|
'fastllm.responses._response_base': ('responses.html#_response_base', 'fastllm/responses.py'),
|
|
321
332
|
'fastllm.responses._server_call': ('responses.html#_server_call', 'fastllm/responses.py'),
|
|
322
333
|
'fastllm.responses._text_content': ('responses.html#_text_content', 'fastllm/responses.py'),
|
|
@@ -390,6 +401,7 @@ d = { 'settings': { 'branch': 'main',
|
|
|
390
401
|
'fastllm.types.payload_kwargs': ('types.html#payload_kwargs', 'fastllm/types.py'),
|
|
391
402
|
'fastllm.types.price_tier': ('types.html#price_tier', 'fastllm/types.py'),
|
|
392
403
|
'fastllm.types.provider_req': ('types.html#provider_req', 'fastllm/types.py'),
|
|
404
|
+
'fastllm.types.rebuild_parts': ('types.html#rebuild_parts', 'fastllm/types.py'),
|
|
393
405
|
'fastllm.types.register_model_info': ('types.html#register_model_info', 'fastllm/types.py'),
|
|
394
406
|
'fastllm.types.register_sub_model': ('types.html#register_sub_model', 'fastllm/types.py'),
|
|
395
407
|
'fastllm.types.resize_b64': ('types.html#resize_b64', 'fastllm/types.py'),
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03_acomplete.ipynb.
|
|
4
4
|
|
|
5
5
|
# %% auto #0
|
|
6
|
-
__all__ = ['vendor_mapping', 'defaults', 'split_vendor', 'mk_client', 'ContextWindowExceededError', 'acomplete']
|
|
6
|
+
__all__ = ['vendor_mapping', 'vendor_doms', 'defaults', 'split_vendor', 'mk_client', 'ContextWindowExceededError', 'acomplete']
|
|
7
7
|
|
|
8
8
|
# %% ../nbs/03_acomplete.ipynb #f2f57253
|
|
9
9
|
import asyncio,json,httpx2
|
|
@@ -40,6 +40,7 @@ vendor_mapping = {
|
|
|
40
40
|
"minimax": ('anthropic', "https://api.minimax.io/anthropic", "MINIMAX_API_KEY"),
|
|
41
41
|
"meta": ('openai_chat', "https://api.meta.ai/v1", "META_API_KEY"),
|
|
42
42
|
"zai": ('openai_chat', "https://api.z.ai/api/paas/v4", "ZAI_API_KEY")}
|
|
43
|
+
vendor_doms = tuple(dict.fromkeys(httpx2.URL(u).host for _,u,_ in vendor_mapping.values())) # hosts of every vendor's API, for cachy
|
|
43
44
|
|
|
44
45
|
# %% ../nbs/03_acomplete.ipynb #e3ed40fb
|
|
45
46
|
def split_vendor(model):
|
|
@@ -134,6 +135,32 @@ async def _retry_stream(mk_gen, retries=2, retry_delay=0.5):
|
|
|
134
135
|
except APIError as e: await _raise_if_done(e, n, retries, retry_delay, yielded=yielded)
|
|
135
136
|
|
|
136
137
|
# %% ../nbs/03_acomplete.ipynb #2379ec94
|
|
138
|
+
_TOOL_TEXT_MAX = 1 << 20
|
|
139
|
+
|
|
140
|
+
def _limit_tool_text(text):
|
|
141
|
+
scalar = isinstance(text, str)
|
|
142
|
+
parts = [Text(text)] if scalar else text
|
|
143
|
+
if not isinstance(parts, list): return text
|
|
144
|
+
size = sum(len(p.text.encode()) for p in parts if isinstance(p, Text) and isinstance(p.text, str))
|
|
145
|
+
if size <= _TOOL_TEXT_MAX: return text
|
|
146
|
+
cut,limited = size-_TOOL_TEXT_MAX,[]
|
|
147
|
+
for p in parts:
|
|
148
|
+
if isinstance(p, Text) and isinstance(p.text, str):
|
|
149
|
+
data = p.text.encode()
|
|
150
|
+
kept = data[min(cut, len(data)):].decode(errors='ignore')
|
|
151
|
+
cut = max(0, cut-len(data))
|
|
152
|
+
p = p.replace(text=type(p.text)(kept))
|
|
153
|
+
limited.append(p)
|
|
154
|
+
removed = size-sum(len(p.text.encode()) for p in limited if isinstance(p, Text) and isinstance(p.text, str))
|
|
155
|
+
at = next(i for i,p in enumerate(limited) if isinstance(p, Text) and p.text)
|
|
156
|
+
limited[at] = limited[at].replace(text=type(limited[at].text)(f'[{removed} bytes of output removed]\n'+limited[at].text))
|
|
157
|
+
return limited[0].text if scalar else limited
|
|
158
|
+
|
|
159
|
+
def _tool_messages(msgs):
|
|
160
|
+
if not msgs: return msgs
|
|
161
|
+
return [Msg(m.role, [p.replace(text=_limit_tool_text(p.text)) if isinstance(p, ToolResult) else p for p in m.content],
|
|
162
|
+
raw=m.raw, raw_api=m.raw_api) if any(isinstance(p, ToolResult) for p in m.content) else m for m in msgs]
|
|
163
|
+
|
|
137
164
|
@delegates(payload_kwargs)
|
|
138
165
|
async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None, oauth_token=None, base_url=None, endpoint=None, xtra_body=None, xtra_hdrs=None,
|
|
139
166
|
stream=False, previous_response_id=None, stop_callables=None, retries=2, retry_delay=0.5, **kwargs):
|
|
@@ -158,7 +185,7 @@ async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None,
|
|
|
158
185
|
if not getattr(api, 'supports_previous_response_id', False):
|
|
159
186
|
raise ValueError('previous_response_id is not supported by this transport')
|
|
160
187
|
kwargs['previous_response_id'] = previous_response_id
|
|
161
|
-
payload = api.mk_payload(msgs, model, stream=stream, **kwargs)
|
|
188
|
+
payload = api.mk_payload(_tool_messages(msgs), model, stream=stream, **kwargs)
|
|
162
189
|
if fix := getattr(api, 'fix_payload', None): fix(payload, model, vendor_name)
|
|
163
190
|
body = payload | ifnone(xtra_body, {})
|
|
164
191
|
if not own:
|
|
@@ -11,7 +11,6 @@ __all__ = ['ant_tc_types', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'f
|
|
|
11
11
|
|
|
12
12
|
# %% ../nbs/06_anthropic.ipynb #02afd3d7
|
|
13
13
|
import json, copy
|
|
14
|
-
from collections import Counter
|
|
15
14
|
from fastcore.utils import *
|
|
16
15
|
from fastcore.meta import *
|
|
17
16
|
from fasttransport.errors import api_error_from_event
|
|
@@ -187,12 +186,12 @@ def denorm_tool_use(p:ToolUse):
|
|
|
187
186
|
return _ant_cc(dict(type='tool_use', id=_sanid(p.id or ''), name=p.name or '', input=p.arguments or {}), p)
|
|
188
187
|
|
|
189
188
|
def denorm_assistant(m:Msg):
|
|
190
|
-
"The message as the API sent it, else its text and tool calls; a cache mark on the last part is copied onto the last raw block"
|
|
191
|
-
if m.raw:
|
|
189
|
+
"The message as the API sent it, else its text and client tool calls; a cache mark on the last part is copied onto the last raw block"
|
|
190
|
+
if m.raw and m.raw_api == 'anthropic':
|
|
192
191
|
cc = [p.cache_control for p in m.content if p.cache_control]
|
|
193
192
|
if not cc: return m.raw
|
|
194
193
|
return m.raw | dict(content=m.raw['content'][:-1] + [m.raw['content'][-1] | dict(cache_control=cc[-1])])
|
|
195
|
-
blocks = [_ant_cc(dict(type='text', text=p.text or ''), p) if isinstance(p, Text) else denorm_tool_use(p) for p in m
|
|
194
|
+
blocks = [_ant_cc(dict(type='text', text=p.text or ''), p) if isinstance(p, Text) else denorm_tool_use(p) for p in rebuild_parts(m)]
|
|
196
195
|
return dict(role='assistant', content=blocks)
|
|
197
196
|
|
|
198
197
|
def denorm_tool(m:Msg):
|
|
@@ -305,11 +304,11 @@ def _add_cache_control(
|
|
|
305
304
|
return msg
|
|
306
305
|
|
|
307
306
|
# %% ../nbs/06_anthropic.ipynb #6bc52ef5
|
|
308
|
-
def
|
|
309
|
-
"
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
307
|
+
def _copy_msg(m):
|
|
308
|
+
"A copy of `m` with copied parts, to mark without changing `m`"
|
|
309
|
+
m = copy.copy(m)
|
|
310
|
+
m.content = [copy.copy(p) for p in m.content]
|
|
311
|
+
return m
|
|
313
312
|
|
|
314
313
|
def apply_cache_idxs(
|
|
315
314
|
msgs, # Canonical Msgs to mark
|
|
@@ -317,27 +316,28 @@ def apply_cache_idxs(
|
|
|
317
316
|
sp=None, # System prompt: str, Msg, or Part
|
|
318
317
|
ttl=None # Cache TTL: '5m' (default) or '1h'
|
|
319
318
|
):
|
|
320
|
-
"
|
|
321
|
-
|
|
322
|
-
_clear_cache_control(sp)
|
|
319
|
+
"`msgs` and `sp` with breakpoints added at `cache_idxs`, marking copies"
|
|
320
|
+
msgs = list(msgs)
|
|
323
321
|
if sp:
|
|
324
322
|
if 0 in cache_idxs:
|
|
325
323
|
if isinstance(sp, str): sp = Text(sp)
|
|
326
|
-
if isinstance(sp,
|
|
327
|
-
_add_cache_control(sp, ttl)
|
|
324
|
+
sp = _add_cache_control(_copy_msg(sp if isinstance(sp, Msg) else Msg('', [sp])), ttl)
|
|
328
325
|
cache_idxs = [o-1 if o>0 else o for o in cache_idxs if o]
|
|
329
326
|
ms = [j for j,m in enumerate(msgs) if m.role != 'tool']
|
|
330
327
|
for i in cache_idxs:
|
|
331
|
-
try:
|
|
328
|
+
try: j = ms[i]
|
|
332
329
|
except IndexError: continue
|
|
333
|
-
|
|
330
|
+
msgs[j] = _add_cache_control(_copy_msg(msgs[j]), ttl)
|
|
331
|
+
return msgs,sp
|
|
334
332
|
|
|
335
333
|
# %% ../nbs/06_anthropic.ipynb #587d4fe5
|
|
336
334
|
@delegates(payload_kwargs)
|
|
337
335
|
def mk_payload(msgs, model, **kwargs):
|
|
338
336
|
sp = kwargs.get('system')
|
|
339
|
-
if
|
|
340
|
-
|
|
337
|
+
if ci := kwargs.get('cache_idxs'): msgs,sp = apply_cache_idxs(msgs, ci, sp, kwargs.get('ttl'))
|
|
338
|
+
max_tokens = kwargs.get('max_tokens')
|
|
339
|
+
if max_tokens is None: max_tokens = get_model_info(model, 'anthropic').get('max_output_tokens') or 1024
|
|
340
|
+
payload = dict(model=model, messages=denorm_msgs(msgs), max_tokens=max_tokens)
|
|
341
341
|
if kwargs.get('stream'): payload['stream'] = True
|
|
342
342
|
if sp: payload['system'] = denorm_system(sp)
|
|
343
343
|
if tools:=kwargs.get('tools'): payload['tools'] = denorm_tool_schs(tools)
|
|
@@ -358,17 +358,13 @@ def get_hdrs(api_key=None, oauth_token=None):
|
|
|
358
358
|
# %% ../nbs/06_anthropic.ipynb #0d03643a
|
|
359
359
|
def cost(usage, m):
|
|
360
360
|
raw = usage.raw
|
|
361
|
-
in_tok = raw['input_tokens']
|
|
362
|
-
cache_read = raw.get('cache_read_input_tokens', 0)
|
|
363
361
|
cc = raw.get('cache_creation', {}) or {}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
cost += cache_1h * m.get('cache_creation_input_token_cost_above_1hr', 0)
|
|
371
|
-
return cost
|
|
362
|
+
prompt = dict(input_cost_per_token=raw['input_tokens'], cache_read_input_token_cost=raw.get('cache_read_input_tokens', 0),
|
|
363
|
+
cache_creation_input_token_cost=cc.get('ephemeral_5m_input_tokens', raw.get('cache_creation_input_tokens', 0)),
|
|
364
|
+
cache_creation_input_token_cost_above_1hr=cc.get('ephemeral_1h_input_tokens', 0))
|
|
365
|
+
tier = price_tier(m, sum(prompt.values()))
|
|
366
|
+
toks = prompt | dict(output_cost_per_token=raw['output_tokens'])
|
|
367
|
+
return sum(n * tier_rate(m, k, tier) for k,n in toks.items())
|
|
372
368
|
|
|
373
369
|
# %% ../nbs/06_anthropic.ipynb #f7c0b989
|
|
374
370
|
api_registry.register('anthropic', norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/09_chat.ipynb.
|
|
4
4
|
|
|
5
5
|
# %% auto #0
|
|
6
|
-
__all__ = ['effort', 'contents', 'stop_reason', 'cite_footnote', 'postproc', 'Refresh', '
|
|
6
|
+
__all__ = ['effort', 'contents', 'stop_reason', 'cite_footnote', 'postproc', 'Refresh', 'lite_mk_func', 'structured',
|
|
7
7
|
'search_count', 'UsageStats', 'AsyncChat', 'astream_with_complete', 'ChatCallback', 'DeepseekMsgsCallback',
|
|
8
8
|
'add_warning', 'StopReasonCallback', 'run_fence_tool', 'FenceToolCallback', 'ToolReminderCallback',
|
|
9
9
|
'StopSequencesCallback', 'StreamAccum', 'adisplay_stream']
|
|
@@ -16,8 +16,8 @@ from fastcore.utils import *
|
|
|
16
16
|
from fastcore.meta import delegates
|
|
17
17
|
|
|
18
18
|
from .types import *
|
|
19
|
-
from aidialog.msg_parts import (Msg, Part,
|
|
20
|
-
StopResponse, ToolResponse, display_list,
|
|
19
|
+
from aidialog.msg_parts import (Msg, Part, Text, Thinking, ToolUse, ToolResult, tool_text, mk_tool_res_msg, msg2dict,
|
|
20
|
+
StopResponse, ToolResponse, display_list, hist2fmt, mk_result_fence, split_fence_msgs, extract_fence_call,
|
|
21
21
|
trunc_str, Completion, mk_msg, mk_msgs, part_txt)
|
|
22
22
|
from .acomplete import *
|
|
23
23
|
from .streaming import FenceToolStop, stop_sequences
|
|
@@ -53,11 +53,6 @@ class Refresh:
|
|
|
53
53
|
"Stream signal: state-holding renderers should re-render in full (e.g. via `AsyncChat.full`)"
|
|
54
54
|
formatted = ''
|
|
55
55
|
|
|
56
|
-
def think_sep(last, typ, sep='\n\n'):
|
|
57
|
-
"Separator to insert between `formatted` items when crossing a 🧠↔text boundary"
|
|
58
|
-
think = {PartType.thinking, 'status'}
|
|
59
|
-
return sep if last and typ and (typ in think) != (last in think) else ''
|
|
60
|
-
|
|
61
56
|
# %% ../nbs/09_chat.ipynb #0e7d3980
|
|
62
57
|
def lite_mk_func(f):
|
|
63
58
|
if isinstance(f, dict): return f
|
|
@@ -65,9 +60,9 @@ def lite_mk_func(f):
|
|
|
65
60
|
|
|
66
61
|
# %% ../nbs/09_chat.ipynb #bba6fd58
|
|
67
62
|
def _mk_tool_result(res):
|
|
68
|
-
"
|
|
63
|
+
"Preserve canonical results, unwrap `ToolResponse`, or format other values as text"
|
|
69
64
|
if isinstance(res, ToolResponse): return res.content
|
|
70
|
-
return tool_text(res)
|
|
65
|
+
return res if isinstance(res, ToolResult) else tool_text(res)
|
|
71
66
|
|
|
72
67
|
# %% ../nbs/09_chat.ipynb #a0fcc96e
|
|
73
68
|
def _call_func(tc:ToolUse, tool_schemas, ns, callf):
|
|
@@ -155,11 +150,6 @@ class UsageStats:
|
|
|
155
150
|
if self.model: parts.append(self.model.split('/')[-1])
|
|
156
151
|
return ' | '.join(parts)
|
|
157
152
|
|
|
158
|
-
def fmt(self):
|
|
159
|
-
if not self.total_tokens: return ''
|
|
160
|
-
d = {k:v for k,v in self.__dict__.items() if v and not k.startswith('_')}
|
|
161
|
-
return "\n\n" + fenced(dumps(d, ensure_ascii=False), usage_info) + "\n"
|
|
162
|
-
|
|
163
153
|
# %% ../nbs/09_chat.ipynb #cb3d7e77
|
|
164
154
|
class AsyncChat:
|
|
165
155
|
def __init__(
|
|
@@ -280,7 +270,7 @@ async def astream_with_complete(self, agen, postproc=noop):
|
|
|
280
270
|
# %% ../nbs/09_chat.ipynb #a049cf52
|
|
281
271
|
@patch
|
|
282
272
|
@delegates(acomplete)
|
|
283
|
-
async def _call(self:AsyncChat, msg=None, step=1, search=None, tool_choice=None,
|
|
273
|
+
async def _call(self:AsyncChat, msg=None, step=1, search=None, tool_choice=None, **kwargs):
|
|
284
274
|
"One model turn plus its tool round, recursing for the next; turn-constant options come from `turn_opts`"
|
|
285
275
|
t = AttrDict(self.turn_opts)
|
|
286
276
|
if step>t.max_steps+1: return
|
|
@@ -291,12 +281,10 @@ async def _call(self:AsyncChat, msg=None, step=1, search=None, tool_choice=None,
|
|
|
291
281
|
|
|
292
282
|
self.turn_kwargs, self.stream = kwargs, t.stream
|
|
293
283
|
async for o in self._call_cbs('before_acomplete'): yield o
|
|
294
|
-
kw = self.turn_kwargs
|
|
295
|
-
if initial_body: kw = merge(kw, dict(xtra_body=merge(kw.get('xtra_body'), initial_body)))
|
|
296
284
|
res = await acomplete(self.turn_msgs, self.model, system=self.turn_sysp, stream=t.stream, tools=self.tool_schemas,
|
|
297
285
|
tool_choice=tool_choice, max_tokens=int(max_tokens), temperature=None if t.think else ifnone(t.temp,self.temp),
|
|
298
286
|
previous_response_id=self.response_id if self.use_previous_response_id else None,
|
|
299
|
-
cache_idxs=self.cache_idxs if self.cache else [], ttl=self.ttl, **
|
|
287
|
+
cache_idxs=self.cache_idxs if self.cache else [], ttl=self.ttl, **self.turn_kwargs)
|
|
300
288
|
if t.stream:
|
|
301
289
|
res = astream_with_complete(res, postproc=postproc)
|
|
302
290
|
async for chunk in res:
|
|
@@ -346,7 +334,6 @@ async def __call__(
|
|
|
346
334
|
stream=False, # Stream results
|
|
347
335
|
max_steps=2, # Maximum number of tool calls
|
|
348
336
|
final_prompt=_final_prompt, # Final prompt when tool calls have ran out
|
|
349
|
-
initial_body=None, # Extra body fields sent with the first request only; tool-loop continuations use the standard payload
|
|
350
337
|
return_all=False, # Returns all intermediate ModelResponses if not streaming and has tool calls
|
|
351
338
|
tool_choice=None, # Force ('required'/a tool name) or disable ('none') tool use
|
|
352
339
|
max_tokens=None, # Response token limit (default: the model's maximum)
|
|
@@ -360,17 +347,38 @@ async def __call__(
|
|
|
360
347
|
self._turn_start = len(self.hist)
|
|
361
348
|
self.turn_opts = dict(temp=temp, think=think, stream=stream, max_steps=max_steps, final_prompt=final_prompt,
|
|
362
349
|
max_tokens=max_tokens, n_workers=n_workers, pause=pause, tc_timeout=tc_timeout)
|
|
363
|
-
result_gen = self._call(msg, 1, search, tool_choice,
|
|
350
|
+
result_gen = self._call(msg, 1, search, tool_choice, **kwargs)
|
|
364
351
|
if stream or return_all: return result_gen
|
|
365
352
|
async for res in result_gen: pass
|
|
366
353
|
return res # normal chat behavior only return last msg
|
|
367
354
|
|
|
368
355
|
# %% ../nbs/09_chat.ipynb #5a6b6554
|
|
356
|
+
@patch
|
|
357
|
+
def reply_msgs(self:AsyncChat):
|
|
358
|
+
"Completed assistant and tool messages from the current top-level turn"
|
|
359
|
+
return [m for m in self.hist[self._turn_start:] if m.role in ('assistant','tool')]
|
|
360
|
+
|
|
369
361
|
@patch
|
|
370
362
|
def full(self:AsyncChat, mx=2000, showthink=None):
|
|
371
|
-
"Formatted text of the current turn so far
|
|
372
|
-
|
|
373
|
-
|
|
363
|
+
"Formatted text of the current turn so far, with completed tool calls as `{.tool}` blocks"
|
|
364
|
+
return hist2fmt(self.reply_msgs(), mx=mx, showthink=ifnone(showthink, self.showthink))
|
|
365
|
+
|
|
366
|
+
# %% ../nbs/09_chat.ipynb #4f1fba78
|
|
367
|
+
def _cut(p, mx):
|
|
368
|
+
"`p` with its tool or server result cut to `mx`, losing `raw` if that changes it"
|
|
369
|
+
if not (p.text and (isinstance(p, ToolResult) or isinstance(p, ToolUse) and p.server)): return p
|
|
370
|
+
t = trunc_str(tool_text(p.text), mx=mx)
|
|
371
|
+
return p if t == p.text else p.replace(text=t, raw=None)
|
|
372
|
+
|
|
373
|
+
def _trim(m, mx):
|
|
374
|
+
"`m` without thinking, with tool and server results cut to `mx`"
|
|
375
|
+
parts = [_cut(p, mx) for p in m.content if not isinstance(p, Thinking)]
|
|
376
|
+
return m if parts == m.content else Msg(m.role, parts)
|
|
377
|
+
|
|
378
|
+
@patch
|
|
379
|
+
def dumps(self:AsyncChat, mx=None):
|
|
380
|
+
"`hist` as JSON, one `msg2dict` dict per message; `mx` drops thinking and cuts tool and server results to `mx` characters"
|
|
381
|
+
return dumps([msg2dict(m if mx is None else _trim(m, mx)) for m in self.hist], ensure_ascii=False)
|
|
374
382
|
|
|
375
383
|
# %% ../nbs/09_chat.ipynb #a4bbd2ce
|
|
376
384
|
class ChatCallback(GetAttr):
|
|
@@ -433,7 +441,9 @@ async def run_fence_tool(lang, code, ns):
|
|
|
433
441
|
tname = _lang2tool[lang]
|
|
434
442
|
arg = dict(code=code) if lang == 'py' else dict(cmd=code)
|
|
435
443
|
res = _mk_tool_result(await call_func_async(tname, arg, ns=ns, raise_on_err=False))
|
|
436
|
-
|
|
444
|
+
mx = None if isinstance(res, ToolResult) and res.full else 2000
|
|
445
|
+
if isinstance(res, ToolResult): res = res.text
|
|
446
|
+
return mk_result_fence(trunc_str(tool_text(res), mx=mx))
|
|
437
447
|
|
|
438
448
|
# %% ../nbs/09_chat.ipynb #740ee3a4
|
|
439
449
|
class FenceToolCallback(ChatCallback):
|
|
@@ -497,16 +507,13 @@ defaults.chat_callbacks = [FenceToolCallback, ToolReminderCallback, StopReasonCa
|
|
|
497
507
|
# %% ../nbs/09_chat.ipynb #ead4b56d
|
|
498
508
|
class StreamAccum:
|
|
499
509
|
"Accumulate formatted parts; refresh from `chat.full()` on `Refresh` or `Completion`"
|
|
500
|
-
def __init__(self, chat=None, mx=2000): self.chat,self.mx,self.txt
|
|
510
|
+
def __init__(self, chat=None, mx=2000): self.chat,self.mx,self.txt = chat,mx,''
|
|
501
511
|
def __call__(self, o):
|
|
502
512
|
"Feed one stream item; True if the display text changed"
|
|
503
513
|
if isinstance(o, (Refresh, Completion)):
|
|
504
514
|
if self.chat is None: return False
|
|
505
|
-
self.txt
|
|
506
|
-
elif f := getattr(o, 'formatted', ''):
|
|
507
|
-
typ = getattr(o, 'type', None)
|
|
508
|
-
self.txt += think_sep(self.last, typ) + f
|
|
509
|
-
self.last = typ
|
|
515
|
+
self.txt = self.chat.full(mx=self.mx)+'\n\n'
|
|
516
|
+
elif f := getattr(o, 'formatted', ''): self.txt += f
|
|
510
517
|
else: return False
|
|
511
518
|
return True
|
|
512
519
|
|
|
@@ -516,6 +523,8 @@ async def adisplay_stream(rs, chat=None, mx=2000):
|
|
|
516
523
|
try: from IPython.display import display, Markdown
|
|
517
524
|
except ModuleNotFoundError: raise ModuleNotFoundError("This function requires ipython. Please run `pip install ipython` to use.")
|
|
518
525
|
acc,h = StreamAccum(chat, mx=mx), display(Markdown(' '), display_id=True)
|
|
526
|
+
upd = Debounce(lambda: h.update(Markdown(acc.txt)), wait=0.05, max_wait=0.2)
|
|
519
527
|
async for o in rs:
|
|
520
|
-
if acc(o) and acc.txt:
|
|
528
|
+
if acc(o) and acc.txt: upd()
|
|
529
|
+
await upd.flush()
|
|
521
530
|
return acc.txt
|
|
@@ -152,15 +152,15 @@ async def acollect_stream(resp, **kwargs):
|
|
|
152
152
|
|
|
153
153
|
# %% ../nbs/07_gemini.ipynb #58d0cb74
|
|
154
154
|
def denorm_tool_use(p:ToolUse):
|
|
155
|
-
"Convert canonical tool_use Part to Gemini functionCall part."
|
|
155
|
+
"Convert canonical tool_use Part to a Gemini functionCall part, signed with Gemini's validator bypass."
|
|
156
156
|
fc = dict(name=p.name or '', args=p.arguments or {})
|
|
157
157
|
if p.id: fc['id'] = p.id
|
|
158
|
-
return dict(functionCall=fc)
|
|
158
|
+
return dict(functionCall=fc, thoughtSignature='skip_thought_signature_validator')
|
|
159
159
|
|
|
160
160
|
def denorm_assistant(m:Msg):
|
|
161
|
-
"The content as the API sent it, else its text and tool calls"
|
|
162
|
-
if m.raw: return m.raw
|
|
163
|
-
parts = [dict(text=p.text or '') if isinstance(p, Text) else denorm_tool_use(p) for p in m
|
|
161
|
+
"The content as the API sent it, else its text and client tool calls"
|
|
162
|
+
if m.raw and m.raw_api == 'gemini': return m.raw
|
|
163
|
+
parts = [dict(text=p.text or '') if isinstance(p, Text) else denorm_tool_use(p) for p in rebuild_parts(m)]
|
|
164
164
|
return dict(role='model', parts=parts)
|
|
165
165
|
|
|
166
166
|
def denorm_tool(m:Msg):
|