python-fastllm 0.0.49__py3-none-any.whl → 0.0.51__py3-none-any.whl

fastllm/__init__.py

@@ -2,4 +2,4 @@
 
 - `fastllm.responses`: Provider-independent Responses resources and streaming over FastLLM"""
 
-__version__ = "0.0.49"
+__version__ = "0.0.51"

fastllm/_modidx.py

@@ -25,6 +25,7 @@ d = { 'settings': { 'branch': 'main',
                                    'fastllm.anthropic._sanid': ('anthropic.html#_sanid', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.acollect_stream': ('anthropic.html#acollect_stream', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.apply_cache_idxs': ('anthropic.html#apply_cache_idxs', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic.collate_raw': ('anthropic.html#collate_raw', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.cost': ('anthropic.html#cost', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.delta_index_fn': ('anthropic.html#delta_index_fn', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.denorm_assistant': ('anthropic.html#denorm_assistant', 'fastllm/anthropic.py'),
@@ -49,7 +50,9 @@ d = { 'settings': { 'branch': 'main',
                                    'fastllm.anthropic.norm_tool_call': ('anthropic.html#norm_tool_call', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.norm_tool_calls': ('anthropic.html#norm_tool_calls', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.norm_tr_parts': ('anthropic.html#norm_tr_parts', 'fastllm/anthropic.py'),
-                                   'fastllm.anthropic.norm_usage': ('anthropic.html#norm_usage', 'fastllm/anthropic.py')},
+                                   'fastllm.anthropic.norm_usage': ('anthropic.html#norm_usage', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic.raw_msg': ('anthropic.html#raw_msg', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic.search_hits': ('anthropic.html#search_hits', 'fastllm/anthropic.py')},
             'fastllm.chat': { 'fastllm.chat.AsyncChat': ('chat.html#asyncchat', 'fastllm/chat.py'),
                               'fastllm.chat.AsyncChat.__call__': ('chat.html#asyncchat.__call__', 'fastllm/chat.py'),
                               'fastllm.chat.AsyncChat.__init__': ('chat.html#asyncchat.__init__', 'fastllm/chat.py'),
@@ -139,6 +142,7 @@ d = { 'settings': { 'branch': 'main',
             'fastllm.gemini': { 'fastllm.gemini._gem_filter_sch': ('gemini.html#_gem_filter_sch', 'fastllm/gemini.py'),
                                 'fastllm.gemini._gem_part_type': ('gemini.html#_gem_part_type', 'fastllm/gemini.py'),
                                 'fastllm.gemini.acollect_stream': ('gemini.html#acollect_stream', 'fastllm/gemini.py'),
+                                'fastllm.gemini.collate_raw': ('gemini.html#collate_raw', 'fastllm/gemini.py'),
                                 'fastllm.gemini.cost': ('gemini.html#cost', 'fastllm/gemini.py'),
                                 'fastllm.gemini.delta_index_fn': ('gemini.html#delta_index_fn', 'fastllm/gemini.py'),
                                 'fastllm.gemini.denorm_assistant': ('gemini.html#denorm_assistant', 'fastllm/gemini.py'),
@@ -163,8 +167,10 @@ d = { 'settings': { 'branch': 'main',
                                 'fastllm.gemini.norm_parts': ('gemini.html#norm_parts', 'fastllm/gemini.py'),
                                 'fastllm.gemini.norm_sse_event': ('gemini.html#norm_sse_event', 'fastllm/gemini.py'),
                                 'fastllm.gemini.norm_tool_calls': ('gemini.html#norm_tool_calls', 'fastllm/gemini.py'),
-                                'fastllm.gemini.norm_usage': ('gemini.html#norm_usage', 'fastllm/gemini.py')},
+                                'fastllm.gemini.norm_usage': ('gemini.html#norm_usage', 'fastllm/gemini.py'),
+                                'fastllm.gemini.raw_msg': ('gemini.html#raw_msg', 'fastllm/gemini.py')},
             'fastllm.openai_chat': { 'fastllm.openai_chat.acollect_stream': ('oai_chat.html#acollect_stream', 'fastllm/openai_chat.py'),
+                                     'fastllm.openai_chat.collate_raw': ('oai_chat.html#collate_raw', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.cost': ('oai_chat.html#cost', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.delta_index_fn': ('oai_chat.html#delta_index_fn', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.denorm_assistant': ('oai_chat.html#denorm_assistant', 'fastllm/openai_chat.py'),
@@ -189,14 +195,15 @@ d = { 'settings': { 'branch': 'main',
                                      'fastllm.openai_chat.norm_finish': ('oai_chat.html#norm_finish', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.norm_parts': ('oai_chat.html#norm_parts', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.norm_sse_event': ('oai_chat.html#norm_sse_event', 'fastllm/openai_chat.py'),
-                                     'fastllm.openai_chat.norm_tool_calls': ('oai_chat.html#norm_tool_calls', 'fastllm/openai_chat.py')},
+                                     'fastllm.openai_chat.norm_tool_calls': ('oai_chat.html#norm_tool_calls', 'fastllm/openai_chat.py'),
+                                     'fastllm.openai_chat.raw_msg': ('oai_chat.html#raw_msg', 'fastllm/openai_chat.py')},
             'fastllm.openai_responses': { 'fastllm.openai_responses.Completion.response_id': ( 'oai_responses.html#completion.response_id',
                                                                                                'fastllm/openai_responses.py'),
-                                          'fastllm.openai_responses.Completion.response_id_reusable': ( 'oai_responses.html#completion.response_id_reusable',
-                                                                                                        'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses._sanid': ('oai_responses.html#_sanid', 'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.acollect_stream': ( 'oai_responses.html#acollect_stream',
                                                                                         'fastllm/openai_responses.py'),
+                                          'fastllm.openai_responses.collate_raw': ( 'oai_responses.html#collate_raw',
+                                                                                    'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.cost': ('oai_responses.html#cost', 'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.delta_index_fn': ( 'oai_responses.html#delta_index_fn',
                                                                                        'fastllm/openai_responses.py'),
@@ -245,13 +252,19 @@ d = { 'settings': { 'branch': 'main',
                                           'fastllm.openai_responses.norm_tool_calls': ( 'oai_responses.html#norm_tool_calls',
                                                                                         'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.norm_usage': ( 'oai_responses.html#norm_usage',
-                                                                                   'fastllm/openai_responses.py')},
+                                                                                   'fastllm/openai_responses.py'),
+                                          'fastllm.openai_responses.raw_msg': ( 'oai_responses.html#raw_msg',
+                                                                                'fastllm/openai_responses.py')},
             'fastllm.responses': { 'fastllm.responses.AsyncResponses': ('responses.html#asyncresponses', 'fastllm/responses.py'),
+                                   'fastllm.responses.AsyncResponses.__init__': ( 'responses.html#asyncresponses.__init__',
+                                                                                  'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.call': ('responses.html#asyncresponses.call', 'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.events': ( 'responses.html#asyncresponses.events',
                                                                                 'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.prepare': ( 'responses.html#asyncresponses.prepare',
                                                                                  'fastllm/responses.py'),
+                                   'fastllm.responses.AsyncResponses.serve': ( 'responses.html#asyncresponses.serve',
+                                                                               'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.server_results': ( 'responses.html#asyncresponses.server_results',
                                                                                         'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.server_turn': ( 'responses.html#asyncresponses.server_turn',

fastllm/anthropic.py

@@ -1,21 +1,21 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_anthropic.ipynb.
 
 # %% auto #0
-__all__ = ['ant_tc_types', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'finalize_usage', 'norm_finish', 'norm_parts',
-           'norm_tr_parts', 'norm_sse_event', 'delta_index_fn', 'acollect_stream', 'denorm_tool_use',
-           'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice',
-           'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user', 'denorm_image', 'denorm_file',
-           'denorm_tool_result', 'apply_cache_idxs', 'mk_payload', 'get_hdrs', 'cost']
+__all__ = ['ant_tc_types', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'finalize_usage', 'norm_finish', 'search_hits',
+           'norm_parts', 'raw_msg', 'norm_tr_parts', 'norm_sse_event', 'delta_index_fn', 'collate_raw',
+           'acollect_stream', 'denorm_tool_use', 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs',
+           'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user',
+           'denorm_image', 'denorm_file', 'denorm_tool_result', 'apply_cache_idxs', 'mk_payload', 'get_hdrs', 'cost']
 
 # %% ../nbs/04_anthropic.ipynb #02afd3d7
-import json
+import json, copy
 from collections import Counter
 from fastcore.utils import *
 from fastcore.meta import *
 from fasttransport.errors import api_error_from_event
 
 from .types import *
-from aidialog.msg_parts import (Part, PartType, mk_part, Msg, Text, Thinking, ToolUse, ToolResult, tool_text, ServerToolResult,
+from aidialog.msg_parts import (Part, PartType, mk_part, Msg, Completion, Text, Thinking, ToolUse, ToolResult, tool_text,
                                 InputImage, InputAudio, InputVideo, InputFile, data_url)
 from .streaming import *
 from .streaming import mk_acollect_stream
@@ -47,11 +47,11 @@ def norm_usage(resp):
                  cached_tokens=cached, cache_creation_tokens=cache_creation, reasoning_tokens=0, raw=usg)
 
 def finalize_usage(usg, parts):
-    "Adjust usage using finalized Anthropic content parts."
+    "Split reasoning tokens out of `output_tokens`: the reported `thinking_tokens`, else an estimate from the thinking text."
     if not usg: return usg
     rc = '\n'.join(p.text or '' for p in parts if isinstance(p, Thinking))
     ct = int(usg.raw.get('output_tokens', usg.completion_tokens) or 0)
-    rt = min(int(len(rc.split())*1.5), ct) if rc else 0
+    rt = int(nested_idx(usg.raw, 'output_tokens_details', 'thinking_tokens') or 0) or (min(int(len(rc.split())*1.5), ct) if rc else 0)
     return Usage(prompt_tokens=usg.prompt_tokens, completion_tokens=ct-rt, total_tokens=usg.prompt_tokens+ct,
         cached_tokens=usg.cached_tokens, cache_creation_tokens=usg.cache_creation_tokens, reasoning_tokens=rt, raw=usg.raw)
 
@@ -67,20 +67,31 @@ def norm_finish(resp, tcs=None):
 def _ant_part_type(typ):
     "Map Anthropic content block type to canonical PartType."
     ant_parts_mapping = dict(text=PartType.text, thinking=PartType.thinking, redacted_thinking=PartType.thinking, tool_use=PartType.tool_use, server_tool_use=PartType.tool_use, mcp_tool_use=PartType.tool_use)
-    if typ in ant_parts_mapping: return ant_parts_mapping[typ]
-    if typ.endswith('_tool_result'): return PartType.server_tool_result
-    return typ
+    return ant_parts_mapping.get(typ, typ)
+
+def search_hits(parts, blocks):
+    "Set each server call's `text` to the `title: url` lines of its result block, when the response has one"
+    calls = {p.id: p for p in parts if isinstance(p, ToolUse)}
+    for b in blocks:
+        if b.get('type', '').endswith('_tool_result') and (tc := calls.get(b.get('tool_use_id'))):
+            tc.text = '\n'.join(f"{r.get('title', '')}: {r.get('url', '')}" for r in b.get('content') or [] if isinstance(r, dict) and r.get('url'))
+    return parts
 
 def norm_parts(resp):
+    "Canonical parts of a response; a server tool's result block lives in the raw message, and as the call's `text`"
     parts = []
     for b in resp.get("content", []):
         typ = _ant_part_type(b.get("type", "text"))
         if   typ == PartType.thinking: parts.append(Thinking(b.get("thinking", ""), raw=b))
         elif typ == PartType.tool_use:
             if tc := norm_tool_call(b): parts.append(tc)
-        elif typ == PartType.server_tool_result: parts.append(ServerToolResult(b.get("text", ""), raw=b))
-        else: parts.append(Text(b.get("text", ""), raw=b))
-    return parts
+        elif typ == PartType.text: parts.append(Text(b.get("text", ""), raw=b))
+    return search_hits(parts, resp.get("content", []))
+
+# %% ../nbs/04_anthropic.ipynb #f24e52db
+def raw_msg(resp):
+    "The assistant message as the API sent it"
+    return dict(role='assistant', content=resp['content'])
 
 # %% ../nbs/04_anthropic.ipynb #5d1440b7
 def norm_tr_parts(content):
@@ -103,7 +114,6 @@ def norm_sse_event(ev, **kwargs):
     text, thinking, tcs, citations = None, None, [], None
     if typ == "content_block_start":
         cb = ev.get("content_block", {})
-        if cb.get("type", "").endswith("_tool_result"): return Delta(server_tool_result=cb, raw=ev, **kwargs)
         if tc := norm_tool_call(cb):
             if not tc.arguments: tc.arguments = {'_delta': ''}
             tcs = [tc]
@@ -127,11 +137,30 @@ def delta_index_fn(d, typ, last_typ, last_idx):
     'Returns accumulation index for current delta and updated last idx'
     return nested_idx(d, 'raw', 'index'), None
 
+# %% ../nbs/04_anthropic.ipynb #de900e34
+def collate_raw(deltas):
+    "The assistant message rebuilt from a stream's events"
+    blocks = {}
+    for ev in L(deltas).attrgot('raw'):
+        typ,i = ev.get('type'),ev.get('index')
+        if typ == 'content_block_start': blocks[i] = copy.deepcopy(ev['content_block'])
+        elif typ == 'content_block_delta':
+            d,b = ev['delta'],blocks[i]
+            if   d['type'] == 'text_delta':       b['text'] += d['text']
+            elif d['type'] == 'thinking_delta':   b['thinking'] += d['thinking']
+            elif d['type'] == 'signature_delta':  b['signature'] = d['signature']
+            elif d['type'] == 'input_json_delta': b['_json'] = b.get('_json', '') + d['partial_json']
+            elif d['type'] == 'citations_delta':  b['citations'] = (b.get('citations') or []) + [d['citation']]
+        elif typ == 'content_block_stop' and '_json' in blocks[i]: blocks[i]['input'] = json.loads(blocks[i].pop('_json') or '{}')
+    return dict(role='assistant', content=list(blocks.values()))
+
 # %% ../nbs/04_anthropic.ipynb #c994adb1
 @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
 async def acollect_stream(resp, **kwargs):
     res = mk_acollect_stream(norm_and_yield(resp, norm_sse_event), index_fn=delta_index_fn, api_name='anthropic', **kwargs)
-    async for o in res: yield o
+    async for o in res:
+        if isinstance(o, Completion): search_hits(o.message.content, o.message.raw['content'])
+        yield o
 
 # %% ../nbs/04_anthropic.ipynb #bb11ab62
 def _ant_cc(block, p):
@@ -145,44 +174,29 @@ def _sanid(id_str): return re.sub(r'[^a-zA-Z0-9_-]', '_', id_str or '')
 # %% ../nbs/04_anthropic.ipynb #6ec772cb
 def denorm_tool_use(p:ToolUse):
     "Convert canonical tool_use Part to Anthropic tool_use content block."
-    block = dict(type='tool_use', id=_sanid(p.id or ''), name=p.name or '', input=p.arguments or {})
-    if caller := (p.raw or {}).get('caller'): block['caller'] = caller
-    return _ant_cc(block, p)
+    return _ant_cc(dict(type='tool_use', id=_sanid(p.id or ''), name=p.name or '', input=p.arguments or {}), p)
 
 def denorm_assistant(m:Msg):
-    "Convert canonical assistant Msg to Anthropic assistant message + synthetic tool results for non-Anthropic server tools."
-    blocks, srv_results, srv_call_id = [], [], None
-    for p in m.content:
-        if isinstance(p, (Thinking, Text)):
-            if srv_call_id:
-                srv_results.append(dict(type='tool_result', tool_use_id=srv_call_id, content=p.text or ''))
-                srv_call_id = None
-            elif isinstance(p, Thinking) and (sig:=(p.raw or {}).get('signature','')):
-                blocks.append(dict(type='thinking', thinking=p.text or '', signature=sig))
-            else: blocks.append(_ant_cc(dict(type='text', text=p.text or ''), p))
-        elif isinstance(p, ToolUse):
-            if p.server and (p.id or '').startswith('srvtoolu_'):
-                blocks.append(dict(type='server_tool_use', id=p.id, name=p.name or '', input=p.arguments or {}))
-            else:
-                blocks.append(denorm_tool_use(p))
-                if p.server: srv_call_id = p.id or ''
-        elif isinstance(p, ServerToolResult): blocks.append(p.raw or dict(type='server_tool_result'))
-    res = [dict(role='assistant', content=blocks)]
-    if srv_results: res.append(dict(role='user', content=srv_results))
-    return res
+    "The message as the API sent it, else its text and tool calls; a cache mark on the last part is copied onto the last raw block"
+    if m.raw:
+        cc = [p.cache_control for p in m.content if p.cache_control]
+        if not cc: return m.raw
+        return m.raw | dict(content=m.raw['content'][:-1] + [m.raw['content'][-1] | dict(cache_control=cc[-1])])
+    blocks = [_ant_cc(dict(type='text', text=p.text or ''), p) if isinstance(p, Text) else denorm_tool_use(p) for p in m.content if isinstance(p, (Text, ToolUse))]
+    return dict(role='assistant', content=blocks)
 
 def denorm_tool(m:Msg):
     "Convert canonical tool Msg to Anthropic user message with tool_result blocks."
     blocks = [denorm_tool_result(p) for p in m.content if isinstance(p, ToolResult)]
-    return [dict(role='user', content=blocks)]
+    return dict(role='user', content=blocks)
 
 def denorm_msgs(msgs:list[Msg]):
     "Convert list of canonical Msgs to Anthropic messages."
     res = []
     for m in msgs:
         if   m.role == 'user':      res.append(denorm_user(m))
-        elif m.role == 'assistant': res.extend(denorm_assistant(m))
-        elif m.role == 'tool':      res.extend(denorm_tool(m))
+        elif m.role == 'assistant': res.append(denorm_assistant(m))
+        elif m.role == 'tool':      res.append(denorm_tool(m))
     return res
 
 # %% ../nbs/04_anthropic.ipynb #f762a6f2
@@ -346,5 +360,6 @@ def cost(usage, m):
 
 # %% ../nbs/04_anthropic.ipynb #f7c0b989
 api_registry.register('anthropic', norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
-    finalize_usage=finalize_usage, acollect_stream=acollect_stream, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs,
+    finalize_usage=finalize_usage, raw_msg=raw_msg, collate_raw=collate_raw, acollect_stream=acollect_stream, mk_payload=mk_payload,
+    cost=cost, get_hdrs=get_hdrs,
     endpoint='/v1/messages')

fastllm/gemini.py

@@ -1,11 +1,11 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/05_gemini.ipynb.
 
 # %% auto #0
-__all__ = ['api_ns', 'endpoint', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'norm_sse_event', 'delta_index_fn',
-           'acollect_stream', 'denorm_tool_use', 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs',
-           'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user',
-           'denorm_image', 'denorm_audio', 'denorm_video', 'denorm_file', 'denorm_tool_result', 'mk_payload',
-           'get_hdrs', 'cost']
+__all__ = ['api_ns', 'endpoint', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'raw_msg', 'norm_sse_event',
+           'delta_index_fn', 'collate_raw', 'acollect_stream', 'denorm_tool_use', 'denorm_assistant', 'denorm_tool',
+           'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search',
+           'denorm_system', 'denorm_user', 'denorm_image', 'denorm_audio', 'denorm_video', 'denorm_file',
+           'denorm_tool_result', 'mk_payload', 'get_hdrs', 'cost']
 
 # %% ../nbs/05_gemini.ipynb #02afd3d7
 import json
@@ -27,12 +27,14 @@ def endpoint(model, stream):
 
 # %% ../nbs/05_gemini.ipynb #f1eb32f3
 def norm_tool_calls(resp):
-    "Extract Gemini functionCall parts as normalized tool calls."
+    "Tool calls of a response: `functionCall` parts, and a `toolCall` part for a search the API ran itself"
     out = []
     for i,p in enumerate(nested_idx(resp, 'candidates', 0, 'content', 'parts') or []):
-        if not (fc:=p.get("functionCall")): continue
-        raw = {k:v for k,v in p.items() if k != "functionCall"}
-        out.append(ToolUse(id=fc.get("id", f"call_{i}"), name=fc.get("name", ""), arguments=fc.get("args") or {}, raw=raw))
+        if fc := p.get("functionCall"):
+            raw = {k:v for k,v in p.items() if k != "functionCall"}
+            out.append(ToolUse(id=fc.get("id", f"call_{i}"), name=fc.get("name", ""), arguments=fc.get("args") or {}, raw=raw))
+        elif tc := p.get("toolCall"):
+            out.append(ToolUse(id=tc.get("id", f"call_{i}"), name=tc.get("toolType", "").lower(), arguments=tc.get("args") or {}, server=True))
     return out
 
 # %% ../nbs/05_gemini.ipynb #90504163
@@ -48,7 +50,7 @@ def norm_usage(resp):
     cand = nested_idx(resp, 'candidates', 0) or {}
     stu = {}
     if any("executableCode" in p for p in parts): stu["code_execution"] = sum(1 for p in parts if "executableCode" in p)
-    if "groundingMetadata" in cand: stu["google_search"] = 1
+    if n := ("groundingMetadata" in cand) + sum(1 for p in parts if 'toolCall' in p): stu["google_search"] = n
     if stu: usg['server_tool_use'] = stu
     return Usage(prompt_tokens=pt, completion_tokens=ct, total_tokens=tt,
                  cached_tokens=cached, reasoning_tokens=reasoning, raw=usg)
@@ -76,6 +78,7 @@ def norm_parts(resp):
     tc_map = {tc.id: tc for tc in norm_tool_calls(resp)}
     parts = []
     for p in nested_idx(c0, 'content', 'parts') or []:
+        if 'toolResponse' in p: continue
         typ = _gem_part_type(p)
         if typ == PartType.tool_use:
             fc = p.get('functionCall') or p.get('toolCall') or {}
@@ -88,6 +91,11 @@ def norm_parts(resp):
             if isinstance(p, Text): p.citations = citations
     return parts
 
+# %% ../nbs/05_gemini.ipynb #3acc9538
+def raw_msg(resp):
+    "The model content as the API sent it"
+    return nested_idx(resp, 'candidates', 0, 'content')
+
 # %% ../nbs/05_gemini.ipynb #9a5024ee
 def norm_sse_event(ev, **kwargs):
     "Normalize Gemini stream event into Delta."
@@ -107,6 +115,19 @@ def delta_index_fn(d, typ, last_typ, last_idx):
     if not (last_typ or last_idx): return 0,0
     return last_idx + 1, last_idx + 1
 
+# %% ../nbs/05_gemini.ipynb #33506d56
+def collate_raw(deltas):
+    "The model content rebuilt from a stream's chunks"
+    parts = []
+    for ev in L(deltas).attrgot('raw'):
+        for p in nested_idx(ev, 'candidates', 0, 'content', 'parts') or []:
+            last = parts[-1] if parts else None
+            if 'text' in p and last and 'text' in last and last.get('thought') == p.get('thought') and 'thoughtSignature' not in last:
+                last['text'] += p['text']
+                if 'thoughtSignature' in p: last['thoughtSignature'] = p['thoughtSignature']
+            else: parts.append(dict(p))
+    return dict(role='model', parts=parts)
+
 # %% ../nbs/05_gemini.ipynb #328af4d5
 @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
 async def acollect_stream(resp, **kwargs):
@@ -118,17 +139,12 @@ def denorm_tool_use(p:ToolUse):
     "Convert canonical tool_use Part to Gemini functionCall part."
     fc = dict(name=p.name or '', args=p.arguments or {})
     if p.id: fc['id'] = p.id
-    part = dict(functionCall=fc)
-    part['thoughtSignature'] = (p.raw or {}).get('thoughtSignature', 'skip_thought_signature_validator')
-    return part
+    return dict(functionCall=fc)
 
 def denorm_assistant(m:Msg):
-    "Convert canonical assistant Msg to Gemini model content."
-    parts = []
-    for p in m.content:
-        if   isinstance(p, Thinking): parts.append(dict(text=p.text or '', thought=True))
-        elif isinstance(p, ToolUse):  parts.append(denorm_tool_use(p))
-        elif isinstance(p, Text):     parts.append(dict(text=p.text or ''))
+    "The content as the API sent it, else its text and tool calls"
+    if m.raw: return m.raw
+    parts = [dict(text=p.text or '') if isinstance(p, Text) else denorm_tool_use(p) for p in m.content if isinstance(p, (Text, ToolUse))]
     return dict(role='model', parts=parts)
 
 def denorm_tool(m:Msg):
@@ -301,6 +317,8 @@ api_ns = dict(norm_tool_calls=norm_tool_calls,
                 norm_finish=norm_finish,
                 norm_usage=norm_usage,
                 acollect_stream=acollect_stream,
+                raw_msg=raw_msg,
+                collate_raw=collate_raw,
                 mk_payload=mk_payload,
                 cost=cost,
                 get_hdrs=get_hdrs,

fastllm/openai_chat.py

@@ -1,8 +1,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03_oai_chat.ipynb.
 
 # %% auto #0
-__all__ = ['api_ns', 'norm_tool_calls', 'norm_finish', 'norm_parts', 'norm_sse_event', 'delta_index_fn', 'acollect_stream',
-           'denorm_tool_use', 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs',
+__all__ = ['api_ns', 'norm_tool_calls', 'norm_finish', 'norm_parts', 'raw_msg', 'norm_sse_event', 'delta_index_fn', 'collate_raw',
+           'acollect_stream', 'denorm_tool_use', 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs',
            'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user',
            'denorm_image', 'denorm_audio', 'denorm_file', 'denorm_tool_result', 'mk_payload', 'get_hdrs', 'cost',
            'fix_payload']
@@ -50,6 +50,11 @@ def norm_parts(resp):
     if ref := msg.get('refusal'): parts.append(Refusal(ref))
     return parts + norm_tool_calls(resp)
 
+# %% ../nbs/03_oai_chat.ipynb #a30e4324
+def raw_msg(resp):
+    "The assistant message as the API sent it"
+    return nested_idx(resp, 'choices', 0, 'message')
+
 # %% ../nbs/03_oai_chat.ipynb #9baac40e
 def norm_sse_event(ev, **kwargs):
     "Normalize a chat completion stream event."
@@ -73,6 +78,23 @@ def delta_index_fn(d, typ, last_typ, last_idx):
     if typ == last_typ: return last_idx, last_idx
     return last_idx + 1, last_idx + 1
 
+# %% ../nbs/03_oai_chat.ipynb #b7a19b8d
+def collate_raw(deltas):
+    "The assistant message rebuilt from a stream's chunks"
+    msg, tcs = dict(role='assistant', content=None), {}
+    for ev in L(deltas).attrgot('raw'):
+        d = nested_idx(ev, 'choices', 0, 'delta') or {}
+        for k in ('content', 'reasoning_content', 'refusal'):
+            if d.get(k): msg[k] = (msg.get(k) or '') + d[k]
+        for tc in d.get('tool_calls') or []:
+            t = tcs.setdefault(tc['index'], dict(id=tc.get('id'), type='function', function=dict(name='', arguments='')))
+            if tc.get('id'): t['id'] = tc['id']
+            fn = tc.get('function') or {}
+            t['function']['name'] += fn.get('name') or ''
+            t['function']['arguments'] += fn.get('arguments') or ''
+    if tcs: msg['tool_calls'] = [tcs[i] for i in sorted(tcs)]
+    return msg
+
 # %% ../nbs/03_oai_chat.ipynb #e7f8965b
 @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
 async def acollect_stream(resp, **kwargs):
@@ -85,21 +107,12 @@ def denorm_tool_use(p:ToolUse):
     return dict(id=p.id, type='function', function=dict(name=p.name, arguments=json.dumps(p.arguments)))
 
 def denorm_assistant(m:Msg):
-    "Convert canonical assistant Msg to OpenAI Chat assistant message + synthetic tool responses for server tools."
-    tcs, srv_responses, texts = [], [], []
-    for p in m.content:
-        if isinstance(p, ToolUse):
-            tcs.append(denorm_tool_use(p))
-            if p.server:
-                srv_txt = f"[Server tool `{p.name}` executed successfully, results are generated]"
-                srv_responses.append(dict(role='tool', tool_call_id=p.id, content=srv_txt))
-        elif isinstance(p, Text): texts.append(p)
-    msg = dict(role='assistant')
+    "The message as the API sent it, else its text and tool calls"
+    if m.raw: return m.raw
+    msg, texts = dict(role='assistant'), [p for p in m.content if isinstance(p, Text)]
     if texts: msg['content'] = texts[0].text if len(texts)==1 else [dict(type='text', text=p.text or '') for p in texts]
-    if tcs: msg['tool_calls'] = tcs
-    thinking = [p for p in m.content if isinstance(p, Thinking)]
-    if thinking: msg['reasoning_content'] = ''.join(p.text or '' for p in thinking)
-    return [msg] + srv_responses
+    if tcs := [denorm_tool_use(p) for p in m.content if isinstance(p, ToolUse)]: msg['tool_calls'] = tcs
+    return msg
 
 def denorm_tool(m:Msg):
     "Convert canonical tool Msg to list of OpenAI Chat tool messages."
@@ -110,7 +123,7 @@ def denorm_msgs(msgs:list[Msg]):
     res = []
     for m in msgs:
         if   m.role == 'user':      res.append(denorm_user(m))
-        elif m.role == 'assistant': res.extend(denorm_assistant(m))
+        elif m.role == 'assistant': res.append(denorm_assistant(m))
         elif m.role == 'tool':      res.extend(denorm_tool(m))
     return res
 
@@ -226,6 +239,6 @@ def fix_payload(payload, model, vendor_name):
 
 # %% ../nbs/03_oai_chat.ipynb #e2b0908e
 api_ns = dict(norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
-    acollect_stream=acollect_stream, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs, fix_payload=fix_payload,
+    raw_msg=raw_msg, collate_raw=collate_raw, acollect_stream=acollect_stream, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs, fix_payload=fix_payload,
     endpoint='/chat/completions')
 api_registry.register('openai_chat', **api_ns)

fastllm/openai_responses.py

@@ -1,11 +1,11 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_oai_responses.ipynb.
 
 # %% auto #0
-__all__ = ['api_ns', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'norm_sse_event',
-           'delta_index_fn', 'acollect_stream', 'denorm_tool_use', 'denorm_tool', 'denorm_assistant', 'denorm_msgs',
-           'denorm_tool_schs', 'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search', 'denorm_system',
-           'denorm_user', 'denorm_image', 'denorm_video', 'denorm_file', 'denorm_tool_result', 'mk_payload', 'get_hdrs',
-           'cost', 'fix_payload']
+__all__ = ['api_ns', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'raw_msg', 'norm_sse_event',
+           'delta_index_fn', 'collate_raw', 'acollect_stream', 'denorm_tool_use', 'denorm_tool', 'denorm_assistant',
+           'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search',
+           'denorm_system', 'denorm_user', 'denorm_image', 'denorm_video', 'denorm_file', 'denorm_tool_result',
+           'mk_payload', 'get_hdrs', 'cost', 'fix_payload']
 
 # %% ../nbs/02_oai_responses.ipynb #591f55b5
 import json
@@ -78,6 +78,11 @@ def norm_parts(resp):
         elif typ == "function_call" or typ.endswith("_call"): parts.append(norm_tool_call(item))
     return parts
 
+# %% ../nbs/02_oai_responses.ipynb #2aaab962
+def raw_msg(resp):
+    "The output items as the API sent them"
+    return resp['output']
+
 # %% ../nbs/02_oai_responses.ipynb #ef2d19f3
 @patch(as_prop=True)
 def response_id(self:Completion):
@@ -89,11 +94,6 @@ def response_id(self:Completion):
     for d in reversed(self.raw.get('deltas', [])):
         if rid := nested_idx(d.raw, 'response', 'id'): return rid
 
-@patch(as_prop=True)
-def response_id_reusable(self:Completion):
-    "Can the provider continuation identifier be consumed more than once?"
-    return self.raw.get('response_id_reusable', True)
-
 # %% ../nbs/02_oai_responses.ipynb #7cd48aa5
 def norm_sse_event(ev, **kwargs):
     "Normalize OpenAI Responses API stream event into Delta."
@@ -107,6 +107,7 @@ def norm_sse_event(ev, **kwargs):
         item = ev.get("item", {})
         itype = item.get("type", "")
         if itype == "function_call" or itype.endswith("_call"): return Delta(tool_calls=[norm_tool_call(item)], raw=ev, **kwargs)
+        return Delta(raw=ev, **kwargs)
     if typ in ("response.completed", "response.incomplete"):
         resp = ev.get("response", {})
         return Delta(finish_reason=norm_finish(resp), usage=norm_usage(resp), raw=ev, **kwargs)
@@ -118,6 +119,11 @@ def delta_index_fn(d, typ, last_typ, last_idx):
     idx = nested_idx(d, 'raw', 'output_index')
     return idx, idx
 
+# %% ../nbs/02_oai_responses.ipynb #689e31b0
+def collate_raw(deltas):
+    "The output items rebuilt from a stream's events"
+    return [d.raw['item'] for d in deltas if d.raw.get('type') == 'response.output_item.done']
+
 # %% ../nbs/02_oai_responses.ipynb #021b5257
 @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
 async def acollect_stream(resp, **kwargs):
@@ -137,20 +143,12 @@ def denorm_tool(m:Msg): return [denorm_tool_result(p) for p in m.content if isin
 
 # %% ../nbs/02_oai_responses.ipynb #67c8de13
 def denorm_assistant(m:Msg):
-    items, srv_call_id = [], None
+    "The output items as the API sent them, else the message's text and tool calls as items"
+    if m.raw: return m.raw
+    items = []
     for p in m.content:
-        if isinstance(p, ToolUse):
-            items.append(denorm_tool_use(p))
-            # TODO: Server tcs are excluded during deserialization in lisette, so this
-            # is not used. Anthropic checks for `srvtoolu` and requires the full tool result metadata
-            # which is a lot to store.
-            # Instead of reconstructing server tool calls as actual tool uses we can make it a regular msg
-            # telling AI a server tool was executed
-            if p.server:
-                srv_txt = f"[Server tool `{p.name}` executed successfully, results are generated below]"
-                items.append(dict(type='function_call_output', call_id=p.id, output=srv_txt))    
-        elif isinstance(p, (Text, Thinking)):
-            items.append(dict(type="message", role="assistant", content=[dict(type="output_text", text=p.text)]))
+        if isinstance(p, ToolUse): items.append(denorm_tool_use(p))
+        elif isinstance(p, Text): items.append(dict(type="message", role="assistant", content=[dict(type="output_text", text=p.text or '')]))
     return items
 
 # %% ../nbs/02_oai_responses.ipynb #64883b99
@@ -280,6 +278,7 @@ def fix_payload(payload, model, vendor_name):
 
 # %% ../nbs/02_oai_responses.ipynb #07114b55
 api_ns = dict(norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
-    supports_previous_response_id=True, acollect_stream=acollect_stream, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs, fix_payload=fix_payload,
+    supports_previous_response_id=True, raw_msg=raw_msg, collate_raw=collate_raw, acollect_stream=acollect_stream, mk_payload=mk_payload,
+    cost=cost, get_hdrs=get_hdrs, fix_payload=fix_payload,
     endpoint='/responses')
 api_registry.register('openai', **api_ns)

fastllm/responses.py

@@ -143,7 +143,7 @@ def response_object(rid, body, comp, previous_response_id=None, created_at=None)
 # %% ../nbs/06a_responses.ipynb #73a2fc16
 class ResponseState(BasicRepr):
     "Canonical history and optional provider continuation metadata."
-    def __init__(self, id, model, history, provider_response_id=None, provider_response_reusable=True): store_attr()
+    def __init__(self, id, model, history, provider_response_id=None): store_attr()
 
 # %% ../nbs/06a_responses.ipynb #c9ca3744
 class ResponseTurn(BasicRepr):
@@ -160,6 +160,7 @@ class ResponseTurn(BasicRepr):
 # %% ../nbs/06a_responses.ipynb #a648297f
 class AsyncResponses:
     "Run individual Responses turns over FastLLM."
+    def __init__(self): self.tasks = set()   # served turns still running
     def prepare(self, body, previous=None, server_tools=None, **kwargs):
         if not isinstance(body, dict): raise ResponsesError('Request body must be an object')
         model = body.get('model')
@@ -203,7 +204,7 @@ def state(self:AsyncResponses, turn, comp, results=()):
     "Build the next continuation state; `results` are server tool results answering `comp`'s calls."
     normalize_call_ids(comp)
     history = (*turn.history, comp.message, *([Msg('tool', list(results))] if results else []))
-    return ResponseState(turn.id, turn.body['model'], history, comp.response_id, comp.response_id_reusable)
+    return ResponseState(turn.id, turn.body['model'], history, comp.response_id)
 
 
 # %% ../nbs/06a_responses.ipynb #7615b274
@@ -343,3 +344,21 @@ async def events(self:AsyncResponses, turn, finalize=None, stream=None):
     except ResponsesError as exc: yield state.event('error', code=exc.code, message=str(exc), param=exc.param)
     except APIError as exc: yield state.event('error', code=exc.code or 'provider_error', message=str(exc), param=None)
 
+
+# %% ../nbs/06a_responses.ipynb #cc32ce40
+@patch
+def serve(self:AsyncResponses, turn, finalize=None, stream=None, timeout=None):
+    "The frames of `events`, from a task that outlives its reader, so a disconnected client still gets its turn finalized"
+    q = asyncio.Queue()
+    async def _run():
+        try:
+            async with asyncio.timeout(timeout):
+                async for frame in self.events(turn, finalize=finalize, stream=stream): q.put_nowait(frame)
+        except TimeoutError: q.put_nowait(response_event('error', 0, code='timeout', message=f'turn exceeded {timeout}s', param=None))
+        finally: q.put_nowait(None)
+    t = asyncio.create_task(_run())
+    self.tasks.add(t)
+    t.add_done_callback(self.tasks.discard)
+    async def _frames():
+        while (frame := await q.get()) is not None: yield frame
+    return _frames()

fastllm/streaming.py

@@ -12,13 +12,12 @@ from fastcore.utils import *
 from fastcore.meta import delegates
 from fasttransport.errors import *
 from .types import *
-from aidialog.msg_parts import Part, PartType, Msg, Text, Thinking, ToolUse, ToolResult, ServerToolResult, mk_part, fence_call_re, Completion
+from aidialog.msg_parts import Part, PartType, Msg, Text, Thinking, ToolUse, ToolResult, mk_part, fence_call_re, Completion
 
 # %% ../nbs/01_streaming.ipynb #400d628a
 class Delta(BasicRepr):
     "Normalized streaming delta event."
-    def __init__(self, text='', thinking='', refusal='', tool_calls=None, citations=None,
-        server_tool_result=None, finish_reason=None, usage=None, raw=None):
+    def __init__(self, text='', thinking='', refusal='', tool_calls=None, citations=None, finish_reason=None, usage=None, raw=None):
         if tool_calls is None: tool_calls = []
         if citations is None: citations = []
         if raw is None: raw = {}
@@ -172,11 +171,7 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
                     except json.JSONDecodeError: continue
                 acc.arguments = args
                 yield acc
-                # Server tool results for anthropic are yielded in d.server_tool_result by checking injected dummy `_delta`
                 if acc.server: yield ToolResult(id=acc.id, name=acc.name, arguments=args, server=True, text="Server tool call executed.")
-        if d.server_tool_result:
-            idx = _fidx(d, 'server_tool_result')
-            part_accum.parts[idx] = ServerToolResult(raw=d.server_tool_result)
         r = _proc(d, 'refusal')
         if r[0]: yield _mk_delta_part('refusal', r[0]['refusal'])
         if d.finish_reason: fin = d.finish_reason
@@ -190,6 +185,6 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
     fin = FinishReason.tool_calls if fin==FinishReason.stop and any(~L(tcs).attrgot('server')) else fin # recheck tool calls post collation
     # tool calls and non-anthropic citations are yielded at the end
     yield Completion(model,
-            message=Msg(role="assistant", content=parts),
+            message=Msg(role="assistant", content=parts, raw=api_registry[api_name].collate_raw(deltas) if api_name else None),
             finish_reason=fin, usage=usg, api_name=api_name, vendor_name=vendor_name,
             raw={'deltas':deltas})

fastllm/types.py

@@ -3,14 +3,14 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_types.ipynb.
 
 # %% auto #0
-__all__ = ['FinishReason', 'api_registry', 'model_prices_url', 'sample_img_url', 'haik45', 'sonn45', 'sonn46', 'sonn', 'sonn5',
-           'opus46', 'opus48', 'opus', 'opus5', 'fable', 'fable5', 'fable51', 'gpt54', 'gpt54m', 'gpt55', 'codex54',
-           'codex54m', 'codex55', 'codex53spark', 'model_info_registry', 'modern_llm', 'effort_codes',
-           'deepseek_v4_common', 'deepseek_v4_flash_prices', 'mimo_v25_common', 'codex_pricing', 'sol', 'terra', 'luna',
-           'gpt56s', 'Usage', 'APIRegistry', 'mk_completion', 'fn_schema', 'payload_kwargs', 'provider_req',
-           'get_api_key', 'wrap_typed', 'unwrap_typed', 'resize_b64', 'model_prices_meta', 'infer_api_name',
-           'get_model_meta', 'register_model_info', 'get_model_info', 'effort_code', 'effort_levels', 'resolve_effort',
-           'get_model_pricing', 'approx_pricing', 'is_deepseek_peak_hour', 'price_tier', 'tier_rate']
+__all__ = ['FinishReason', 'api_registry', 'model_prices_url', 'sample_img_url', 'sample_doms', 'haik45', 'sonn45', 'sonn46',
+           'sonn', 'sonn5', 'opus46', 'opus48', 'opus', 'opus5', 'fable', 'fable5', 'fable51', 'gpt54', 'gpt54m',
+           'gpt55', 'codex54', 'codex54m', 'codex55', 'codex53spark', 'model_info_registry', 'modern_llm',
+           'effort_codes', 'deepseek_v4_common', 'deepseek_v4_flash_prices', 'mimo_v25_common', 'codex_pricing', 'sol',
+           'terra', 'luna', 'gpt56s', 'Usage', 'APIRegistry', 'mk_completion', 'fn_schema', 'payload_kwargs',
+           'provider_req', 'get_api_key', 'wrap_typed', 'unwrap_typed', 'resize_b64', 'model_prices_meta',
+           'infer_api_name', 'get_model_meta', 'register_model_info', 'get_model_info', 'effort_code', 'effort_levels',
+           'resolve_effort', 'get_model_pricing', 'approx_pricing', 'is_deepseek_peak_hour', 'price_tier', 'tier_rate']
 
 # %% ../nbs/00_types.ipynb #b4d047fd
 import httpx2, base64, io
@@ -68,7 +68,8 @@ FinishReason = str_enum('finish_reason', 'stop', 'tool_calls', 'length', 'conten
 # %% ../nbs/00_types.ipynb #fc681c52
 class APIRegistry(dict):
     _eps = None
-    def register(self, name, finalize_usage=noop, **kwargs): self[name] = SimpleNamespace(finalize_usage=finalize_usage, **kwargs)
+    def register(self, name, finalize_usage=noop, raw_msg=noop, collate_raw=noop, **kwargs):
+        self[name] = SimpleNamespace(finalize_usage=finalize_usage, raw_msg=raw_msg, collate_raw=collate_raw, **kwargs)
     def _load(self, name):
         "Import the plugin providing api `name` from the `fastllm.apis` entry-point group; its import registers the api."
         if self._eps is None: self._eps = {o.name:o for o in entry_points(group='fastllm.apis')}
@@ -90,7 +91,7 @@ def mk_completion(resp, model, api_name, vendor_name):
     tcs = api.norm_tool_calls(resp)
     parts = api.norm_parts(resp)
     usg = api.finalize_usage(api.norm_usage(resp), parts)
-    return Completion(model=model, message=Msg(role="assistant", content=parts), finish_reason=api.norm_finish(resp, tcs),
+    return Completion(model=model, message=Msg(role="assistant", content=parts, raw=api.raw_msg(resp)), finish_reason=api.norm_finish(resp, tcs),
         usage=usg, api_name=api_name, vendor_name=vendor_name, raw=resp)
 
 # %% ../nbs/00_types.ipynb #8a8e468b
@@ -169,6 +170,7 @@ def model_prices_meta():
 
 # %% ../nbs/00_types.ipynb #019882a3
 sample_img_url = "https://img.magnific.com/free-photo/mountain-range-body-water_53876-139760.jpg?semt=ais_hybrid&w=740&q=80"
+sample_doms = ('img.magnific.com', 'arxiv.org', 'openaiassets.blob.core.windows.net')   # hosts of the sample media the lessons fetch, for cachy
 
 # %% ../nbs/00_types.ipynb #68e488d8
 def infer_api_name(model):

{python_fastllm-0.0.49.dist-info → python_fastllm-0.0.51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-fastllm
-Version: 0.0.49
+Version: 0.0.51
 Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
 License: Apache-2.0
 Project-URL: Repository, https://github.com/AnswerDotAI/fastllm
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: fastcore>=2.1.18
-Requires-Dist: aidialog>=0.0.10
+Requires-Dist: aidialog>=0.0.25
 Requires-Dist: fasttransport>=0.0.2
 Requires-Dist: pillow
 
@@ -21,13 +21,14 @@ Requires-Dist: pillow
 
 ## Install
 
-Clone and install locally into your `aai-ws` env
+Run `pip install python-fastllm` or clone from github and install locally.
 
 ## Setup
 
 ``` python
 from aidialog.msg_parts import Msg, Part, Text, Thinking, ToolUse, InputImage, mk_tool_res_msg, Completion
 from fastllm.acomplete import acomplete
+from fastllm.types import sample_img_url, sample_doms
 import asyncio, json
 
 # Helpers
@@ -56,7 +57,7 @@ The same `acomplete` call works with Claude, GPT, Gemini, and Kimi. Just change
 
 ``` python
 models = [
-    ('claude-sonnet-4-20250514', {}),
+    ('claude-sonnet-5', {}),
     ('gpt-4o-mini', {}),
     ('models/gemini-3-flash-preview', {}),
     ('accounts/fireworks/models/kimi-k2p5', dict(vendor_name='fireworks_ai'))
@@ -80,46 +81,6 @@ for name, kw in models:
 
     The response should be simple and direct.
 
-## Multi-Turn — Swap Providers Mid-Conversation
-
-Build a conversation with one provider, then seamlessly continue it with another. `fastllm` translates between every provider’s native format automatically:
-
-``` python
-# Turn 1: Claude starts the conversation
-msgs = [user("Name the 3 largest planets in our solar system. One sentence.")]
-print("Claude: ", end='')
-r1 = await stream(msgs, model='claude-sonnet-4-20250514', max_tokens=mtok)
-```
-
-    Claude: The three largest planets in our solar system are Jupiter, Saturn, and Neptune.
-
-``` python
-# Turn 2: Switch to GPT — just change the model string
-msgs += [r1.message, user("Which one has the most moons?")]
-print("GPT: ", end='')
-r2 = await stream(msgs, model='gpt-4o-mini', max_tokens=mtok)
-```
-
-    GPT: As of now, Saturn has the most moons, with over 80 confirmed moons.
-
-``` python
-# Turn 3: Switch to Gemini — same msgs, different provider
-msgs += [r2.message, user("Summarize our conversation in one sentence.")]
-print("Gemini: ", end='')
-r3 = await stream(msgs, model='models/gemini-3-flash-preview', max_tokens=mtok)
-```
-
-    Gemini: The conversation identified the three largest planets in the solar system and noted that Saturn currently has the most moons.
-
-``` python
-# Turn 4: Switch to Kimi — works the same way
-msgs += [r3.message, user("Thanks! What's one surprising fact about Saturn?")]
-print("Kimi: ", end='')
-r4 = await stream(msgs, model='accounts/fireworks/models/kimi-k2p5', vendor_name='fireworks_ai', max_tokens=mtok)
-```
-
-    Kimi: 🧠🧠🧠🧠🧠🧠🧠🧠🧠🧠Saturn is less dense than water, meaning it would theoretically float if you could find a bathtub large enough to hold it.
-
 ## System Prompts
 
 Pass a system prompt to any provider — `fastllm` maps it to each API’s native mechanism (Anthropic `system`, OpenAI `instructions`, Gemini `system_instruction`):
@@ -128,7 +89,7 @@ Pass a system prompt to any provider — `fastllm` maps it to each API’s nativ
 sys = "You are a pirate chef. Always respond in pirate speak and mention food."
 
 print("Claude: ", end='')
-r = await stream([user("What should I do today?")], model='claude-sonnet-4-20250514', system=sys, max_tokens=mtok)
+r = await stream([user("What should I do today?")], model='claude-sonnet-5', system=sys, max_tokens=mtok)
 
 print("Gemini: ", end='')
 r = await stream([user("What should I do today?")], model='models/gemini-3-flash-preview', system=sys, max_tokens=mtok)
@@ -161,7 +122,7 @@ r = await stream([user("What should I do today?")], model='models/gemini-3-flash
 
 ## Tool Calling — Define Once, Use Anywhere
 
-Define tools in a single canonical format. `fastllm` translates to each provider’s native tool schema automatically. Here we start a tool-use conversation with Claude, provide the result, then switch to GPT and Gemini to continue:
+Define tools in a single canonical format. `fastllm` translates to each provider’s native tool schema automatically. Here Claude requests the tool, receives the result, and answers:
 
 ``` python
 tools = [{"type": "function", "function": {
@@ -172,7 +133,7 @@ tools = [{"type": "function", "function": {
 
 # Turn 1: Claude requests the tool
 msgs = [user("What's the weather in Paris?")]
-r1 = await stream(msgs, model='claude-sonnet-4-20250514', tools=tools, max_tokens=mtok)
+r1 = await stream(msgs, model='claude-sonnet-5', tools=tools, max_tokens=mtok)
 print("Tool calls:", r1.tool_calls)
 ```
 
@@ -182,41 +143,26 @@ print("Tool calls:", r1.tool_calls)
     Tool calls: [ToolUse(raw={'caller': {'type': 'direct'}}, cache_control=None, id='toolu_01RMN1WM7vPBT3ovv5Ex6VzC', name='get_weather', arguments={'city': 'Paris'}, server=False, text=None)]
 
 ``` python
-# Provide the tool result
-msgs += [r1.message, mk_tool_res_msg(r1.tool_calls, ['22°C, sunny with light clouds'])]
-
-# Turn 2: Switch to GPT to interpret the result
-msgs.append(user("Should I bring a jacket?"))
-print("GPT: ", end='')
-r2 = await stream(msgs, model='gpt-4o-mini', tools=tools, max_tokens=mtok)
+# Provide the tool result, then continue the same conversation
+msgs += [r1.message, mk_tool_res_msg(r1.tool_calls, ['22°C, sunny with light clouds']), user("Should I bring a jacket?")]
+print("Claude: ", end='')
+r2 = await stream(msgs, model='claude-sonnet-5', tools=tools, max_tokens=mtok)
 ```
 
     GPT: With the temperature at 22°C and sunny with light clouds, a jacket isn't necessary for most people. However, if you tend to feel cold easily or if you plan to be out in the evening when it might cool down a bit, it could be a good idea to bring a light jacket.
 
-``` python
-# Turn 3: Gemini sees the full cross-provider tool history
-msgs += [r2.message, user("How about tomorrow — will it rain?")]
-r3 = await stream(msgs, model='models/gemini-3-flash-preview', tools=tools, max_tokens=mtok, web_search_options={})
-```
-
-    Tomorrow in Paris (Saturday, June 13), the weather is expected to be mostly pleasant and sunny. 
-
-    While there is a small chance of light, passing showers (around a 20-25% chance), most forecasts indicate a dry day with mostly clear or scattered clouds. Temperatures will likely reach a comfortable high of around 22°C to 27°C (72°F to 80°F), making it a great day for being outdoors. 
-
-    So, while you might see a stray shower, you likely won't need to worry about heavy rain!
-
 ## Tool Choice
 
 Control whether the model must use tools, can’t use tools, or decides on its own:
 
 ``` python
 # Force the model to call a tool (even for a greeting)
-r = await stream([user("Hello there!")], model='claude-sonnet-4-20250514', tools=tools, tool_choice='required', max_tokens=mtok)
+r = await stream([user("Hello there!")], model='claude-sonnet-5', tools=tools, tool_choice='required', max_tokens=mtok)
 print("Forced:", r.tool_calls)
 
 # Prevent tool use (model must answer directly)
 print("\nNo tools: ", end='')
-r = await stream([user("What's the weather?")], model='claude-sonnet-4-20250514', tools=tools, tool_choice='none', max_tokens=mtok)
+r = await stream([user("What's the weather?")], model='claude-sonnet-5', tools=tools, tool_choice='none', max_tokens=mtok)
 ```
 
     - ⏳ `get_weather(city="<UNKNOWN>")` ⏳
@@ -327,11 +273,11 @@ long_ctx = "You are an expert on the solar system. " * 200
 system = Text(long_ctx, cache_control={'type': 'ephemeral'})
 
 print("Call 1: ", end='')
-r1 = await stream([user("What is Jupiter's mass?")], model='claude-sonnet-4-20250514', system=system, max_tokens=mtok)
+r1 = await stream([user("What is Jupiter's mass?")], model='claude-sonnet-5', system=system, max_tokens=mtok)
 print(f"Usage: {r1.usage}")
 
 print("Call 2: ", end='')
-r2 = await stream([user("How about Saturn?")], model='claude-sonnet-4-20250514', system=system, max_tokens=mtok)
+r2 = await stream([user("How about Saturn?")], model='claude-sonnet-5', system=system, max_tokens=mtok)
 print(f"Usage: {r2.usage}")
 ```
 
@@ -381,13 +327,12 @@ print(f"Usage: {r2.usage}")
 Send images to any provider that supports them. The canonical `input_image` part works everywhere:
 
 ``` python
-img_url = "https://img.freepik.com/free-photo/mountain-range-body-water_53876-139760.jpg?semt=ais_hybrid&w=740&q=80"
 img_msg = Msg(role='user', content=[
-    InputImage(img_url),
+    InputImage(sample_img_url),
     Text("What do you see in this image?")
 ])
 
-for name, kw in [('claude-sonnet-4-20250514', {}), ('gpt-4o-mini', {}), ('models/gemini-3-flash-preview', {})]:
+for name, kw in [('claude-sonnet-5', {}), ('gpt-4o-mini', {}), ('models/gemini-3-flash-preview', {})]:
     print(f"{name:>30s}: ", end='')
     r = await stream([img_msg], model=name, max_tokens=mtok, **kw)
 ```

python_fastllm-0.0.51.dist-info/RECORD

@@ -0,0 +1,17 @@
+fastllm/__init__.py,sha256=9Ir0J8R-z0_JnZDe2PsJohUmaNENqlehYHPTO3403g8,131
+fastllm/_modidx.py,sha256=fQ6pg5H_5Pp8l7k12Fb5ca6uWmQewKtGtZBDi8hCuFg,44277
+fastllm/acomplete.py,sha256=aP1ff9a7WKNhBd1kwKE2TJ-YyihLcdhd9-1Izi4wQP4,10772
+fastllm/anthropic.py,sha256=k0QeuRnl_CalfCq74WXN6llRQqRiAQ9d8ohn_0YYC7A,18386
+fastllm/chat.py,sha256=klji3SOYs5xyzrSFbNhy3uoQ2TSZ3a-8HTjvdRtaG5Q,25036
+fastllm/codex.py,sha256=r6auZ8egia83aGkWcF4132AzW4SsBvKeDwg_CxyXj9E,5808
+fastllm/gemini.py,sha256=zbxNmI1cMUJJ10xIfBqPqh38hAQ0exsFKkhTXvG2-eE,16281
+fastllm/openai_chat.py,sha256=n5INfDCsAZIiKUImWeaX6SJ_OAJ_uMhw20HUkgFgdgM,12379
+fastllm/openai_responses.py,sha256=zsaOjkQUZ-7iuGPc-UpvnM6oAyqW09BDYwZ_ma4-1Qo,14346
+fastllm/responses.py,sha256=mFFaBpKNsq85sJjXIK5rYKb6WO7nlImEfowqDPf_yzA,19633
+fastllm/streaming.py,sha256=i_FU_Mt-3F2lBEImYav8I5pEjUSKYk3JFIW3lkiXEb8,8813
+fastllm/types.py,sha256=BIZHwlzsbkZELjAib2oSFRab8gKBa216DNNeE_P68go,19525
+python_fastllm-0.0.51.dist-info/METADATA,sha256=Btmnr0P4mLlxL6J39LJEBg6Zw4KkTLu1-1o4ulP3RFY,18537
+python_fastllm-0.0.51.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+python_fastllm-0.0.51.dist-info/entry_points.txt,sha256=WkWd04F3xVYFP5bsexaQe9Se6uelwE5ULoTtHWreO0s,133
+python_fastllm-0.0.51.dist-info/top_level.txt,sha256=F8qodL7nEGUHGmzzqfhNKCTIr1i0D6cvudOnm3z7o0Y,8
+python_fastllm-0.0.51.dist-info/RECORD,,

python_fastllm-0.0.49.dist-info/RECORD

@@ -1,17 +0,0 @@
-fastllm/__init__.py,sha256=udUKGtrdGuO2q1CvwAXrPId3ToCsgVd8MI9FczPGC08,131
-fastllm/_modidx.py,sha256=UExnE14vuPnwPFZ1RcoK2iJawaLmlYJDsOKbpPx-F7o,42832
-fastllm/acomplete.py,sha256=aP1ff9a7WKNhBd1kwKE2TJ-YyihLcdhd9-1Izi4wQP4,10772
-fastllm/anthropic.py,sha256=RQpOpwIpXxpZe5httO7zgWautF7lJEPSzotcUP-eDVE,17265
-fastllm/chat.py,sha256=klji3SOYs5xyzrSFbNhy3uoQ2TSZ3a-8HTjvdRtaG5Q,25036
-fastllm/codex.py,sha256=r6auZ8egia83aGkWcF4132AzW4SsBvKeDwg_CxyXj9E,5808
-fastllm/gemini.py,sha256=d0nLkxPr5k3X6A8CZNaAuiRrWxmJrc6FfNylJoKNpIc,15291
-fastllm/openai_chat.py,sha256=mUOyos5bdmXukPRMD_sEDiVt4WXDRBzvYiCQskQUHPI,11769
-fastllm/openai_responses.py,sha256=pWg1rA53u5A0aTvnBczxvTK-aBIHm01U9FeIg3-lLbE,14618
-fastllm/responses.py,sha256=jcbKUwbEIsy7ngNcHUtv2bfYeDgqSysK_qWM-0LC9Mg,18767
-fastllm/streaming.py,sha256=8qVtWeZ7NlP-9rup_4Ho0ol2dx3-XuGcIMXpZ1GYdEo,9079
-fastllm/types.py,sha256=IYHA3Nfs2ghXUhgYy2bbbCAcfqU-guTOjI5gF3NW5NY,19260
-python_fastllm-0.0.49.dist-info/METADATA,sha256=VYlgLNW8x53M1VvmBcW7qyq3-A6FF5a7Twh_z8-q3l0,21143
-python_fastllm-0.0.49.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-python_fastllm-0.0.49.dist-info/entry_points.txt,sha256=WkWd04F3xVYFP5bsexaQe9Se6uelwE5ULoTtHWreO0s,133
-python_fastllm-0.0.49.dist-info/top_level.txt,sha256=F8qodL7nEGUHGmzzqfhNKCTIr1i0D6cvudOnm3z7o0Y,8
-python_fastllm-0.0.49.dist-info/RECORD,,