python-fastllm 0.0.48__py3-none-any.whl → 0.0.50__py3-none-any.whl

fastllm/__init__.py

@@ -2,4 +2,4 @@
 
 - `fastllm.responses`: Provider-independent Responses resources and streaming over FastLLM"""
 
-__version__ = "0.0.48"
+__version__ = "0.0.50"

fastllm/_modidx.py

@@ -25,6 +25,7 @@ d = { 'settings': { 'branch': 'main',
                                    'fastllm.anthropic._sanid': ('anthropic.html#_sanid', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.acollect_stream': ('anthropic.html#acollect_stream', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.apply_cache_idxs': ('anthropic.html#apply_cache_idxs', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic.collate_raw': ('anthropic.html#collate_raw', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.cost': ('anthropic.html#cost', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.delta_index_fn': ('anthropic.html#delta_index_fn', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.denorm_assistant': ('anthropic.html#denorm_assistant', 'fastllm/anthropic.py'),
@@ -49,7 +50,8 @@ d = { 'settings': { 'branch': 'main',
                                    'fastllm.anthropic.norm_tool_call': ('anthropic.html#norm_tool_call', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.norm_tool_calls': ('anthropic.html#norm_tool_calls', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.norm_tr_parts': ('anthropic.html#norm_tr_parts', 'fastllm/anthropic.py'),
-                                   'fastllm.anthropic.norm_usage': ('anthropic.html#norm_usage', 'fastllm/anthropic.py')},
+                                   'fastllm.anthropic.norm_usage': ('anthropic.html#norm_usage', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic.raw_msg': ('anthropic.html#raw_msg', 'fastllm/anthropic.py')},
             'fastllm.chat': { 'fastllm.chat.AsyncChat': ('chat.html#asyncchat', 'fastllm/chat.py'),
                               'fastllm.chat.AsyncChat.__call__': ('chat.html#asyncchat.__call__', 'fastllm/chat.py'),
                               'fastllm.chat.AsyncChat.__init__': ('chat.html#asyncchat.__init__', 'fastllm/chat.py'),
@@ -139,6 +141,7 @@ d = { 'settings': { 'branch': 'main',
             'fastllm.gemini': { 'fastllm.gemini._gem_filter_sch': ('gemini.html#_gem_filter_sch', 'fastllm/gemini.py'),
                                 'fastllm.gemini._gem_part_type': ('gemini.html#_gem_part_type', 'fastllm/gemini.py'),
                                 'fastllm.gemini.acollect_stream': ('gemini.html#acollect_stream', 'fastllm/gemini.py'),
+                                'fastllm.gemini.collate_raw': ('gemini.html#collate_raw', 'fastllm/gemini.py'),
                                 'fastllm.gemini.cost': ('gemini.html#cost', 'fastllm/gemini.py'),
                                 'fastllm.gemini.delta_index_fn': ('gemini.html#delta_index_fn', 'fastllm/gemini.py'),
                                 'fastllm.gemini.denorm_assistant': ('gemini.html#denorm_assistant', 'fastllm/gemini.py'),
@@ -163,8 +166,10 @@ d = { 'settings': { 'branch': 'main',
                                 'fastllm.gemini.norm_parts': ('gemini.html#norm_parts', 'fastllm/gemini.py'),
                                 'fastllm.gemini.norm_sse_event': ('gemini.html#norm_sse_event', 'fastllm/gemini.py'),
                                 'fastllm.gemini.norm_tool_calls': ('gemini.html#norm_tool_calls', 'fastllm/gemini.py'),
-                                'fastllm.gemini.norm_usage': ('gemini.html#norm_usage', 'fastllm/gemini.py')},
+                                'fastllm.gemini.norm_usage': ('gemini.html#norm_usage', 'fastllm/gemini.py'),
+                                'fastllm.gemini.raw_msg': ('gemini.html#raw_msg', 'fastllm/gemini.py')},
             'fastllm.openai_chat': { 'fastllm.openai_chat.acollect_stream': ('oai_chat.html#acollect_stream', 'fastllm/openai_chat.py'),
+                                     'fastllm.openai_chat.collate_raw': ('oai_chat.html#collate_raw', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.cost': ('oai_chat.html#cost', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.delta_index_fn': ('oai_chat.html#delta_index_fn', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.denorm_assistant': ('oai_chat.html#denorm_assistant', 'fastllm/openai_chat.py'),
@@ -189,14 +194,15 @@ d = { 'settings': { 'branch': 'main',
                                      'fastllm.openai_chat.norm_finish': ('oai_chat.html#norm_finish', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.norm_parts': ('oai_chat.html#norm_parts', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.norm_sse_event': ('oai_chat.html#norm_sse_event', 'fastllm/openai_chat.py'),
-                                     'fastllm.openai_chat.norm_tool_calls': ('oai_chat.html#norm_tool_calls', 'fastllm/openai_chat.py')},
+                                     'fastllm.openai_chat.norm_tool_calls': ('oai_chat.html#norm_tool_calls', 'fastllm/openai_chat.py'),
+                                     'fastllm.openai_chat.raw_msg': ('oai_chat.html#raw_msg', 'fastllm/openai_chat.py')},
             'fastllm.openai_responses': { 'fastllm.openai_responses.Completion.response_id': ( 'oai_responses.html#completion.response_id',
                                                                                                'fastllm/openai_responses.py'),
-                                          'fastllm.openai_responses.Completion.response_id_reusable': ( 'oai_responses.html#completion.response_id_reusable',
-                                                                                                        'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses._sanid': ('oai_responses.html#_sanid', 'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.acollect_stream': ( 'oai_responses.html#acollect_stream',
                                                                                         'fastllm/openai_responses.py'),
+                                          'fastllm.openai_responses.collate_raw': ( 'oai_responses.html#collate_raw',
+                                                                                    'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.cost': ('oai_responses.html#cost', 'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.delta_index_fn': ( 'oai_responses.html#delta_index_fn',
                                                                                        'fastllm/openai_responses.py'),
@@ -245,7 +251,9 @@ d = { 'settings': { 'branch': 'main',
                                           'fastllm.openai_responses.norm_tool_calls': ( 'oai_responses.html#norm_tool_calls',
                                                                                         'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.norm_usage': ( 'oai_responses.html#norm_usage',
-                                                                                   'fastllm/openai_responses.py')},
+                                                                                   'fastllm/openai_responses.py'),
+                                          'fastllm.openai_responses.raw_msg': ( 'oai_responses.html#raw_msg',
+                                                                                'fastllm/openai_responses.py')},
             'fastllm.responses': { 'fastllm.responses.AsyncResponses': ('responses.html#asyncresponses', 'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.call': ('responses.html#asyncresponses.call', 'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.events': ( 'responses.html#asyncresponses.events',
@@ -338,7 +346,7 @@ d = { 'settings': { 'branch': 'main',
                                'fastllm.types.Usage.__add__': ('types.html#usage.__add__', 'fastllm/types.py'),
                                'fastllm.types.Usage.__eq__': ('types.html#usage.__eq__', 'fastllm/types.py'),
                                'fastllm.types.Usage.__init__': ('types.html#usage.__init__', 'fastllm/types.py'),
-                               'fastllm.types._rm_ctx_tiers': ('types.html#_rm_ctx_tiers', 'fastllm/types.py'),
+                               'fastllm.types._flat_rates': ('types.html#_flat_rates', 'fastllm/types.py'),
                                'fastllm.types.approx_pricing': ('types.html#approx_pricing', 'fastllm/types.py'),
                                'fastllm.types.effort_code': ('types.html#effort_code', 'fastllm/types.py'),
                                'fastllm.types.effort_levels': ('types.html#effort_levels', 'fastllm/types.py'),

fastllm/acomplete.py

@@ -67,10 +67,10 @@ def _codex_mtime_policy():
 
 # %% ../nbs/06_acomplete.ipynb #79075d95
 @flexicache(_codex_mtime_policy())
-def mk_client(model=None, vendor_name=None, api_name=None, api_key=None, base_url=None, xtra_hdrs=None,
+def mk_client(model=None, vendor_name=None, api_name=None, api_key=None, oauth_token=None, base_url=None, xtra_hdrs=None,
     timeout=httpx2.Timeout(connect=30, read=300, write=30, pool=10)):
     err_msg = f"please pass a valid one vendor: {', '.join(list(vendor_mapping))} or pass `api_name`,`base_url` and `api_key`"
-    if not vendor_name and not api_name and not (base_url and api_key):
+    if not vendor_name and not api_name and not (base_url and (api_key or oauth_token)):
         v, m = split_vendor(model)
         if v in vendor_mapping: vendor_name, model = v, m
     if vendor_name:
@@ -78,17 +78,17 @@ def mk_client(model=None, vendor_name=None, api_name=None, api_key=None, base_ur
         try:
             api_name, base_url, env_api_nm, *auth_json = vendor_mapping[vendor_name]
             base_url = override_base_url or base_url
-            if auth_json and not api_key and not os.getenv(env_api_nm):
+            if auth_json and not api_key and not oauth_token and not os.getenv(env_api_nm):
                 fn,keys = auth_json[0]  # pyright: ignore[reportAssignmentType]
                 auth_fn = Path(fn).expanduser()
                 if auth_fn.exists(): api_key = nested_idx(json.loads(auth_fn.read_text()), *keys)
-            api_key = get_api_key(api_key, env_api_nm)
+            if not oauth_token: api_key = get_api_key(api_key, env_api_nm)
         except KeyError: raise ValueError(f"Unknown vendor '{vendor_name}', {err_msg}")
-    elif base_url and api_key: vendor_name, api_name = ifnone(vendor_name, 'custom'), ifnone(api_name, 'openai_chat')
+    elif base_url and (api_key or oauth_token): vendor_name, api_name = ifnone(vendor_name, 'custom'), ifnone(api_name, 'openai_chat')
     elif (api_name:=infer_api_name(model)):  base_url, vendor_name = vendor_mapping[api_name][1], api_name
     else: raise ValueError(f"Model {model} can't be auto resolved, {err_msg}")
     api = api_registry[api_name]
-    hdrs = merge(api.get_hdrs(api_key), ifnone(xtra_hdrs, {}))
+    hdrs = merge(api.get_hdrs(api_key, oauth_token), ifnone(xtra_hdrs, {}))
     cli = AsyncHttpCli(base_url, base_headers=hdrs, timeout=timeout)
     return cli, api_name, vendor_name
 
@@ -155,18 +155,19 @@ async def _retry_stream(mk_gen, retries=2, retry_delay=0.5):
 
 # %% ../nbs/06_acomplete.ipynb #2379ec94
 @delegates(payload_kwargs)
-async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None, base_url=None, endpoint=None, xtra_body=None, xtra_hdrs=None,
+async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None, oauth_token=None, base_url=None, endpoint=None, xtra_body=None, xtra_hdrs=None,
     stream=False, previous_response_id=None, stop_callables=None, retries=2, retry_delay=0.5, **kwargs):
     "Unified completion across different APIs."
-    if not vendor_name and not api_name and not (base_url and api_key):
+    if api_key and oauth_token: raise ValueError('Pass `api_key` or `oauth_token`, not both: a request has one identity')
+    if not vendor_name and not api_name and not (base_url and (api_key or oauth_token)):
         v, m = split_vendor(model)
         if v in vendor_mapping: vendor_name, model = v, m
         elif v: api_name, model = v, m  # a registered transport api (e.g. claude_code): not an HTTP vendor
     own = api_name in api_registry and not hasattr(api_registry[api_name], 'endpoint') if api_name else False
     if own:
         vendor_name = ifnone(vendor_name, api_name)  # an own-transport api: no HTTP client to build
-        if api_key: kwargs['api_key'] = api_key  # its `mk_payload` applies a per-user credential itself
-    else: cli, api_name, vendor_name = mk_client(model=model, vendor_name=vendor_name, api_name=api_name, api_key=api_key, base_url=base_url, xtra_hdrs=xtra_hdrs)
+        if oauth_token: kwargs['oauth_token'] = oauth_token  # its `mk_payload` applies the user's token itself
+    else: cli, api_name, vendor_name = mk_client(model=model, vendor_name=vendor_name, api_name=api_name, api_key=api_key, oauth_token=oauth_token, base_url=base_url, xtra_hdrs=xtra_hdrs)
     api = api_registry[api_name]
     if previous_response_id is not None:
         if not getattr(api, 'supports_previous_response_id', False):

fastllm/anthropic.py

@@ -2,20 +2,20 @@
 
 # %% auto #0
 __all__ = ['ant_tc_types', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'finalize_usage', 'norm_finish', 'norm_parts',
-           'norm_tr_parts', 'norm_sse_event', 'delta_index_fn', 'acollect_stream', 'denorm_tool_use',
-           'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice',
-           'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user', 'denorm_image', 'denorm_file',
-           'denorm_tool_result', 'apply_cache_idxs', 'mk_payload', 'get_hdrs', 'cost']
+           'raw_msg', 'norm_tr_parts', 'norm_sse_event', 'delta_index_fn', 'collate_raw', 'acollect_stream',
+           'denorm_tool_use', 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs',
+           'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user',
+           'denorm_image', 'denorm_file', 'denorm_tool_result', 'apply_cache_idxs', 'mk_payload', 'get_hdrs', 'cost']
 
 # %% ../nbs/04_anthropic.ipynb #02afd3d7
-import json
+import json, copy
 from collections import Counter
 from fastcore.utils import *
 from fastcore.meta import *
 from fasttransport.errors import api_error_from_event
 
 from .types import *
-from aidialog.msg_parts import (Part, PartType, mk_part, Msg, Text, Thinking, ToolUse, ToolResult, tool_text, ServerToolResult,
+from aidialog.msg_parts import (Part, PartType, mk_part, Msg, Text, Thinking, ToolUse, ToolResult, tool_text,
                                 InputImage, InputAudio, InputVideo, InputFile, data_url)
 from .streaming import *
 from .streaming import mk_acollect_stream
@@ -47,11 +47,11 @@ def norm_usage(resp):
                  cached_tokens=cached, cache_creation_tokens=cache_creation, reasoning_tokens=0, raw=usg)
 
 def finalize_usage(usg, parts):
-    "Adjust usage using finalized Anthropic content parts."
+    "Split reasoning tokens out of `output_tokens`: the reported `thinking_tokens`, else an estimate from the thinking text."
     if not usg: return usg
     rc = '\n'.join(p.text or '' for p in parts if isinstance(p, Thinking))
     ct = int(usg.raw.get('output_tokens', usg.completion_tokens) or 0)
-    rt = min(int(len(rc.split())*1.5), ct) if rc else 0
+    rt = int(nested_idx(usg.raw, 'output_tokens_details', 'thinking_tokens') or 0) or (min(int(len(rc.split())*1.5), ct) if rc else 0)
     return Usage(prompt_tokens=usg.prompt_tokens, completion_tokens=ct-rt, total_tokens=usg.prompt_tokens+ct,
         cached_tokens=usg.cached_tokens, cache_creation_tokens=usg.cache_creation_tokens, reasoning_tokens=rt, raw=usg.raw)
 
@@ -67,21 +67,24 @@ def norm_finish(resp, tcs=None):
 def _ant_part_type(typ):
     "Map Anthropic content block type to canonical PartType."
     ant_parts_mapping = dict(text=PartType.text, thinking=PartType.thinking, redacted_thinking=PartType.thinking, tool_use=PartType.tool_use, server_tool_use=PartType.tool_use, mcp_tool_use=PartType.tool_use)
-    if typ in ant_parts_mapping: return ant_parts_mapping[typ]
-    if typ.endswith('_tool_result'): return PartType.server_tool_result
-    return typ
+    return ant_parts_mapping.get(typ, typ)
 
 def norm_parts(resp):
+    "Canonical parts of a response; a server tool's result block lives only in the raw message"
     parts = []
     for b in resp.get("content", []):
         typ = _ant_part_type(b.get("type", "text"))
         if   typ == PartType.thinking: parts.append(Thinking(b.get("thinking", ""), raw=b))
         elif typ == PartType.tool_use:
             if tc := norm_tool_call(b): parts.append(tc)
-        elif typ == PartType.server_tool_result: parts.append(ServerToolResult(b.get("text", ""), raw=b))
-        else: parts.append(Text(b.get("text", ""), raw=b))
+        elif typ == PartType.text: parts.append(Text(b.get("text", ""), raw=b))
     return parts
 
+# %% ../nbs/04_anthropic.ipynb #f24e52db
+def raw_msg(resp):
+    "The assistant message as the API sent it"
+    return dict(role='assistant', content=resp['content'])
+
 # %% ../nbs/04_anthropic.ipynb #5d1440b7
 def norm_tr_parts(content):
     "Normalize an Anthropic `tool_result` content str or block list to canonical `Part`s"
@@ -103,7 +106,6 @@ def norm_sse_event(ev, **kwargs):
     text, thinking, tcs, citations = None, None, [], None
     if typ == "content_block_start":
         cb = ev.get("content_block", {})
-        if cb.get("type", "").endswith("_tool_result"): return Delta(server_tool_result=cb, raw=ev, **kwargs)
         if tc := norm_tool_call(cb):
             if not tc.arguments: tc.arguments = {'_delta': ''}
             tcs = [tc]
@@ -127,6 +129,23 @@ def delta_index_fn(d, typ, last_typ, last_idx):
     'Returns accumulation index for current delta and updated last idx'
     return nested_idx(d, 'raw', 'index'), None
 
+# %% ../nbs/04_anthropic.ipynb #de900e34
+def collate_raw(deltas):
+    "The assistant message rebuilt from a stream's events"
+    blocks = {}
+    for ev in L(deltas).attrgot('raw'):
+        typ,i = ev.get('type'),ev.get('index')
+        if typ == 'content_block_start': blocks[i] = copy.deepcopy(ev['content_block'])
+        elif typ == 'content_block_delta':
+            d,b = ev['delta'],blocks[i]
+            if   d['type'] == 'text_delta':       b['text'] += d['text']
+            elif d['type'] == 'thinking_delta':   b['thinking'] += d['thinking']
+            elif d['type'] == 'signature_delta':  b['signature'] = d['signature']
+            elif d['type'] == 'input_json_delta': b['_json'] = b.get('_json', '') + d['partial_json']
+            elif d['type'] == 'citations_delta':  b['citations'] = (b.get('citations') or []) + [d['citation']]
+        elif typ == 'content_block_stop' and '_json' in blocks[i]: blocks[i]['input'] = json.loads(blocks[i].pop('_json') or '{}')
+    return dict(role='assistant', content=list(blocks.values()))
+
 # %% ../nbs/04_anthropic.ipynb #c994adb1
 @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
 async def acollect_stream(resp, **kwargs):
@@ -145,44 +164,29 @@ def _sanid(id_str): return re.sub(r'[^a-zA-Z0-9_-]', '_', id_str or '')
 # %% ../nbs/04_anthropic.ipynb #6ec772cb
 def denorm_tool_use(p:ToolUse):
     "Convert canonical tool_use Part to Anthropic tool_use content block."
-    block = dict(type='tool_use', id=_sanid(p.id or ''), name=p.name or '', input=p.arguments or {})
-    if caller := (p.raw or {}).get('caller'): block['caller'] = caller
-    return _ant_cc(block, p)
+    return _ant_cc(dict(type='tool_use', id=_sanid(p.id or ''), name=p.name or '', input=p.arguments or {}), p)
 
 def denorm_assistant(m:Msg):
-    "Convert canonical assistant Msg to Anthropic assistant message + synthetic tool results for non-Anthropic server tools."
-    blocks, srv_results, srv_call_id = [], [], None
-    for p in m.content:
-        if isinstance(p, (Thinking, Text)):
-            if srv_call_id:
-                srv_results.append(dict(type='tool_result', tool_use_id=srv_call_id, content=p.text or ''))
-                srv_call_id = None
-            elif isinstance(p, Thinking) and (sig:=(p.raw or {}).get('signature','')):
-                blocks.append(dict(type='thinking', thinking=p.text or '', signature=sig))
-            else: blocks.append(_ant_cc(dict(type='text', text=p.text or ''), p))
-        elif isinstance(p, ToolUse):
-            if p.server and (p.id or '').startswith('srvtoolu_'):
-                blocks.append(dict(type='server_tool_use', id=p.id, name=p.name or '', input=p.arguments or {}))
-            else:
-                blocks.append(denorm_tool_use(p))
-                if p.server: srv_call_id = p.id or ''
-        elif isinstance(p, ServerToolResult): blocks.append(p.raw or dict(type='server_tool_result'))
-    res = [dict(role='assistant', content=blocks)]
-    if srv_results: res.append(dict(role='user', content=srv_results))
-    return res
+    "The message as the API sent it, else its text and tool calls; a cache mark on the last part is copied onto the last raw block"
+    if m.raw:
+        cc = [p.cache_control for p in m.content if p.cache_control]
+        if not cc: return m.raw
+        return m.raw | dict(content=m.raw['content'][:-1] + [m.raw['content'][-1] | dict(cache_control=cc[-1])])
+    blocks = [_ant_cc(dict(type='text', text=p.text or ''), p) if isinstance(p, Text) else denorm_tool_use(p) for p in m.content if isinstance(p, (Text, ToolUse))]
+    return dict(role='assistant', content=blocks)
 
 def denorm_tool(m:Msg):
     "Convert canonical tool Msg to Anthropic user message with tool_result blocks."
     blocks = [denorm_tool_result(p) for p in m.content if isinstance(p, ToolResult)]
-    return [dict(role='user', content=blocks)]
+    return dict(role='user', content=blocks)
 
 def denorm_msgs(msgs:list[Msg]):
     "Convert list of canonical Msgs to Anthropic messages."
     res = []
     for m in msgs:
         if   m.role == 'user':      res.append(denorm_user(m))
-        elif m.role == 'assistant': res.extend(denorm_assistant(m))
-        elif m.role == 'tool':      res.extend(denorm_tool(m))
+        elif m.role == 'assistant': res.append(denorm_assistant(m))
+        elif m.role == 'tool':      res.append(denorm_tool(m))
     return res
 
 # %% ../nbs/04_anthropic.ipynb #f762a6f2
@@ -322,8 +326,12 @@ def mk_payload(msgs, model, **kwargs):
     return payload
 
 # %% ../nbs/04_anthropic.ipynb #60d52c1f
-def get_hdrs(api_key=None):
-    return {"x-api-key": get_api_key(api_key, 'ANTHROPIC_API_KEY'), "anthropic-version": "2023-06-01"}
+def get_hdrs(api_key=None, oauth_token=None):
+    "An OAuth token travels as a bearer under Anthropic's OAuth beta; an API key as `x-api-key`"
+    hdrs = {"anthropic-version": "2023-06-01"}
+    if oauth_token: return hdrs | {"Authorization": f"Bearer {oauth_token}", "anthropic-beta": "oauth-2025-04-20"}
+    return hdrs | {"x-api-key": get_api_key(api_key, 'ANTHROPIC_API_KEY')}
+
 
 # %% ../nbs/04_anthropic.ipynb #0d03643a
 def cost(usage, m):
@@ -342,5 +350,6 @@ def cost(usage, m):
 
 # %% ../nbs/04_anthropic.ipynb #f7c0b989
 api_registry.register('anthropic', norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
-    finalize_usage=finalize_usage, acollect_stream=acollect_stream, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs,
+    finalize_usage=finalize_usage, raw_msg=raw_msg, collate_raw=collate_raw, acollect_stream=acollect_stream, mk_payload=mk_payload,
+    cost=cost, get_hdrs=get_hdrs,
     endpoint='/v1/messages')

fastllm/chat.py

@@ -168,6 +168,7 @@ class AsyncChat:
         api_name=None,            # API to use, one of ApiName: openai (responses), openai_chat, anthropic, gemini
         vendor_name=None,         # Vendor name, one of vendor_mapping which resolves api_base/api_key automatically
         api_key=None,             # API key when model can't be resolved or vendor_name is not known or codex
+        oauth_token=None,         # A subscription's OAuth token (Codex, Claude Code) in place of an API key
         base_url=None,            # API base url when model can't be resolved or vendor_name is not known
         endpoint=None,            # Override the transport's request path, for a server mounting Responses at a custom location
         extra_headers=None,       # Extra HTTP headers for custom providers
@@ -178,7 +179,7 @@ class AsyncChat:
         default_cbs=True          # Whether to include default callbacks
     ):
         "LiteLLM chat client."
-        if not vendor_name and not api_name and not (base_url and api_key):
+        if not vendor_name and not api_name and not (base_url and (api_key or oauth_token)):
             v, model = split_vendor(model)
             if v in vendor_mapping: vendor_name = v
             elif v: api_name = v  # a registered transport api (e.g. claude_code)
@@ -248,6 +249,7 @@ def _prep_call(self:AsyncChat, search, max_tokens, kwargs, stream=False, think=N
     if self.api_name:      kwargs['api_name'] = self.api_name
     if self.vendor_name:   kwargs['vendor_name'] = self.vendor_name
     if self.api_key:       kwargs['api_key'] = self.api_key
+    if self.oauth_token:   kwargs['oauth_token'] = self.oauth_token
     if self.base_url:      kwargs['base_url'] = self.base_url
     if self.endpoint:      kwargs['endpoint'] = self.endpoint
     if self.extra_headers: kwargs['xtra_hdrs'] = self.extra_headers

fastllm/gemini.py

@@ -1,11 +1,11 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/05_gemini.ipynb.
 
 # %% auto #0
-__all__ = ['api_ns', 'endpoint', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'norm_sse_event', 'delta_index_fn',
-           'acollect_stream', 'denorm_tool_use', 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs',
-           'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user',
-           'denorm_image', 'denorm_audio', 'denorm_video', 'denorm_file', 'denorm_tool_result', 'mk_payload',
-           'get_hdrs', 'cost']
+__all__ = ['api_ns', 'endpoint', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'raw_msg', 'norm_sse_event',
+           'delta_index_fn', 'collate_raw', 'acollect_stream', 'denorm_tool_use', 'denorm_assistant', 'denorm_tool',
+           'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search',
+           'denorm_system', 'denorm_user', 'denorm_image', 'denorm_audio', 'denorm_video', 'denorm_file',
+           'denorm_tool_result', 'mk_payload', 'get_hdrs', 'cost']
 
 # %% ../nbs/05_gemini.ipynb #02afd3d7
 import json
@@ -27,12 +27,14 @@ def endpoint(model, stream):
 
 # %% ../nbs/05_gemini.ipynb #f1eb32f3
 def norm_tool_calls(resp):
-    "Extract Gemini functionCall parts as normalized tool calls."
+    "Tool calls of a response: `functionCall` parts, and a `toolCall` part for a search the API ran itself"
     out = []
     for i,p in enumerate(nested_idx(resp, 'candidates', 0, 'content', 'parts') or []):
-        if not (fc:=p.get("functionCall")): continue
-        raw = {k:v for k,v in p.items() if k != "functionCall"}
-        out.append(ToolUse(id=fc.get("id", f"call_{i}"), name=fc.get("name", ""), arguments=fc.get("args") or {}, raw=raw))
+        if fc := p.get("functionCall"):
+            raw = {k:v for k,v in p.items() if k != "functionCall"}
+            out.append(ToolUse(id=fc.get("id", f"call_{i}"), name=fc.get("name", ""), arguments=fc.get("args") or {}, raw=raw))
+        elif tc := p.get("toolCall"):
+            out.append(ToolUse(id=tc.get("id", f"call_{i}"), name=tc.get("toolType", "").lower(), arguments=tc.get("args") or {}, server=True))
     return out
 
 # %% ../nbs/05_gemini.ipynb #90504163
@@ -48,7 +50,7 @@ def norm_usage(resp):
     cand = nested_idx(resp, 'candidates', 0) or {}
     stu = {}
     if any("executableCode" in p for p in parts): stu["code_execution"] = sum(1 for p in parts if "executableCode" in p)
-    if "groundingMetadata" in cand: stu["google_search"] = 1
+    if n := ("groundingMetadata" in cand) + sum(1 for p in parts if 'toolCall' in p): stu["google_search"] = n
     if stu: usg['server_tool_use'] = stu
     return Usage(prompt_tokens=pt, completion_tokens=ct, total_tokens=tt,
                  cached_tokens=cached, reasoning_tokens=reasoning, raw=usg)
@@ -76,6 +78,7 @@ def norm_parts(resp):
     tc_map = {tc.id: tc for tc in norm_tool_calls(resp)}
     parts = []
     for p in nested_idx(c0, 'content', 'parts') or []:
+        if 'toolResponse' in p: continue
         typ = _gem_part_type(p)
         if typ == PartType.tool_use:
             fc = p.get('functionCall') or p.get('toolCall') or {}
@@ -88,6 +91,11 @@ def norm_parts(resp):
             if isinstance(p, Text): p.citations = citations
     return parts
 
+# %% ../nbs/05_gemini.ipynb #3acc9538
+def raw_msg(resp):
+    "The model content as the API sent it"
+    return nested_idx(resp, 'candidates', 0, 'content')
+
 # %% ../nbs/05_gemini.ipynb #9a5024ee
 def norm_sse_event(ev, **kwargs):
     "Normalize Gemini stream event into Delta."
@@ -107,6 +115,19 @@ def delta_index_fn(d, typ, last_typ, last_idx):
     if not (last_typ or last_idx): return 0,0
     return last_idx + 1, last_idx + 1
 
+# %% ../nbs/05_gemini.ipynb #33506d56
+def collate_raw(deltas):
+    "The model content rebuilt from a stream's chunks"
+    parts = []
+    for ev in L(deltas).attrgot('raw'):
+        for p in nested_idx(ev, 'candidates', 0, 'content', 'parts') or []:
+            last = parts[-1] if parts else None
+            if 'text' in p and last and 'text' in last and last.get('thought') == p.get('thought') and 'thoughtSignature' not in last:
+                last['text'] += p['text']
+                if 'thoughtSignature' in p: last['thoughtSignature'] = p['thoughtSignature']
+            else: parts.append(dict(p))
+    return dict(role='model', parts=parts)
+
 # %% ../nbs/05_gemini.ipynb #328af4d5
 @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
 async def acollect_stream(resp, **kwargs):
@@ -118,17 +139,12 @@ def denorm_tool_use(p:ToolUse):
     "Convert canonical tool_use Part to Gemini functionCall part."
     fc = dict(name=p.name or '', args=p.arguments or {})
     if p.id: fc['id'] = p.id
-    part = dict(functionCall=fc)
-    part['thoughtSignature'] = (p.raw or {}).get('thoughtSignature', 'skip_thought_signature_validator')
-    return part
+    return dict(functionCall=fc)
 
 def denorm_assistant(m:Msg):
-    "Convert canonical assistant Msg to Gemini model content."
-    parts = []
-    for p in m.content:
-        if   isinstance(p, Thinking): parts.append(dict(text=p.text or '', thought=True))
-        elif isinstance(p, ToolUse):  parts.append(denorm_tool_use(p))
-        elif isinstance(p, Text):     parts.append(dict(text=p.text or ''))
+    "The content as the API sent it, else its text and tool calls"
+    if m.raw: return m.raw
+    parts = [dict(text=p.text or '') if isinstance(p, Text) else denorm_tool_use(p) for p in m.content if isinstance(p, (Text, ToolUse))]
     return dict(role='model', parts=parts)
 
 def denorm_tool(m:Msg):
@@ -267,9 +283,12 @@ def mk_payload(msgs, model, **kwargs):
     return payload
 
 # %% ../nbs/05_gemini.ipynb #60d52c1f
-def get_hdrs(api_key=None):
+def get_hdrs(api_key=None, oauth_token=None):
+    "Bearer auth for an OAuth token; otherwise the API key header"
+    if oauth_token: return {"Authorization": f"Bearer {oauth_token}"}
     return {"x-goog-api-key": get_api_key(api_key, 'GEMINI_API_KEY')}
 
+
 # %% ../nbs/05_gemini.ipynb #4ee3891f
 def cost(usage, m):
     raw = usage.raw
@@ -298,6 +317,8 @@ api_ns = dict(norm_tool_calls=norm_tool_calls,
                 norm_finish=norm_finish,
                 norm_usage=norm_usage,
                 acollect_stream=acollect_stream,
+                raw_msg=raw_msg,
+                collate_raw=collate_raw,
                 mk_payload=mk_payload,
                 cost=cost,
                 get_hdrs=get_hdrs,

fastllm/openai_chat.py

@@ -1,8 +1,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03_oai_chat.ipynb.
 
 # %% auto #0
-__all__ = ['api_ns', 'norm_tool_calls', 'norm_finish', 'norm_parts', 'norm_sse_event', 'delta_index_fn', 'acollect_stream',
-           'denorm_tool_use', 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs',
+__all__ = ['api_ns', 'norm_tool_calls', 'norm_finish', 'norm_parts', 'raw_msg', 'norm_sse_event', 'delta_index_fn', 'collate_raw',
+           'acollect_stream', 'denorm_tool_use', 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs',
            'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user',
            'denorm_image', 'denorm_audio', 'denorm_file', 'denorm_tool_result', 'mk_payload', 'get_hdrs', 'cost',
            'fix_payload']
@@ -50,6 +50,11 @@ def norm_parts(resp):
     if ref := msg.get('refusal'): parts.append(Refusal(ref))
     return parts + norm_tool_calls(resp)
 
+# %% ../nbs/03_oai_chat.ipynb #a30e4324
+def raw_msg(resp):
+    "The assistant message as the API sent it"
+    return nested_idx(resp, 'choices', 0, 'message')
+
 # %% ../nbs/03_oai_chat.ipynb #9baac40e
 def norm_sse_event(ev, **kwargs):
     "Normalize a chat completion stream event."
@@ -73,6 +78,23 @@ def delta_index_fn(d, typ, last_typ, last_idx):
     if typ == last_typ: return last_idx, last_idx
     return last_idx + 1, last_idx + 1
 
+# %% ../nbs/03_oai_chat.ipynb #b7a19b8d
+def collate_raw(deltas):
+    "The assistant message rebuilt from a stream's chunks"
+    msg, tcs = dict(role='assistant', content=None), {}
+    for ev in L(deltas).attrgot('raw'):
+        d = nested_idx(ev, 'choices', 0, 'delta') or {}
+        for k in ('content', 'reasoning_content', 'refusal'):
+            if d.get(k): msg[k] = (msg.get(k) or '') + d[k]
+        for tc in d.get('tool_calls') or []:
+            t = tcs.setdefault(tc['index'], dict(id=tc.get('id'), type='function', function=dict(name='', arguments='')))
+            if tc.get('id'): t['id'] = tc['id']
+            fn = tc.get('function') or {}
+            t['function']['name'] += fn.get('name') or ''
+            t['function']['arguments'] += fn.get('arguments') or ''
+    if tcs: msg['tool_calls'] = [tcs[i] for i in sorted(tcs)]
+    return msg
+
 # %% ../nbs/03_oai_chat.ipynb #e7f8965b
 @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
 async def acollect_stream(resp, **kwargs):
@@ -85,21 +107,12 @@ def denorm_tool_use(p:ToolUse):
     return dict(id=p.id, type='function', function=dict(name=p.name, arguments=json.dumps(p.arguments)))
 
 def denorm_assistant(m:Msg):
-    "Convert canonical assistant Msg to OpenAI Chat assistant message + synthetic tool responses for server tools."
-    tcs, srv_responses, texts = [], [], []
-    for p in m.content:
-        if isinstance(p, ToolUse):
-            tcs.append(denorm_tool_use(p))
-            if p.server:
-                srv_txt = f"[Server tool `{p.name}` executed successfully, results are generated]"
-                srv_responses.append(dict(role='tool', tool_call_id=p.id, content=srv_txt))
-        elif isinstance(p, Text): texts.append(p)
-    msg = dict(role='assistant')
+    "The message as the API sent it, else its text and tool calls"
+    if m.raw: return m.raw
+    msg, texts = dict(role='assistant'), [p for p in m.content if isinstance(p, Text)]
     if texts: msg['content'] = texts[0].text if len(texts)==1 else [dict(type='text', text=p.text or '') for p in texts]
-    if tcs: msg['tool_calls'] = tcs
-    thinking = [p for p in m.content if isinstance(p, Thinking)]
-    if thinking: msg['reasoning_content'] = ''.join(p.text or '' for p in thinking)
-    return [msg] + srv_responses
+    if tcs := [denorm_tool_use(p) for p in m.content if isinstance(p, ToolUse)]: msg['tool_calls'] = tcs
+    return msg
 
 def denorm_tool(m:Msg):
     "Convert canonical tool Msg to list of OpenAI Chat tool messages."
@@ -110,7 +123,7 @@ def denorm_msgs(msgs:list[Msg]):
     res = []
     for m in msgs:
         if   m.role == 'user':      res.append(denorm_user(m))
-        elif m.role == 'assistant': res.extend(denorm_assistant(m))
+        elif m.role == 'assistant': res.append(denorm_assistant(m))
         elif m.role == 'tool':      res.extend(denorm_tool(m))
     return res
 
@@ -198,7 +211,10 @@ def mk_payload(msgs, model, **kwargs):
     return payload
 
 # %% ../nbs/03_oai_chat.ipynb #16e813d2
-def get_hdrs(api_key=None): return {"Authorization": f"Bearer {get_api_key(api_key, 'OPENAI_API_KEY')}"}
+def get_hdrs(api_key=None, oauth_token=None):
+    "Bearer auth: an OAuth access token, or else an API key from the argument or the environment"
+    return {"Authorization": f"Bearer {oauth_token or get_api_key(api_key, 'OPENAI_API_KEY')}"}
+
 
 # %% ../nbs/03_oai_chat.ipynb #f89e2bf6
 def cost(usage, m):
@@ -223,6 +239,6 @@ def fix_payload(payload, model, vendor_name):
 
 # %% ../nbs/03_oai_chat.ipynb #e2b0908e
 api_ns = dict(norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
-    acollect_stream=acollect_stream, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs, fix_payload=fix_payload,
+    raw_msg=raw_msg, collate_raw=collate_raw, acollect_stream=acollect_stream, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs, fix_payload=fix_payload,
     endpoint='/chat/completions')
 api_registry.register('openai_chat', **api_ns)

fastllm/openai_responses.py

@@ -1,11 +1,11 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_oai_responses.ipynb.
 
 # %% auto #0
-__all__ = ['api_ns', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'norm_sse_event',
-           'delta_index_fn', 'acollect_stream', 'denorm_tool_use', 'denorm_tool', 'denorm_assistant', 'denorm_msgs',
-           'denorm_tool_schs', 'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search', 'denorm_system',
-           'denorm_user', 'denorm_image', 'denorm_video', 'denorm_file', 'denorm_tool_result', 'mk_payload', 'get_hdrs',
-           'cost', 'fix_payload']
+__all__ = ['api_ns', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'norm_finish', 'norm_parts', 'raw_msg', 'norm_sse_event',
+           'delta_index_fn', 'collate_raw', 'acollect_stream', 'denorm_tool_use', 'denorm_tool', 'denorm_assistant',
+           'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search',
+           'denorm_system', 'denorm_user', 'denorm_image', 'denorm_video', 'denorm_file', 'denorm_tool_result',
+           'mk_payload', 'get_hdrs', 'cost', 'fix_payload']
 
 # %% ../nbs/02_oai_responses.ipynb #591f55b5
 import json
@@ -78,6 +78,11 @@ def norm_parts(resp):
         elif typ == "function_call" or typ.endswith("_call"): parts.append(norm_tool_call(item))
     return parts
 
+# %% ../nbs/02_oai_responses.ipynb #2aaab962
+def raw_msg(resp):
+    "The output items as the API sent them"
+    return resp['output']
+
 # %% ../nbs/02_oai_responses.ipynb #ef2d19f3
 @patch(as_prop=True)
 def response_id(self:Completion):
@@ -89,11 +94,6 @@ def response_id(self:Completion):
     for d in reversed(self.raw.get('deltas', [])):
         if rid := nested_idx(d.raw, 'response', 'id'): return rid
 
-@patch(as_prop=True)
-def response_id_reusable(self:Completion):
-    "Can the provider continuation identifier be consumed more than once?"
-    return self.raw.get('response_id_reusable', True)
-
 # %% ../nbs/02_oai_responses.ipynb #7cd48aa5
 def norm_sse_event(ev, **kwargs):
     "Normalize OpenAI Responses API stream event into Delta."
@@ -107,6 +107,7 @@ def norm_sse_event(ev, **kwargs):
         item = ev.get("item", {})
         itype = item.get("type", "")
         if itype == "function_call" or itype.endswith("_call"): return Delta(tool_calls=[norm_tool_call(item)], raw=ev, **kwargs)
+        return Delta(raw=ev, **kwargs)
     if typ in ("response.completed", "response.incomplete"):
         resp = ev.get("response", {})
         return Delta(finish_reason=norm_finish(resp), usage=norm_usage(resp), raw=ev, **kwargs)
@@ -118,6 +119,11 @@ def delta_index_fn(d, typ, last_typ, last_idx):
     idx = nested_idx(d, 'raw', 'output_index')
     return idx, idx
 
+# %% ../nbs/02_oai_responses.ipynb #689e31b0
+def collate_raw(deltas):
+    "The output items rebuilt from a stream's events"
+    return [d.raw['item'] for d in deltas if d.raw.get('type') == 'response.output_item.done']
+
 # %% ../nbs/02_oai_responses.ipynb #021b5257
 @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
 async def acollect_stream(resp, **kwargs):
@@ -137,20 +143,12 @@ def denorm_tool(m:Msg): return [denorm_tool_result(p) for p in m.content if isin
 
 # %% ../nbs/02_oai_responses.ipynb #67c8de13
 def denorm_assistant(m:Msg):
-    items, srv_call_id = [], None
+    "The output items as the API sent them, else the message's text and tool calls as items"
+    if m.raw: return m.raw
+    items = []
     for p in m.content:
-        if isinstance(p, ToolUse):
-            items.append(denorm_tool_use(p))
-            # TODO: Server tcs are excluded during deserialization in lisette, so this
-            # is not used. Anthropic checks for `srvtoolu` and requires the full tool result metadata
-            # which is a lot to store.
-            # Instead of reconstructing server tool calls as actual tool uses we can make it a regular msg
-            # telling AI a server tool was executed
-            if p.server:
-                srv_txt = f"[Server tool `{p.name}` executed successfully, results are generated below]"
-                items.append(dict(type='function_call_output', call_id=p.id, output=srv_txt))    
-        elif isinstance(p, (Text, Thinking)):
-            items.append(dict(type="message", role="assistant", content=[dict(type="output_text", text=p.text)]))
+        if isinstance(p, ToolUse): items.append(denorm_tool_use(p))
+        elif isinstance(p, Text): items.append(dict(type="message", role="assistant", content=[dict(type="output_text", text=p.text or '')]))
     return items
 
 # %% ../nbs/02_oai_responses.ipynb #64883b99
@@ -256,7 +254,10 @@ def mk_payload(msgs, model, **kwargs):
     return payload
 
 # %% ../nbs/02_oai_responses.ipynb #529fd4bc
-def get_hdrs(api_key=None): return {"Authorization": f"Bearer {get_api_key(api_key, 'OPENAI_API_KEY')}"}
+def get_hdrs(api_key=None, oauth_token=None):
+    "Bearer auth: an OAuth access token, or else an API key from the argument or the environment"
+    return {"Authorization": f"Bearer {oauth_token or get_api_key(api_key, 'OPENAI_API_KEY')}"}
+
 
 # %% ../nbs/02_oai_responses.ipynb #a907ffa8
 def cost(usage, m):
@@ -277,6 +278,7 @@ def fix_payload(payload, model, vendor_name):
 
 # %% ../nbs/02_oai_responses.ipynb #07114b55
 api_ns = dict(norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
-    supports_previous_response_id=True, acollect_stream=acollect_stream, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs, fix_payload=fix_payload,
+    supports_previous_response_id=True, raw_msg=raw_msg, collate_raw=collate_raw, acollect_stream=acollect_stream, mk_payload=mk_payload,
+    cost=cost, get_hdrs=get_hdrs, fix_payload=fix_payload,
     endpoint='/responses')
 api_registry.register('openai', **api_ns)

fastllm/responses.py

@@ -143,7 +143,7 @@ def response_object(rid, body, comp, previous_response_id=None, created_at=None)
 # %% ../nbs/06a_responses.ipynb #73a2fc16
 class ResponseState(BasicRepr):
     "Canonical history and optional provider continuation metadata."
-    def __init__(self, id, model, history, provider_response_id=None, provider_response_reusable=True): store_attr()
+    def __init__(self, id, model, history, provider_response_id=None): store_attr()
 
 # %% ../nbs/06a_responses.ipynb #c9ca3744
 class ResponseTurn(BasicRepr):
@@ -203,7 +203,7 @@ def state(self:AsyncResponses, turn, comp, results=()):
     "Build the next continuation state; `results` are server tool results answering `comp`'s calls."
     normalize_call_ids(comp)
     history = (*turn.history, comp.message, *([Msg('tool', list(results))] if results else []))
-    return ResponseState(turn.id, turn.body['model'], history, comp.response_id, comp.response_id_reusable)
+    return ResponseState(turn.id, turn.body['model'], history, comp.response_id)
 
 
 # %% ../nbs/06a_responses.ipynb #7615b274

fastllm/streaming.py

@@ -12,13 +12,12 @@ from fastcore.utils import *
 from fastcore.meta import delegates
 from fasttransport.errors import *
 from .types import *
-from aidialog.msg_parts import Part, PartType, Msg, Text, Thinking, ToolUse, ToolResult, ServerToolResult, mk_part, fence_call_re, Completion
+from aidialog.msg_parts import Part, PartType, Msg, Text, Thinking, ToolUse, ToolResult, mk_part, fence_call_re, Completion
 
 # %% ../nbs/01_streaming.ipynb #400d628a
 class Delta(BasicRepr):
     "Normalized streaming delta event."
-    def __init__(self, text='', thinking='', refusal='', tool_calls=None, citations=None,
-        server_tool_result=None, finish_reason=None, usage=None, raw=None):
+    def __init__(self, text='', thinking='', refusal='', tool_calls=None, citations=None, finish_reason=None, usage=None, raw=None):
         if tool_calls is None: tool_calls = []
         if citations is None: citations = []
         if raw is None: raw = {}
@@ -172,11 +171,7 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
                     except json.JSONDecodeError: continue
                 acc.arguments = args
                 yield acc
-                # Server tool results for anthropic are yielded in d.server_tool_result by checking injected dummy `_delta`
                 if acc.server: yield ToolResult(id=acc.id, name=acc.name, arguments=args, server=True, text="Server tool call executed.")
-        if d.server_tool_result:
-            idx = _fidx(d, 'server_tool_result')
-            part_accum.parts[idx] = ServerToolResult(raw=d.server_tool_result)
         r = _proc(d, 'refusal')
         if r[0]: yield _mk_delta_part('refusal', r[0]['refusal'])
         if d.finish_reason: fin = d.finish_reason
@@ -190,6 +185,6 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
     fin = FinishReason.tool_calls if fin==FinishReason.stop and any(~L(tcs).attrgot('server')) else fin # recheck tool calls post collation
     # tool calls and non-anthropic citations are yielded at the end
     yield Completion(model,
-            message=Msg(role="assistant", content=parts),
+            message=Msg(role="assistant", content=parts, raw=api_registry[api_name].collate_raw(deltas) if api_name else None),
             finish_reason=fin, usage=usg, api_name=api_name, vendor_name=vendor_name,
             raw={'deltas':deltas})

fastllm/types.py

@@ -68,7 +68,8 @@ FinishReason = str_enum('finish_reason', 'stop', 'tool_calls', 'length', 'conten
 # %% ../nbs/00_types.ipynb #fc681c52
 class APIRegistry(dict):
     _eps = None
-    def register(self, name, finalize_usage=noop, **kwargs): self[name] = SimpleNamespace(finalize_usage=finalize_usage, **kwargs)
+    def register(self, name, finalize_usage=noop, raw_msg=noop, collate_raw=noop, **kwargs):
+        self[name] = SimpleNamespace(finalize_usage=finalize_usage, raw_msg=raw_msg, collate_raw=collate_raw, **kwargs)
     def _load(self, name):
         "Import the plugin providing api `name` from the `fastllm.apis` entry-point group; its import registers the api."
         if self._eps is None: self._eps = {o.name:o for o in entry_points(group='fastllm.apis')}
@@ -90,7 +91,7 @@ def mk_completion(resp, model, api_name, vendor_name):
     tcs = api.norm_tool_calls(resp)
     parts = api.norm_parts(resp)
     usg = api.finalize_usage(api.norm_usage(resp), parts)
-    return Completion(model=model, message=Msg(role="assistant", content=parts), finish_reason=api.norm_finish(resp, tcs),
+    return Completion(model=model, message=Msg(role="assistant", content=parts, raw=api.raw_msg(resp)), finish_reason=api.norm_finish(resp, tcs),
         usage=usg, api_name=api_name, vendor_name=vendor_name, raw=resp)
 
 # %% ../nbs/00_types.ipynb #8a8e468b
@@ -300,14 +301,14 @@ register_model_info('muse-spark-1.1', vendor_name='meta_ai', **modern_llm, max_i
 codex_pricing = dict(input_cost_per_token = 0.10/1_000_000, output_cost_per_token = 0.50/1_000_000,
     cache_creation_input_token_cost = 0.10/1_000_000, cache_read_input_token_cost = 0.10/1_000_000)
 
-def _rm_ctx_tiers(vendor_name, model):
-    "Drop upstream context-tier rates, which would otherwise override flat subscription pricing"
+def _flat_rates(vendor_name, model):
+    "Keep only the flat subscription rates: drop the upstream context, cache-TTL and service-tier variants that would otherwise apply"
     info = model_info_registry[vendor_name, model]
-    for k in [k for k in info if re.search(r'_above_\d+k_tokens', k)]: del info[k]
+    for k in [k for k in info if 'cost' in k and k not in codex_pricing and k != 'search_context_cost_per_query']: del info[k]
 
 for model in (codex54, codex54m, codex55):
     register_model_info(model, 'codex', base=model, base_vendor_name='chatgpt', supports_web_search=True, max_input_tokens=256000, **codex_pricing)
-    _rm_ctx_tiers('codex', model)
+    _flat_rates('codex', model)
 
 register_model_info(codex53spark, 'codex', **codex_pricing,
     supports_vision=False, supports_image_input=False, supports_web_search=True, supports_reasoning=True, supports_function_calling=True,
@@ -318,6 +319,7 @@ register_model_info(codex53spark, 'codex', **codex_pricing,
 register_model_info(fable51, 'anthropic', base=fable5)
 for model in (haik45, sonn45, sonn46, sonn5, opus46, opus48, opus5, fable5, fable51):
     register_model_info(model, 'claude_code', base=model, base_vendor_name='anthropic', **codex_pricing)
+    _flat_rates('claude_code', model)
 
 # %% ../nbs/00_types.ipynb #bb0c4c2a
 sol,terra,luna = gpt56s = 'gpt-5.6-sol gpt-5.6-terra gpt-5.6-luna'.split()
@@ -325,7 +327,7 @@ sol,terra,luna = gpt56s = 'gpt-5.6-sol gpt-5.6-terra gpt-5.6-luna'.split()
 # Its window is smaller than the API's: 371,331 input tokens is accepted and 371,981 is not, on all three.
 for model in gpt56s:
     register_model_info(model, 'codex', base=model, base_vendor_name='openai', max_input_tokens=371_000, **codex_pricing)
-    _rm_ctx_tiers('codex', model)
+    _flat_rates('codex', model)
 
 # %% ../nbs/00_types.ipynb #24cc47ec
 def get_model_pricing(mn, vendor_name, million=True):

{python_fastllm-0.0.48.dist-info → python_fastllm-0.0.50.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-fastllm
-Version: 0.0.48
+Version: 0.0.50
 Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
 License: Apache-2.0
 Project-URL: Repository, https://github.com/AnswerDotAI/fastllm
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: fastcore>=2.1.18
-Requires-Dist: aidialog>=0.0.10
+Requires-Dist: aidialog>=0.0.24
 Requires-Dist: fasttransport>=0.0.2
 Requires-Dist: pillow
 
@@ -21,13 +21,14 @@ Requires-Dist: pillow
 
 ## Install
 
-Clone and install locally into your `aai-ws` env
+Run `pip install python-fastllm` or clone from github and install locally.
 
 ## Setup
 
 ``` python
 from aidialog.msg_parts import Msg, Part, Text, Thinking, ToolUse, InputImage, mk_tool_res_msg, Completion
 from fastllm.acomplete import acomplete
+from fastllm.types import sample_img_url
 import asyncio, json
 
 # Helpers
@@ -219,6 +220,7 @@ print("\nNo tools: ", end='')
 r = await stream([user("What's the weather?")], model='claude-sonnet-4-20250514', tools=tools, tool_choice='none', max_tokens=mtok)
 ```
 
+
     - ⏳ `get_weather(city="<UNKNOWN>")` ⏳
 
     Forced: [ToolUse(raw={'caller': {'type': 'direct'}}, cache_control=None, id='toolu_01U7tAXjXAtwjp6xNSMbsyPU', name='get_weather', arguments={'city': '<UNKNOWN>'}, server=False, text=None)]
@@ -381,9 +383,8 @@ print(f"Usage: {r2.usage}")
 Send images to any provider that supports them. The canonical `input_image` part works everywhere:
 
 ``` python
-img_url = "https://img.freepik.com/free-photo/mountain-range-body-water_53876-139760.jpg?semt=ais_hybrid&w=740&q=80"
 img_msg = Msg(role='user', content=[
-    InputImage(img_url),
+    InputImage(sample_img_url),
     Text("What do you see in this image?")
 ])
 

python_fastllm-0.0.50.dist-info/RECORD

@@ -0,0 +1,17 @@
+fastllm/__init__.py,sha256=49eWk0NL1c-ZrseVvC8ayVNYEyk7v6rCK0Q5Z4EQB64,131
+fastllm/_modidx.py,sha256=o9foY201ZEAz0AR-9VkRUdpALNA8Jv6WYDs797YRyyw,43700
+fastllm/acomplete.py,sha256=aP1ff9a7WKNhBd1kwKE2TJ-YyihLcdhd9-1Izi4wQP4,10772
+fastllm/anthropic.py,sha256=0mmTjn9ZoEP6bBJki3YaCE1Ck97Q5qT06BZNGH1OMtg,17706
+fastllm/chat.py,sha256=klji3SOYs5xyzrSFbNhy3uoQ2TSZ3a-8HTjvdRtaG5Q,25036
+fastllm/codex.py,sha256=r6auZ8egia83aGkWcF4132AzW4SsBvKeDwg_CxyXj9E,5808
+fastllm/gemini.py,sha256=zbxNmI1cMUJJ10xIfBqPqh38hAQ0exsFKkhTXvG2-eE,16281
+fastllm/openai_chat.py,sha256=n5INfDCsAZIiKUImWeaX6SJ_OAJ_uMhw20HUkgFgdgM,12379
+fastllm/openai_responses.py,sha256=zsaOjkQUZ-7iuGPc-UpvnM6oAyqW09BDYwZ_ma4-1Qo,14346
+fastllm/responses.py,sha256=kRKXISv2c36wIM8EOF68p7c41VZ_Rpn9LdDmByPRf-Q,18707
+fastllm/streaming.py,sha256=i_FU_Mt-3F2lBEImYav8I5pEjUSKYk3JFIW3lkiXEb8,8813
+fastllm/types.py,sha256=MR8ukG3cuBXTbExBKHsZCa1GzCzzJ1Fgu-L9yxOnjtk,19365
+python_fastllm-0.0.50.dist-info/METADATA,sha256=BTPOAD6e-hgHew_i7ITJMwqfY6C0t-H58y1cmEQKAjo,21101
+python_fastllm-0.0.50.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+python_fastllm-0.0.50.dist-info/entry_points.txt,sha256=WkWd04F3xVYFP5bsexaQe9Se6uelwE5ULoTtHWreO0s,133
+python_fastllm-0.0.50.dist-info/top_level.txt,sha256=F8qodL7nEGUHGmzzqfhNKCTIr1i0D6cvudOnm3z7o0Y,8
+python_fastllm-0.0.50.dist-info/RECORD,,

python_fastllm-0.0.48.dist-info/RECORD

@@ -1,17 +0,0 @@
-fastllm/__init__.py,sha256=ioW91VJRYfSiEMjolhjzoNX1vPYfkbDKdrHXp79B9cQ,131
-fastllm/_modidx.py,sha256=jF1SdJd1qbfI_ksSmsrjKUgRHGrANOpxyqTn_hcZ2LY,42836
-fastllm/acomplete.py,sha256=Fb1GWkU6F7-bllbEEp7TPOzqmoWp309i5sJqYJbU6PM,10478
-fastllm/anthropic.py,sha256=udojlLuritp90JQlH-dGHcWRpzgnO1mZYFYtbt2F5IA,17015
-fastllm/chat.py,sha256=W8PPpZ95Az7wzLgQ15u3HMroWBVXDoaUz871f0x7cEk,24842
-fastllm/codex.py,sha256=r6auZ8egia83aGkWcF4132AzW4SsBvKeDwg_CxyXj9E,5808
-fastllm/gemini.py,sha256=DveWJNd78Pf2omC_t46iOzT84a-Lo3kWxchCKR3WsCA,15135
-fastllm/openai_chat.py,sha256=f11U-iaxGbBq0-Rjz17GhA1Vame5n9kMqsk1gkfDtIQ,11633
-fastllm/openai_responses.py,sha256=MUIBZY447tJ4Lc6saBPrVolXe8cdQdMK58YOtaJUHYk,14482
-fastllm/responses.py,sha256=jcbKUwbEIsy7ngNcHUtv2bfYeDgqSysK_qWM-0LC9Mg,18767
-fastllm/streaming.py,sha256=8qVtWeZ7NlP-9rup_4Ho0ol2dx3-XuGcIMXpZ1GYdEo,9079
-fastllm/types.py,sha256=3qXyiS431AlSSn8GyGCNDflZxMWPeIWirp8TIyhIu_A,19146
-python_fastllm-0.0.48.dist-info/METADATA,sha256=meHtTkX0PRsmljM1a72zXLGwlFHm1IKkjroJKnAmc1g,21143
-python_fastllm-0.0.48.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-python_fastllm-0.0.48.dist-info/entry_points.txt,sha256=WkWd04F3xVYFP5bsexaQe9Se6uelwE5ULoTtHWreO0s,133
-python_fastllm-0.0.48.dist-info/top_level.txt,sha256=F8qodL7nEGUHGmzzqfhNKCTIr1i0D6cvudOnm3z7o0Y,8
-python_fastllm-0.0.48.dist-info/RECORD,,