python-fastllm 0.0.59__py3-none-any.whl → 0.0.61__py3-none-any.whl

fastllm/__init__.py

@@ -2,4 +2,4 @@
 
 - `fastllm.responses`: Provider-independent Responses resources and streaming over FastLLM"""
 
-__version__ = "0.0.59"
+__version__ = "0.0.61"

fastllm/_modidx.py

@@ -22,6 +22,8 @@ d = { 'settings': { 'branch': 'main',
             'fastllm.anthropic': { 'fastllm.anthropic._add_cache_control': ('anthropic.html#_add_cache_control', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic._ant_cc': ('anthropic.html#_ant_cc', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic._ant_part_type': ('anthropic.html#_ant_part_type', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic._clear_cache_control': ( 'anthropic.html#_clear_cache_control',
+                                                                               'fastllm/anthropic.py'),
                                    'fastllm.anthropic._sanid': ('anthropic.html#_sanid', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic._url_cite': ('anthropic.html#_url_cite', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.acollect_stream': ('anthropic.html#acollect_stream', 'fastllm/anthropic.py'),
@@ -140,7 +142,8 @@ d = { 'settings': { 'branch': 'main',
                                'fastllm.codex.main': ('codex.html#main', 'fastllm/codex.py'),
                                'fastllm.codex.mk_jwt': ('codex.html#mk_jwt', 'fastllm/codex.py'),
                                'fastllm.codex.token_expired': ('codex.html#token_expired', 'fastllm/codex.py')},
-            'fastllm.gemini': { 'fastllm.gemini._fn_calls': ('gemini.html#_fn_calls', 'fastllm/gemini.py'),
+            'fastllm.gemini': { 'fastllm.gemini._fn_call': ('gemini.html#_fn_call', 'fastllm/gemini.py'),
+                                'fastllm.gemini._fn_calls': ('gemini.html#_fn_calls', 'fastllm/gemini.py'),
                                 'fastllm.gemini._gem_filter_sch': ('gemini.html#_gem_filter_sch', 'fastllm/gemini.py'),
                                 'fastllm.gemini._gem_part_type': ('gemini.html#_gem_part_type', 'fastllm/gemini.py'),
                                 'fastllm.gemini._url_cite': ('gemini.html#_url_cite', 'fastllm/gemini.py'),
@@ -315,7 +318,9 @@ d = { 'settings': { 'branch': 'main',
                                    'fastllm.responses._msg_parts': ('responses.html#_msg_parts', 'fastllm/responses.py'),
                                    'fastllm.responses._new_id': ('responses.html#_new_id', 'fastllm/responses.py'),
                                    'fastllm.responses._response_base': ('responses.html#_response_base', 'fastllm/responses.py'),
+                                   'fastllm.responses._server_call': ('responses.html#_server_call', 'fastllm/responses.py'),
                                    'fastllm.responses._text_content': ('responses.html#_text_content', 'fastllm/responses.py'),
+                                   'fastllm.responses._visible_parts': ('responses.html#_visible_parts', 'fastllm/responses.py'),
                                    'fastllm.responses.normalize_call_ids': ('responses.html#normalize_call_ids', 'fastllm/responses.py'),
                                    'fastllm.responses.pending_response': ('responses.html#pending_response', 'fastllm/responses.py'),
                                    'fastllm.responses.response_event': ('responses.html#response_event', 'fastllm/responses.py'),

fastllm/acomplete.py

@@ -1,4 +1,4 @@
-"""A high level unified function make api calls"""
+"""A unified async API for model completions and streaming"""
 
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/06_acomplete.ipynb.
 
@@ -33,6 +33,7 @@ vendor_mapping = {
     "codex":        ('openai', 'https://chatgpt.com/backend-api/codex', 'CODEX_AUTH_TOKEN', _codex_json),
     "moonshot":     ('openai_chat', "https://api.moonshot.ai/v1", "MOONSHOT_API_KEY"),
     "deepseek":     ('openai_chat', "https://api.deepseek.com/v1", "DEEPSEEK_API_KEY"),
+    "inception":    ('openai_chat', "https://api.inceptionlabs.ai/v1", "INCEPTION_API_KEY"),
     "mimo":         ('anthropic', "https://api.xiaomimimo.com/anthropic", "MIMO_API_KEY"),
     "openrouter":   ('openai_chat', "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"),
     "together":     ('openai_chat', "https://api.together.xyz/v1", "TOGETHER_API_KEY"),

fastllm/anthropic.py

@@ -1,3 +1,5 @@
+"""Translate Anthropic Messages API requests, responses, and streams to and from fastllm's shared message types, with tool replay, media, and prompt caching"""
+
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_anthropic.ipynb.
 
 # %% auto #0
@@ -16,7 +18,7 @@ from fasttransport.errors import api_error_from_event
 
 from .types import *
 from aidialog.msg_parts import (MdStr, Part, PartType, mk_part, Msg, Completion, Text, Thinking, ToolUse, ToolResult, tool_text,
-                                InputImage, InputAudio, InputVideo, InputFile, data_url)
+    InputImage, InputAudio, InputVideo, InputFile, data_url)
 from .streaming import *
 from .streaming import mk_acollect_stream
 
@@ -44,8 +46,8 @@ def norm_usage(resp):
     pt = int(usg.get("input_tokens", 0) or 0) + cached + cache_creation
     ct = int(usg.get("output_tokens", 0) or 0)
     return Usage(prompt_tokens=pt, completion_tokens=ct, total_tokens=pt + ct,
-                 cached_tokens=cached, cache_creation_tokens=cache_creation, reasoning_tokens=0, raw=usg,
-                 search_queries=int(nested_idx(usg, 'server_tool_use', 'web_search_requests') or 0))
+        cached_tokens=cached, cache_creation_tokens=cache_creation, reasoning_tokens=0, raw=usg,
+        search_queries=int(nested_idx(usg, 'server_tool_use', 'web_search_requests') or 0))
 
 def finalize_usage(usg, parts):
     "Split reasoning tokens out of `output_tokens`: the reported `thinking_tokens`, else an estimate from the thinking text."
@@ -110,7 +112,9 @@ def norm_tr_parts(content):
         if typ=='text': parts.append(Text(b.get('text',''), raw=b))
         elif typ=='image':
             s = b['source']
-            parts.append(InputImage(f"data:{s['media_type']};base64,{s['data']}", mime=s['media_type'], raw=b))
+            if s['type']=='url': parts.append(InputImage(s['url'], raw=b))
+            elif s['type']=='base64': parts.append(InputImage(f"data:{s['media_type']};base64,{s['data']}", mime=s['media_type'], raw=b))
+            else: raise ValueError(f"Unsupported Anthropic image source type: {s['type']!r}")
         else: parts.append(mk_part(**b))
     return parts
 
@@ -238,13 +242,11 @@ def denorm_reasoning(v):
 # %% ../nbs/04_anthropic.ipynb #8fa9fbb8
 def denorm_web_search(v):
     "Map canonical web_search_options to Anthropic hosted web_search tool."
-    _max_uses = {"low": 1, "medium": 5, "high": 10}
+    _max_uses = dict(low=1, medium=5, high=10)
     t = {"type": "web_search_20250305", "name": "web_search"}
     if (typ := (v or {}).get("type")): t["type"] = typ
-    if (s := (v or {}).get("search_context_size")):
-        t["max_uses"] = _max_uses.get(s, 5)
-    if (u := (v or {}).get("user_location", {}).get("approximate")):
-        t["user_location"] = {"type": "approximate", **u}
+    if (s := (v or {}).get("search_context_size")): t["max_uses"] = _max_uses.get(s, 5)
+    if (u := (v or {}).get("user_location", {}).get("approximate")): t["user_location"] = {"type": "approximate", **u}
     return t
 
 # %% ../nbs/04_anthropic.ipynb #6b485275
@@ -269,12 +271,12 @@ def denorm_user(m:Msg):
 def denorm_image(p, max_sz=None):
     if (b64:=data_url(p.text)):
         data = resize_b64(b64[1], max_sz) if max_sz else b64[1]
-        return {"type": "image", "source": {"type": "base64", "media_type": b64[0], "data": data}}
+        return {"type": "image", "source": dict(type="base64", media_type=b64[0], data=data)}
     return {"type": "image", "source": {"type": "url", "url": p.text}}
 
 # %% ../nbs/04_anthropic.ipynb #fc6bbdfc
 def denorm_file(p):
-    if (b64:=data_url(p.text)): return {"type": "document", "source": {"type": "base64", "media_type": b64[0], "data": b64[1]}}
+    if (b64:=data_url(p.text)): return {"type": "document", "source": dict(type="base64", media_type=b64[0], data=b64[1])}
     return {"type": "document", "source": {"type": "url", "url": p.text}}
 
 # %% ../nbs/04_anthropic.ipynb #37b63cc2
@@ -303,15 +305,21 @@ def _add_cache_control(
     return msg
 
 # %% ../nbs/04_anthropic.ipynb #6bc52ef5
+def _clear_cache_control(obj):
+    "Clear canonical cache marks from a message or part."
+    if isinstance(obj, Msg):
+        for p in obj.content: _clear_cache_control(p)
+    elif isinstance(obj, Part): obj.cache_control = None
+
 def apply_cache_idxs(
     msgs,            # Canonical Msgs to mark
     cache_idxs=[-1], # Breakpoint idxs into `msgs` (tool-role msgs excluded); 0 is `sp` when given
     sp=None,         # System prompt: str, Msg, or Part
     ttl=None         # Cache TTL: '5m' (default) or '1h'
 ):
-    "Strip all cache breakpoints from `msgs`, then mark `cache_idxs`; returns `sp`, marked if 0 is included"
-    for m in msgs:
-        for p in m.content: p.cache_control = None
+    "Replace cache breakpoints in `msgs` and `sp`, then return the system prompt."
+    for m in msgs: _clear_cache_control(m)
+    _clear_cache_control(sp)
     if sp:
         if 0 in cache_idxs:
             if isinstance(sp, str): sp = Text(sp)
@@ -335,10 +343,8 @@ def mk_payload(msgs, model, **kwargs):
     if tools:=kwargs.get('tools'):          payload['tools'] = denorm_tool_schs(tools)
     if tchc:=kwargs.get('tool_choice'):     payload['tool_choice'] = denorm_tool_choice(tchc)
     if thk:=kwargs.get('reasoning_effort'): payload.update(denorm_reasoning(thk))
-    if (wopts:=kwargs.get('web_search_options')) is not None: 
-        payload.setdefault('tools', []).append(denorm_web_search(wopts))
-    if (temp:=kwargs.get('temperature')) is not None: 
-        payload['temperature'] = temp
+    if (wopts:=kwargs.get('web_search_options')) is not None: payload.setdefault('tools', []).append(denorm_web_search(wopts))
+    if (temp:=kwargs.get('temperature')) is not None: payload['temperature'] = temp
     return payload
 
 # %% ../nbs/04_anthropic.ipynb #60d52c1f
@@ -367,5 +373,4 @@ def cost(usage, m):
 # %% ../nbs/04_anthropic.ipynb #f7c0b989
 api_registry.register('anthropic', norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
     finalize_usage=finalize_usage, raw_msg=raw_msg, collate_raw=collate_raw, acollect_stream=acollect_stream, mk_payload=mk_payload,
-    cost=cost, get_hdrs=get_hdrs,
-    endpoint='/v1/messages')
+    cost=cost, get_hdrs=get_hdrs, endpoint='/v1/messages')

fastllm/chat.py

@@ -1,4 +1,4 @@
-"""High level chat api for fastllm similar to lisette"""
+"""A high-level chat API for fastllm, similar to Lisette"""
 
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/07_chat.ipynb.
 
@@ -112,9 +112,12 @@ _cwe_msg = "ContextWindowExceededError: Do no more tool calls and complete your
 
 # %% ../nbs/07_chat.ipynb #05c20a94
 def search_count(r):
-    if cnt := nested_idx(r.usage.raw, 'server_tool_use', 'web_search_requests'): return cnt # Anthropic
-    if cnt := nested_idx(r.usage.raw, 'server_tool_use', 'google_search'): return cnt # Gemini
-    if cnt := nested_idx(r.usage.raw, 'web_search_requests'): return cnt # streaming with `include_usage`
+    raw = r.usage.raw or {}
+    if (cnt := raw.get('web_search_requests')) is not None: return cnt # host aggregate
+    if cnt := r.usage.search_queries: return cnt
+    legacy = raw.get('server_tool_use') or {}
+    if (cnt := legacy.get('web_search_requests')) is not None: return cnt # Anthropic
+    if (cnt := legacy.get('google_search')) is not None: return cnt # Gemini
     return 0
 
 # %% ../nbs/07_chat.ipynb #61395e0d
@@ -288,8 +291,9 @@ async def _call(self:AsyncChat, msg=None, step=1, search=None, tool_choice=None,
     async for o in self._call_cbs('after_msgs'): yield o
 
     self.turn_kwargs, self.stream = kwargs, t.stream
-    kw = merge(kwargs, dict(xtra_body=merge(kwargs.get('xtra_body'), initial_body))) if initial_body else kwargs
     async for o in self._call_cbs('before_acomplete'): yield o
+    kw = self.turn_kwargs
+    if initial_body: kw = merge(kw, dict(xtra_body=merge(kw.get('xtra_body'), initial_body)))
     res = await acomplete(self.turn_msgs, self.model, system=self.turn_sysp, stream=t.stream, tools=self.tool_schemas,
         tool_choice=tool_choice, max_tokens=int(max_tokens), temperature=None if t.think else ifnone(t.temp,self.temp),
         previous_response_id=self.response_id if self.use_previous_response_id else None,
@@ -413,7 +417,7 @@ class StopReasonCallback(ChatCallback):
         action,msg = _handle_stop_reason(self.turn_res)
         if action != 'warning': return
         add_warning(self.chat.turn_res, msg)
-        if self.stream: yield dict(text=f"warning: {msg}\n\n")
+        if self.stream: yield Text(f'\n\nwarning: {msg}\n\n')
 
 # %% ../nbs/07_chat.ipynb #aa7630b2
 _lang2tool = dict(py='python', bash='bash')
@@ -493,11 +497,11 @@ defaults.chat_callbacks = [FenceToolCallback, ToolReminderCallback, StopReasonCa
 
 # %% ../nbs/07_chat.ipynb #ead4b56d
 class StreamAccum:
-    "Accumulate a formatted stream: append each item's `formatted`, swap in `chat.full()` on `Refresh`"
+    "Accumulate formatted parts; refresh from `chat.full()` on `Refresh` or `Completion`"
     def __init__(self, chat=None, mx=2000): self.chat,self.mx,self.txt,self.last = chat,mx,'',None
     def __call__(self, o):
         "Feed one stream item; True if the display text changed"
-        if isinstance(o, Refresh):
+        if isinstance(o, (Refresh, Completion)):
             if self.chat is None: return False
             self.txt,self.last = self.chat.full(mx=self.mx)+'\n\n',None
         elif f := getattr(o, 'formatted', ''):
@@ -509,7 +513,7 @@ class StreamAccum:
 
 # %% ../nbs/07_chat.ipynb #944bcd25
 async def adisplay_stream(rs, chat=None, mx=2000):
-    "Use IPython.display to markdown display the response stream, refreshing from `chat.full()` on `Refresh`."
+    "Display a stream as Markdown, refreshing from `chat.full()` on `Refresh` or `Completion`."
     try: from IPython.display import display, Markdown
     except ModuleNotFoundError: raise ModuleNotFoundError("This function requires ipython. Please run `pip install ipython` to use.")
     acc,h = StreamAccum(chat, mx=mx), display(Markdown(' '), display_id=True)

fastllm/gemini.py

@@ -1,3 +1,5 @@
+"""Translate Gemini generateContent requests, responses, and streams to and from fastllm's shared message types, including tools, thought signatures, search grounding, and media"""
+
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/05_gemini.ipynb.
 
 # %% auto #0
@@ -16,7 +18,7 @@ from fasttransport.errors import api_error_from_event
 
 from .types import *
 from aidialog.msg_parts import (MdStr, Part, PartType, Msg, Completion, Text, Thinking, ToolUse, ToolResult, tool_text,
-                                InputImage, InputAudio, InputVideo, InputFile, sys_text, part_txt, data_url, url_mime)
+    InputImage, InputAudio, InputVideo, InputFile, sys_text, part_txt, data_url, url_mime)
 from .streaming import *
 from .streaming import mk_acollect_stream
 
@@ -41,9 +43,12 @@ def grounding(cand):
     return cites, call
 
 # %% ../nbs/05_gemini.ipynb #48de3125
-def _fn_calls(parts):
-    return [ToolUse(id=fc.get("id", f"call_{i}"), name=fc.get("name", ""), arguments=fc.get("args") or {}, raw={k:v for k,v in p.items() if k != "functionCall"})
-        for i,p in enumerate(parts) if (fc := p.get("functionCall"))]
+def _fn_call(p, i):
+    fc = p['functionCall']
+    return ToolUse(id=fc.get("id", f"call_{i}"), name=fc.get("name", ""), arguments=fc.get("args") or {},
+        raw={k:v for k,v in p.items() if k != "functionCall"})
+
+def _fn_calls(parts): return [_fn_call(p, i) for i,p in enumerate(parts) if p.get('functionCall')]
 
 def norm_tool_calls(resp):
     "Tool calls of a response: the `google_search` call its grounding describes, then its `functionCall` parts"
@@ -67,7 +72,7 @@ def norm_usage(resp):
     if call := grounding(cand)[1]: stu['google_search'] = len(call.arguments['queries'] or [1])
     if stu: usg['server_tool_use'] = stu
     return Usage(prompt_tokens=pt, completion_tokens=ct, total_tokens=tt,
-                 cached_tokens=cached, reasoning_tokens=reasoning, search_queries=stu.get('google_search', 0), raw=usg)
+        cached_tokens=cached, reasoning_tokens=reasoning, search_queries=stu.get('google_search', 0), raw=usg)
 
 # %% ../nbs/05_gemini.ipynb #7a8b1f8f
 def norm_finish(resp, tcs=None):
@@ -90,13 +95,11 @@ def norm_parts(resp):
     "Normalize Gemini generateContent response; a search's `toolCall` and `toolResponse` parts give way to the call `grounding` builds"
     c0 = nested_idx(resp, 'candidates', 0) or {}
     cites,call = grounding(c0)
-    tc_map = {tc.id: tc for tc in norm_tool_calls(resp)}
     parts = [call] if call else []
-    for p in nested_idx(c0, 'content', 'parts') or []:
+    for i,p in enumerate(nested_idx(c0, 'content', 'parts') or []):
         if 'toolCall' in p or 'toolResponse' in p: continue
         typ = _gem_part_type(p)
-        if typ == PartType.tool_use:
-            if tc := tc_map.get(p['functionCall'].get('id')): parts.append(tc)
+        if typ == PartType.tool_use:      parts.append(_fn_call(p, i))
         elif typ == PartType.thinking:    parts.append(Thinking(p.get("text",""), raw=p))
         elif typ == PartType.tool_result: parts.append(ToolResult(text=p.get("text",""), raw=p))
         else:                             parts.append(Text(p.get("text",""), raw=p))
@@ -175,15 +178,20 @@ def denorm_msgs(msgs:list[Msg]):
     return res
 
 # %% ../nbs/05_gemini.ipynb #adb525db
-_valid_gemini_sch = {'type', 'format', 'title', 'description', 'nullable', 'default',
-    'items', 'minItems', 'maxItems', 'enum', 'properties', 'propertyOrdering',
-    'required', 'minProperties', 'maxProperties', 'minimum', 'maximum',
-    'minLength', 'maxLength', 'pattern', 'example', 'anyOf'}
+_valid_gemini_sch = {'type', 'format', 'title', 'description', 'nullable', 'default', 'items', 'minItems', 'maxItems', 'enum', 'properties',
+    'propertyOrdering', 'required', 'minProperties', 'maxProperties', 'minimum', 'maximum', 'minLength', 'maxLength', 'pattern', 'example',
+    'anyOf'}
 
 def _gem_filter_sch(s):
     if isinstance(s, list): return [_gem_filter_sch(x) for x in s]
     if not isinstance(s, dict): return s
-    return {k: _gem_filter_sch(v) for k,v in s.items() if k in _valid_gemini_sch}
+    res = {}
+    for k,v in s.items():
+        if k not in _valid_gemini_sch: continue
+        if k == 'properties': res[k] = {name:_gem_filter_sch(prop) for name,prop in v.items()}
+        elif k in ('items', 'anyOf'): res[k] = _gem_filter_sch(v)
+        else: res[k] = v
+    return res
 
 def denorm_tool_schs(tools):
     "Convert canonical tools to Gemini format."
@@ -192,8 +200,7 @@ def denorm_tool_schs(tools):
         fn = fn_schema(t)
         if fn is None: other.append(t); continue
         name, desc, params = fn
-        params = {k:v for k,v in params.items() if k in _valid_gemini_sch}
-        params['properties'] = {k:_gem_filter_sch(v) for k,v in params['properties'].items()}
+        params = _gem_filter_sch(params)
         fn_decls.append(dict(name=name, description=desc, parameters=params))
     out = other[:]
     if fn_decls: out.insert(0, dict(functionDeclarations=fn_decls))
@@ -214,13 +221,11 @@ _gem_think_levels  = dict(minimal='low', low='low', medium='medium', high='high'
 
 def denorm_reasoning(v, model=''):
     "Map canonical reasoning_effort to Gemini thinkingConfig (uses thinkingLevel for Gemini 3+)."
-    err = ValueError(f"Invalid reasoning effort for Gemini: {v}, accepted string values are: {list(_gem_think_budgets)} and dicts are passthrough")
-    if v is None: return None
-    elif isinstance(v, dict): return v
-    elif isinstance(v, str) and v in _gem_think_budgets:
-        # defaults to includeThoughts same as litellm
-        if 'gemini-3' in model: return {'thinkingLevel': _gem_think_levels.get(v, 'medium'), 'includeThoughts': True}
-        return {'thinkingBudget': _gem_think_budgets.get(str(v).lower(), 1024), 'includeThoughts': True}
+    if v is None or isinstance(v, dict): return v
+    if not isinstance(v, str) or v not in _gem_think_budgets:
+        raise ValueError(f"Invalid reasoning effort for Gemini: {v!r}; expected one of {list(_gem_think_budgets)}, a dict, or None")
+    if 'gemini-3' in model: return {'thinkingLevel': _gem_think_levels[v], 'includeThoughts': True}
+    return {'thinkingBudget': _gem_think_budgets[v], 'includeThoughts': True}
 
 # %% ../nbs/05_gemini.ipynb #8fa9fbb8
 def denorm_web_search(v): return {"googleSearch": {}}
@@ -325,15 +330,7 @@ def cost(usage, m):
     return cost
 
 # %% ../nbs/05_gemini.ipynb #f7c0b989
-api_ns = dict(norm_tool_calls=norm_tool_calls,
-                norm_parts=norm_parts,
-                norm_finish=norm_finish,
-                norm_usage=norm_usage,
-                acollect_stream=acollect_stream,
-                raw_msg=raw_msg,
-                collate_raw=collate_raw,
-                mk_payload=mk_payload,
-                cost=cost,
-                get_hdrs=get_hdrs,
-                endpoint=endpoint)
+api_ns = dict(norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
+    acollect_stream=acollect_stream, raw_msg=raw_msg, collate_raw=collate_raw, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs,
+    endpoint=endpoint)
 api_registry.register('gemini', **api_ns)