python-fastllm 0.0.9__tar.gz → 0.0.11__tar.gz

{python_fastllm-0.0.9 → python_fastllm-0.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-fastllm
-Version: 0.0.9
+Version: 0.0.11
 Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
 License: Apache-2.0
 Project-URL: Repository, https://github.com/AnswerDotAI/fastllm

python_fastllm-0.0.11/fastllm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.11"

{python_fastllm-0.0.9 → python_fastllm-0.0.11}/fastllm/_modidx.py

@@ -12,6 +12,9 @@ d = { 'settings': { 'branch': 'main',
                                                                                  'fastllm/acomplete.py'),
                                    'fastllm.acomplete._debug_print': ('acomplete.html#_debug_print', 'fastllm/acomplete.py'),
                                    'fastllm.acomplete._is_ctx_exceeded': ('acomplete.html#_is_ctx_exceeded', 'fastllm/acomplete.py'),
+                                   'fastllm.acomplete._raise_if_done': ('acomplete.html#_raise_if_done', 'fastllm/acomplete.py'),
+                                   'fastllm.acomplete._retry_call': ('acomplete.html#_retry_call', 'fastllm/acomplete.py'),
+                                   'fastllm.acomplete._retry_stream': ('acomplete.html#_retry_stream', 'fastllm/acomplete.py'),
                                    'fastllm.acomplete.acomplete': ('acomplete.html#acomplete', 'fastllm/acomplete.py'),
                                    'fastllm.acomplete.mk_client': ('acomplete.html#mk_client', 'fastllm/acomplete.py')},
             'fastllm.anthropic': { 'fastllm.anthropic._ant_cc': ('anthropic.html#_ant_cc', 'fastllm/anthropic.py'),
@@ -74,6 +77,8 @@ d = { 'settings': { 'branch': 'main',
                               'fastllm.chat.FenceToolStop.__call__': ('chat.html#fencetoolstop.__call__', 'fastllm/chat.py'),
                               'fastllm.chat.FenceToolStop.__init__': ('chat.html#fencetoolstop.__init__', 'fastllm/chat.py'),
                               'fastllm.chat.FullResponse': ('chat.html#fullresponse', 'fastllm/chat.py'),
+                              'fastllm.chat.MediaUrl': ('chat.html#mediaurl', 'fastllm/chat.py'),
+                              'fastllm.chat.MediaUrl.__init__': ('chat.html#mediaurl.__init__', 'fastllm/chat.py'),
                               'fastllm.chat.Msg.text': ('chat.html#msg.text', 'fastllm/chat.py'),
                               'fastllm.chat.StopReasonCallback': ('chat.html#stopreasoncallback', 'fastllm/chat.py'),
                               'fastllm.chat.StopReasonCallback.after_acomplete': ( 'chat.html#stopreasoncallback.after_acomplete',
@@ -113,19 +118,19 @@ d = { 'settings': { 'branch': 'main',
                               'fastllm.chat._has_stop': ('chat.html#_has_stop', 'fastllm/chat.py'),
                               'fastllm.chat._inject_tool_reminder': ('chat.html#_inject_tool_reminder', 'fastllm/chat.py'),
                               'fastllm.chat._lite_call_func': ('chat.html#_lite_call_func', 'fastllm/chat.py'),
+                              'fastllm.chat._mime2part_type': ('chat.html#_mime2part_type', 'fastllm/chat.py'),
                               'fastllm.chat._mk_content': ('chat.html#_mk_content', 'fastllm/chat.py'),
                               'fastllm.chat._mk_prefill': ('chat.html#_mk_prefill', 'fastllm/chat.py'),
                               'fastllm.chat._mk_result_fence': ('chat.html#_mk_result_fence', 'fastllm/chat.py'),
                               'fastllm.chat._mk_tool_result': ('chat.html#_mk_tool_result', 'fastllm/chat.py'),
                               'fastllm.chat._split_fence_msgs': ('chat.html#_split_fence_msgs', 'fastllm/chat.py'),
                               'fastllm.chat._split_msg_on_fences': ('chat.html#_split_msg_on_fences', 'fastllm/chat.py'),
-                              'fastllm.chat._srv_tc_summary': ('chat.html#_srv_tc_summary', 'fastllm/chat.py'),
-                              'fastllm.chat._srvtools': ('chat.html#_srvtools', 'fastllm/chat.py'),
                               'fastllm.chat._tc_summary': ('chat.html#_tc_summary', 'fastllm/chat.py'),
                               'fastllm.chat._think_kw': ('chat.html#_think_kw', 'fastllm/chat.py'),
                               'fastllm.chat._trunc_content': ('chat.html#_trunc_content', 'fastllm/chat.py'),
                               'fastllm.chat._trunc_param': ('chat.html#_trunc_param', 'fastllm/chat.py'),
                               'fastllm.chat._trunc_str': ('chat.html#_trunc_str', 'fastllm/chat.py'),
+                              'fastllm.chat._url2content': ('chat.html#_url2content', 'fastllm/chat.py'),
                               'fastllm.chat._usrtools': ('chat.html#_usrtools', 'fastllm/chat.py'),
                               'fastllm.chat.add_warning': ('chat.html#add_warning', 'fastllm/chat.py'),
                               'fastllm.chat.adisplay_stream': ('chat.html#adisplay_stream', 'fastllm/chat.py'),
@@ -137,7 +142,6 @@ d = { 'settings': { 'branch': 'main',
                               'fastllm.chat.lite_mk_func': ('chat.html#lite_mk_func', 'fastllm/chat.py'),
                               'fastllm.chat.mk_msg': ('chat.html#mk_msg', 'fastllm/chat.py'),
                               'fastllm.chat.mk_msgs': ('chat.html#mk_msgs', 'fastllm/chat.py'),
-                              'fastllm.chat.mk_srv_tc_details': ('chat.html#mk_srv_tc_details', 'fastllm/chat.py'),
                               'fastllm.chat.mk_tr_details': ('chat.html#mk_tr_details', 'fastllm/chat.py'),
                               'fastllm.chat.postproc': ('chat.html#postproc', 'fastllm/chat.py'),
                               'fastllm.chat.remove_cache_ckpts': ('chat.html#remove_cache_ckpts', 'fastllm/chat.py'),
@@ -270,6 +274,7 @@ d = { 'settings': { 'branch': 'main',
                                'fastllm.types.ToolCall': ('types.html#toolcall', 'fastllm/types.py'),
                                'fastllm.types.ToolCall._repr_markdown_': ('types.html#toolcall._repr_markdown_', 'fastllm/types.py'),
                                'fastllm.types.Usage': ('types.html#usage', 'fastllm/types.py'),
+                               'fastllm.types._fetch_url_partial': ('types.html#_fetch_url_partial', 'fastllm/types.py'),
                                'fastllm.types._trunc_strs': ('types.html#_trunc_strs', 'fastllm/types.py'),
                                'fastllm.types.approx_pricing': ('types.html#approx_pricing', 'fastllm/types.py'),
                                'fastllm.types.data_url': ('types.html#data_url', 'fastllm/types.py'),

{python_fastllm-0.0.9 → python_fastllm-0.0.11}/fastllm/acomplete.py

@@ -7,7 +7,7 @@ __all__ = ['specs_path', 'ant_spec', 'oai_spec', 'gem_spec', 'vendor_mapping', '
            'ContextWindowExceededError', 'acomplete']
 
 # %% ../nbs/06_acomplete.ipynb #f2f57253
-import json
+import asyncio,json,httpx
 from importlib.resources import files
 from fastcore.utils import *
 from fastcore.meta import *
@@ -38,6 +38,7 @@ vendor_mapping = {
     "codex":        ('openai', 'https://chatgpt.com/backend-api/codex', 'CODEX_AUTH_TOKEN', _codex_json),
     "moonshot":     ('openai_chat', "https://api.moonshot.ai/v1", "MOONSHOT_API_KEY"),
     "deepseek":     ('openai_chat', "https://api.deepseek.com/v1", "DEEPSEEK_API_KEY"),
+    "mimo":         ('openai_chat', "https://api.xiaomimimo.com/v1", "MIMO_API_KEY"),
     "openrouter":   ('openai_chat', "https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"),
     "together":     ('openai_chat', "https://api.together.xyz/v1", "TOGETHER_API_KEY"),
     "fireworks_ai": ('openai_chat', "https://api.fireworks.ai/inference/v1", "FIREWORKS_API_KEY"),
@@ -49,7 +50,8 @@ api2spec = {'openai':oai_spec, 'openai_chat':oai_spec, 'anthropic':ant_spec, 'ge
 
 # %% ../nbs/06_acomplete.ipynb #79075d95
 @flexicache()
-def mk_client(model=None, vendor_name=None, api_name=None, api_key=None, base_url=None, xtra_hdrs=None):
+def mk_client(model=None, vendor_name=None, api_name=None, api_key=None, base_url=None, xtra_hdrs=None,
+    timeout=httpx.Timeout(connect=30, read=300, write=30, pool=10)):
     err_msg = f"please pass a valid one vendor: {', '.join(list(vendor_mapping))} or pass `api_name`,`base_url` and `api_key`"
     if vendor_name:
         override_base_url = base_url
@@ -67,7 +69,7 @@ def mk_client(model=None, vendor_name=None, api_name=None, api_key=None, base_ur
     else: raise ValueError(f"Model {model} can't be auto resolved, {err_msg}")
     api = api_registry.apis[api_name]
     spec, hdrs = api2spec[api_name], api.get_hdrs(api_key)
-    cli = OpenAPIClient(spec, headers=merge(hdrs, ifnone(xtra_hdrs, {})))
+    cli = OpenAPIClient(spec, headers=merge(hdrs, ifnone(xtra_hdrs, {})), timeout=timeout)
     if base_url is not None:
         for op in cli.ops: op.base_url = base_url  # pyright: ignore[reportAttributeAccessIssue]
     return cli, api_name, vendor_name
@@ -81,7 +83,7 @@ def _is_ctx_exceeded(code, msg):
     if str(code or "").lower() == "context_length_exceeded": return True
     return any(s in m for s in ("exceed context limit", "maximum context length", "maximum context limit",
     "longer than the model's context length", "input tokens exceed the configured limit",
-    "exceeds the maximum number of tokens allowed", "prompt is too long"))
+    "exceeds the maximum number of tokens allowed", "prompt is too long", "exceeds the context window"))
 
 def _classify_error(exc):
     "Upgrade generic `APIError` to a specific subclass if applicable."
@@ -113,14 +115,36 @@ def _debug_print(model, api_name, vendor_name, payload, func):
     print(f"\033[1;33mpayload:\033[0m\n{pformat(p, width=120, sort_dicts=False)}")
     print('━'*60)
 
+# %% ../nbs/06_acomplete.ipynb #497c8565
+async def _raise_if_done(e, n, retries, retry_delay, yielded=False):
+    e = _classify_error(e)
+    if yielded or not e.retryable or n == retries: raise e
+    await asyncio.sleep(retry_delay*2**n)
+
+async def _retry_call(f, retries=2, retry_delay=0.5):
+    for n in range(retries+1):
+        try: return await f()
+        except APIError as e: await _raise_if_done(e, n, retries, retry_delay)
+
+async def _retry_stream(mk_gen, retries=2, retry_delay=0.5):
+    for n in range(retries+1):
+        yielded = False
+        try:
+            async for o in mk_gen():
+                yielded = True
+                yield o
+            return
+        except APIError as e: await _raise_if_done(e, n, retries, retry_delay, yielded=yielded)
+
 # %% ../nbs/06_acomplete.ipynb #2379ec94
 @delegates(payload_kwargs)
-async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None, base_url=None, xtra_body=None, xtra_hdrs=None,
-    stream=False, stop_callables=None, stop_sequences=None, **kwargs):
+async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None,
+                    base_url=None, xtra_body=None, xtra_hdrs=None, stream=False,
+                    stop_callables=None, retries=2, retry_delay=0.5, **kwargs):
     "Unified completion across different APIs."
     cli, api_name, vendor_name = mk_client(model, vendor_name, api_name, api_key, base_url, xtra_hdrs)
     api = api_registry.apis[api_name]
-    payload = api.mk_payload(msgs, model, stream=stream, stop_callables=stop_callables, **kwargs)
+    payload = api.mk_payload(msgs, model, stream=stream, **kwargs)
     payload = merge(payload, ifnone(xtra_body, {}))
     if vendor_name == 'codex':
         for k in 'temperature max_tokens max_output_tokens max_completion_tokens metadata'.split(): payload.pop(k, None)
@@ -130,7 +154,13 @@ async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None,
         if vendor_name == 'moonshot' and 'kimi' in model: payload['messages'][-1]['partial'] = True
     func = attrgetter(api.op_path[stream])(cli)
     if defaults.debug_mode: _debug_print(model, api_name, vendor_name, payload, func)
-    try: resp = await func(**payload)
-    except APIError as e: raise _classify_error(e) from e
-    if stream: return _classify_error_stream(api.acollect_stream(resp, model=model, vendor_name=vendor_name, stop_callables=stop_callables))
-    return mk_completion(resp, model=model, api_name=api_name, vendor_name=vendor_name)
+    async def _call(): return await func(**payload)
+    if not stream:
+        resp = await _retry_call(_call, retries, retry_delay)
+        return mk_completion(resp, model=model, api_name=api_name, vendor_name=vendor_name)
+
+    async def _mk_gen():
+        resp = await _call()
+        async for o in api.acollect_stream(resp, model=model, vendor_name=vendor_name, stop_callables=stop_callables): yield o
+
+    return _retry_stream(_mk_gen, retries, retry_delay)

{python_fastllm-0.0.9 → python_fastllm-0.0.11}/fastllm/anthropic.py

@@ -90,7 +90,9 @@ def norm_sse_event(ev, **kwargs):
     if typ == "content_block_start":
         cb = ev.get("content_block", {})
         if cb.get("type", "").endswith("_tool_result"): return Delta(server_tool_result=cb, raw=ev, **kwargs)
-        if tc := norm_tool_call(cb): tcs = [tc]
+        if tc := norm_tool_call(cb):
+            if not tc.arguments: tc.arguments = {'_delta': ''}
+            tcs = [tc]
     elif typ == "content_block_delta":
         d = ev.get("delta", {})
         dtyp = d.get("type")
@@ -285,7 +287,7 @@ def cost(usage, m):
     in_tok = raw['input_tokens']
     cache_read = raw.get('cache_read_input_tokens', 0)
     cc = raw.get('cache_creation', {}) or {}
-    cache_5m  = cc.get('ephemeral_5m_input_tokens', 0)
+    cache_5m  = cc.get('ephemeral_5m_input_tokens', raw.get('cache_creation_input_tokens', 0))
     cache_1h  = cc.get('ephemeral_1h_input_tokens', 0)
     cost  = in_tok     * m.input_cost_per_token
     cost += raw['output_tokens'] * m.output_cost_per_token

{python_fastllm-0.0.9 → python_fastllm-0.0.11}/fastllm/chat.py

@@ -3,13 +3,13 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/07_chat.ipynb.
 
 # %% auto #0
-__all__ = ['tool_dtls_tag', 're_tools', 'token_dtls_tag', 're_token', 'effort', 'remove_cache_ckpts', 'contents', 'stop_reason',
-           'mk_msg', 'FenceToolStop', 'extract_fence_call', 'split_tools', 'fmt2hist', 'mk_msgs', 'cite_footnote',
-           'postproc', 'lite_mk_func', 'ToolResponse', 'structured', 'StopResponse', 'FullResponse', 'search_count',
-           'UsageStats', 'AsyncChat', 'astream_with_complete', 'ChatCallback', 'DeepseekMsgsCallback',
+__all__ = ['tool_dtls_tag', 're_tools', 'token_dtls_tag', 're_token', 'effort', 'MediaUrl', 'remove_cache_ckpts', 'contents',
+           'stop_reason', 'mk_msg', 'FenceToolStop', 'extract_fence_call', 'split_tools', 'fmt2hist', 'mk_msgs',
+           'cite_footnote', 'postproc', 'lite_mk_func', 'ToolResponse', 'structured', 'StopResponse', 'FullResponse',
+           'search_count', 'UsageStats', 'AsyncChat', 'astream_with_complete', 'ChatCallback', 'DeepseekMsgsCallback',
            'DeepseekPrefillCallback', 'add_warning', 'StopReasonCallback', 'run_fence_tool', 'FenceToolCallback',
-           'ToolReminderCallback', 'stop_sequences', 'StopSequencesCallback', 'mk_tr_details', 'mk_srv_tc_details',
-           'StreamFormatter', 'AsyncStreamFormatter', 'adisplay_stream']
+           'ToolReminderCallback', 'stop_sequences', 'StopSequencesCallback', 'mk_tr_details', 'StreamFormatter',
+           'AsyncStreamFormatter', 'adisplay_stream']
 
 # %% ../nbs/07_chat.ipynb #d5a3bc1f
 import asyncio, base64, json, mimetypes, random, string, ast, warnings
@@ -25,15 +25,30 @@ from dataclasses import dataclass
 from .types import *
 from .acomplete import *
 
-# %% ../nbs/07_chat.ipynb #90f55ad4
+# %% ../nbs/07_chat.ipynb #1b75c262
+class MediaUrl(BasicRepr):
+    "Direct URL media reference"
+    def __init__(self, url, mime=None): self.url, self.mime = url, ifnone(mime, url_mime(url))
+
+# %% ../nbs/07_chat.ipynb #eb557831
+def _mime2part_type(mime):
+    "Map MIME string to canonical PartType"
+    if mime.startswith('image/'): return PartType.input_image
+    if mime.startswith('audio/'): return PartType.input_audio
+    if mime.startswith('video/'): return PartType.input_video
+    return PartType.input_file
+
 def _bytes2content(data):
-    "Convert bytes to litellm content dict (image, pdf, audio, video)"
+    "Convert bytes to fastllm canonical content"
     mtype = detect_mime(data)
     if not mtype: raise ValueError(f'Data must be a supported file type, got {data[:10]}')
-    encoded = base64.b64encode(data).decode("utf-8")    
-    if mtype.startswith('image/'): return Part(type=PartType.input_image, text=f'data:{mtype};base64,{encoded}')
-    return Part(type=PartType.input_file, text=f'data:{mtype};base64,{encoded}')
+    encoded = base64.b64encode(data).decode("utf-8")
+    return Part(type=_mime2part_type(mtype), text=f'data:{mtype};base64,{encoded}')
 
+def _url2content(o):
+    "Convert MediaUrl to fastllm canonical content"
+    mime = o.mime or url_mime(o.url)
+    return Part(type=_mime2part_type(mime), text=o.url, data=dict(mime=mime))
 
 # %% ../nbs/07_chat.ipynb #48c78e48
 def _add_cache_control(msg,          # LiteLLM formatted msg
@@ -57,8 +72,9 @@ def remove_cache_ckpts(msg):
     return msg
 
 def _mk_content(o):
-    if isinstance(o, str): return Part(type=PartType.text, text=o)
-    elif isinstance(o,bytes): return _bytes2content(o)
+    if isinstance(o, str):        return Part(type=PartType.text, text=o)
+    elif isinstance(o, bytes):    return _bytes2content(o)
+    elif isinstance(o, MediaUrl): return _url2content(o)
     return o
 
 def contents(c):
@@ -100,7 +116,7 @@ re_token = re.compile(fr"^{re.escape(token_dtls_tag)}\n*<summary>.*?</summary>\n
 _fence_back = '`````'
 _fence_re = re.compile(f'^{_fence_back}(py|bash)\n(.*?)\n{_fence_back}$', re.DOTALL | re.MULTILINE)
 _result_re = re.compile(f'\n{_fence_back}result\n(.*?)\n{_fence_back}\n', re.DOTALL)
-_lang2tool = dict(py='python', bash='bash')
+_lang2tool = dict(py='pyrun', bash='bash')
 
 class FenceToolStop:
     def __init__(self, langs): self.langs = langs
@@ -188,6 +204,7 @@ def fmt2hist(outp:str)->list[Msg]:
     for msg in hist:
         if msg.role == 'assistant': result.extend(_split_msg_on_fences(msg))
         else: result.append(msg)
+    if result[-1].role == 'tool': result.append(Msg(role='assistant', content=[Part(type=PartType.text, text='.')])) 
     return result
 
 # %% ../nbs/07_chat.ipynb #8de5ce8d
@@ -427,7 +444,6 @@ class AsyncChat:
         return self
 
 # %% ../nbs/07_chat.ipynb #2e469ea1
-def _srvtools(tcs): return L(tcs).filter(lambda o: o.server) if tcs else None
 def _usrtools(tcs): return L(tcs).filter(lambda o: not o.server) if tcs else None
 
 # %% ../nbs/07_chat.ipynb #19b87f53
@@ -511,8 +527,6 @@ async def _call(self:AsyncChat, msg=None, prefill=None, temp=None, think=None, s
 
     self.toolloop, self.prompt, tmsg = False, None, None
     async for o in self._call_cbs('before_tool_calls'): yield o
-    if stcs:= _srvtools(res.tool_calls): 
-        for tc in stcs: yield tc
     if tcs := _usrtools(res.tool_calls):
         tres = await parallel_async(_alite_call_func, tcs, timeout=tc_timeout, n_workers=n_workers, pause=pause, **self.tcdict)
         tmsg = mk_tool_res_msg(tcs, tres)
@@ -622,7 +636,7 @@ def _active_fence_langs(tool_schemas):
 async def run_fence_tool(lang, code, ns):
     "Run the mapped tool for `lang` with the code, return result fence"
     tname = _lang2tool[lang]
-    arg = dict(code=code) if lang == 'py' else dict(command=code)
+    arg = dict(code=code) if lang == 'py' else dict(cmd=code)
     res = _mk_tool_result(await call_func_async(tname, arg, ns=ns, raise_on_err=False))
     return _mk_result_fence(_trunc_str(str(res)))
 
@@ -703,15 +717,9 @@ def _trunc_param(v, mx=40):
 def _tc_summary(tr):
     "Format tool call as func(params) → result string"
     params = ', '.join(f"{k}={_trunc_param(v)}" for k,v in tr.data['arguments'].items())
-    res = f"→{_trunc_param(tr.text)}"
+    res = f"→{_trunc_param(tr.text)}" if tr.text else ''
     return '<code>'+escape(f"{tr.data['name']}({params}){res}")+'</code>'
 
-# %% ../nbs/07_chat.ipynb #91beb26c
-def _srv_tc_summary(tc):
-    "Format tool call as func(params) → result string"
-    params = ', '.join(f"{k}={_trunc_param(v)}" for k,v in tc.arguments.items())
-    return '<code>'+escape(f"{tc.name}({params})")+'</code>'
-
 # %% ../nbs/07_chat.ipynb #80f344cc
 def _trunc_content(content, mx):
     "Truncate tool result content, respecting '_full' flag"
@@ -722,23 +730,13 @@ def _trunc_content(content, mx):
 def mk_tr_details(tr, mx=2000):
     "Create <details> block for tool call as JSON"
     args = {k:_trunc_str(v, mx=mx*5) for k,v in tr.data['arguments'].items()}
-    res = {'id':tr.data['id'], 'server':False,
+    res = {'id':tr.data['id'], 'server':tr.data.get('server', False),
            'call':{'function': tr.data['name'], 'arguments': args},
            'result':_trunc_content(tr.text, mx=mx),}
     summ = f"<summary>{_tc_summary(tr)}</summary>"
     return f"\n\n{tool_dtls_tag}\n{summ}\n\n```json\n{dumps(res, indent=2, ensure_ascii=False)}\n```\n\n</details>\n\n"
 
-# %% ../nbs/07_chat.ipynb #3049001c
-def mk_srv_tc_details(tc, mx=2000):
-    "Create <details> block for tool call as JSON"
-    args = {k:_trunc_str(v, mx=mx*5) for k,v in tc.arguments.items()}
-    res = {'id':tc.id, 'server':True, 'call':{'function': tc.name, 'arguments': args}, 'result':"Server tool call executed."}
-    summ = f"<summary>{_srv_tc_summary(tc)}</summary>"
-    return f"\n\n{tool_dtls_tag}\n{summ}\n\n```json\n{dumps(res, indent=2, ensure_ascii=False)}\n```\n\n</details>\n\n"
-
 # %% ../nbs/07_chat.ipynb #f0d984ec
-# status_re = re.compile(r'^- ⏳ <code>(.*)</code> ⏳$|^🧠+$', re.MULTILINE) # TODO: Need to yield tool calls as they are done collated in fastllm `_acollect_stream`
-
 class StreamFormatter:
     def __init__(self, mx=2000, debug=False, showthink=False):
         self.outp,self.tcs = '',{}
@@ -754,8 +752,8 @@ class StreamFormatter:
                 res+= '🧠' if not self.outp or self.outp[-1]=='🧠' else '\n\n🧠'
             elif self.outp and self.outp[-1] == '🧠': res+= '\n\n'
             if txt:=o.get('text'): res+=f"\n\n{txt}" if res and res[-1] == '🧠' else txt
-        if isinstance(o, ToolCall):
-            res += mk_srv_tc_details(o)
+        if isinstance(o, Part) and o.type==PartType.tool_use:
+            res += f"\n- ⏳ {_tc_summary(o)} ⏳\n"
         if isinstance(o, Part) and o.type == PartType.tool_result:
             res += mk_tr_details(o,mx=self.mx)
         self.outp+=res

{python_fastllm-0.0.9 → python_fastllm-0.0.11}/fastllm/openai_chat.py

@@ -195,7 +195,7 @@ def get_hdrs(api_key=None):
 # %% ../nbs/03_oai_chat.ipynb #f89e2bf6
 def cost(usage, m):
     raw = usage.raw
-    pd, cd = raw.get('prompt_tokens_details', {}), raw.get('completion_tokens_details', {})
+    pd,cd = raw.get('prompt_tokens_details') or {},raw.get('completion_tokens_details') or {}
     cached = pd.get('cached_tokens', 0)
     in_audio, out_audio = pd.get('audio_tokens', 0), cd.get('audio_tokens', 0)
     in_txt  = raw['prompt_tokens']     - cached - in_audio

{python_fastllm-0.0.9 → python_fastllm-0.0.11}/fastllm/streaming.py

@@ -105,14 +105,14 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
         idx,last_idx = index_fn(d, typ, last_typ, last_idx)
         return idx
     def _proc(d, name, pt=None, kw='txt', ret=None):
-        if not ret and not (val := getattr(d, name)): return
+        if not ret and not (val := getattr(d, name)): return None, None
         idx = _fidx(d, name, pt)
         part_accum.append(typ, idx, **(ret or {kw: val}))
-        return ret or {name: val}
+        return ret or {name: val}, idx
     def _yield_parts(d):
         for args in [('text',), ('thinking',), ('citations', 'text', 'citations')]:
-            if (r := _proc(d, args[0], pt=args[1] if len(args)>1 else None, kw=args[2] if len(args)>2 else 'txt')):
-                yield r
+            r = _proc(d, args[0], pt=args[1] if len(args)>1 else None, kw=args[2] if len(args)>2 else 'txt')
+            if r[0]: yield r[0]
     stop, stop_yielded = False, False
     async for d in it:
         # Check stop condition and yield stop delta
@@ -127,11 +127,26 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
         # Rest incl. tools, finish reason, usage is processed independently
         for tc in d.tool_calls:
             args = tc.arguments.get('_delta', tc.arguments)
-            _proc(d, 'tool_use', ret=dict(id=tc.id, name=tc.name, arguments=args, server=tc.server, extra=tc.extra))
+            _, idx = _proc(d, 'tool_use', ret=dict(id=tc.id, name=tc.name, arguments=args, server=tc.server, extra=tc.extra))
+            if (isinstance(args, str) and args.endswith('}')) or (isinstance(args, dict) and '_delta' not in tc.arguments): # tool call ready
+                if isinstance(args, str):
+                    try: args = json.loads(part_accum.parts[idx].arguments) if args else {}
+                    except json.JSONDecodeError: continue
+                acc = part_accum.parts[idx]
+                acc.arguments = args
+                data = {**acc.extra, 'id':acc.id, 'name':acc.name, 'arguments':args, 'server':acc.server}
+                yield Part(type=PartType.tool_use, data=data)
+                # Server tool results for anthropic are yielded in d.server_tool_result by checking injected dummy `_delta`
+                if acc.server and '_delta' not in tc.arguments: yield Part(type=PartType.tool_result, text="Server tool call executed.", data=data)
         if d.server_tool_result:
             idx = _fidx(d, 'server_tool_result')
             part_accum.parts[idx] = Part(type=typ, data=d.server_tool_result)
-        if (r:=_proc(d, 'refusal')): yield r
+            srv_tc = next((p for p in reversed(list(part_accum.parts.values())) if isinstance(p, ToolCall) and p.server), None)
+            if srv_tc:
+                data = {**srv_tc.extra, 'id':srv_tc.id, 'name':srv_tc.name, 'arguments':srv_tc.arguments, 'server':True}
+                yield Part(type=PartType.tool_result, text="Server tool call executed.", data=data)
+        r = _proc(d, 'refusal')
+        if r[0]: yield r[0]
         if d.finish_reason: fin = d.finish_reason
         if d.usage: usg = d.usage
         last_typ = typ
@@ -142,8 +157,7 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
     if stop: fin = FinishReason.stop
     fin = FinishReason.tool_calls if fin==FinishReason.stop and any(~L(tcs).attrgot('server')) else fin # recheck tool calls post collation
     # tool calls and non-anthropic citations are yielded at the end
-    yield Completion(d.raw.get('model', model),
+    yield Completion(model,
             message=Msg(role="assistant", content=part_accum.parts),
             finish_reason=fin, usage=usg, tool_calls=tcs, api_name=api_name, vendor_name=vendor_name,
             raw={'deltas':deltas})
-

{python_fastllm-0.0.9 → python_fastllm-0.0.11}/fastllm/types.py

@@ -5,12 +5,14 @@
 # %% auto #0
 __all__ = ['PartType', 'FinishReason', 'api_registry', 'model_prices_url', 'haik45', 'sonn45', 'sonn', 'sonn46', 'opus46', 'opus',
            'gpt54', 'gpt54m', 'gpt55', 'codex54', 'codex54m', 'codex55', 'codex53spark', 'model_info_registry',
-           'deepseek_v4_common', 'codex_pricing', 'Part', 'Msg', 'ToolCall', 'display_list', 'Usage', 'Completion',
-           'APIRegistry', 'mk_completion', 'mk_tool_res_msg', 'fn_schema', 'sys_text', 'part_txt', 'data_url',
-           'url_mime', 'payload_kwargs', 'get_api_key', 'model_prices_meta', 'infer_api_name', 'get_model_meta',
-           'register_model_info', 'get_model_info', 'get_model_pricing', 'approx_pricing']
+           'modern_llm', 'deepseek_v4_common', 'mimo_v25_common', 'codex_pricing', 'Part', 'Msg', 'ToolCall',
+           'display_list', 'Usage', 'Completion', 'APIRegistry', 'mk_completion', 'mk_tool_res_msg', 'fn_schema',
+           'sys_text', 'part_txt', 'data_url', 'url_mime', 'payload_kwargs', 'get_api_key', 'model_prices_meta',
+           'infer_api_name', 'get_model_meta', 'register_model_info', 'get_model_info', 'get_model_pricing',
+           'approx_pricing']
 
 # %% ../nbs/00_types.ipynb #b4d047fd
+import httpx
 from dataclasses import dataclass, field
 from fastcore.net import urljson
 from fastcore.utils import *
@@ -28,14 +30,16 @@ PartType = str_enum('PartType', 'text', 'thinking', 'refusal', 'tool_use', 'serv
                     'input_image', 'input_audio', 'input_video', 'input_file')
 
 # %% ../nbs/00_types.ipynb #2eeff103
-def _trunc_strs(d, n=200):
-    "Return copy of dict `d` with str values >n chars truncated to first 10 chars + '...'"
-    if not d: return d
-    return {k: (v[:10]+'...' if isinstance(v,str) and len(v)>n else v) for k,v in d.items()}
+def _trunc_strs(o, n=200):
+    "Truncate str or dict"
+    if not o: return o
+    if isinstance(o,str) and len(o)>n: return o[:100]+'...'
+    if isinstance(o,dict): return {k: (v[:100]+'...' if isinstance(v,str) and len(v)>n else v) for k,v in o.items()}
+    return o
 
 @patch
 def _repr_markdown_(self: Part):
-    body = self.text if self.text else ''
+    body = _trunc_strs(self.text) if self.text else ''
     data = _trunc_strs(self.data)
     return f"""**Part** (`{self.type}`)
 
@@ -161,7 +165,7 @@ def mk_completion(resp, model, api_name, vendor_name):
     parts = api.norm_parts(resp)
     usg = api.finalize_usage(api.norm_usage(resp), parts)
     return Completion(
-        model=resp.get("model") or model,
+        model=model,
         message=Msg(role="assistant", content=parts),
         finish_reason=api.norm_finish(resp, tcs),
         usage=usg,
@@ -170,7 +174,6 @@ def mk_completion(resp, model, api_name, vendor_name):
         vendor_name=vendor_name,
         raw=resp)
 
-
 # %% ../nbs/00_types.ipynb #d5322db5
 def mk_tool_res_msg(tool_calls:list[ToolCall], results:list[str|list]):
     'A util to prepare parallel tool call with str or media list results'
@@ -198,7 +201,17 @@ def sys_text(system):
 
 def part_txt(p): return p.text if isinstance(p,Part) else p
 
-# %% ../nbs/00_types.ipynb #dc2b75a0
+# %% ../nbs/00_types.ipynb #f3deb055
+@flexicache(time_policy(24*3600))
+def _fetch_url_partial(url, nbytes=512): 
+    "Fetch remote media bytes, optionally only first `nbytes`."
+    try:
+        with httpx.stream('GET', url, headers={'Range': f'bytes=0-{nbytes-1}'}, follow_redirects=True) as r:
+            if r.status_code not in (200, 206): return
+            return r.read()
+    except (httpx.HTTPError, httpx.InvalidURL): return
+
+# %% ../nbs/00_types.ipynb #70a9a0c3
 _ext_mime = {
     '.jpg':'image/jpeg', '.jpeg':'image/jpeg', '.png':'image/png', '.gif':'image/gif', '.webp':'image/webp',
     '.pdf':'application/pdf',
@@ -214,18 +227,19 @@ def data_url(url):
     return header[5:].split(';',1)[0].strip() or 'application/octet-stream', body
 
 def url_mime(url, default='application/octet-stream'):
-    "Guess mime from URL extension."
+    "Guess mime from URL extension, and optional bytes fallback."
+    if "youtube.com" in url or "youtu.be" in url: return "video/mp4"
     ext = '.' + url.rsplit('.', 1)[-1].split('?')[0].lower() if '.' in url.split('?')[0].split('/')[-1] else ''
-    return _ext_mime.get(ext, default)
+    if (mime:=_ext_mime.get(ext)) is None: return detect_mime(_fetch_url_partial(url))
+    return ifnone(mime, default)
 
 # %% ../nbs/00_types.ipynb #28c698fe
-def payload_kwargs(msgs, model, stream=False, system=None, max_tokens=None, temperature=None, tools=None, tool_choice=None, reasoning_effort=None, web_search_options=None, stop_callables=None, stop_sequences=None): pass
+def payload_kwargs(msgs, model, stream=False, system=None, max_tokens=None, temperature=None, tools=None, tool_choice=None, reasoning_effort=None, web_search_options=None, stop_callables=None): pass
 
 # %% ../nbs/00_types.ipynb #c2a2cb49
 def get_api_key(api_key, default):
-    err = ValueError(f"Missing API key: make sure to have the expected env var name or pass `api_key`")
     key = api_key or os.getenv(default)
-    if not key: raise err
+    if not key: raise ValueError(f"Missing API key: set environment variable '{default}' or pass `api_key` parameter")
     return key
 
 # %% ../nbs/00_types.ipynb #852adecd
@@ -257,7 +271,7 @@ haik45 = "claude-haiku-4-5"
 sonn45 = "claude-sonnet-4-5"
 sonn = sonn46 = "claude-sonnet-4-6"
 opus46 = "claude-opus-4-6"
-opus = "claude-opus-4-7"
+opus = "claude-opus-4-8"
 gpt54 = "gpt-5.4"
 gpt54m = "gpt-5.4-mini"
 gpt55 = "gpt-5.5"
@@ -273,6 +287,8 @@ def register_model_info(model, vendor_name=None, base=None, base_vendor_name=Non
     "Register model metadata, optionally starting from `base`."
     info = dict(get_model_info(base, base_vendor_name or vendor_name)) if base else {}
     info.update(overrides)
+    if isinstance(c := info.get('search_context_cost_per_query'), (int,float)):
+        info['search_context_cost_per_query'] = {f'search_context_size_{s}':c for s in ('low','medium','high')}
     model_info_registry[vendor_name, model] = info
 
 def get_model_info(mn, vendor_name=None):
@@ -280,11 +296,17 @@ def get_model_info(mn, vendor_name=None):
     if 'search_context_cost_per_query' in info: info['supports_web_search'] = True
     return dict2obj(info)
 
+# %% ../nbs/00_types.ipynb #331c5d0a
+register_model_info("claude-opus-4-8", vendor_name='anthropic', base="claude-opus-4-6")
+
+# %% ../nbs/00_types.ipynb #b36178d4
+modern_llm = dict(supports_function_calling=True, supports_tool_choice=True, supports_prompt_caching=True,
+    supports_parallel_function_calling=True, supports_native_streaming=True, supports_native_structured_output=True,
+    supports_reasoning=True, supports_response_schema=True, supports_system_messages=True)
+
 # %% ../nbs/00_types.ipynb #8261dcd0
-register_model_info('accounts/fireworks/models/qwen3p6-plus', vendor_name='fireworks_ai',
-    supports_vision=True, supports_reasoning=True, supports_function_calling=True, supports_tool_choice=True,
-    supports_system_messages=True, supports_response_schema=True, supports_parallel_function_calling=True,
-    supports_prompt_caching=True, supports_native_streaming=True, supports_native_structured_output=True,
+register_model_info('accounts/fireworks/models/qwen3p6-plus', vendor_name='fireworks_ai', **modern_llm,
+    supports_vision=True,
     max_tokens=1000000, max_input_tokens=1000000, max_output_tokens=65536,
     input_cost_per_token=0.5e-6, cache_read_input_token_cost=0.1e-6, output_cost_per_token=3.0e-6)
 
@@ -308,9 +330,7 @@ for model in ('accounts/fireworks/models/kimi-k2p5', 'accounts/fireworks/models/
         input_cost_per_token=0.95e-6, cache_read_input_token_cost=0.16e-6, output_cost_per_token=4.0e-6)
 
 # %% ../nbs/00_types.ipynb #948d55d0
-deepseek_v4_common = dict(
-    supports_assistant_prefill=True, supports_function_calling=True, supports_prompt_caching=True,
-    supports_reasoning=True, supports_tool_choice=True,
+deepseek_v4_common = dict(**modern_llm, supports_assistant_prefill=True,
     max_input_tokens=1048576, max_output_tokens=393216, max_tokens=393216)
 
 register_model_info('deepseek-v4-flash', vendor_name='deepseek', base='deepseek/deepseek-v3.2', **deepseek_v4_common,
@@ -320,19 +340,26 @@ register_model_info('deepseek-v4-pro', vendor_name='deepseek', base='deepseek/de
     input_cost_per_token=4.35e-07, input_cost_per_token_cache_hit=3.625e-09,
     output_cost_per_token=8.7e-07, cache_read_input_token_cost=4.35e-07/10)
 
+mimo_v25_common = dict(**modern_llm, supports_web_search=True, max_input_tokens=1048576, max_output_tokens=131072, max_tokens=131072)
+
+register_model_info('mimo-v2.5-pro', vendor_name='mimo', **mimo_v25_common, base='deepseek/deepseek-v4-pro',
+    input_cost_per_token=0.435e-6, output_cost_per_token=0.87e-6, cache_read_input_token_cost=0.0036e-6, search_context_cost_per_query=0.005)
+register_model_info('mimo-v2.5', vendor_name='mimo', **mimo_v25_common, base='deepseek/deepseek-v4',
+    input_cost_per_token=0.14e-6,  output_cost_per_token=0.28e-6, cache_read_input_token_cost=0.0028e-6, search_context_cost_per_query=0.005,
+    supports_vision=True, supports_image_input=True)
+
 # %% ../nbs/00_types.ipynb #2c23d11e
 codex_pricing = dict(
     input_cost_per_token = 0.10/1_000_000, output_cost_per_token = 0.50/1_000_000,
     cache_creation_input_token_cost = 0.10/1_000_000, cache_read_input_token_cost = 0.10/1_000_000)
 
 for model in (codex54, codex54m, codex55):
-    register_model_info(model, 'codex', base=model, base_vendor_name='chatgpt', supports_web_search=True, **codex_pricing)
+    register_model_info(model, 'codex', base=model, base_vendor_name='chatgpt', supports_web_search=True, max_input_tokens=256000, **codex_pricing)
 
 register_model_info(codex53spark, 'codex', **codex_pricing,
-    supports_vision=False, supports_image_input=False, supports_web_search=True, supports_reasoning=True,
+    supports_vision=False, supports_image_input=False, supports_web_search=True, supports_reasoning=True, supports_function_calling=True,
     max_tokens=128000, max_input_tokens=128000, max_output_tokens=128000)
 
-
 # %% ../nbs/00_types.ipynb #24cc47ec
 def get_model_pricing(mn, vendor_name, million=True):
     return {k:round(v * (1e6 if million else 1), 6)
@@ -345,7 +372,7 @@ def approx_pricing(nm, vendor_name, out=10, cache=80, inp=10, markup=0):
     p = get_model_pricing(nm, vendor_name)
     ic = p.get('cache_creation_input_token_cost', p['input_cost_per_token'])
     res = (p['output_cost_per_token']*out + p['cache_read_input_token_cost']*cache + ic*inp) / (out+cache+inp)
-    if nm=='claude-opus-4-7': res *= 1.5
+    if nm in ('claude-opus-4-7','claude-opus-4-8'): res *= 1.5
     return res*(1+markup)
 
 # %% ../nbs/00_types.ipynb #8bfca02d

{python_fastllm-0.0.9 → python_fastllm-0.0.11}/python_fastllm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-fastllm
-Version: 0.0.9
+Version: 0.0.11
 Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
 License: Apache-2.0
 Project-URL: Repository, https://github.com/AnswerDotAI/fastllm

python_fastllm-0.0.9/fastllm/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.9"