python-fastllm 0.0.8__tar.gz → 0.0.10__tar.gz

{python_fastllm-0.0.8 → python_fastllm-0.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-fastllm
-Version: 0.0.8
+Version: 0.0.10
 Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
 License: Apache-2.0
 Project-URL: Repository, https://github.com/AnswerDotAI/fastllm

python_fastllm-0.0.10/fastllm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.10"

{python_fastllm-0.0.8 → python_fastllm-0.0.10}/fastllm/_modidx.py

@@ -12,6 +12,9 @@ d = { 'settings': { 'branch': 'main',
                                                                                  'fastllm/acomplete.py'),
                                    'fastllm.acomplete._debug_print': ('acomplete.html#_debug_print', 'fastllm/acomplete.py'),
                                    'fastllm.acomplete._is_ctx_exceeded': ('acomplete.html#_is_ctx_exceeded', 'fastllm/acomplete.py'),
+                                   'fastllm.acomplete._raise_if_done': ('acomplete.html#_raise_if_done', 'fastllm/acomplete.py'),
+                                   'fastllm.acomplete._retry_call': ('acomplete.html#_retry_call', 'fastllm/acomplete.py'),
+                                   'fastllm.acomplete._retry_stream': ('acomplete.html#_retry_stream', 'fastllm/acomplete.py'),
                                    'fastllm.acomplete.acomplete': ('acomplete.html#acomplete', 'fastllm/acomplete.py'),
                                    'fastllm.acomplete.mk_client': ('acomplete.html#mk_client', 'fastllm/acomplete.py')},
             'fastllm.anthropic': { 'fastllm.anthropic._ant_cc': ('anthropic.html#_ant_cc', 'fastllm/anthropic.py'),
@@ -74,6 +77,8 @@ d = { 'settings': { 'branch': 'main',
                               'fastllm.chat.FenceToolStop.__call__': ('chat.html#fencetoolstop.__call__', 'fastllm/chat.py'),
                               'fastllm.chat.FenceToolStop.__init__': ('chat.html#fencetoolstop.__init__', 'fastllm/chat.py'),
                               'fastllm.chat.FullResponse': ('chat.html#fullresponse', 'fastllm/chat.py'),
+                              'fastllm.chat.MediaUrl': ('chat.html#mediaurl', 'fastllm/chat.py'),
+                              'fastllm.chat.MediaUrl.__init__': ('chat.html#mediaurl.__init__', 'fastllm/chat.py'),
                               'fastllm.chat.Msg.text': ('chat.html#msg.text', 'fastllm/chat.py'),
                               'fastllm.chat.StopReasonCallback': ('chat.html#stopreasoncallback', 'fastllm/chat.py'),
                               'fastllm.chat.StopReasonCallback.after_acomplete': ( 'chat.html#stopreasoncallback.after_acomplete',
@@ -113,19 +118,19 @@ d = { 'settings': { 'branch': 'main',
                               'fastllm.chat._has_stop': ('chat.html#_has_stop', 'fastllm/chat.py'),
                               'fastllm.chat._inject_tool_reminder': ('chat.html#_inject_tool_reminder', 'fastllm/chat.py'),
                               'fastllm.chat._lite_call_func': ('chat.html#_lite_call_func', 'fastllm/chat.py'),
+                              'fastllm.chat._mime2part_type': ('chat.html#_mime2part_type', 'fastllm/chat.py'),
                               'fastllm.chat._mk_content': ('chat.html#_mk_content', 'fastllm/chat.py'),
                               'fastllm.chat._mk_prefill': ('chat.html#_mk_prefill', 'fastllm/chat.py'),
                               'fastllm.chat._mk_result_fence': ('chat.html#_mk_result_fence', 'fastllm/chat.py'),
                               'fastllm.chat._mk_tool_result': ('chat.html#_mk_tool_result', 'fastllm/chat.py'),
                               'fastllm.chat._split_fence_msgs': ('chat.html#_split_fence_msgs', 'fastllm/chat.py'),
                               'fastllm.chat._split_msg_on_fences': ('chat.html#_split_msg_on_fences', 'fastllm/chat.py'),
-                              'fastllm.chat._srv_tc_summary': ('chat.html#_srv_tc_summary', 'fastllm/chat.py'),
-                              'fastllm.chat._srvtools': ('chat.html#_srvtools', 'fastllm/chat.py'),
                               'fastllm.chat._tc_summary': ('chat.html#_tc_summary', 'fastllm/chat.py'),
                               'fastllm.chat._think_kw': ('chat.html#_think_kw', 'fastllm/chat.py'),
                               'fastllm.chat._trunc_content': ('chat.html#_trunc_content', 'fastllm/chat.py'),
                               'fastllm.chat._trunc_param': ('chat.html#_trunc_param', 'fastllm/chat.py'),
                               'fastllm.chat._trunc_str': ('chat.html#_trunc_str', 'fastllm/chat.py'),
+                              'fastllm.chat._url2content': ('chat.html#_url2content', 'fastllm/chat.py'),
                               'fastllm.chat._usrtools': ('chat.html#_usrtools', 'fastllm/chat.py'),
                               'fastllm.chat.add_warning': ('chat.html#add_warning', 'fastllm/chat.py'),
                               'fastllm.chat.adisplay_stream': ('chat.html#adisplay_stream', 'fastllm/chat.py'),
@@ -137,7 +142,6 @@ d = { 'settings': { 'branch': 'main',
                               'fastllm.chat.lite_mk_func': ('chat.html#lite_mk_func', 'fastllm/chat.py'),
                               'fastllm.chat.mk_msg': ('chat.html#mk_msg', 'fastllm/chat.py'),
                               'fastllm.chat.mk_msgs': ('chat.html#mk_msgs', 'fastllm/chat.py'),
-                              'fastllm.chat.mk_srv_tc_details': ('chat.html#mk_srv_tc_details', 'fastllm/chat.py'),
                               'fastllm.chat.mk_tr_details': ('chat.html#mk_tr_details', 'fastllm/chat.py'),
                               'fastllm.chat.postproc': ('chat.html#postproc', 'fastllm/chat.py'),
                               'fastllm.chat.remove_cache_ckpts': ('chat.html#remove_cache_ckpts', 'fastllm/chat.py'),
@@ -270,18 +274,22 @@ d = { 'settings': { 'branch': 'main',
                                'fastllm.types.ToolCall': ('types.html#toolcall', 'fastllm/types.py'),
                                'fastllm.types.ToolCall._repr_markdown_': ('types.html#toolcall._repr_markdown_', 'fastllm/types.py'),
                                'fastllm.types.Usage': ('types.html#usage', 'fastllm/types.py'),
+                               'fastllm.types._fetch_url_partial': ('types.html#_fetch_url_partial', 'fastllm/types.py'),
                                'fastllm.types._trunc_strs': ('types.html#_trunc_strs', 'fastllm/types.py'),
+                               'fastllm.types.approx_pricing': ('types.html#approx_pricing', 'fastllm/types.py'),
                                'fastllm.types.data_url': ('types.html#data_url', 'fastllm/types.py'),
                                'fastllm.types.display_list': ('types.html#display_list', 'fastllm/types.py'),
                                'fastllm.types.fn_schema': ('types.html#fn_schema', 'fastllm/types.py'),
                                'fastllm.types.get_api_key': ('types.html#get_api_key', 'fastllm/types.py'),
                                'fastllm.types.get_model_info': ('types.html#get_model_info', 'fastllm/types.py'),
                                'fastllm.types.get_model_meta': ('types.html#get_model_meta', 'fastllm/types.py'),
+                               'fastllm.types.get_model_pricing': ('types.html#get_model_pricing', 'fastllm/types.py'),
                                'fastllm.types.infer_api_name': ('types.html#infer_api_name', 'fastllm/types.py'),
                                'fastllm.types.mk_completion': ('types.html#mk_completion', 'fastllm/types.py'),
                                'fastllm.types.mk_tool_res_msg': ('types.html#mk_tool_res_msg', 'fastllm/types.py'),
                                'fastllm.types.model_prices_meta': ('types.html#model_prices_meta', 'fastllm/types.py'),
                                'fastllm.types.part_txt': ('types.html#part_txt', 'fastllm/types.py'),
                                'fastllm.types.payload_kwargs': ('types.html#payload_kwargs', 'fastllm/types.py'),
+                               'fastllm.types.register_model_info': ('types.html#register_model_info', 'fastllm/types.py'),
                                'fastllm.types.sys_text': ('types.html#sys_text', 'fastllm/types.py'),
                                'fastllm.types.url_mime': ('types.html#url_mime', 'fastllm/types.py')}}}

{python_fastllm-0.0.8 → python_fastllm-0.0.10}/fastllm/acomplete.py

@@ -7,7 +7,7 @@ __all__ = ['specs_path', 'ant_spec', 'oai_spec', 'gem_spec', 'vendor_mapping', '
            'ContextWindowExceededError', 'acomplete']
 
 # %% ../nbs/06_acomplete.ipynb #f2f57253
-import json
+import asyncio,json,httpx
 from importlib.resources import files
 from fastcore.utils import *
 from fastcore.meta import *
@@ -49,7 +49,8 @@ api2spec = {'openai':oai_spec, 'openai_chat':oai_spec, 'anthropic':ant_spec, 'ge
 
 # %% ../nbs/06_acomplete.ipynb #79075d95
 @flexicache()
-def mk_client(model=None, vendor_name=None, api_name=None, api_key=None, base_url=None, xtra_hdrs=None):
+def mk_client(model=None, vendor_name=None, api_name=None, api_key=None, base_url=None, xtra_hdrs=None,
+    timeout=httpx.Timeout(connect=30, read=300, write=30, pool=10)):
     err_msg = f"please pass a valid one vendor: {', '.join(list(vendor_mapping))} or pass `api_name`,`base_url` and `api_key`"
     if vendor_name:
         override_base_url = base_url
@@ -67,7 +68,7 @@ def mk_client(model=None, vendor_name=None, api_name=None, api_key=None, base_ur
     else: raise ValueError(f"Model {model} can't be auto resolved, {err_msg}")
     api = api_registry.apis[api_name]
     spec, hdrs = api2spec[api_name], api.get_hdrs(api_key)
-    cli = OpenAPIClient(spec, headers=merge(hdrs, ifnone(xtra_hdrs, {})))
+    cli = OpenAPIClient(spec, headers=merge(hdrs, ifnone(xtra_hdrs, {})), timeout=timeout)
     if base_url is not None:
         for op in cli.ops: op.base_url = base_url  # pyright: ignore[reportAttributeAccessIssue]
     return cli, api_name, vendor_name
@@ -113,14 +114,36 @@ def _debug_print(model, api_name, vendor_name, payload, func):
     print(f"\033[1;33mpayload:\033[0m\n{pformat(p, width=120, sort_dicts=False)}")
     print('━'*60)
 
+# %% ../nbs/06_acomplete.ipynb #497c8565
+async def _raise_if_done(e, n, retries, retry_delay, yielded=False):
+    e = _classify_error(e)
+    if yielded or not e.retryable or n == retries: raise e
+    await asyncio.sleep(retry_delay*2**n)
+
+async def _retry_call(f, retries=2, retry_delay=0.5):
+    for n in range(retries+1):
+        try: return await f()
+        except APIError as e: await _raise_if_done(e, n, retries, retry_delay)
+
+async def _retry_stream(mk_gen, retries=2, retry_delay=0.5):
+    for n in range(retries+1):
+        yielded = False
+        try:
+            async for o in mk_gen():
+                yielded = True
+                yield o
+            return
+        except APIError as e: await _raise_if_done(e, n, retries, retry_delay, yielded=yielded)
+
 # %% ../nbs/06_acomplete.ipynb #2379ec94
 @delegates(payload_kwargs)
-async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None, base_url=None, xtra_body=None, xtra_hdrs=None,
-    stream=False, stop_callables=None, stop_sequences=None, **kwargs):
+async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None,
+                    base_url=None, xtra_body=None, xtra_hdrs=None, stream=False,
+                    stop_callables=None, retries=2, retry_delay=0.5, **kwargs):
     "Unified completion across different APIs."
     cli, api_name, vendor_name = mk_client(model, vendor_name, api_name, api_key, base_url, xtra_hdrs)
     api = api_registry.apis[api_name]
-    payload = api.mk_payload(msgs, model, stream=stream, stop_callables=stop_callables, **kwargs)
+    payload = api.mk_payload(msgs, model, stream=stream, **kwargs)
     payload = merge(payload, ifnone(xtra_body, {}))
     if vendor_name == 'codex':
         for k in 'temperature max_tokens max_output_tokens max_completion_tokens metadata'.split(): payload.pop(k, None)
@@ -130,7 +153,13 @@ async def acomplete(msgs, model, api_name=None, vendor_name=None, api_key=None,
         if vendor_name == 'moonshot' and 'kimi' in model: payload['messages'][-1]['partial'] = True
     func = attrgetter(api.op_path[stream])(cli)
     if defaults.debug_mode: _debug_print(model, api_name, vendor_name, payload, func)
-    try: resp = await func(**payload)
-    except APIError as e: raise _classify_error(e) from e
-    if stream: return _classify_error_stream(api.acollect_stream(resp, model=model, vendor_name=vendor_name, stop_callables=stop_callables))
-    return mk_completion(resp, model=model, api_name=api_name, vendor_name=vendor_name)
+    async def _call(): return await func(**payload)
+    if not stream:
+        resp = await _retry_call(_call, retries, retry_delay)
+        return mk_completion(resp, model=model, api_name=api_name, vendor_name=vendor_name)
+
+    async def _mk_gen():
+        resp = await _call()
+        async for o in api.acollect_stream(resp, model=model, vendor_name=vendor_name, stop_callables=stop_callables): yield o
+
+    return _retry_stream(_mk_gen, retries, retry_delay)

{python_fastllm-0.0.8 → python_fastllm-0.0.10}/fastllm/anthropic.py

@@ -50,10 +50,8 @@ def finalize_usage(usg, parts):
     rc = '\n'.join(p.text or '' for p in parts if p.type == PartType.thinking)
     ct = int(usg.raw.get('output_tokens', usg.completion_tokens) or 0)
     rt = min(int(len(rc.split())*1.5), ct) if rc else 0
-    res = Usage(prompt_tokens=usg.prompt_tokens, completion_tokens=ct-rt, total_tokens=usg.prompt_tokens+ct,
-                 cached_tokens=usg.cached_tokens, cache_creation_tokens=usg.cache_creation_tokens, reasoning_tokens=rt, raw=usg.raw)
-    print(res)
-    return res
+    return Usage(prompt_tokens=usg.prompt_tokens, completion_tokens=ct-rt, total_tokens=usg.prompt_tokens+ct,
+        cached_tokens=usg.cached_tokens, cache_creation_tokens=usg.cache_creation_tokens, reasoning_tokens=rt, raw=usg.raw)
 
 # %% ../nbs/04_anthropic.ipynb #7a8b1f8f
 def norm_finish(resp, tcs=None):
@@ -92,7 +90,9 @@ def norm_sse_event(ev, **kwargs):
     if typ == "content_block_start":
         cb = ev.get("content_block", {})
         if cb.get("type", "").endswith("_tool_result"): return Delta(server_tool_result=cb, raw=ev, **kwargs)
-        if tc := norm_tool_call(cb): tcs = [tc]
+        if tc := norm_tool_call(cb):
+            if not tc.arguments: tc.arguments = {'_delta': ''}
+            tcs = [tc]
     elif typ == "content_block_delta":
         d = ev.get("delta", {})
         dtyp = d.get("type")

{python_fastllm-0.0.8 → python_fastllm-0.0.10}/fastllm/chat.py

@@ -3,13 +3,13 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/07_chat.ipynb.
 
 # %% auto #0
-__all__ = ['tool_dtls_tag', 're_tools', 'token_dtls_tag', 're_token', 'effort', 'remove_cache_ckpts', 'contents', 'stop_reason',
-           'mk_msg', 'FenceToolStop', 'extract_fence_call', 'split_tools', 'fmt2hist', 'mk_msgs', 'cite_footnote',
-           'postproc', 'lite_mk_func', 'ToolResponse', 'structured', 'StopResponse', 'FullResponse', 'search_count',
-           'UsageStats', 'AsyncChat', 'astream_with_complete', 'ChatCallback', 'DeepseekMsgsCallback',
+__all__ = ['tool_dtls_tag', 're_tools', 'token_dtls_tag', 're_token', 'effort', 'MediaUrl', 'remove_cache_ckpts', 'contents',
+           'stop_reason', 'mk_msg', 'FenceToolStop', 'extract_fence_call', 'split_tools', 'fmt2hist', 'mk_msgs',
+           'cite_footnote', 'postproc', 'lite_mk_func', 'ToolResponse', 'structured', 'StopResponse', 'FullResponse',
+           'search_count', 'UsageStats', 'AsyncChat', 'astream_with_complete', 'ChatCallback', 'DeepseekMsgsCallback',
            'DeepseekPrefillCallback', 'add_warning', 'StopReasonCallback', 'run_fence_tool', 'FenceToolCallback',
-           'ToolReminderCallback', 'stop_sequences', 'StopSequencesCallback', 'mk_tr_details', 'mk_srv_tc_details',
-           'StreamFormatter', 'AsyncStreamFormatter', 'adisplay_stream']
+           'ToolReminderCallback', 'stop_sequences', 'StopSequencesCallback', 'mk_tr_details', 'StreamFormatter',
+           'AsyncStreamFormatter', 'adisplay_stream']
 
 # %% ../nbs/07_chat.ipynb #d5a3bc1f
 import asyncio, base64, json, mimetypes, random, string, ast, warnings
@@ -25,15 +25,30 @@ from dataclasses import dataclass
 from .types import *
 from .acomplete import *
 
-# %% ../nbs/07_chat.ipynb #90f55ad4
+# %% ../nbs/07_chat.ipynb #1b75c262
+class MediaUrl(BasicRepr):
+    "Direct URL media reference"
+    def __init__(self, url, mime=None): self.url, self.mime = url, ifnone(mime, url_mime(url))
+
+# %% ../nbs/07_chat.ipynb #eb557831
+def _mime2part_type(mime):
+    "Map MIME string to canonical PartType"
+    if mime.startswith('image/'): return PartType.input_image
+    if mime.startswith('audio/'): return PartType.input_audio
+    if mime.startswith('video/'): return PartType.input_video
+    return PartType.input_file
+
 def _bytes2content(data):
-    "Convert bytes to litellm content dict (image, pdf, audio, video)"
+    "Convert bytes to fastllm canonical content"
     mtype = detect_mime(data)
     if not mtype: raise ValueError(f'Data must be a supported file type, got {data[:10]}')
-    encoded = base64.b64encode(data).decode("utf-8")    
-    if mtype.startswith('image/'): return Part(type=PartType.input_image, text=f'data:{mtype};base64,{encoded}')
-    return Part(type=PartType.input_file, text=f'data:{mtype};base64,{encoded}')
+    encoded = base64.b64encode(data).decode("utf-8")
+    return Part(type=_mime2part_type(mtype), text=f'data:{mtype};base64,{encoded}')
 
+def _url2content(o):
+    "Convert MediaUrl to fastllm canonical content"
+    mime = o.mime or url_mime(o.url)
+    return Part(type=_mime2part_type(mime), text=o.url, data=dict(mime=mime))
 
 # %% ../nbs/07_chat.ipynb #48c78e48
 def _add_cache_control(msg,          # LiteLLM formatted msg
@@ -57,8 +72,9 @@ def remove_cache_ckpts(msg):
     return msg
 
 def _mk_content(o):
-    if isinstance(o, str): return Part(type=PartType.text, text=o)
-    elif isinstance(o,bytes): return _bytes2content(o)
+    if isinstance(o, str):        return Part(type=PartType.text, text=o)
+    elif isinstance(o, bytes):    return _bytes2content(o)
+    elif isinstance(o, MediaUrl): return _url2content(o)
     return o
 
 def contents(c):
@@ -427,7 +443,6 @@ class AsyncChat:
         return self
 
 # %% ../nbs/07_chat.ipynb #2e469ea1
-def _srvtools(tcs): return L(tcs).filter(lambda o: o.server) if tcs else None
 def _usrtools(tcs): return L(tcs).filter(lambda o: not o.server) if tcs else None
 
 # %% ../nbs/07_chat.ipynb #19b87f53
@@ -511,8 +526,6 @@ async def _call(self:AsyncChat, msg=None, prefill=None, temp=None, think=None, s
 
     self.toolloop, self.prompt, tmsg = False, None, None
     async for o in self._call_cbs('before_tool_calls'): yield o
-    if stcs:= _srvtools(res.tool_calls): 
-        for tc in stcs: yield tc
     if tcs := _usrtools(res.tool_calls):
         tres = await parallel_async(_alite_call_func, tcs, timeout=tc_timeout, n_workers=n_workers, pause=pause, **self.tcdict)
         tmsg = mk_tool_res_msg(tcs, tres)
@@ -703,15 +716,9 @@ def _trunc_param(v, mx=40):
 def _tc_summary(tr):
     "Format tool call as func(params) → result string"
     params = ', '.join(f"{k}={_trunc_param(v)}" for k,v in tr.data['arguments'].items())
-    res = f"→{_trunc_param(tr.text)}"
+    res = f"→{_trunc_param(tr.text)}" if tr.text else ''
     return '<code>'+escape(f"{tr.data['name']}({params}){res}")+'</code>'
 
-# %% ../nbs/07_chat.ipynb #91beb26c
-def _srv_tc_summary(tc):
-    "Format tool call as func(params) → result string"
-    params = ', '.join(f"{k}={_trunc_param(v)}" for k,v in tc.arguments.items())
-    return '<code>'+escape(f"{tc.name}({params})")+'</code>'
-
 # %% ../nbs/07_chat.ipynb #80f344cc
 def _trunc_content(content, mx):
     "Truncate tool result content, respecting '_full' flag"
@@ -722,23 +729,13 @@ def _trunc_content(content, mx):
 def mk_tr_details(tr, mx=2000):
     "Create <details> block for tool call as JSON"
     args = {k:_trunc_str(v, mx=mx*5) for k,v in tr.data['arguments'].items()}
-    res = {'id':tr.data['id'], 'server':False,
+    res = {'id':tr.data['id'], 'server':tr.data.get('server', False),
            'call':{'function': tr.data['name'], 'arguments': args},
            'result':_trunc_content(tr.text, mx=mx),}
     summ = f"<summary>{_tc_summary(tr)}</summary>"
     return f"\n\n{tool_dtls_tag}\n{summ}\n\n```json\n{dumps(res, indent=2, ensure_ascii=False)}\n```\n\n</details>\n\n"
 
-# %% ../nbs/07_chat.ipynb #3049001c
-def mk_srv_tc_details(tc, mx=2000):
-    "Create <details> block for tool call as JSON"
-    args = {k:_trunc_str(v, mx=mx*5) for k,v in tc.arguments.items()}
-    res = {'id':tc.id, 'server':True, 'call':{'function': tc.name, 'arguments': args}, 'result':"Server tool call executed."}
-    summ = f"<summary>{_srv_tc_summary(tc)}</summary>"
-    return f"\n\n{tool_dtls_tag}\n{summ}\n\n```json\n{dumps(res, indent=2, ensure_ascii=False)}\n```\n\n</details>\n\n"
-
 # %% ../nbs/07_chat.ipynb #f0d984ec
-# status_re = re.compile(r'^- ⏳ <code>(.*)</code> ⏳$|^🧠+$', re.MULTILINE) # TODO: Need to yield tool calls as they are done collated in fastllm `_acollect_stream`
-
 class StreamFormatter:
     def __init__(self, mx=2000, debug=False, showthink=False):
         self.outp,self.tcs = '',{}
@@ -754,8 +751,8 @@ class StreamFormatter:
                 res+= '🧠' if not self.outp or self.outp[-1]=='🧠' else '\n\n🧠'
             elif self.outp and self.outp[-1] == '🧠': res+= '\n\n'
             if txt:=o.get('text'): res+=f"\n\n{txt}" if res and res[-1] == '🧠' else txt
-        if isinstance(o, ToolCall):
-            res += mk_srv_tc_details(o)
+        if isinstance(o, Part) and o.type==PartType.tool_use:
+            res += f"\n- ⏳ {_tc_summary(o)} ⏳\n"
         if isinstance(o, Part) and o.type == PartType.tool_result:
             res += mk_tr_details(o,mx=self.mx)
         self.outp+=res

{python_fastllm-0.0.8 → python_fastllm-0.0.10}/fastllm/streaming.py

@@ -105,14 +105,14 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
         idx,last_idx = index_fn(d, typ, last_typ, last_idx)
         return idx
     def _proc(d, name, pt=None, kw='txt', ret=None):
-        if not ret and not (val := getattr(d, name)): return
+        if not ret and not (val := getattr(d, name)): return None, None
         idx = _fidx(d, name, pt)
         part_accum.append(typ, idx, **(ret or {kw: val}))
-        return ret or {name: val}
+        return ret or {name: val}, idx
     def _yield_parts(d):
         for args in [('text',), ('thinking',), ('citations', 'text', 'citations')]:
-            if (r := _proc(d, args[0], pt=args[1] if len(args)>1 else None, kw=args[2] if len(args)>2 else 'txt')):
-                yield r
+            r = _proc(d, args[0], pt=args[1] if len(args)>1 else None, kw=args[2] if len(args)>2 else 'txt')
+            if r[0]: yield r[0]
     stop, stop_yielded = False, False
     async for d in it:
         # Check stop condition and yield stop delta
@@ -127,11 +127,26 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
         # Rest incl. tools, finish reason, usage is processed independently
         for tc in d.tool_calls:
             args = tc.arguments.get('_delta', tc.arguments)
-            _proc(d, 'tool_use', ret=dict(id=tc.id, name=tc.name, arguments=args, server=tc.server, extra=tc.extra))
+            _, idx = _proc(d, 'tool_use', ret=dict(id=tc.id, name=tc.name, arguments=args, server=tc.server, extra=tc.extra))
+            if (isinstance(args, str) and args.endswith('}')) or (isinstance(args, dict) and '_delta' not in tc.arguments): # tool call ready
+                if isinstance(args, str):
+                    try: args = json.loads(part_accum.parts[idx].arguments) if args else {}
+                    except json.JSONDecodeError: continue
+                acc = part_accum.parts[idx]
+                acc.arguments = args
+                data = {**acc.extra, 'id':acc.id, 'name':acc.name, 'arguments':args, 'server':acc.server}
+                yield Part(type=PartType.tool_use, data=data)
+                # Server tool results for anthropic are yielded in d.server_tool_result by checking injected dummy `_delta`
+                if acc.server and '_delta' not in tc.arguments: yield Part(type=PartType.tool_result, text="Server tool call executed.", data=data)
         if d.server_tool_result:
             idx = _fidx(d, 'server_tool_result')
             part_accum.parts[idx] = Part(type=typ, data=d.server_tool_result)
-        if (r:=_proc(d, 'refusal')): yield r
+            srv_tc = next((p for p in reversed(list(part_accum.parts.values())) if isinstance(p, ToolCall) and p.server), None)
+            if srv_tc:
+                data = {**srv_tc.extra, 'id':srv_tc.id, 'name':srv_tc.name, 'arguments':srv_tc.arguments, 'server':True}
+                yield Part(type=PartType.tool_result, text="Server tool call executed.", data=data)
+        r = _proc(d, 'refusal')
+        if r[0]: yield r[0]
         if d.finish_reason: fin = d.finish_reason
         if d.usage: usg = d.usage
         last_typ = typ
@@ -146,4 +161,3 @@ async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_nam
             message=Msg(role="assistant", content=part_accum.parts),
             finish_reason=fin, usage=usg, tool_calls=tcs, api_name=api_name, vendor_name=vendor_name,
             raw={'deltas':deltas})
-

{python_fastllm-0.0.8 → python_fastllm-0.0.10}/fastllm/types.py

@@ -4,12 +4,14 @@
 
 # %% auto #0
 __all__ = ['PartType', 'FinishReason', 'api_registry', 'model_prices_url', 'haik45', 'sonn45', 'sonn', 'sonn46', 'opus46', 'opus',
-           'gpt54', 'gpt54m', 'codex54', 'codex55', 'codex53spark', 'codex_pricing', 'Part', 'Msg', 'ToolCall',
-           'display_list', 'Usage', 'Completion', 'APIRegistry', 'mk_completion', 'mk_tool_res_msg', 'fn_schema',
-           'sys_text', 'part_txt', 'data_url', 'url_mime', 'payload_kwargs', 'get_api_key', 'model_prices_meta',
-           'infer_api_name', 'get_model_meta', 'get_model_info']
+           'gpt54', 'gpt54m', 'gpt55', 'codex54', 'codex54m', 'codex55', 'codex53spark', 'model_info_registry',
+           'deepseek_v4_common', 'codex_pricing', 'Part', 'Msg', 'ToolCall', 'display_list', 'Usage', 'Completion',
+           'APIRegistry', 'mk_completion', 'mk_tool_res_msg', 'fn_schema', 'sys_text', 'part_txt', 'data_url',
+           'url_mime', 'payload_kwargs', 'get_api_key', 'model_prices_meta', 'infer_api_name', 'get_model_meta',
+           'register_model_info', 'get_model_info', 'get_model_pricing', 'approx_pricing']
 
 # %% ../nbs/00_types.ipynb #b4d047fd
+import httpx
 from dataclasses import dataclass, field
 from fastcore.net import urljson
 from fastcore.utils import *
@@ -27,14 +29,16 @@ PartType = str_enum('PartType', 'text', 'thinking', 'refusal', 'tool_use', 'serv
                     'input_image', 'input_audio', 'input_video', 'input_file')
 
 # %% ../nbs/00_types.ipynb #2eeff103
-def _trunc_strs(d, n=200):
-    "Return copy of dict `d` with str values >n chars truncated to first 10 chars + '...'"
-    if not d: return d
-    return {k: (v[:10]+'...' if isinstance(v,str) and len(v)>n else v) for k,v in d.items()}
+def _trunc_strs(o, n=200):
+    "Truncate str or dict"
+    if not o: return o
+    if isinstance(o,str) and len(o)>n: return o[:100]+'...'
+    if isinstance(o,dict): return {k: (v[:100]+'...' if isinstance(v,str) and len(v)>n else v) for k,v in o.items()}
+    return o
 
 @patch
 def _repr_markdown_(self: Part):
-    body = self.text if self.text else ''
+    body = _trunc_strs(self.text) if self.text else ''
     data = _trunc_strs(self.data)
     return f"""**Part** (`{self.type}`)
 
@@ -197,7 +201,17 @@ def sys_text(system):
 
 def part_txt(p): return p.text if isinstance(p,Part) else p
 
-# %% ../nbs/00_types.ipynb #dc2b75a0
+# %% ../nbs/00_types.ipynb #f3deb055
+@flexicache(time_policy(24*3600))
+def _fetch_url_partial(url, nbytes=512): 
+    "Fetch remote media bytes, optionally only first `nbytes`."
+    try:
+        with httpx.stream('GET', url, headers={'Range': f'bytes=0-{nbytes-1}'}, follow_redirects=True) as r:
+            if r.status_code not in (200, 206): return
+            return r.read()
+    except (httpx.HTTPError, httpx.InvalidURL): return
+
+# %% ../nbs/00_types.ipynb #70a9a0c3
 _ext_mime = {
     '.jpg':'image/jpeg', '.jpeg':'image/jpeg', '.png':'image/png', '.gif':'image/gif', '.webp':'image/webp',
     '.pdf':'application/pdf',
@@ -213,22 +227,24 @@ def data_url(url):
     return header[5:].split(';',1)[0].strip() or 'application/octet-stream', body
 
 def url_mime(url, default='application/octet-stream'):
-    "Guess mime from URL extension."
+    "Guess mime from URL extension, and optional bytes fallback."
+    if "youtube.com" in url or "youtu.be" in url: return "video/mp4"
     ext = '.' + url.rsplit('.', 1)[-1].split('?')[0].lower() if '.' in url.split('?')[0].split('/')[-1] else ''
-    return _ext_mime.get(ext, default)
+    if (mime:=_ext_mime.get(ext)) is None: return detect_mime(_fetch_url_partial(url))
+    return ifnone(mime, default)
 
 # %% ../nbs/00_types.ipynb #28c698fe
-def payload_kwargs(msgs, model, stream=False, system=None, max_tokens=None, temperature=None, tools=None, tool_choice=None, reasoning_effort=None, web_search_options=None, stop_callables=None, stop_sequences=None): pass
+def payload_kwargs(msgs, model, stream=False, system=None, max_tokens=None, temperature=None, tools=None, tool_choice=None, reasoning_effort=None, web_search_options=None, stop_callables=None): pass
 
 # %% ../nbs/00_types.ipynb #c2a2cb49
 def get_api_key(api_key, default):
-    err = ValueError(f"Missing API key: make sure to have the expected env var name or pass `api_key`")
     key = api_key or os.getenv(default)
-    if not key: raise err
+    if not key: raise ValueError(f"Missing API key: set environment variable '{default}' or pass `api_key` parameter")
     return key
 
 # %% ../nbs/00_types.ipynb #852adecd
 model_prices_url = 'https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json'
+
 @flexicache(time_policy(24*60*60))
 def model_prices_meta(): return urljson(model_prices_url)
 
@@ -258,63 +274,94 @@ opus46 = "claude-opus-4-6"
 opus = "claude-opus-4-7"
 gpt54 = "gpt-5.4"
 gpt54m = "gpt-5.4-mini"
+gpt55 = "gpt-5.5"
 codex54 = "gpt-5.4"
+codex54m = "gpt-5.4-mini"
 codex55 = "gpt-5.5"
 codex53spark = "gpt-5.3-codex-spark"
 
-# %% ../nbs/00_types.ipynb #d6d5b98c
-codex_pricing = {
-    "input_cost_per_token": 0.10 / 1_000_000,
-    "cache_creation_input_token_cost": 0.10 / 1_000_000,
-    "cache_read_input_token_cost": 0.10 / 1_000_000,
-    "output_cost_per_token": 0.50 / 1_000_000,
-}
+# %% ../nbs/00_types.ipynb #583e017b
+model_info_registry = {}
 
-_codex_overrides = {
-    codex53spark: dict(
-        supports_vision=False, supports_image_input=False, supports_web_search=True, supports_reasoning=True,
-        max_tokens=128000, max_input_tokens=128000, max_output_tokens=128000)
-}
+def register_model_info(model, vendor_name=None, base=None, base_vendor_name=None, **overrides):
+    "Register model metadata, optionally starting from `base`."
+    info = dict(get_model_info(base, base_vendor_name or vendor_name)) if base else {}
+    info.update(overrides)
+    model_info_registry[vendor_name, model] = info
 
-# %% ../nbs/00_types.ipynb #fbfdeb0a
-def get_model_info(mn, vendor_name=None, strict=False):
-    info = get_model_meta(mn, 'chatgpt' if vendor_name=='codex' else vendor_name)
-    # anthropic web search 
+def get_model_info(mn, vendor_name=None):
+    info = model_info_registry.get((vendor_name, mn)) or get_model_meta(mn, vendor_name)
     if 'search_context_cost_per_query' in info: info['supports_web_search'] = True
-    # kimi
-    if 'kimi' in mn: 
-        if 'k2p6' in mn: info = get_model_meta(mn.replace('k2p6', 'k2p5'), vendor_name)
-        info['supports_reasoning'] = True
-        info['supports_vision'] = True
-        if vendor_name == 'moonshot': info['supports_assistant_prefill'] = True
-    # gpt web search
-    if mn in ("gpt-5.4", "gpt-5.4-mini"):
-        info['supports_web_search'] = True
-        info.pop('mode', None)
-    # codex updates
-    if vendor_name == 'codex':
-        info = merge(info, codex_pricing)
-        info |= _codex_overrides.get(mn, {})
-    # deepseek v4
-    if vendor_name == 'deepseek' and mn in ("deepseek-v4-flash", "deepseek-v4-pro"):
-        info = dict(get_model_meta("deepseek/deepseek-v3.2"))
-        info |= dict(supports_assistant_prefill=True, supports_function_calling=True, supports_prompt_caching=True,
-            supports_reasoning=True, supports_tool_choice=True)
-        info.update(input_cost_per_token=1.4e-07, input_cost_per_token_cache_hit=2.8e-09, output_cost_per_token=2.8e-07,
-            max_input_tokens=1048576, max_output_tokens=393216, max_tokens=393216)
-        if 'pro' in mn: info = {**info, 'input_cost_per_token': 4.35e-07, 'input_cost_per_token_cache_hit': 3.625e-09, 'output_cost_per_token': 8.7e-07}
-    # qwen 3p6
-    if vendor_name == 'fireworks_ai' and mn == 'accounts/fireworks/models/qwen3p6-plus':
-        info = dict(supports_vision=True, supports_reasoning=True, supports_function_calling=True, supports_tool_choice=True,
-                    supports_system_messages=True, supports_response_schema=True, supports_parallel_function_calling=True,
-                    supports_prompt_caching=True, supports_native_streaming=True, supports_native_structured_output=True,
-                    max_tokens=1000000, max_input_tokens=1000000, max_output_tokens=65536,
-                    input_cost_per_token=0.5e-6, cache_read_input_token_cost=0.1e-6, output_cost_per_token=3.0e-6)
-    
-    # unresolved models
-    if not info and not strict: info = info | codex_pricing
     return dict2obj(info)
 
+# %% ../nbs/00_types.ipynb #8261dcd0
+register_model_info('accounts/fireworks/models/qwen3p6-plus', vendor_name='fireworks_ai',
+    supports_vision=True, supports_reasoning=True, supports_function_calling=True, supports_tool_choice=True,
+    supports_system_messages=True, supports_response_schema=True, supports_parallel_function_calling=True,
+    supports_prompt_caching=True, supports_native_streaming=True, supports_native_structured_output=True,
+    max_tokens=1000000, max_input_tokens=1000000, max_output_tokens=65536,
+    input_cost_per_token=0.5e-6, cache_read_input_token_cost=0.1e-6, output_cost_per_token=3.0e-6)
+
+register_model_info('gemini-3.5-flash', vendor_name='gemini', base='gemini-3-flash-preview',
+    input_cost_per_token=1.5e-6, output_cost_per_token=9e-6,
+    output_cost_per_reasoning_token=9e-6, cache_read_input_token_cost=1.5e-7)
+
+for model in ('gpt-5.4', 'gpt-5.4-mini'):
+    register_model_info(model, vendor_name='openai', base=model, supports_web_search=True, mode=None)
+
+for model in ('kimi-k2.5', 'kimi-k2.6'):
+    register_model_info(model, vendor_name='moonshot', base=f'moonshot/{model}', base_vendor_name=None,
+        supports_reasoning=True, supports_vision=True, supports_assistant_prefill=True)
+
+register_model_info('gemini-3.1-flash-lite', vendor_name='gemini', base='gemini-3.1-flash-lite-preview')
+register_model_info('models/gemini-3.1-flash-lite', vendor_name='gemini', base='gemini-3.1-flash-lite-preview')
+
+for model in ('accounts/fireworks/models/kimi-k2p5', 'accounts/fireworks/models/kimi-k2p6'):
+    register_model_info(model, vendor_name='fireworks_ai', base=model.replace('k2p6', 'k2p5'),
+        supports_reasoning=True, supports_vision=True,
+        input_cost_per_token=0.95e-6, cache_read_input_token_cost=0.16e-6, output_cost_per_token=4.0e-6)
+
+# %% ../nbs/00_types.ipynb #948d55d0
+deepseek_v4_common = dict(
+    supports_assistant_prefill=True, supports_function_calling=True, supports_prompt_caching=True,
+    supports_reasoning=True, supports_tool_choice=True,
+    max_input_tokens=1048576, max_output_tokens=393216, max_tokens=393216)
+
+register_model_info('deepseek-v4-flash', vendor_name='deepseek', base='deepseek/deepseek-v3.2', **deepseek_v4_common,
+    input_cost_per_token=1.4e-07, input_cost_per_token_cache_hit=2.8e-09,
+    output_cost_per_token=2.8e-07, cache_read_input_token_cost=1.4e-07/10)
+register_model_info('deepseek-v4-pro', vendor_name='deepseek', base='deepseek/deepseek-v3.2', **deepseek_v4_common,
+    input_cost_per_token=4.35e-07, input_cost_per_token_cache_hit=3.625e-09,
+    output_cost_per_token=8.7e-07, cache_read_input_token_cost=4.35e-07/10)
+
+# %% ../nbs/00_types.ipynb #2c23d11e
+codex_pricing = dict(
+    input_cost_per_token = 0.10/1_000_000, output_cost_per_token = 0.50/1_000_000,
+    cache_creation_input_token_cost = 0.10/1_000_000, cache_read_input_token_cost = 0.10/1_000_000)
+
+for model in (codex54, codex54m, codex55):
+    register_model_info(model, 'codex', base=model, base_vendor_name='chatgpt', supports_web_search=True, **codex_pricing)
+
+register_model_info(codex53spark, 'codex', **codex_pricing,
+    supports_vision=False, supports_image_input=False, supports_web_search=True, supports_reasoning=True,
+    max_tokens=128000, max_input_tokens=128000, max_output_tokens=128000)
+
+
+# %% ../nbs/00_types.ipynb #24cc47ec
+def get_model_pricing(mn, vendor_name, million=True):
+    return {k:round(v * (1e6 if million else 1), 6)
+        for k,v in get_model_info(mn, vendor_name).items()
+        if 'cost' in k and isinstance(v,float) and 'priority' not in k}
+
+# %% ../nbs/00_types.ipynb #79304cd9
+def approx_pricing(nm, vendor_name, out=10, cache=80, inp=10, markup=0):
+    "Approx cost per million tokens with given output/cache/input proportions"
+    p = get_model_pricing(nm, vendor_name)
+    ic = p.get('cache_creation_input_token_cost', p['input_cost_per_token'])
+    res = (p['output_cost_per_token']*out + p['cache_read_input_token_cost']*cache + ic*inp) / (out+cache+inp)
+    if nm=='claude-opus-4-7': res *= 1.5
+    return res*(1+markup)
+
 # %% ../nbs/00_types.ipynb #8bfca02d
 @patch(as_prop=True)
 def cost(self:Completion):

{python_fastllm-0.0.8 → python_fastllm-0.0.10}/python_fastllm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-fastllm
-Version: 0.0.8
+Version: 0.0.10
 Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
 License: Apache-2.0
 Project-URL: Repository, https://github.com/AnswerDotAI/fastllm

python_fastllm-0.0.8/fastllm/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.8"