python-fastllm 0.0.1__py3-none-any.whl

fastllm/specs/spec_manifest.json

@@ -0,0 +1,17 @@
+{
+  "generated_at": "2026-03-30T14:27:11+00:00",
+  "specs_dir": "/Users/keremturgutlu/aai-ws/fastllm/specs",
+  "items": [
+    {
+      "provider": "anthropic",
+      "source_url": "https://storage.googleapis.com/stainless-sdk-openapi-specs/anthropic%2Fanthropic-dd2dcd00a757075370a7e4a7f469a1e2d067c2118684c3b70d7906a8f5cf518b.yml",
+      "target": "/Users/keremturgutlu/aai-ws/fastllm/specs/anthropic.yml",
+      "bytes": 724122,
+      "sha256": "dd2dcd00a757075370a7e4a7f469a1e2d067c2118684c3b70d7906a8f5cf518b",
+      "content_type": "text/yaml",
+      "fetched_at": "2026-03-30T14:27:11+00:00",
+      "changed": true,
+      "wrote": true
+    }
+  ]
+}

fastllm/streaming.py

@@ -0,0 +1,162 @@
+"""Streaming helpers for lossless event collation."""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_streaming.ipynb.
+
+# %% auto #0
+__all__ = ['Delta', 'norm_and_yield', 'PartAccum', 'accum_completion', 'completion_text', 'stop_sequences', 'mk_acollect_stream',
+           'fake_stream']
+
+# %% ../nbs/01_streaming.ipynb #0df5c926
+import json
+from dataclasses import dataclass, field, fields
+from fastcore.utils import *
+from fastcore.meta import delegates
+from fastspec.errors import *
+from .types import *
+
+# %% ../nbs/01_streaming.ipynb #400d628a
+@dataclass
+class Delta:
+    "Normalized streaming delta event."
+    text: str = ""
+    thinking: str = ""
+    refusal: str = ""
+    tool_calls: list[ToolCall] = field(default_factory=list)
+    citations: list = field(default_factory=list)
+    server_tool_result: dict = None
+    finish_reason: str = None
+    usage: Usage = None
+    raw: dict = field(default_factory=dict)
+
+# %% ../nbs/01_streaming.ipynb #f59f837b
+async def norm_and_yield(resp, norm_func):
+    async for ev in resp:
+        if (d := norm_func(ev)) is not None: yield d
+
+# %% ../nbs/01_streaming.ipynb #a7f1738a
+@dataclass
+class PartAccum:
+    parts: dict[Part|ToolCall] = field(default_factory=dict)
+    tool_calls: list[ToolCall] = field(default_factory=list)
+    
+    def append(self, typ, index, txt='', citations=None, **tc_kwargs):
+        'Create and accumulate same type sequential parts'
+        if index not in self.parts: 
+            if typ==PartType.tool_use: self.parts[index] = ToolCall(**tc_kwargs)
+            else:                      self.parts[index] = Part(type=typ, text=txt, data=dict(citations=citations or []))
+        else:
+            if typ==PartType.tool_use:
+                new_args = tc_kwargs.get('arguments', '')
+                cur_args = self.parts[index].arguments
+                if isinstance(new_args, str) and isinstance(cur_args, str): self.parts[index].arguments += new_args
+                elif isinstance(new_args, str) and isinstance(cur_args, dict): self.parts[index].arguments = new_args
+                else: self.parts[index].arguments = new_args
+            else: 
+                self.parts[index].text += txt
+                # anthropic citations have matching idx
+                self.parts[index].data['citations'].extend(citations or [])
+    
+    def finalize(self):
+        for idx,tc in self.parts.items():
+            if isinstance(tc, ToolCall):
+                if isinstance(tc.arguments, str): tc.arguments = json.loads(tc.arguments) if tc.arguments else {}
+                self.tool_calls.append(tc)
+                data = {**tc.extra, 'id':tc.id, 'name':tc.name, 'arguments':tc.arguments, 'server':tc.server}
+                self.parts[idx] = Part(type=PartType.tool_use, data=data)
+        
+        merged = []
+        for p in self.parts.values():
+            if merged and merged[-1].type == p.type and p.type in (PartType.text, PartType.thinking): merged[-1].text += p.text
+            else: merged.append(p)        
+        self.parts = merged
+
+# %% ../nbs/01_streaming.ipynb #0e8ca58e
+def accum_completion(pa, raw, fin, usg, deltas, model=None, api_name=None, vendor_name=None, delta=None):
+    "Build a Completion snapshot from in-progress PartAccum state"
+    parts = [p for p in pa.parts.values() if isinstance(p, Part)]
+    if delta and delta.text:
+        parts = parts.copy()
+        if parts and parts[-1].type==PartType.text:
+            p = parts[-1]
+            parts[-1] = Part(type=p.type, text=(p.text or '') + delta.text, data=p.data)
+        else: parts.append(Part(type=PartType.text, text=delta.text))
+    return Completion(raw.get('model', model), Msg(role="assistant", content=parts),
+                      fin, usg, api_name=api_name, vendor_name=vendor_name, raw={'deltas':deltas})
+
+# %% ../nbs/01_streaming.ipynb #c28f706f
+def completion_text(c):
+    "Combined text from a Completion."
+    return ''.join(p.text or '' for p in c.message.content if p.type==PartType.text)
+
+# %% ../nbs/01_streaming.ipynb #b2b9f7ca
+def stop_sequences(seqs):
+    "Stop when any sequence appears in the accumulated completion text."
+    seqs = L(seqs)
+    def _stop(c):
+        txt = completion_text(c)
+        for s in seqs:
+            if s in txt: return s
+    return _stop
+
+# %% ../nbs/01_streaming.ipynb #931f686b
+def _trim_delta(d, cur, s):
+    "Trim `d.text` so accumulated text in `cur` stops just before stop sequence `s`."
+    txt,dt = completion_text(cur), d.text or ''
+    i = txt.find(s)
+    if i>=0: d.text = dt[:max(0, i-(len(txt)-len(dt)))]
+
+# %% ../nbs/01_streaming.ipynb #fc71790b
+async def mk_acollect_stream(it, index_fn, model=None, api_name=None, vendor_name=None, stop_callables=None):
+    "Collect a Delta stream, yielding incremental chunks and a final Completion."
+    part_accum,deltas,stop_callables = PartAccum(), [], L(stop_callables)
+    fin, usg, typ, last_typ, last_idx = [None]*5
+    def _fidx(d, name, pt=None):
+        nonlocal typ, last_idx
+        typ = getattr(PartType, pt or name)
+        idx,last_idx = index_fn(d, typ, last_typ, last_idx)
+        return idx
+    def _proc(d, name, pt=None, kw='txt', ret=None):
+        if not ret and not (val := getattr(d, name)): return
+        idx = _fidx(d, name, pt)
+        part_accum.append(typ, idx, **(ret or {kw: val}))
+        return ret or {name: val}
+    async for d in it:
+        stop = False
+        if stop_callables:
+            cur = accum_completion(part_accum, d.raw, fin, usg, deltas+[d], model, api_name=api_name, vendor_name=vendor_name, delta=d)
+            for f in stop_callables:
+                if res:=f(cur):
+                    if isinstance(res, str): _trim_delta(d, cur, res)
+                    stop = True
+                    break
+        if (r:=_proc(d, 'text')): yield r
+        if (r:=_proc(d, 'thinking')): yield r
+        if (r:=_proc(d, 'citations', pt='text', kw='citations')): yield r
+        for tc in d.tool_calls:
+            args = tc.arguments.get('_delta', tc.arguments)
+            _proc(d, 'tool_use', ret=dict(id=tc.id, name=tc.name, arguments=args, server=tc.server, extra=tc.extra))
+        if d.server_tool_result:
+            idx = _fidx(d, 'server_tool_result')
+            part_accum.parts[idx] = Part(type=typ, data=d.server_tool_result)
+        if (r:=_proc(d, 'refusal')): yield r
+        if d.finish_reason: fin = d.finish_reason
+        if d.usage: usg = d.usage
+        last_typ = typ
+        deltas.append(d)
+        if stop:
+            fin = fin or FinishReason.stop
+            await it.aclose()
+            break
+    part_accum.finalize()
+    # need to recheck for tool calls post collation for streaming
+    tcs = part_accum.tool_calls
+    fin = FinishReason.tool_calls if fin==FinishReason.stop and any(~L(tcs).attrgot('server')) else fin
+    # tool calls and non-anthropic citations are yielded at the end
+    yield Completion(d.raw.get('model', model),
+            message=Msg(role="assistant", content=part_accum.parts),
+            finish_reason=fin, usage=usg, tool_calls=tcs, api_name=api_name, vendor_name=vendor_name,
+            raw={'deltas':deltas})
+
+# %% ../nbs/01_streaming.ipynb #f79d3b99
+async def fake_stream(*ss):
+    for s in ss: yield Delta(text=s, raw={'model':'fake'})

fastllm/types.py

@@ -0,0 +1,301 @@
+"""Core internal types."""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_types.ipynb.
+
+# %% auto #0
+__all__ = ['PartType', 'FinishReason', 'api_registry', 'model_prices_url', 'codex_pricing', 'Part', 'Msg', 'ToolCall',
+           'display_list', 'Usage', 'Completion', 'APIRegistry', 'mk_completion', 'mk_tool_res_msg', 'fn_schema',
+           'sys_text', 'part_txt', 'data_url', 'url_mime', 'payload_kwargs', 'get_api_key', 'model_prices_meta',
+           'infer_api_name', 'get_model_meta', 'get_model_info']
+
+# %% ../nbs/00_types.ipynb #b4d047fd
+from dataclasses import dataclass, field
+from fastcore.net import urljson
+from fastcore.utils import *
+
+# %% ../nbs/00_types.ipynb #e568bade
+@dataclass
+class Part:
+    "A normalized content part."
+    type: str
+    text: str = None
+    data: dict = None
+
+# %% ../nbs/00_types.ipynb #aa3cfc23
+PartType = str_enum('PartType', 'text', 'thinking', 'refusal', 'tool_use', 'server_tool_result', 'tool_result',
+                    'input_image', 'input_audio', 'input_video', 'input_file')
+
+# %% ../nbs/00_types.ipynb #2eeff103
+def _trunc_strs(d, n=200):
+    "Return copy of dict `d` with str values >n chars truncated to first 10 chars + '...'"
+    if not d: return d
+    return {k: (v[:10]+'...' if isinstance(v,str) and len(v)>n else v) for k,v in d.items()}
+
+@patch
+def _repr_markdown_(self: Part):
+    body = self.text if self.text else ''
+    data = _trunc_strs(self.data)
+    return f"""**Part** (`{self.type}`)
+
+{body}
+
+<details>
+
+- data: `{data}`
+
+</details>"""
+
+# %% ../nbs/00_types.ipynb #afeb9eef
+@dataclass
+class Msg:
+    "A normalized message."
+    role: str
+    content: List[Part]
+
+    def _repr_markdown_(self):
+        return f"""**Msg**
+
+- role: `{self.role}`
+
+<contents>
+
+{'\n\n'.join(p._repr_markdown_() for p in self.content)}
+
+</contents>"""
+
+# %% ../nbs/00_types.ipynb #f6d04920
+@dataclass
+class ToolCall:
+    "Normalized tool call."
+    id: str
+    name: str
+    arguments: dict = field(default_factory=dict)
+    server: bool = False
+    extra: dict = field(default_factory=dict)
+
+# %% ../nbs/00_types.ipynb #509cfc99
+@patch
+def _repr_markdown_(self: ToolCall):
+    extra = _trunc_strs(self.extra)
+    return f"""🔧 **{self.name}**(`{self.arguments}`)
+
+<details>
+
+- id: `{self.id}`
+- server: `{self.server}`
+- extra: `{extra}`
+
+</details>"""
+
+def display_list(l): 
+    from IPython.display import Markdown, display
+    display(Markdown('\n\n'.join(o._repr_markdown_() for o in l)))
+
+# %% ../nbs/00_types.ipynb #802ad832
+@dataclass(frozen=True)
+class Usage:
+    "Normalized usage."
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+    cached_tokens: int = 0
+    cache_creation_tokens: int = 0
+    reasoning_tokens: int = 0
+    raw: dict = field(default_factory=dict)
+
+
+# %% ../nbs/00_types.ipynb #297a1672
+@dataclass(frozen=True)
+class Completion:
+    "Normalized completion response."
+    model: str
+    message: Msg
+    finish_reason: str = None
+    usage: Usage = None
+    tool_calls: List[ToolCall] = field(default_factory=list)
+    api_name: str = None
+    vendor_name: str = None
+    raw: dict = field(default_factory=dict)
+
+# %% ../nbs/00_types.ipynb #1b0dbab0
+@patch
+def _repr_markdown_(self: Completion):
+    message = self.message
+    content = ''
+    for p in message.content:
+        if p.type == PartType.thinking:
+            if p.text: content += f"<details><summary>Thinking</summary>\n\n{p.text}\n\n</details>\n\n"
+        elif txt := p.text: content += txt
+    if self.tool_calls:
+        tool_calls = [f"\n\n🔧 {tc.name}({tc.arguments})\n" for tc in self.tool_calls]
+        content += "\n".join(tool_calls)
+    # for img in getattr(message, 'images', []): content += f"\n\n![generated image]({nested_idx(img, 'image_url', 'url')})" # TODO
+    details = [f"model: `{self.model}`", f"finish_reason: `{self.finish_reason}`", f"usage: `{self.usage}`"]
+    det_str = '\n- '.join(details)    
+    return f"""{content}
+
+<details>
+
+- {det_str}
+
+</details>"""
+
+# %% ../nbs/00_types.ipynb #ce59e431
+FinishReason = str_enum('finish_reason', 'stop', 'tool_calls', 'length', 'content_filter')
+
+# %% ../nbs/00_types.ipynb #fc681c52
+class APIRegistry:
+    def __init__(self): self.apis = {}
+    def register(self, name, **kwargs): self.apis[name] = SimpleNamespace(**kwargs)
+
+api_registry = APIRegistry()
+
+# %% ../nbs/00_types.ipynb #d58a5f96
+#COMMON
+def mk_completion(resp, model, api_name, vendor_name):
+    "Normalize an api response into Completion."
+    api = api_registry.apis[api_name]
+    tcs = api.norm_tool_calls(resp)
+    return Completion(
+        model=resp.get("model") or model,
+        message=Msg(role="assistant", content=api.norm_parts(resp)),
+        finish_reason=api.norm_finish(resp, tcs),
+        usage=api.norm_usage(resp),
+        tool_calls=tcs,
+        api_name=api_name,
+        vendor_name=vendor_name,
+        raw=resp)
+
+# %% ../nbs/00_types.ipynb #d5322db5
+#COMMON
+def mk_tool_res_msg(tool_calls:list[ToolCall], results:list[str|list]):
+    'A util to prepare parallel tool call with str or media list results'
+    parts = []
+    for tc,res in zip(tool_calls, results):
+        data = dict(id=tc.id, name=tc.name, arguments=tc.arguments, server=tc.server)
+        parts.append(Part(type=PartType.tool_result, text=res, data=data))
+    return Msg(role="tool", content=parts)
+
+# %% ../nbs/00_types.ipynb #8a8e468b
+#COMMON
+def fn_schema(t):
+    "Extract (name, description, parameters) from any tool format."
+    if not isinstance(t, dict): return None
+    fn = t.get('function')
+    if isinstance(fn, dict): return fn.get('name',''), fn.get('description',''), fn.get('parameters',{})
+    if 'name' in t and ('parameters' in t or 'input_schema' in t):
+        return t.get('name',''), t.get('description',''), t.get('parameters', t.get('input_schema',{}))
+    return None
+
+# %% ../nbs/00_types.ipynb #d1d48d91
+#COMMON
+def sys_text(system):
+    "Extract text from system (str or Part)."
+    if system is None: return None
+    return system if isinstance(system, str) else system.text
+
+def part_txt(p): return p.text if isinstance(p,Part) else p
+
+# %% ../nbs/00_types.ipynb #dc2b75a0
+#COMMON
+_ext_mime = {
+    '.jpg':'image/jpeg', '.jpeg':'image/jpeg', '.png':'image/png', '.gif':'image/gif', '.webp':'image/webp',
+    '.pdf':'application/pdf',
+    '.mp3':'audio/mpeg', '.wav':'audio/wav', '.ogg':'audio/ogg', '.flac':'audio/flac', '.m4a':'audio/mp4',
+    '.mp4':'video/mp4', '.mov':'video/quicktime', '.webm':'video/webm',
+}
+
+def data_url(url):
+    "Parse data:mime;base64,data URL into (mime, b64_data), or None."
+    if not isinstance(url, str) or not url.startswith('data:') or ',' not in url: return None
+    header, body = url.split(',', 1)
+    if ';base64' not in header or not body: return None
+    return header[5:].split(';',1)[0].strip() or 'application/octet-stream', body
+
+def url_mime(url, default='application/octet-stream'):
+    "Guess mime from URL extension."
+    ext = '.' + url.rsplit('.', 1)[-1].split('?')[0].lower() if '.' in url.split('?')[0].split('/')[-1] else ''
+    return _ext_mime.get(ext, default)
+
+# %% ../nbs/00_types.ipynb #28c698fe
+def payload_kwargs(msgs, model, stream=False, system=None, max_tokens=None, temperature=None, tools=None, tool_choice=None, reasoning_effort=None, web_search_options=None, stop_callables=None, stop_sequences=None): pass
+
+# %% ../nbs/00_types.ipynb #c2a2cb49
+def get_api_key(api_key, default):
+    err = ValueError(f"Missing API key: make sure to have the expected env var name or pass `api_key`")
+    key = api_key or os.getenv(default)
+    if not key: raise err
+    return key
+
+# %% ../nbs/00_types.ipynb #852adecd
+model_prices_url = 'https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json'
+@flexicache(time_policy(24*60*60))
+def model_prices_meta(): return urljson(model_prices_url)
+
+# %% ../nbs/00_types.ipynb #68e488d8
+def infer_api_name(model):
+    "Infer api_name from model"
+    if "claude" in model: return 'anthropic'
+    if "gemini" in model: return 'gemini'
+    if any(o in model for o in ('gpt','o3-','o4-')): return 'openai'
+
+# %% ../nbs/00_types.ipynb #2f0720c2
+@flexicache(time_policy(24*60*60))
+def get_model_meta(model, vendor_name=None, tfm=noop):
+    "Look up cost metadata for `model` from litellm price map, using `vendor_name` prefix if needed."
+    vendor_name = ifnone(vendor_name, infer_api_name(model))
+    mp = model_prices_meta()
+    if model in mp: key = model
+    elif vendor_name=='gemini' and model.startswith('models/'): key = f"gemini/{model.removeprefix('models/')}"
+    elif vendor_name:                                           key = f"{vendor_name}/{model}"
+    return dict2obj(tfm(mp.get(key), model, vendor_name))
+
+# %% ../nbs/00_types.ipynb #fbfdeb0a
+codex_pricing = {
+    "input_cost_per_token": 0.10 / 1_000_000,
+    "cache_creation_input_token_cost": 0.10 / 1_000_000,
+    "cache_read_input_token_cost": 0.10 / 1_000_000,
+    "output_cost_per_token": 0.50 / 1_000_000,
+}
+
+def get_model_info(mn, vendor_name=None):
+    info = get_model_meta(mn, vendor_name)
+    # anthropic web search 
+    if 'search_context_cost_per_query' in info:
+        info['supports_web_search'] = True
+    # add reasoning to kimi
+    if 'kimi' in mn: 
+        if 'k2p6' in mn: info = get_model_meta(mn.replace('k2p6', 'k2p5'), vendor_name)
+        info['supports_reasoning'] = True
+        info['supports_vision'] = True
+        if vendor_name == 'moonshot': info['supports_assistant_prefill'] = True
+    # add web search to gpt
+    if mn in ("gpt-5.4", "gpt-5.4-mini"):
+        info['supports_web_search'] = True
+        info.pop('mode', None)
+    # codex pricing
+    if vendor_name == 'codex': info = merge(info, codex_pricing)
+    # deepseek v4
+    if vendor_name == 'deepseek' and mn in ("deepseek-v4-flash", "deepseek-v4-pro"):
+        info = dict(get_model_meta("deepseek/deepseek-v3.2"))
+        info |= dict(supports_assistant_prefill=True, supports_function_calling=True, supports_prompt_caching=True,
+            supports_reasoning=True, supports_tool_choice=True)
+        info.update(input_cost_per_token=1.4e-07, input_cost_per_token_cache_hit=2.8e-09, output_cost_per_token=2.8e-07,
+            max_input_tokens=1048576, max_output_tokens=393216, max_tokens=393216)
+        if 'pro' in mn: info = {**info, 'input_cost_per_token': 4.35e-07, 'input_cost_per_token_cache_hit': 3.625e-09, 'output_cost_per_token': 8.7e-07}
+    # qwen 3p6
+    if vendor_name == 'fireworks_ai' and mn == 'accounts/fireworks/models/qwen3p6-plus':
+        info = dict(supports_vision=True, supports_reasoning=True, supports_function_calling=True, supports_tool_choice=True,
+                    supports_system_messages=True, supports_response_schema=True, supports_parallel_function_calling=True,
+                    supports_prompt_caching=True, supports_native_streaming=True, supports_native_structured_output=True,
+                    max_tokens=1000000, max_input_tokens=1000000, max_output_tokens=65536,
+                    input_cost_per_token=0.5e-6, cache_read_input_token_cost=0.1e-6, output_cost_per_token=3.0e-6)
+    return dict2obj(info)
+
+# %% ../nbs/00_types.ipynb #8bfca02d
+@patch(as_prop=True)
+def cost(self:Completion):
+    meta = dict2obj(get_model_info(self.model, self.vendor_name))
+    api = api_registry.apis[self.api_name]
+    if not hasattr(api, 'cost'): raise NotImplementedError(f"API: {self.api_name} doesn't have a registered `cost` function in ns")
+    return api.cost(self.usage, meta)