python-fastllm 0.0.38__tar.gz → 0.0.39__tar.gz

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-fastllm
-Version: 0.0.38
+Version: 0.0.39
 Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
 License: Apache-2.0
 Project-URL: Repository, https://github.com/AnswerDotAI/fastllm
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: fastcore>=2.1.18
-Requires-Dist: aidialog>=0.0.8
+Requires-Dist: aidialog>=0.0.10
 Requires-Dist: fastspec>=0.0.11
 Requires-Dist: pillow
 

python_fastllm-0.0.39/fastllm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.39"

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/fastllm/_modidx.py

@@ -19,10 +19,12 @@ d = { 'settings': { 'branch': 'main',
                                    'fastllm.acomplete.acomplete': ('acomplete.html#acomplete', 'fastllm/acomplete.py'),
                                    'fastllm.acomplete.mk_client': ('acomplete.html#mk_client', 'fastllm/acomplete.py'),
                                    'fastllm.acomplete.split_vendor': ('acomplete.html#split_vendor', 'fastllm/acomplete.py')},
-            'fastllm.anthropic': { 'fastllm.anthropic._ant_cc': ('anthropic.html#_ant_cc', 'fastllm/anthropic.py'),
+            'fastllm.anthropic': { 'fastllm.anthropic._add_cache_control': ('anthropic.html#_add_cache_control', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic._ant_cc': ('anthropic.html#_ant_cc', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic._ant_part_type': ('anthropic.html#_ant_part_type', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic._sanid': ('anthropic.html#_sanid', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.acollect_stream': ('anthropic.html#acollect_stream', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic.apply_cache_idxs': ('anthropic.html#apply_cache_idxs', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.cost': ('anthropic.html#cost', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.delta_index_fn': ('anthropic.html#delta_index_fn', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.denorm_assistant': ('anthropic.html#denorm_assistant', 'fastllm/anthropic.py'),
@@ -66,9 +68,6 @@ d = { 'settings': { 'branch': 'main',
                               'fastllm.chat.DeepseekMsgsCallback': ('chat.html#deepseekmsgscallback', 'fastllm/chat.py'),
                               'fastllm.chat.DeepseekMsgsCallback.after_msgs': ( 'chat.html#deepseekmsgscallback.after_msgs',
                                                                                 'fastllm/chat.py'),
-                              'fastllm.chat.DeepseekPrefillCallback': ('chat.html#deepseekprefillcallback', 'fastllm/chat.py'),
-                              'fastllm.chat.DeepseekPrefillCallback.before_acomplete': ( 'chat.html#deepseekprefillcallback.before_acomplete',
-                                                                                         'fastllm/chat.py'),
                               'fastllm.chat.FenceToolCallback': ('chat.html#fencetoolcallback', 'fastllm/chat.py'),
                               'fastllm.chat.FenceToolCallback.after_msgs': ('chat.html#fencetoolcallback.after_msgs', 'fastllm/chat.py'),
                               'fastllm.chat.FenceToolCallback.before_acomplete': ( 'chat.html#fencetoolcallback.before_acomplete',
@@ -99,17 +98,13 @@ d = { 'settings': { 'branch': 'main',
                               'fastllm.chat.UsageStats.fmt': ('chat.html#usagestats.fmt', 'fastllm/chat.py'),
                               'fastllm.chat.UsageStats.from_response': ('chat.html#usagestats.from_response', 'fastllm/chat.py'),
                               'fastllm.chat._active_fence_langs': ('chat.html#_active_fence_langs', 'fastllm/chat.py'),
-                              'fastllm.chat._add_cache_control': ('chat.html#_add_cache_control', 'fastllm/chat.py'),
                               'fastllm.chat._alite_call_func': ('chat.html#_alite_call_func', 'fastllm/chat.py'),
-                              'fastllm.chat._apply_cache_idxs': ('chat.html#_apply_cache_idxs', 'fastllm/chat.py'),
                               'fastllm.chat._call_func': ('chat.html#_call_func', 'fastllm/chat.py'),
                               'fastllm.chat._handle_stop_reason': ('chat.html#_handle_stop_reason', 'fastllm/chat.py'),
-                              'fastllm.chat._has_cache': ('chat.html#_has_cache', 'fastllm/chat.py'),
                               'fastllm.chat._has_search': ('chat.html#_has_search', 'fastllm/chat.py'),
                               'fastllm.chat._has_stop': ('chat.html#_has_stop', 'fastllm/chat.py'),
                               'fastllm.chat._inject_tool_reminder': ('chat.html#_inject_tool_reminder', 'fastllm/chat.py'),
                               'fastllm.chat._lite_call_func': ('chat.html#_lite_call_func', 'fastllm/chat.py'),
-                              'fastllm.chat._mk_prefill': ('chat.html#_mk_prefill', 'fastllm/chat.py'),
                               'fastllm.chat._mk_tool_result': ('chat.html#_mk_tool_result', 'fastllm/chat.py'),
                               'fastllm.chat._think_kw': ('chat.html#_think_kw', 'fastllm/chat.py'),
                               'fastllm.chat._usrtools': ('chat.html#_usrtools', 'fastllm/chat.py'),
@@ -119,10 +114,7 @@ d = { 'settings': { 'branch': 'main',
                               'fastllm.chat.cite_footnote': ('chat.html#cite_footnote', 'fastllm/chat.py'),
                               'fastllm.chat.contents': ('chat.html#contents', 'fastllm/chat.py'),
                               'fastllm.chat.lite_mk_func': ('chat.html#lite_mk_func', 'fastllm/chat.py'),
-                              'fastllm.chat.mk_msg': ('chat.html#mk_msg', 'fastllm/chat.py'),
-                              'fastllm.chat.mk_msgs': ('chat.html#mk_msgs', 'fastllm/chat.py'),
                               'fastllm.chat.postproc': ('chat.html#postproc', 'fastllm/chat.py'),
-                              'fastllm.chat.remove_cache_ckpts': ('chat.html#remove_cache_ckpts', 'fastllm/chat.py'),
                               'fastllm.chat.run_fence_tool': ('chat.html#run_fence_tool', 'fastllm/chat.py'),
                               'fastllm.chat.search_count': ('chat.html#search_count', 'fastllm/chat.py'),
                               'fastllm.chat.stop_reason': ('chat.html#stop_reason', 'fastllm/chat.py'),
@@ -232,12 +224,18 @@ d = { 'settings': { 'branch': 'main',
                                           'fastllm.openai_responses.norm_usage': ( 'oai_responses.html#norm_usage',
                                                                                    'fastllm/openai_responses.py')},
             'fastllm.streaming': { 'fastllm.streaming.Delta': ('streaming.html#delta', 'fastllm/streaming.py'),
+                                   'fastllm.streaming.Delta.__eq__': ('streaming.html#delta.__eq__', 'fastllm/streaming.py'),
+                                   'fastllm.streaming.Delta.__hash__': ('streaming.html#delta.__hash__', 'fastllm/streaming.py'),
+                                   'fastllm.streaming.Delta.__init__': ('streaming.html#delta.__init__', 'fastllm/streaming.py'),
                                    'fastllm.streaming.FenceToolStop': ('streaming.html#fencetoolstop', 'fastllm/streaming.py'),
                                    'fastllm.streaming.FenceToolStop.__call__': ( 'streaming.html#fencetoolstop.__call__',
                                                                                  'fastllm/streaming.py'),
                                    'fastllm.streaming.FenceToolStop.__init__': ( 'streaming.html#fencetoolstop.__init__',
                                                                                  'fastllm/streaming.py'),
                                    'fastllm.streaming.PartAccum': ('streaming.html#partaccum', 'fastllm/streaming.py'),
+                                   'fastllm.streaming.PartAccum.__eq__': ('streaming.html#partaccum.__eq__', 'fastllm/streaming.py'),
+                                   'fastllm.streaming.PartAccum.__hash__': ('streaming.html#partaccum.__hash__', 'fastllm/streaming.py'),
+                                   'fastllm.streaming.PartAccum.__init__': ('streaming.html#partaccum.__init__', 'fastllm/streaming.py'),
                                    'fastllm.streaming.PartAccum.append': ('streaming.html#partaccum.append', 'fastllm/streaming.py'),
                                    'fastllm.streaming.PartAccum.get_merged': ( 'streaming.html#partaccum.get_merged',
                                                                                'fastllm/streaming.py'),
@@ -247,6 +245,9 @@ d = { 'settings': { 'branch': 'main',
                                    'fastllm.streaming.PrintStream.__init__': ( 'streaming.html#printstream.__init__',
                                                                                'fastllm/streaming.py'),
                                    'fastllm.streaming.Status': ('streaming.html#status', 'fastllm/streaming.py'),
+                                   'fastllm.streaming.Status.__eq__': ('streaming.html#status.__eq__', 'fastllm/streaming.py'),
+                                   'fastllm.streaming.Status.__hash__': ('streaming.html#status.__hash__', 'fastllm/streaming.py'),
+                                   'fastllm.streaming.Status.__init__': ('streaming.html#status.__init__', 'fastllm/streaming.py'),
                                    'fastllm.streaming.Status.formatted': ('streaming.html#status.formatted', 'fastllm/streaming.py'),
                                    'fastllm.streaming._mk_delta_part': ('streaming.html#_mk_delta_part', 'fastllm/streaming.py'),
                                    'fastllm.streaming._trim_delta': ('streaming.html#_trim_delta', 'fastllm/streaming.py'),
@@ -259,12 +260,12 @@ d = { 'settings': { 'branch': 'main',
                                'fastllm.types.APIRegistry.__missing__': ('types.html#apiregistry.__missing__', 'fastllm/types.py'),
                                'fastllm.types.APIRegistry._load': ('types.html#apiregistry._load', 'fastllm/types.py'),
                                'fastllm.types.APIRegistry.register': ('types.html#apiregistry.register', 'fastllm/types.py'),
-                               'fastllm.types.Completion': ('types.html#completion', 'fastllm/types.py'),
                                'fastllm.types.Completion._repr_markdown_': ('types.html#completion._repr_markdown_', 'fastllm/types.py'),
                                'fastllm.types.Completion.cost': ('types.html#completion.cost', 'fastllm/types.py'),
                                'fastllm.types.Completion.formatted': ('types.html#completion.formatted', 'fastllm/types.py'),
-                               'fastllm.types.Completion.tool_calls': ('types.html#completion.tool_calls', 'fastllm/types.py'),
                                'fastllm.types.Usage': ('types.html#usage', 'fastllm/types.py'),
+                               'fastllm.types.Usage.__eq__': ('types.html#usage.__eq__', 'fastllm/types.py'),
+                               'fastllm.types.Usage.__init__': ('types.html#usage.__init__', 'fastllm/types.py'),
                                'fastllm.types._rm_ctx_tiers': ('types.html#_rm_ctx_tiers', 'fastllm/types.py'),
                                'fastllm.types.approx_pricing': ('types.html#approx_pricing', 'fastllm/types.py'),
                                'fastllm.types.fn_schema': ('types.html#fn_schema', 'fastllm/types.py'),

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/fastllm/acomplete.py

@@ -16,7 +16,8 @@ from fastspec.oapi import *
 from fastspec.errors import APIError
 
 from .types import *
-from aidialog.msg_parts import Part, PartType, Msg, Text, Thinking, ToolUse, ToolResult, InputImage, InputAudio, InputVideo, InputFile, mk_tool_res_msg
+from aidialog.msg_parts import (Part, PartType, Msg, Text, Thinking, ToolUse, ToolResult, InputImage, InputAudio, InputVideo,
+    InputFile, mk_tool_res_msg, Completion)
 from .streaming import *
 from .openai_responses import *
 from .openai_chat import *

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/fastllm/anthropic.py

@@ -5,7 +5,7 @@ __all__ = ['ant_tc_types', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'f
            'norm_tr_parts', 'norm_sse_event', 'delta_index_fn', 'acollect_stream', 'denorm_tool_use',
            'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice',
            'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user', 'denorm_image', 'denorm_file',
-           'denorm_tool_result', 'mk_payload', 'get_hdrs', 'cost']
+           'denorm_tool_result', 'apply_cache_idxs', 'mk_payload', 'get_hdrs', 'cost']
 
 # %% ../nbs/04_anthropic.ipynb #02afd3d7
 import json
@@ -271,12 +271,47 @@ def denorm_tool_result(p:ToolResult):
         return _ant_cc(dict(type='tool_result', tool_use_id=tid, content=blocks), p)
     return _ant_cc(dict(type='tool_result', tool_use_id=tid, content=tool_text(p.text)), p)
 
+# %% ../nbs/04_anthropic.ipynb #e4eb8d3b
+def _add_cache_control(
+    msg,     # Canonical Msg to mark
+    ttl=None # Cache TTL: '5m' (default) or '1h'
+):
+    "Set a cache breakpoint on the last `Text` or `ToolUse` part of `msg`"
+    cc = {"type": "ephemeral"} | ({"ttl": ttl} if ttl else {})
+    idxs = [i for i,p in enumerate(msg.content) if isinstance(p, (Text, ToolUse))]
+    if idxs: msg.content[idxs[-1]].cache_control = cc
+    return msg
+
+# %% ../nbs/04_anthropic.ipynb #6bc52ef5
+def apply_cache_idxs(
+    msgs,            # Canonical Msgs to mark
+    cache_idxs=[-1], # Breakpoint idxs into `msgs` (tool-role msgs excluded); 0 is `sp` when given
+    sp=None,         # System prompt: str, Msg, or Part
+    ttl=None         # Cache TTL: '5m' (default) or '1h'
+):
+    "Strip all cache breakpoints from `msgs`, then mark `cache_idxs`; returns `sp`, marked if 0 is included"
+    for m in msgs:
+        for p in m.content: p.cache_control = None
+    if sp:
+        if 0 in cache_idxs:
+            if isinstance(sp, str): sp = Text(sp)
+            if isinstance(sp, Part): sp = Msg('', [sp])
+            _add_cache_control(sp, ttl)
+        cache_idxs = [o-1 if o>0 else o for o in cache_idxs if o]
+    ms = [j for j,m in enumerate(msgs) if m.role != 'tool']
+    for i in cache_idxs:
+        try: _add_cache_control(msgs[ms[i]], ttl)
+        except IndexError: continue
+    return sp
+
 # %% ../nbs/04_anthropic.ipynb #587d4fe5
 @delegates(payload_kwargs)
 def mk_payload(msgs, model, **kwargs):
+    sp = kwargs.get('system')
+    if (ci:=kwargs.get('cache_idxs')) is not None: sp = apply_cache_idxs(msgs, ci, sp, kwargs.get('ttl'))
     payload = dict(model=model, messages=denorm_msgs(msgs), max_tokens=kwargs.get('max_tokens') or 1024)
     if kwargs.get('stream'):                payload['stream'] = True
-    if sp:=kwargs.get('system'):            payload['system'] = denorm_system(sp)
+    if sp:                              payload['system'] = denorm_system(sp)
     if tools:=kwargs.get('tools'):          payload['tools'] = denorm_tool_schs(tools)
     if tchc:=kwargs.get('tool_choice'):     payload['tool_choice'] = denorm_tool_choice(tchc)
     if thk:=kwargs.get('reasoning_effort'): payload.update(denorm_reasoning(thk))

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/fastllm/chat.py

@@ -3,49 +3,24 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/07_chat.ipynb.
 
 # %% auto #0
-__all__ = ['effort', 'remove_cache_ckpts', 'contents', 'stop_reason', 'mk_msg', 'mk_msgs', 'cite_footnote', 'postproc', 'Refresh',
-           'think_sep', 'lite_mk_func', 'structured', 'search_count', 'UsageStats', 'AsyncChat',
-           'astream_with_complete', 'ChatCallback', 'DeepseekMsgsCallback', 'DeepseekPrefillCallback', 'add_warning',
-           'StopReasonCallback', 'run_fence_tool', 'FenceToolCallback', 'ToolReminderCallback', 'StopSequencesCallback',
-           'StreamAccum', 'adisplay_stream']
+__all__ = ['effort', 'contents', 'stop_reason', 'cite_footnote', 'postproc', 'Refresh', 'think_sep', 'lite_mk_func', 'structured',
+           'search_count', 'UsageStats', 'AsyncChat', 'astream_with_complete', 'ChatCallback', 'DeepseekMsgsCallback',
+           'add_warning', 'StopReasonCallback', 'run_fence_tool', 'FenceToolCallback', 'ToolReminderCallback',
+           'StopSequencesCallback', 'StreamAccum', 'adisplay_stream']
 
 # %% ../nbs/07_chat.ipynb #d5a3bc1f
 from typing import Optional,Callable
 from fastcore.funccall import mk_ns, call_func, call_func_async, get_schema
 from fastcore.utils import *
 from fastcore.meta import delegates
-from dataclasses import dataclass
 
 from .types import *
-from aidialog.msg_parts import (Msg, Part, PartType, Text, Thinking, ToolUse, ToolResult, InputImage, tool_text, mk_tool_res_msg, mk_tr_details,
-    tool_info, usage_info, think_start, think_end, StopResponse, ToolResponse, display_list, fmt2hist, hist2fmt,
-    mk_content, mk_result_fence, split_fence_msgs, extract_fence_call, trunc_str)
+from aidialog.msg_parts import (Msg, Part, PartType, Text, Thinking, ToolUse, tool_text, mk_tool_res_msg, tool_info, usage_info,
+    StopResponse, ToolResponse, display_list, fmt2hist, hist2fmt, mk_result_fence, split_fence_msgs, extract_fence_call,
+    trunc_str, Completion, mk_msg, mk_msgs)
 from .acomplete import *
 from .streaming import FenceToolStop, stop_sequences
 
-# %% ../nbs/07_chat.ipynb #43a55cda
-def _add_cache_control(
-    msg,          # LiteLLM formatted msg
-    ttl=None # Cache TTL: '5m' (default) or '1h'
-):
-    "cache `msg` with default time-to-live (ttl) of 5minutes ('5m'), but can be set to '1h'."
-    cc = {"type": "ephemeral"} | ({"ttl": ttl} if ttl else {})
-    cache_idx = None
-    for idx, part in enumerate(msg.content):
-        if isinstance(part, (Text, ToolUse)): cache_idx = idx
-    msg.content[idx].cache_control = cc
-    return msg
-
-def _has_cache(msg):
-    "Check if msg has cache_control set"
-    return any(part.cache_control for part in msg.content)
-
-def remove_cache_ckpts(msg):
-    "remove cache checkpoints and return msg."
-    for part in msg.content: part.cache_control = None
-    return msg
-
-
 # %% ../nbs/07_chat.ipynb #48c78e48
 def contents(c):
     "Get Msg object from Completion."
@@ -56,51 +31,6 @@ def stop_reason(c):
     if not c.finish_reason: return 'unk'
     return c.finish_reason
 
-# %% ../nbs/07_chat.ipynb #4a1c44ea
-def mk_msg(
-    content,      # Content: str, bytes (image), list of mixed content, or dict w 'role' and 'content' fields
-    role="user",  # Message role if content isn't already a dict/Message
-    cache=False,  # Enable Anthropic caching
-    ttl=None      # Cache TTL: '5m' (default) or '1h'
-):
-    "Create a LiteLLM compatible message."
-    if content is None: return None
-    if isinstance(content, Msg): return content
-    if isinstance(content, Completion): return content.message
-    if isinstance(content, list) and len(content) == 1 and isinstance(content[0], str): parts = [Text(content[0])]
-    elif isinstance(content, list): parts = [mk_content(o) for o in content]
-    elif isinstance(content, dict): return Msg(role=content['role'], content=[Text(content['content'])])
-    else: parts = [Text(content)]
-    msg = Msg(role=role, content=parts)
-    return _add_cache_control(msg, ttl=ttl) if cache else msg
-
-# %% ../nbs/07_chat.ipynb #01bd81de
-def _apply_cache_idxs(msgs, cache_idxs=[-1], ttl=None):
-    "Add cache control to `cache_idxs` after filtering tool-role msgs"
-    ms = [j for j,m in enumerate(msgs) if m.role != 'tool']
-    for i in cache_idxs:
-        try: idx = ms[i]
-        except IndexError: continue
-        _add_cache_control(msgs[idx], ttl)
-
-# %% ../nbs/07_chat.ipynb #fcbb9f5d
-def mk_msgs(
-    msgs,                   # List of messages (each: str, bytes, list, Msg, or Completion)
-    cache=False,            # Enable Anthropic caching
-    cache_idxs=[-1],        # Cache breakpoint idxs
-    ttl=None,               # Cache TTL: '5m' (default) or '1h'
-):
-    "Create a list of fastllm canonical Msgs."
-    if not msgs: return []
-    if not isinstance(msgs, list): msgs = [msgs]
-    msgs = L(msgs).map(lambda m: fmt2hist(m) if isinstance(m,str) and (tool_info in m or usage_info in m) else [m]).concat()
-    res, role = [], 'user'
-    for m in msgs:
-        res.append(msg := remove_cache_ckpts(mk_msg(m, role=role)))
-        role = 'assistant' if msg.role in ('user','tool') else 'user'
-    if cache: _apply_cache_idxs(res, cache_idxs, ttl)
-    return res
-
 # %% ../nbs/07_chat.ipynb #447aed6e
 def cite_footnote(citations):
     'Build citation footnotes for a single Delta'
@@ -172,9 +102,6 @@ def _has_search(info): return bool(info.get('search_context_cost_per_query') or
 effort = AttrDict({o[0]:o for o in ('low','medium','high')})
 effort['x'] = 'max'
 
-# %% ../nbs/07_chat.ipynb #e1facb77
-def _mk_prefill(pf): return Text(pf)
-
 # %% ../nbs/07_chat.ipynb #dc17f844
 def _has_stop(tres_parts): return any(isinstance(p.text, StopResponse) for p in tres_parts)
 
@@ -255,7 +182,7 @@ class AsyncChat:
             if v in vendor_mapping: vendor_name = v
             elif v: api_name = v  # a registered transport api (e.g. claude_code)
         self.model = model
-        hist,tools = mk_msgs(hist,cache,cache_idxs,ttl),listify(tools)
+        hist,tools = mk_msgs(hist),listify(tools)
         if ns is None and tools: ns = mk_ns(tools)
         elif ns is None: ns = globals()
         self.tool_schemas = [lite_mk_func(t) for t in tools] if tools else None
@@ -267,18 +194,11 @@ class AsyncChat:
         if default_cbs: self.add_cbs(defaults.chat_callbacks)
         self.add_cbs(cbs)
     
-    def _prep_msg(self, msg=None, prefill=None):
+    def _prep_msg(self, msg=None):
         "Prepare the system prompt and messages list for the API call"
-        sp = self.sp
-        if sp:
-            if 0 in self.cache_idxs: sp = _add_cache_control(Msg('',[Text(sp)]))
-            cache_idxs = L(self.cache_idxs).filter().map(lambda o: o-1 if o>0 else o)
-        else: cache_idxs = self.cache_idxs
         if msg: self.hist = self.hist+[msg]
-        self.hist = mk_msgs(self.hist, self.cache and 'claude' in self.model, cache_idxs, self.ttl)
-        msgs = self.hist
-        if prefill: msgs = self.hist + [Msg(role='assistant', content=[Text(prefill)])]
-        return sp, msgs
+        self.hist = mk_msgs(self.hist)
+        return self.sp, self.hist
     
     @property
     def tcdict(self): return dict(tool_schemas=self.tool_schemas, ns=self.ns)
@@ -317,11 +237,10 @@ def _think_kw(model, think, vendor_name):
 
 # %% ../nbs/07_chat.ipynb #06e898fd
 @patch
-def _prep_call(self:AsyncChat, prefill, search, max_tokens, kwargs, stream=False, think=None):
-    "Prepare model info, prefill, search, and provider kwargs for a completion call"
+def _prep_call(self:AsyncChat, search, max_tokens, kwargs, stream=False, think=None):
+    "Prepare model info, search, and provider kwargs for a completion call"
     model_info = get_model_info(self.model, self.vendor_name)
     if max_tokens is None: max_tokens = ifnone(model_info.get('max_output_tokens'), 32_000)
-    if not model_info.get("supports_assistant_prefill"): prefill = None
     if _has_search(model_info) and (s:=ifnone(search,self.search)):
         if 'web_search_options' not in kwargs: kwargs['web_search_options'] = {}
         kwargs['web_search_options']['search_context_size'] = effort[s]
@@ -333,7 +252,7 @@ def _prep_call(self:AsyncChat, prefill, search, max_tokens, kwargs, stream=False
     if self.base_url:      kwargs['base_url'] = self.base_url
     if self.extra_headers: kwargs['xtra_headers'] = self.extra_headers
     kwargs.update(_think_kw(self.model, think, self.vendor_name))
-    return prefill, max_tokens
+    return max_tokens
 
 # %% ../nbs/07_chat.ipynb #07951b77
 @patch
@@ -357,26 +276,25 @@ async def astream_with_complete(self, agen, postproc=noop):
 # %% ../nbs/07_chat.ipynb #a049cf52
 @patch
 @delegates(acomplete)
-async def _call(self:AsyncChat, msg=None, prefill=None, temp=None, think=None, search=None, stream=False, max_steps=2, step=1,
+async def _call(self:AsyncChat, msg=None, temp=None, think=None, search=None, stream=False, max_steps=2, step=1,
     final_prompt=None, tool_choice=None, max_tokens=None, n_workers=8, pause=0.001, tc_timeout=7200, **kwargs):
     if step>max_steps+1: return
-    self.prefill, max_tokens = self._prep_call(prefill, search, max_tokens, kwargs, stream=stream, think=think)
-    self.turn_sysp, self.turn_msgs = self._prep_msg(msg, prefill)
+    max_tokens = self._prep_call(search, max_tokens, kwargs, stream=stream, think=think)
+    self.turn_sysp, self.turn_msgs = self._prep_msg(msg)
     async for o in self._call_cbs('after_msgs'): yield o
 
     self.turn_kwargs, self.stream = kwargs, stream
     async for o in self._call_cbs('before_acomplete'): yield o
     res = await acomplete(self.turn_msgs, self.model, system=self.turn_sysp, stream=stream, tools=self.tool_schemas,
-        tool_choice=tool_choice, max_tokens=int(max_tokens), temperature=None if think else ifnone(temp,self.temp), **self.turn_kwargs)
+        tool_choice=tool_choice, max_tokens=int(max_tokens), temperature=None if think else ifnone(temp,self.temp),
+        cache_idxs=self.cache_idxs if self.cache else [], ttl=self.ttl, **self.turn_kwargs)
     if stream:
-        if self.prefill: yield _mk_prefill(self.prefill)
         res = astream_with_complete(res, postproc=postproc)
         async for chunk in res:
             if isinstance(chunk, Thinking) and self.showthink: chunk.showthink = True
             yield chunk
         res = res.value
     self.turn_res, self.turn_msg = res, contents(res)
-    if self.prefill: self.turn_msg.content[0].text = self.prefill + self.turn_msg.content[0].text
     self.hist.append(self.turn_msg)
     async for o in self._call_cbs('after_acomplete'): yield o
     self._track(self.turn_res)
@@ -397,14 +315,14 @@ async def _call(self:AsyncChat, msg=None, prefill=None, temp=None, think=None, s
     if self.toolloop and step <= max_steps:
         try:
             async for result in self._call(
-                self.prompt, None, temp, think, search, stream, max_steps, step+1,
+                self.prompt, temp, think, search, stream, max_steps, step+1,
                 final_prompt, tool_choice=tool_choice, **kwargs): yield result
         except ContextWindowExceededError:
             if tmsg is not None:
                 for p in tmsg.content:
                     if len(p.text)>1000: p.text = _cwe_msg + trunc_str(p.text, mx=1000)
             async for result in self._call(
-                self.prompt, None, temp, think, search, stream, max_steps, step+1,
+                self.prompt, temp, think, search, stream, max_steps, step+1,
                 final_prompt, tool_choice='none', **kwargs): yield result
 
 # %% ../nbs/07_chat.ipynb #1361515a
@@ -413,7 +331,6 @@ async def _call(self:AsyncChat, msg=None, prefill=None, temp=None, think=None, s
 async def __call__(
     self:AsyncChat,
     msg=None,          # Message str, or list of multiple message parts
-    prefill=None,      # Prefill AI response if model supports it
     temp=None,         # Override temp set on chat initialization
     think=None,        # Thinking (l,m,h)
     search=None,       # Override search set on chat initialization (l,m,h)
@@ -425,7 +342,7 @@ async def __call__(
 ):
     self.use = UsageStats()
     self._turn_start = len(self.hist)
-    result_gen = self._call(msg, prefill, temp, think, search, stream, max_steps, 1, final_prompt, **kwargs)
+    result_gen = self._call(msg, temp, think, search, stream, max_steps, 1, final_prompt, **kwargs)
     if stream or return_all: return result_gen
     async for res in result_gen: pass
     return res # normal chat behavior only return last msg
@@ -458,14 +375,6 @@ class DeepseekMsgsCallback(ChatCallback):
             if m.role=='assistant' and not any(isinstance(p, Thinking) for p in m.content): m.content.append(Thinking(''))
         if False: yield
 
-# %% ../nbs/07_chat.ipynb #14baac3e
-class DeepseekPrefillCallback(ChatCallback):
-    order = 10
-    async def before_acomplete(self):
-        if self.prefill and self.vendor_name == 'deepseek' and self.model.startswith("deepseek-"):
-            self.chat.turn_kwargs['base_url'] = 'https://api.deepseek.com/beta'
-        if False: yield
-
 # %% ../nbs/07_chat.ipynb #ce47dc4a
 def add_warning(r, msg):
     wrn = Text(f"<warning>{msg}</warning>")
@@ -565,7 +474,7 @@ class StopSequencesCallback(ChatCallback):
         if False: yield
 
 # %% ../nbs/07_chat.ipynb #318ee856
-defaults.chat_callbacks = [DeepseekPrefillCallback, FenceToolCallback, ToolReminderCallback, StopReasonCallback]
+defaults.chat_callbacks = [FenceToolCallback, ToolReminderCallback, StopReasonCallback]
 
 # %% ../nbs/07_chat.ipynb #ead4b56d
 class StreamAccum:

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/fastllm/streaming.py

@@ -8,26 +8,21 @@ __all__ = ['Delta', 'norm_and_yield', 'PrintStream', 'PartAccum', 'FenceToolStop
 
 # %% ../nbs/01_streaming.ipynb #0df5c926
 import json,copy
-from dataclasses import dataclass, field, fields
 from fastcore.utils import *
 from fastcore.meta import delegates
 from fastspec.errors import *
 from .types import *
-from aidialog.msg_parts import Part, PartType, Msg, Text, Thinking, ToolUse, ToolResult, ServerToolResult, mk_part, fence_call_re
+from aidialog.msg_parts import Part, PartType, Msg, Text, Thinking, ToolUse, ToolResult, ServerToolResult, mk_part, fence_call_re, Completion
 
 # %% ../nbs/01_streaming.ipynb #400d628a
-@dataclass
-class Delta:
+class Delta(BasicRepr):
     "Normalized streaming delta event."
-    text: str = ""
-    thinking: str = ""
-    refusal: str = ""
-    tool_calls: list[ToolUse] = field(default_factory=list)
-    citations: list = field(default_factory=list)
-    server_tool_result: dict = None
-    finish_reason: str = None
-    usage: Usage = None
-    raw: dict = field(default_factory=dict)
+    def __init__(self, text='', thinking='', refusal='', tool_calls=None, citations=None,
+        server_tool_result=None, finish_reason=None, usage=None, raw=None):
+        store_attr()
+        self.tool_calls,self.citations,self.raw = ifnone(tool_calls,[]),ifnone(citations,[]),ifnone(raw,{})
+    def __eq__(self, o): return type(o) is type(self) and self.__dict__ == o.__dict__
+    def __hash__(self): return hash((self.text, self.thinking, self.finish_reason))
 
 # %% ../nbs/01_streaming.ipynb #f59f837b
 async def norm_and_yield(resp, norm_func):
@@ -47,10 +42,13 @@ class PrintStream:
         if f: print(f, end='', flush=True)
 
 # %% ../nbs/01_streaming.ipynb #a7f1738a
-@dataclass
-class PartAccum:
+class PartAccum(BasicRepr):
     "Accumulate streamed part deltas by index"
-    parts: dict = field(default_factory=dict)
+    def __init__(self, parts=None):
+        store_attr()
+        self.parts = ifnone(parts, {})
+    def __eq__(self, o): return type(o) is type(self) and self.__dict__ == o.__dict__
+    def __hash__(self): return hash(len(self.parts))
 
     def append(self, typ, index, txt='', citations=None, **tc_kwargs):
         'Create and accumulate same type sequential parts'
@@ -117,11 +115,12 @@ def stop_sequences(seqs):
     return _stop
 
 # %% ../nbs/01_streaming.ipynb #90f91955
-@dataclass
-class Status:
+class Status(BasicRepr):
     "Stream marker for activity that isn't model content (e.g. draining deltas after a stop sequence)"
-    status: str
     type = 'status'
+    def __init__(self, status): store_attr()
+    def __eq__(self, o): return type(o) is type(self) and self.__dict__ == o.__dict__
+    def __hash__(self): return hash(self.status)
     @property
     def formatted(self): return '🧠'
 

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/fastllm/types.py

@@ -6,51 +6,29 @@
 __all__ = ['FinishReason', 'api_registry', 'model_prices_url', 'haik45', 'sonn45', 'sonn46', 'sonn', 'sonn5', 'opus46', 'opus48',
            'opus', 'opus5', 'fable', 'fable5', 'gpt54', 'gpt54m', 'gpt55', 'codex54', 'codex54m', 'codex55',
            'codex53spark', 'model_info_registry', 'modern_llm', 'deepseek_v4_common', 'mimo_v25_common',
-           'codex_pricing', 'sol', 'terra', 'luna', 'gpt56s', 'Usage', 'Completion', 'APIRegistry', 'mk_completion',
-           'fn_schema', 'payload_kwargs', 'get_api_key', 'resize_b64', 'model_prices_meta', 'infer_api_name',
-           'get_model_meta', 'register_model_info', 'get_model_info', 'get_model_pricing', 'approx_pricing',
-           'is_deepseek_peak_hour', 'price_tier', 'tier_rate']
+           'codex_pricing', 'sol', 'terra', 'luna', 'gpt56s', 'Usage', 'APIRegistry', 'mk_completion', 'fn_schema',
+           'payload_kwargs', 'get_api_key', 'resize_b64', 'model_prices_meta', 'infer_api_name', 'get_model_meta',
+           'register_model_info', 'get_model_info', 'get_model_pricing', 'approx_pricing', 'is_deepseek_peak_hour',
+           'price_tier', 'tier_rate']
 
 # %% ../nbs/00_types.ipynb #b4d047fd
 import httpx, base64, io
 from importlib.metadata import entry_points
-from dataclasses import dataclass, field
 from datetime import datetime, timezone
-from fastcore.net import urljson
 from fastcore.utils import *
 from PIL import Image as PImg
-from aidialog.msg_parts import Part, PartType, Msg, Text, Thinking, ToolUse, ToolResult, InputImage, InputAudio, InputVideo, InputFile, data_url
+from aidialog.msg_parts import Msg, Thinking, ToolUse, Completion
 
 
 # %% ../nbs/00_types.ipynb #802ad832
-@dataclass(frozen=True)
-class Usage:
+class Usage(BasicRepr):
     "Normalized usage."
-    prompt_tokens: int = 0
-    completion_tokens: int = 0
-    total_tokens: int = 0
-    cached_tokens: int = 0
-    cache_creation_tokens: int = 0
-    reasoning_tokens: int = 0
-    raw: dict = field(default_factory=dict)
-
-
-# %% ../nbs/00_types.ipynb #297a1672
-@dataclass(frozen=True)
-class Completion:
-    "Normalized completion response."
-    model: str
-    message: Msg
-    finish_reason: str = None
-    usage: Usage = None
-    api_name: str = None
-    vendor_name: str = None
-    raw: dict = field(default_factory=dict)
-
-    @property
-    def tool_calls(self):
-        "The `ToolUse` parts of `message`: a call lives in the content, not beside it"
-        return [p for p in self.message.content if isinstance(p, ToolUse)]
+    def __init__(self, prompt_tokens=0, completion_tokens=0, total_tokens=0, cached_tokens=0,
+        cache_creation_tokens=0, reasoning_tokens=0, raw=None):
+        store_attr()
+        self.raw = ifnone(raw, {})
+    def __eq__(self, o): return type(o) is type(self) and self.__dict__ == o.__dict__
+
 
 # %% ../nbs/00_types.ipynb #4901e693
 @patch(as_prop=True)
@@ -62,18 +40,18 @@ def _repr_markdown_(self: Completion):
     content = ''
     for p in self.message.content:
         if isinstance(p, Thinking):
-            if p.text: content += f"<details><summary>Thinking</summary>\n\n{p.text}\n\n</details>\n\n"
+            if p.text: content += f"\n::: details\n## Thinking\n\n{p.text}\n\n:::\n\n"
         elif isinstance(p, ToolUse): content += f"\n\n🔧 {p.name}({p.arguments})\n"
         elif txt := p.text: content += txt
     details = [f"model: `{self.model}`", f"finish_reason: `{self.finish_reason}`", f"usage: `{self.usage}`"]
     det_str = '\n- '.join(details)    
     return f"""{content}
 
-<details markdown='1'>
+::: details
 
 - {det_str}
 
-</details>"""
+:::"""
 
 # %% ../nbs/00_types.ipynb #ce59e431
 FinishReason = str_enum('finish_reason', 'stop', 'tool_calls', 'length', 'content_filter')
@@ -123,7 +101,8 @@ def fn_schema(t):
     return None
 
 # %% ../nbs/00_types.ipynb #28c698fe
-def payload_kwargs(msgs, model, stream=False, system=None, max_tokens=None, temperature=None, tools=None, tool_choice=None, reasoning_effort=None, web_search_options=None, stop_callables=None): pass
+def payload_kwargs(msgs, model, stream=False, system=None, max_tokens=None, temperature=None, tools=None, tool_choice=None,
+    reasoning_effort=None, web_search_options=None, cache_idxs=None, ttl=None, stop_callables=None): pass
 
 # %% ../nbs/00_types.ipynb #c2a2cb49
 def get_api_key(api_key, default):
@@ -234,7 +213,7 @@ register_model_info('gpt-5.4-mini', vendor_name='openai', base='gpt-5.4-mini', s
 
 for model in ('kimi-k2.5', 'kimi-k2.6' , 'kimi-k3'):
     register_model_info(model, vendor_name='moonshot', base=f'moonshot/{model}', base_vendor_name=None,
-        supports_reasoning=True, supports_vision=True, supports_assistant_prefill=True)
+        supports_reasoning=True, supports_vision=True)
 
 register_model_info('gemini-3.1-flash-lite', vendor_name='gemini', base='gemini-3.1-flash-lite-preview')
 register_model_info('models/gemini-3.1-flash-lite', vendor_name='gemini', base='gemini-3.1-flash-lite-preview')
@@ -252,8 +231,7 @@ register_model_info('kimi-k3', vendor_name='moonshot', base='kimi-k2.6',
     input_cost_per_token=3e-6, cache_read_input_token_cost=0.30e-6, output_cost_per_token=15.0e-6)
 
 # %% ../nbs/00_types.ipynb #948d55d0
-deepseek_v4_common = dict(**modern_llm, supports_assistant_prefill=True,
-    max_input_tokens=1048576, max_output_tokens=393216, max_tokens=393216)
+deepseek_v4_common = dict(**modern_llm, max_input_tokens=1048576, max_output_tokens=393216, max_tokens=393216)
 
 register_model_info('deepseek-v4-flash', vendor_name='deepseek', base='deepseek/deepseek-v3.2', **deepseek_v4_common,
     input_cost_per_token=1.4e-07, input_cost_per_token_cache_hit=2.8e-09,

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
 ]
-dependencies = ['fastcore>=2.1.18', 'aidialog>=0.0.8', 'fastspec>=0.0.11', 'pillow']
+dependencies = ['fastcore>=2.1.18', 'aidialog>=0.0.10', 'fastspec>=0.0.11', 'pillow']
 
 [project.urls]
 Repository = "https://github.com/AnswerDotAI/fastllm"

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/python_fastllm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-fastllm
-Version: 0.0.38
+Version: 0.0.39
 Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
 License: Apache-2.0
 Project-URL: Repository, https://github.com/AnswerDotAI/fastllm
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: fastcore>=2.1.18
-Requires-Dist: aidialog>=0.0.8
+Requires-Dist: aidialog>=0.0.10
 Requires-Dist: fastspec>=0.0.11
 Requires-Dist: pillow
 

{python_fastllm-0.0.38 → python_fastllm-0.0.39}/python_fastllm.egg-info/requires.txt

@@ -1,4 +1,4 @@
 fastcore>=2.1.18
-aidialog>=0.0.8
+aidialog>=0.0.10
 fastspec>=0.0.11
 pillow

python_fastllm-0.0.38/fastllm/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.38"