python-fastllm 0.0.49__tar.gz → 0.0.51__tar.gz

{python_fastllm-0.0.49 → python_fastllm-0.0.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-fastllm
-Version: 0.0.49
+Version: 0.0.51
 Author-email: Kerem Turgutlu <keremturgutlu@gmail.com>
 License: Apache-2.0
 Project-URL: Repository, https://github.com/AnswerDotAI/fastllm
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: fastcore>=2.1.18
-Requires-Dist: aidialog>=0.0.10
+Requires-Dist: aidialog>=0.0.25
 Requires-Dist: fasttransport>=0.0.2
 Requires-Dist: pillow
 
@@ -21,13 +21,14 @@ Requires-Dist: pillow
 
 ## Install
 
-Clone and install locally into your `aai-ws` env
+Run `pip install python-fastllm` or clone from github and install locally.
 
 ## Setup
 
 ``` python
 from aidialog.msg_parts import Msg, Part, Text, Thinking, ToolUse, InputImage, mk_tool_res_msg, Completion
 from fastllm.acomplete import acomplete
+from fastllm.types import sample_img_url, sample_doms
 import asyncio, json
 
 # Helpers
@@ -56,7 +57,7 @@ The same `acomplete` call works with Claude, GPT, Gemini, and Kimi. Just change
 
 ``` python
 models = [
-    ('claude-sonnet-4-20250514', {}),
+    ('claude-sonnet-5', {}),
     ('gpt-4o-mini', {}),
     ('models/gemini-3-flash-preview', {}),
     ('accounts/fireworks/models/kimi-k2p5', dict(vendor_name='fireworks_ai'))
@@ -80,46 +81,6 @@ for name, kw in models:
 
     The response should be simple and direct.
 
-## Multi-Turn — Swap Providers Mid-Conversation
-
-Build a conversation with one provider, then seamlessly continue it with another. `fastllm` translates between every provider’s native format automatically:
-
-``` python
-# Turn 1: Claude starts the conversation
-msgs = [user("Name the 3 largest planets in our solar system. One sentence.")]
-print("Claude: ", end='')
-r1 = await stream(msgs, model='claude-sonnet-4-20250514', max_tokens=mtok)
-```
-
-    Claude: The three largest planets in our solar system are Jupiter, Saturn, and Neptune.
-
-``` python
-# Turn 2: Switch to GPT — just change the model string
-msgs += [r1.message, user("Which one has the most moons?")]
-print("GPT: ", end='')
-r2 = await stream(msgs, model='gpt-4o-mini', max_tokens=mtok)
-```
-
-    GPT: As of now, Saturn has the most moons, with over 80 confirmed moons.
-
-``` python
-# Turn 3: Switch to Gemini — same msgs, different provider
-msgs += [r2.message, user("Summarize our conversation in one sentence.")]
-print("Gemini: ", end='')
-r3 = await stream(msgs, model='models/gemini-3-flash-preview', max_tokens=mtok)
-```
-
-    Gemini: The conversation identified the three largest planets in the solar system and noted that Saturn currently has the most moons.
-
-``` python
-# Turn 4: Switch to Kimi — works the same way
-msgs += [r3.message, user("Thanks! What's one surprising fact about Saturn?")]
-print("Kimi: ", end='')
-r4 = await stream(msgs, model='accounts/fireworks/models/kimi-k2p5', vendor_name='fireworks_ai', max_tokens=mtok)
-```
-
-    Kimi: 🧠🧠🧠🧠🧠🧠🧠🧠🧠🧠Saturn is less dense than water, meaning it would theoretically float if you could find a bathtub large enough to hold it.
-
 ## System Prompts
 
 Pass a system prompt to any provider — `fastllm` maps it to each API’s native mechanism (Anthropic `system`, OpenAI `instructions`, Gemini `system_instruction`):
@@ -128,7 +89,7 @@ Pass a system prompt to any provider — `fastllm` maps it to each API’s nativ
 sys = "You are a pirate chef. Always respond in pirate speak and mention food."
 
 print("Claude: ", end='')
-r = await stream([user("What should I do today?")], model='claude-sonnet-4-20250514', system=sys, max_tokens=mtok)
+r = await stream([user("What should I do today?")], model='claude-sonnet-5', system=sys, max_tokens=mtok)
 
 print("Gemini: ", end='')
 r = await stream([user("What should I do today?")], model='models/gemini-3-flash-preview', system=sys, max_tokens=mtok)
@@ -161,7 +122,7 @@ r = await stream([user("What should I do today?")], model='models/gemini-3-flash
 
 ## Tool Calling — Define Once, Use Anywhere
 
-Define tools in a single canonical format. `fastllm` translates to each provider’s native tool schema automatically. Here we start a tool-use conversation with Claude, provide the result, then switch to GPT and Gemini to continue:
+Define tools in a single canonical format. `fastllm` translates to each provider’s native tool schema automatically. Here Claude requests the tool, receives the result, and answers:
 
 ``` python
 tools = [{"type": "function", "function": {
@@ -172,7 +133,7 @@ tools = [{"type": "function", "function": {
 
 # Turn 1: Claude requests the tool
 msgs = [user("What's the weather in Paris?")]
-r1 = await stream(msgs, model='claude-sonnet-4-20250514', tools=tools, max_tokens=mtok)
+r1 = await stream(msgs, model='claude-sonnet-5', tools=tools, max_tokens=mtok)
 print("Tool calls:", r1.tool_calls)
 ```
 
@@ -182,41 +143,26 @@ print("Tool calls:", r1.tool_calls)
     Tool calls: [ToolUse(raw={'caller': {'type': 'direct'}}, cache_control=None, id='toolu_01RMN1WM7vPBT3ovv5Ex6VzC', name='get_weather', arguments={'city': 'Paris'}, server=False, text=None)]
 
 ``` python
-# Provide the tool result
-msgs += [r1.message, mk_tool_res_msg(r1.tool_calls, ['22°C, sunny with light clouds'])]
-
-# Turn 2: Switch to GPT to interpret the result
-msgs.append(user("Should I bring a jacket?"))
-print("GPT: ", end='')
-r2 = await stream(msgs, model='gpt-4o-mini', tools=tools, max_tokens=mtok)
+# Provide the tool result, then continue the same conversation
+msgs += [r1.message, mk_tool_res_msg(r1.tool_calls, ['22°C, sunny with light clouds']), user("Should I bring a jacket?")]
+print("Claude: ", end='')
+r2 = await stream(msgs, model='claude-sonnet-5', tools=tools, max_tokens=mtok)
 ```
 
     GPT: With the temperature at 22°C and sunny with light clouds, a jacket isn't necessary for most people. However, if you tend to feel cold easily or if you plan to be out in the evening when it might cool down a bit, it could be a good idea to bring a light jacket.
 
-``` python
-# Turn 3: Gemini sees the full cross-provider tool history
-msgs += [r2.message, user("How about tomorrow — will it rain?")]
-r3 = await stream(msgs, model='models/gemini-3-flash-preview', tools=tools, max_tokens=mtok, web_search_options={})
-```
-
-    Tomorrow in Paris (Saturday, June 13), the weather is expected to be mostly pleasant and sunny. 
-
-    While there is a small chance of light, passing showers (around a 20-25% chance), most forecasts indicate a dry day with mostly clear or scattered clouds. Temperatures will likely reach a comfortable high of around 22°C to 27°C (72°F to 80°F), making it a great day for being outdoors. 
-
-    So, while you might see a stray shower, you likely won't need to worry about heavy rain!
-
 ## Tool Choice
 
 Control whether the model must use tools, can’t use tools, or decides on its own:
 
 ``` python
 # Force the model to call a tool (even for a greeting)
-r = await stream([user("Hello there!")], model='claude-sonnet-4-20250514', tools=tools, tool_choice='required', max_tokens=mtok)
+r = await stream([user("Hello there!")], model='claude-sonnet-5', tools=tools, tool_choice='required', max_tokens=mtok)
 print("Forced:", r.tool_calls)
 
 # Prevent tool use (model must answer directly)
 print("\nNo tools: ", end='')
-r = await stream([user("What's the weather?")], model='claude-sonnet-4-20250514', tools=tools, tool_choice='none', max_tokens=mtok)
+r = await stream([user("What's the weather?")], model='claude-sonnet-5', tools=tools, tool_choice='none', max_tokens=mtok)
 ```
 
     - ⏳ `get_weather(city="<UNKNOWN>")` ⏳
@@ -327,11 +273,11 @@ long_ctx = "You are an expert on the solar system. " * 200
 system = Text(long_ctx, cache_control={'type': 'ephemeral'})
 
 print("Call 1: ", end='')
-r1 = await stream([user("What is Jupiter's mass?")], model='claude-sonnet-4-20250514', system=system, max_tokens=mtok)
+r1 = await stream([user("What is Jupiter's mass?")], model='claude-sonnet-5', system=system, max_tokens=mtok)
 print(f"Usage: {r1.usage}")
 
 print("Call 2: ", end='')
-r2 = await stream([user("How about Saturn?")], model='claude-sonnet-4-20250514', system=system, max_tokens=mtok)
+r2 = await stream([user("How about Saturn?")], model='claude-sonnet-5', system=system, max_tokens=mtok)
 print(f"Usage: {r2.usage}")
 ```
 
@@ -381,13 +327,12 @@ print(f"Usage: {r2.usage}")
 Send images to any provider that supports them. The canonical `input_image` part works everywhere:
 
 ``` python
-img_url = "https://img.freepik.com/free-photo/mountain-range-body-water_53876-139760.jpg?semt=ais_hybrid&w=740&q=80"
 img_msg = Msg(role='user', content=[
-    InputImage(img_url),
+    InputImage(sample_img_url),
     Text("What do you see in this image?")
 ])
 
-for name, kw in [('claude-sonnet-4-20250514', {}), ('gpt-4o-mini', {}), ('models/gemini-3-flash-preview', {})]:
+for name, kw in [('claude-sonnet-5', {}), ('gpt-4o-mini', {}), ('models/gemini-3-flash-preview', {})]:
     print(f"{name:>30s}: ", end='')
     r = await stream([img_msg], model=name, max_tokens=mtok, **kw)
 ```

{python_fastllm-0.0.49 → python_fastllm-0.0.51}/README.md

@@ -5,13 +5,14 @@
 
 ## Install
 
-Clone and install locally into your `aai-ws` env
+Run `pip install python-fastllm` or clone from github and install locally.
 
 ## Setup
 
 ``` python
 from aidialog.msg_parts import Msg, Part, Text, Thinking, ToolUse, InputImage, mk_tool_res_msg, Completion
 from fastllm.acomplete import acomplete
+from fastllm.types import sample_img_url, sample_doms
 import asyncio, json
 
 # Helpers
@@ -40,7 +41,7 @@ The same `acomplete` call works with Claude, GPT, Gemini, and Kimi. Just change
 
 ``` python
 models = [
-    ('claude-sonnet-4-20250514', {}),
+    ('claude-sonnet-5', {}),
     ('gpt-4o-mini', {}),
     ('models/gemini-3-flash-preview', {}),
     ('accounts/fireworks/models/kimi-k2p5', dict(vendor_name='fireworks_ai'))
@@ -64,46 +65,6 @@ for name, kw in models:
 
     The response should be simple and direct.
 
-## Multi-Turn — Swap Providers Mid-Conversation
-
-Build a conversation with one provider, then seamlessly continue it with another. `fastllm` translates between every provider’s native format automatically:
-
-``` python
-# Turn 1: Claude starts the conversation
-msgs = [user("Name the 3 largest planets in our solar system. One sentence.")]
-print("Claude: ", end='')
-r1 = await stream(msgs, model='claude-sonnet-4-20250514', max_tokens=mtok)
-```
-
-    Claude: The three largest planets in our solar system are Jupiter, Saturn, and Neptune.
-
-``` python
-# Turn 2: Switch to GPT — just change the model string
-msgs += [r1.message, user("Which one has the most moons?")]
-print("GPT: ", end='')
-r2 = await stream(msgs, model='gpt-4o-mini', max_tokens=mtok)
-```
-
-    GPT: As of now, Saturn has the most moons, with over 80 confirmed moons.
-
-``` python
-# Turn 3: Switch to Gemini — same msgs, different provider
-msgs += [r2.message, user("Summarize our conversation in one sentence.")]
-print("Gemini: ", end='')
-r3 = await stream(msgs, model='models/gemini-3-flash-preview', max_tokens=mtok)
-```
-
-    Gemini: The conversation identified the three largest planets in the solar system and noted that Saturn currently has the most moons.
-
-``` python
-# Turn 4: Switch to Kimi — works the same way
-msgs += [r3.message, user("Thanks! What's one surprising fact about Saturn?")]
-print("Kimi: ", end='')
-r4 = await stream(msgs, model='accounts/fireworks/models/kimi-k2p5', vendor_name='fireworks_ai', max_tokens=mtok)
-```
-
-    Kimi: 🧠🧠🧠🧠🧠🧠🧠🧠🧠🧠Saturn is less dense than water, meaning it would theoretically float if you could find a bathtub large enough to hold it.
-
 ## System Prompts
 
 Pass a system prompt to any provider — `fastllm` maps it to each API’s native mechanism (Anthropic `system`, OpenAI `instructions`, Gemini `system_instruction`):
@@ -112,7 +73,7 @@ Pass a system prompt to any provider — `fastllm` maps it to each API’s nativ
 sys = "You are a pirate chef. Always respond in pirate speak and mention food."
 
 print("Claude: ", end='')
-r = await stream([user("What should I do today?")], model='claude-sonnet-4-20250514', system=sys, max_tokens=mtok)
+r = await stream([user("What should I do today?")], model='claude-sonnet-5', system=sys, max_tokens=mtok)
 
 print("Gemini: ", end='')
 r = await stream([user("What should I do today?")], model='models/gemini-3-flash-preview', system=sys, max_tokens=mtok)
@@ -145,7 +106,7 @@ r = await stream([user("What should I do today?")], model='models/gemini-3-flash
 
 ## Tool Calling — Define Once, Use Anywhere
 
-Define tools in a single canonical format. `fastllm` translates to each provider’s native tool schema automatically. Here we start a tool-use conversation with Claude, provide the result, then switch to GPT and Gemini to continue:
+Define tools in a single canonical format. `fastllm` translates to each provider’s native tool schema automatically. Here Claude requests the tool, receives the result, and answers:
 
 ``` python
 tools = [{"type": "function", "function": {
@@ -156,7 +117,7 @@ tools = [{"type": "function", "function": {
 
 # Turn 1: Claude requests the tool
 msgs = [user("What's the weather in Paris?")]
-r1 = await stream(msgs, model='claude-sonnet-4-20250514', tools=tools, max_tokens=mtok)
+r1 = await stream(msgs, model='claude-sonnet-5', tools=tools, max_tokens=mtok)
 print("Tool calls:", r1.tool_calls)
 ```
 
@@ -166,41 +127,26 @@ print("Tool calls:", r1.tool_calls)
     Tool calls: [ToolUse(raw={'caller': {'type': 'direct'}}, cache_control=None, id='toolu_01RMN1WM7vPBT3ovv5Ex6VzC', name='get_weather', arguments={'city': 'Paris'}, server=False, text=None)]
 
 ``` python
-# Provide the tool result
-msgs += [r1.message, mk_tool_res_msg(r1.tool_calls, ['22°C, sunny with light clouds'])]
-
-# Turn 2: Switch to GPT to interpret the result
-msgs.append(user("Should I bring a jacket?"))
-print("GPT: ", end='')
-r2 = await stream(msgs, model='gpt-4o-mini', tools=tools, max_tokens=mtok)
+# Provide the tool result, then continue the same conversation
+msgs += [r1.message, mk_tool_res_msg(r1.tool_calls, ['22°C, sunny with light clouds']), user("Should I bring a jacket?")]
+print("Claude: ", end='')
+r2 = await stream(msgs, model='claude-sonnet-5', tools=tools, max_tokens=mtok)
 ```
 
     GPT: With the temperature at 22°C and sunny with light clouds, a jacket isn't necessary for most people. However, if you tend to feel cold easily or if you plan to be out in the evening when it might cool down a bit, it could be a good idea to bring a light jacket.
 
-``` python
-# Turn 3: Gemini sees the full cross-provider tool history
-msgs += [r2.message, user("How about tomorrow — will it rain?")]
-r3 = await stream(msgs, model='models/gemini-3-flash-preview', tools=tools, max_tokens=mtok, web_search_options={})
-```
-
-    Tomorrow in Paris (Saturday, June 13), the weather is expected to be mostly pleasant and sunny. 
-
-    While there is a small chance of light, passing showers (around a 20-25% chance), most forecasts indicate a dry day with mostly clear or scattered clouds. Temperatures will likely reach a comfortable high of around 22°C to 27°C (72°F to 80°F), making it a great day for being outdoors. 
-
-    So, while you might see a stray shower, you likely won't need to worry about heavy rain!
-
 ## Tool Choice
 
 Control whether the model must use tools, can’t use tools, or decides on its own:
 
 ``` python
 # Force the model to call a tool (even for a greeting)
-r = await stream([user("Hello there!")], model='claude-sonnet-4-20250514', tools=tools, tool_choice='required', max_tokens=mtok)
+r = await stream([user("Hello there!")], model='claude-sonnet-5', tools=tools, tool_choice='required', max_tokens=mtok)
 print("Forced:", r.tool_calls)
 
 # Prevent tool use (model must answer directly)
 print("\nNo tools: ", end='')
-r = await stream([user("What's the weather?")], model='claude-sonnet-4-20250514', tools=tools, tool_choice='none', max_tokens=mtok)
+r = await stream([user("What's the weather?")], model='claude-sonnet-5', tools=tools, tool_choice='none', max_tokens=mtok)
 ```
 
     - ⏳ `get_weather(city="<UNKNOWN>")` ⏳
@@ -311,11 +257,11 @@ long_ctx = "You are an expert on the solar system. " * 200
 system = Text(long_ctx, cache_control={'type': 'ephemeral'})
 
 print("Call 1: ", end='')
-r1 = await stream([user("What is Jupiter's mass?")], model='claude-sonnet-4-20250514', system=system, max_tokens=mtok)
+r1 = await stream([user("What is Jupiter's mass?")], model='claude-sonnet-5', system=system, max_tokens=mtok)
 print(f"Usage: {r1.usage}")
 
 print("Call 2: ", end='')
-r2 = await stream([user("How about Saturn?")], model='claude-sonnet-4-20250514', system=system, max_tokens=mtok)
+r2 = await stream([user("How about Saturn?")], model='claude-sonnet-5', system=system, max_tokens=mtok)
 print(f"Usage: {r2.usage}")
 ```
 
@@ -365,13 +311,12 @@ print(f"Usage: {r2.usage}")
 Send images to any provider that supports them. The canonical `input_image` part works everywhere:
 
 ``` python
-img_url = "https://img.freepik.com/free-photo/mountain-range-body-water_53876-139760.jpg?semt=ais_hybrid&w=740&q=80"
 img_msg = Msg(role='user', content=[
-    InputImage(img_url),
+    InputImage(sample_img_url),
     Text("What do you see in this image?")
 ])
 
-for name, kw in [('claude-sonnet-4-20250514', {}), ('gpt-4o-mini', {}), ('models/gemini-3-flash-preview', {})]:
+for name, kw in [('claude-sonnet-5', {}), ('gpt-4o-mini', {}), ('models/gemini-3-flash-preview', {})]:
     print(f"{name:>30s}: ", end='')
     r = await stream([img_msg], model=name, max_tokens=mtok, **kw)
 ```

{python_fastllm-0.0.49 → python_fastllm-0.0.51}/fastllm/__init__.py

@@ -2,4 +2,4 @@
 
 - `fastllm.responses`: Provider-independent Responses resources and streaming over FastLLM"""
 
-__version__ = "0.0.49"
+__version__ = "0.0.51"

{python_fastllm-0.0.49 → python_fastllm-0.0.51}/fastllm/_modidx.py

@@ -25,6 +25,7 @@ d = { 'settings': { 'branch': 'main',
                                    'fastllm.anthropic._sanid': ('anthropic.html#_sanid', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.acollect_stream': ('anthropic.html#acollect_stream', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.apply_cache_idxs': ('anthropic.html#apply_cache_idxs', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic.collate_raw': ('anthropic.html#collate_raw', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.cost': ('anthropic.html#cost', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.delta_index_fn': ('anthropic.html#delta_index_fn', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.denorm_assistant': ('anthropic.html#denorm_assistant', 'fastllm/anthropic.py'),
@@ -49,7 +50,9 @@ d = { 'settings': { 'branch': 'main',
                                    'fastllm.anthropic.norm_tool_call': ('anthropic.html#norm_tool_call', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.norm_tool_calls': ('anthropic.html#norm_tool_calls', 'fastllm/anthropic.py'),
                                    'fastllm.anthropic.norm_tr_parts': ('anthropic.html#norm_tr_parts', 'fastllm/anthropic.py'),
-                                   'fastllm.anthropic.norm_usage': ('anthropic.html#norm_usage', 'fastllm/anthropic.py')},
+                                   'fastllm.anthropic.norm_usage': ('anthropic.html#norm_usage', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic.raw_msg': ('anthropic.html#raw_msg', 'fastllm/anthropic.py'),
+                                   'fastllm.anthropic.search_hits': ('anthropic.html#search_hits', 'fastllm/anthropic.py')},
             'fastllm.chat': { 'fastllm.chat.AsyncChat': ('chat.html#asyncchat', 'fastllm/chat.py'),
                               'fastllm.chat.AsyncChat.__call__': ('chat.html#asyncchat.__call__', 'fastllm/chat.py'),
                               'fastllm.chat.AsyncChat.__init__': ('chat.html#asyncchat.__init__', 'fastllm/chat.py'),
@@ -139,6 +142,7 @@ d = { 'settings': { 'branch': 'main',
             'fastllm.gemini': { 'fastllm.gemini._gem_filter_sch': ('gemini.html#_gem_filter_sch', 'fastllm/gemini.py'),
                                 'fastllm.gemini._gem_part_type': ('gemini.html#_gem_part_type', 'fastllm/gemini.py'),
                                 'fastllm.gemini.acollect_stream': ('gemini.html#acollect_stream', 'fastllm/gemini.py'),
+                                'fastllm.gemini.collate_raw': ('gemini.html#collate_raw', 'fastllm/gemini.py'),
                                 'fastllm.gemini.cost': ('gemini.html#cost', 'fastllm/gemini.py'),
                                 'fastllm.gemini.delta_index_fn': ('gemini.html#delta_index_fn', 'fastllm/gemini.py'),
                                 'fastllm.gemini.denorm_assistant': ('gemini.html#denorm_assistant', 'fastllm/gemini.py'),
@@ -163,8 +167,10 @@ d = { 'settings': { 'branch': 'main',
                                 'fastllm.gemini.norm_parts': ('gemini.html#norm_parts', 'fastllm/gemini.py'),
                                 'fastllm.gemini.norm_sse_event': ('gemini.html#norm_sse_event', 'fastllm/gemini.py'),
                                 'fastllm.gemini.norm_tool_calls': ('gemini.html#norm_tool_calls', 'fastllm/gemini.py'),
-                                'fastllm.gemini.norm_usage': ('gemini.html#norm_usage', 'fastllm/gemini.py')},
+                                'fastllm.gemini.norm_usage': ('gemini.html#norm_usage', 'fastllm/gemini.py'),
+                                'fastllm.gemini.raw_msg': ('gemini.html#raw_msg', 'fastllm/gemini.py')},
             'fastllm.openai_chat': { 'fastllm.openai_chat.acollect_stream': ('oai_chat.html#acollect_stream', 'fastllm/openai_chat.py'),
+                                     'fastllm.openai_chat.collate_raw': ('oai_chat.html#collate_raw', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.cost': ('oai_chat.html#cost', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.delta_index_fn': ('oai_chat.html#delta_index_fn', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.denorm_assistant': ('oai_chat.html#denorm_assistant', 'fastllm/openai_chat.py'),
@@ -189,14 +195,15 @@ d = { 'settings': { 'branch': 'main',
                                      'fastllm.openai_chat.norm_finish': ('oai_chat.html#norm_finish', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.norm_parts': ('oai_chat.html#norm_parts', 'fastllm/openai_chat.py'),
                                      'fastllm.openai_chat.norm_sse_event': ('oai_chat.html#norm_sse_event', 'fastllm/openai_chat.py'),
-                                     'fastllm.openai_chat.norm_tool_calls': ('oai_chat.html#norm_tool_calls', 'fastllm/openai_chat.py')},
+                                     'fastllm.openai_chat.norm_tool_calls': ('oai_chat.html#norm_tool_calls', 'fastllm/openai_chat.py'),
+                                     'fastllm.openai_chat.raw_msg': ('oai_chat.html#raw_msg', 'fastllm/openai_chat.py')},
             'fastllm.openai_responses': { 'fastllm.openai_responses.Completion.response_id': ( 'oai_responses.html#completion.response_id',
                                                                                                'fastllm/openai_responses.py'),
-                                          'fastllm.openai_responses.Completion.response_id_reusable': ( 'oai_responses.html#completion.response_id_reusable',
-                                                                                                        'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses._sanid': ('oai_responses.html#_sanid', 'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.acollect_stream': ( 'oai_responses.html#acollect_stream',
                                                                                         'fastllm/openai_responses.py'),
+                                          'fastllm.openai_responses.collate_raw': ( 'oai_responses.html#collate_raw',
+                                                                                    'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.cost': ('oai_responses.html#cost', 'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.delta_index_fn': ( 'oai_responses.html#delta_index_fn',
                                                                                        'fastllm/openai_responses.py'),
@@ -245,13 +252,19 @@ d = { 'settings': { 'branch': 'main',
                                           'fastllm.openai_responses.norm_tool_calls': ( 'oai_responses.html#norm_tool_calls',
                                                                                         'fastllm/openai_responses.py'),
                                           'fastllm.openai_responses.norm_usage': ( 'oai_responses.html#norm_usage',
-                                                                                   'fastllm/openai_responses.py')},
+                                                                                   'fastllm/openai_responses.py'),
+                                          'fastllm.openai_responses.raw_msg': ( 'oai_responses.html#raw_msg',
+                                                                                'fastllm/openai_responses.py')},
             'fastllm.responses': { 'fastllm.responses.AsyncResponses': ('responses.html#asyncresponses', 'fastllm/responses.py'),
+                                   'fastllm.responses.AsyncResponses.__init__': ( 'responses.html#asyncresponses.__init__',
+                                                                                  'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.call': ('responses.html#asyncresponses.call', 'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.events': ( 'responses.html#asyncresponses.events',
                                                                                 'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.prepare': ( 'responses.html#asyncresponses.prepare',
                                                                                  'fastllm/responses.py'),
+                                   'fastllm.responses.AsyncResponses.serve': ( 'responses.html#asyncresponses.serve',
+                                                                               'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.server_results': ( 'responses.html#asyncresponses.server_results',
                                                                                         'fastllm/responses.py'),
                                    'fastllm.responses.AsyncResponses.server_turn': ( 'responses.html#asyncresponses.server_turn',

{python_fastllm-0.0.49 → python_fastllm-0.0.51}/fastllm/anthropic.py

@@ -1,21 +1,21 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_anthropic.ipynb.
 
 # %% auto #0
-__all__ = ['ant_tc_types', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'finalize_usage', 'norm_finish', 'norm_parts',
-           'norm_tr_parts', 'norm_sse_event', 'delta_index_fn', 'acollect_stream', 'denorm_tool_use',
-           'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs', 'denorm_tool_choice',
-           'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user', 'denorm_image', 'denorm_file',
-           'denorm_tool_result', 'apply_cache_idxs', 'mk_payload', 'get_hdrs', 'cost']
+__all__ = ['ant_tc_types', 'norm_tool_call', 'norm_tool_calls', 'norm_usage', 'finalize_usage', 'norm_finish', 'search_hits',
+           'norm_parts', 'raw_msg', 'norm_tr_parts', 'norm_sse_event', 'delta_index_fn', 'collate_raw',
+           'acollect_stream', 'denorm_tool_use', 'denorm_assistant', 'denorm_tool', 'denorm_msgs', 'denorm_tool_schs',
+           'denorm_tool_choice', 'denorm_reasoning', 'denorm_web_search', 'denorm_system', 'denorm_user',
+           'denorm_image', 'denorm_file', 'denorm_tool_result', 'apply_cache_idxs', 'mk_payload', 'get_hdrs', 'cost']
 
 # %% ../nbs/04_anthropic.ipynb #02afd3d7
-import json
+import json, copy
 from collections import Counter
 from fastcore.utils import *
 from fastcore.meta import *
 from fasttransport.errors import api_error_from_event
 
 from .types import *
-from aidialog.msg_parts import (Part, PartType, mk_part, Msg, Text, Thinking, ToolUse, ToolResult, tool_text, ServerToolResult,
+from aidialog.msg_parts import (Part, PartType, mk_part, Msg, Completion, Text, Thinking, ToolUse, ToolResult, tool_text,
                                 InputImage, InputAudio, InputVideo, InputFile, data_url)
 from .streaming import *
 from .streaming import mk_acollect_stream
@@ -47,11 +47,11 @@ def norm_usage(resp):
                  cached_tokens=cached, cache_creation_tokens=cache_creation, reasoning_tokens=0, raw=usg)
 
 def finalize_usage(usg, parts):
-    "Adjust usage using finalized Anthropic content parts."
+    "Split reasoning tokens out of `output_tokens`: the reported `thinking_tokens`, else an estimate from the thinking text."
     if not usg: return usg
     rc = '\n'.join(p.text or '' for p in parts if isinstance(p, Thinking))
     ct = int(usg.raw.get('output_tokens', usg.completion_tokens) or 0)
-    rt = min(int(len(rc.split())*1.5), ct) if rc else 0
+    rt = int(nested_idx(usg.raw, 'output_tokens_details', 'thinking_tokens') or 0) or (min(int(len(rc.split())*1.5), ct) if rc else 0)
     return Usage(prompt_tokens=usg.prompt_tokens, completion_tokens=ct-rt, total_tokens=usg.prompt_tokens+ct,
         cached_tokens=usg.cached_tokens, cache_creation_tokens=usg.cache_creation_tokens, reasoning_tokens=rt, raw=usg.raw)
 
@@ -67,20 +67,31 @@ def norm_finish(resp, tcs=None):
 def _ant_part_type(typ):
     "Map Anthropic content block type to canonical PartType."
     ant_parts_mapping = dict(text=PartType.text, thinking=PartType.thinking, redacted_thinking=PartType.thinking, tool_use=PartType.tool_use, server_tool_use=PartType.tool_use, mcp_tool_use=PartType.tool_use)
-    if typ in ant_parts_mapping: return ant_parts_mapping[typ]
-    if typ.endswith('_tool_result'): return PartType.server_tool_result
-    return typ
+    return ant_parts_mapping.get(typ, typ)
+
+def search_hits(parts, blocks):
+    "Set each server call's `text` to the `title: url` lines of its result block, when the response has one"
+    calls = {p.id: p for p in parts if isinstance(p, ToolUse)}
+    for b in blocks:
+        if b.get('type', '').endswith('_tool_result') and (tc := calls.get(b.get('tool_use_id'))):
+            tc.text = '\n'.join(f"{r.get('title', '')}: {r.get('url', '')}" for r in b.get('content') or [] if isinstance(r, dict) and r.get('url'))
+    return parts
 
 def norm_parts(resp):
+    "Canonical parts of a response; a server tool's result block lives in the raw message, and as the call's `text`"
     parts = []
     for b in resp.get("content", []):
         typ = _ant_part_type(b.get("type", "text"))
         if   typ == PartType.thinking: parts.append(Thinking(b.get("thinking", ""), raw=b))
         elif typ == PartType.tool_use:
             if tc := norm_tool_call(b): parts.append(tc)
-        elif typ == PartType.server_tool_result: parts.append(ServerToolResult(b.get("text", ""), raw=b))
-        else: parts.append(Text(b.get("text", ""), raw=b))
-    return parts
+        elif typ == PartType.text: parts.append(Text(b.get("text", ""), raw=b))
+    return search_hits(parts, resp.get("content", []))
+
+# %% ../nbs/04_anthropic.ipynb #f24e52db
+def raw_msg(resp):
+    "The assistant message as the API sent it"
+    return dict(role='assistant', content=resp['content'])
 
 # %% ../nbs/04_anthropic.ipynb #5d1440b7
 def norm_tr_parts(content):
@@ -103,7 +114,6 @@ def norm_sse_event(ev, **kwargs):
     text, thinking, tcs, citations = None, None, [], None
     if typ == "content_block_start":
         cb = ev.get("content_block", {})
-        if cb.get("type", "").endswith("_tool_result"): return Delta(server_tool_result=cb, raw=ev, **kwargs)
         if tc := norm_tool_call(cb):
             if not tc.arguments: tc.arguments = {'_delta': ''}
             tcs = [tc]
@@ -127,11 +137,30 @@ def delta_index_fn(d, typ, last_typ, last_idx):
     'Returns accumulation index for current delta and updated last idx'
     return nested_idx(d, 'raw', 'index'), None
 
+# %% ../nbs/04_anthropic.ipynb #de900e34
+def collate_raw(deltas):
+    "The assistant message rebuilt from a stream's events"
+    blocks = {}
+    for ev in L(deltas).attrgot('raw'):
+        typ,i = ev.get('type'),ev.get('index')
+        if typ == 'content_block_start': blocks[i] = copy.deepcopy(ev['content_block'])
+        elif typ == 'content_block_delta':
+            d,b = ev['delta'],blocks[i]
+            if   d['type'] == 'text_delta':       b['text'] += d['text']
+            elif d['type'] == 'thinking_delta':   b['thinking'] += d['thinking']
+            elif d['type'] == 'signature_delta':  b['signature'] = d['signature']
+            elif d['type'] == 'input_json_delta': b['_json'] = b.get('_json', '') + d['partial_json']
+            elif d['type'] == 'citations_delta':  b['citations'] = (b.get('citations') or []) + [d['citation']]
+        elif typ == 'content_block_stop' and '_json' in blocks[i]: blocks[i]['input'] = json.loads(blocks[i].pop('_json') or '{}')
+    return dict(role='assistant', content=list(blocks.values()))
+
 # %% ../nbs/04_anthropic.ipynb #c994adb1
 @delegates(mk_acollect_stream, but=['index_fn', 'api_name'])
 async def acollect_stream(resp, **kwargs):
     res = mk_acollect_stream(norm_and_yield(resp, norm_sse_event), index_fn=delta_index_fn, api_name='anthropic', **kwargs)
-    async for o in res: yield o
+    async for o in res:
+        if isinstance(o, Completion): search_hits(o.message.content, o.message.raw['content'])
+        yield o
 
 # %% ../nbs/04_anthropic.ipynb #bb11ab62
 def _ant_cc(block, p):
@@ -145,44 +174,29 @@ def _sanid(id_str): return re.sub(r'[^a-zA-Z0-9_-]', '_', id_str or '')
 # %% ../nbs/04_anthropic.ipynb #6ec772cb
 def denorm_tool_use(p:ToolUse):
     "Convert canonical tool_use Part to Anthropic tool_use content block."
-    block = dict(type='tool_use', id=_sanid(p.id or ''), name=p.name or '', input=p.arguments or {})
-    if caller := (p.raw or {}).get('caller'): block['caller'] = caller
-    return _ant_cc(block, p)
+    return _ant_cc(dict(type='tool_use', id=_sanid(p.id or ''), name=p.name or '', input=p.arguments or {}), p)
 
 def denorm_assistant(m:Msg):
-    "Convert canonical assistant Msg to Anthropic assistant message + synthetic tool results for non-Anthropic server tools."
-    blocks, srv_results, srv_call_id = [], [], None
-    for p in m.content:
-        if isinstance(p, (Thinking, Text)):
-            if srv_call_id:
-                srv_results.append(dict(type='tool_result', tool_use_id=srv_call_id, content=p.text or ''))
-                srv_call_id = None
-            elif isinstance(p, Thinking) and (sig:=(p.raw or {}).get('signature','')):
-                blocks.append(dict(type='thinking', thinking=p.text or '', signature=sig))
-            else: blocks.append(_ant_cc(dict(type='text', text=p.text or ''), p))
-        elif isinstance(p, ToolUse):
-            if p.server and (p.id or '').startswith('srvtoolu_'):
-                blocks.append(dict(type='server_tool_use', id=p.id, name=p.name or '', input=p.arguments or {}))
-            else:
-                blocks.append(denorm_tool_use(p))
-                if p.server: srv_call_id = p.id or ''
-        elif isinstance(p, ServerToolResult): blocks.append(p.raw or dict(type='server_tool_result'))
-    res = [dict(role='assistant', content=blocks)]
-    if srv_results: res.append(dict(role='user', content=srv_results))
-    return res
+    "The message as the API sent it, else its text and tool calls; a cache mark on the last part is copied onto the last raw block"
+    if m.raw:
+        cc = [p.cache_control for p in m.content if p.cache_control]
+        if not cc: return m.raw
+        return m.raw | dict(content=m.raw['content'][:-1] + [m.raw['content'][-1] | dict(cache_control=cc[-1])])
+    blocks = [_ant_cc(dict(type='text', text=p.text or ''), p) if isinstance(p, Text) else denorm_tool_use(p) for p in m.content if isinstance(p, (Text, ToolUse))]
+    return dict(role='assistant', content=blocks)
 
 def denorm_tool(m:Msg):
     "Convert canonical tool Msg to Anthropic user message with tool_result blocks."
     blocks = [denorm_tool_result(p) for p in m.content if isinstance(p, ToolResult)]
-    return [dict(role='user', content=blocks)]
+    return dict(role='user', content=blocks)
 
 def denorm_msgs(msgs:list[Msg]):
     "Convert list of canonical Msgs to Anthropic messages."
     res = []
     for m in msgs:
         if   m.role == 'user':      res.append(denorm_user(m))
-        elif m.role == 'assistant': res.extend(denorm_assistant(m))
-        elif m.role == 'tool':      res.extend(denorm_tool(m))
+        elif m.role == 'assistant': res.append(denorm_assistant(m))
+        elif m.role == 'tool':      res.append(denorm_tool(m))
     return res
 
 # %% ../nbs/04_anthropic.ipynb #f762a6f2
@@ -346,5 +360,6 @@ def cost(usage, m):
 
 # %% ../nbs/04_anthropic.ipynb #f7c0b989
 api_registry.register('anthropic', norm_tool_calls=norm_tool_calls, norm_parts=norm_parts, norm_finish=norm_finish, norm_usage=norm_usage,
-    finalize_usage=finalize_usage, acollect_stream=acollect_stream, mk_payload=mk_payload, cost=cost, get_hdrs=get_hdrs,
+    finalize_usage=finalize_usage, raw_msg=raw_msg, collate_raw=collate_raw, acollect_stream=acollect_stream, mk_payload=mk_payload,
+    cost=cost, get_hdrs=get_hdrs,
     endpoint='/v1/messages')