inferencesh 0.2.17__py3-none-any.whl → 0.2.18__py3-none-any.whl

inferencesh/models/llm.py

@@ -89,6 +89,8 @@ class LLMInput(BaseAppInput):
     
     # Model specific flags
     reasoning: bool = Field(default=False)
+    
+    tools: List[Dict[str, Any]] = Field(default=[])
 
 class LLMUsage(BaseAppOutput):
     stop_reason: str = ""
@@ -104,6 +106,7 @@ class LLMUsage(BaseAppOutput):
 class LLMOutput(BaseAppOutput):
     response: str
     reasoning: Optional[str] = None
+    tool_calls: Optional[List[Dict[str, Any]]] = None
     usage: Optional[LLMUsage] = None
 
 
@@ -362,6 +365,8 @@ class ResponseTransformer:
 def stream_generate(
     model: Any,
     messages: List[Dict[str, Any]],
+    tools: List[Dict[str, Any]],
+    tool_choice: Dict[str, Any],
     transformer: ResponseTransformer,
     temperature: float = 0.7,
     top_p: float = 0.95,
@@ -379,7 +384,7 @@ def stream_generate(
         max_tokens: Maximum tokens to generate
         stop: Optional list of stop sequences
     """
-    response_queue: Queue[Optional[tuple[str, dict]]] = Queue()
+    response_queue: Queue[Optional[tuple[str, dict, Optional[List[Dict[str, Any]]]]]] = Queue()
     thread_exception = None
     usage_stats = {
         "prompt_tokens": 0,
@@ -397,6 +402,8 @@ def stream_generate(
             try:
                 completion = model.create_chat_completion(
                     messages=messages,
+                    tools=tools,
+                    tool_choice=tool_choice,
                     stream=True,
                     temperature=temperature,
                     top_p=top_p,
@@ -411,18 +418,23 @@ def stream_generate(
                     delta = chunk.get("choices", [{}])[0]
                     content = None
                     finish_reason = None
+                    tool_calls = None
                     
                     if "message" in delta:
-                        content = delta["message"].get("content", "")
+                        message = delta["message"]
+                        content = message.get("content", "")
+                        tool_calls = message.get("tool_calls")
                         finish_reason = delta.get("finish_reason")
                     elif "delta" in delta:
-                        content = delta["delta"].get("content", "")
+                        delta_content = delta["delta"]
+                        content = delta_content.get("content", "")
+                        tool_calls = delta_content.get("tool_calls")
                         finish_reason = delta.get("finish_reason")
                     
-                    if content:
+                    if content or tool_calls:
                         if not timing.first_token_time:
                             timing.mark_first_token()
-                        response_queue.put((content, {}))
+                        response_queue.put((content or "", {}, tool_calls))
                         
                     if finish_reason:
                         usage_stats["stop_reason"] = finish_reason
@@ -438,7 +450,7 @@ def stream_generate(
                     "tokens_per_second": tokens_per_second,
                     "reasoning_time": timing_stats["reasoning_time"],
                     "reasoning_tokens": timing_stats["reasoning_tokens"]
-                }))
+                }, None))
 
         thread = Thread(target=generation_thread, daemon=True)
         thread.start()
@@ -451,7 +463,7 @@ def stream_generate(
                     if thread_exception:
                         raise thread_exception
                     
-                    piece, timing_stats = result
+                    piece, timing_stats, tool_calls = result
                     if piece is None:
                         # Final yield with complete usage stats
                         usage = LLMUsage(
@@ -467,10 +479,14 @@ def stream_generate(
                         
                         buffer, output, _ = transformer(piece or "", buffer)
                         output.usage = usage
+                        if tool_calls:
+                            output.tool_calls = tool_calls
                         yield output
                         break
 
                     buffer, output, _ = transformer(piece, buffer)
+                    if tool_calls:
+                        output.tool_calls = tool_calls
                     yield output
                     
                 except Exception as e:

{inferencesh-0.2.17.dist-info → inferencesh-0.2.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: inferencesh
-Version: 0.2.17
+Version: 0.2.18
 Summary: inference.sh Python SDK
 Author: Inference Shell Inc.
 Author-email: "Inference Shell Inc." <hello@inference.sh>

{inferencesh-0.2.17.dist-info → inferencesh-0.2.18.dist-info}/RECORD

@@ -2,13 +2,13 @@ inferencesh/__init__.py,sha256=WdADtOhfa3HDOunoE9HLFCTFlXRykYstBIH1FpyWvj8,613
 inferencesh/models/__init__.py,sha256=FDwcdtT6c4hbRitymjmN-hZMlQa8RbKSftkZZyjtUXA,536
 inferencesh/models/base.py,sha256=4gZQRi8J7y9U6PrGD9pRIehd1MJVJAqGakPQDs2AKFM,3251
 inferencesh/models/file.py,sha256=5xnpypcRahM1YcEjj64rv9g2gTimxrZb41YT4r440hU,7393
-inferencesh/models/llm.py,sha256=ndRFWbnAD1-BQFTTEb2YShGP-gjVd7w80ItT4XFV1_U,18983
+inferencesh/models/llm.py,sha256=EwluZhgPhzV-WYR2-lREru7dHQOBDRwSFD2dAS-Xwx8,19774
 inferencesh/utils/__init__.py,sha256=-xiD6uo2XzcrPAWFb_fUbaimmnW4KFKc-8IvBzaxNd4,148
 inferencesh/utils/download.py,sha256=7n5twvoNYDcFnKJyefImaj2YfzRI7vddQw4usZbj38c,1521
 inferencesh/utils/storage.py,sha256=E4J8emd4eFKdmdDgAqzz3TpaaDd3n0l8gYlMHuY8yIU,519
-inferencesh-0.2.17.dist-info/licenses/LICENSE,sha256=OsgqEWIh2el_QMj0y8O1A5Q5Dl-dxqqYbFE6fszuR4s,1086
-inferencesh-0.2.17.dist-info/METADATA,sha256=gOHSF2qhb_XT_tXlHqp5HAVblDDZzOjM9kGcbp99JQ4,2757
-inferencesh-0.2.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-inferencesh-0.2.17.dist-info/entry_points.txt,sha256=6IC-fyozAqW3ljsMLGCXxJ0_ui2Jb-2fLHtoH1RTnEE,45
-inferencesh-0.2.17.dist-info/top_level.txt,sha256=TSMHg3T1ThMl1HGAWmzBClwOYH1ump5neof9BfHIwaA,12
-inferencesh-0.2.17.dist-info/RECORD,,
+inferencesh-0.2.18.dist-info/licenses/LICENSE,sha256=OsgqEWIh2el_QMj0y8O1A5Q5Dl-dxqqYbFE6fszuR4s,1086
+inferencesh-0.2.18.dist-info/METADATA,sha256=GnF8E65FD-AxtyciJYin-lGIxNqViw8zTqg58NZIjmc,2757
+inferencesh-0.2.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+inferencesh-0.2.18.dist-info/entry_points.txt,sha256=6IC-fyozAqW3ljsMLGCXxJ0_ui2Jb-2fLHtoH1RTnEE,45
+inferencesh-0.2.18.dist-info/top_level.txt,sha256=TSMHg3T1ThMl1HGAWmzBClwOYH1ump5neof9BfHIwaA,12
+inferencesh-0.2.18.dist-info/RECORD,,