npcpy 1.1.28__py3-none-any.whl → 1.2.32__py3-none-any.whl

npcpy/gen/response.py

@@ -11,14 +11,15 @@ try:
 except ImportError:
     pass
 except OSError:
-    # Handle case where ollama is not installed or not available
+    
     print("Ollama is not installed or not available. Please install it to use this feature.")
 try:
+    import litellm
     from litellm import completion
 except ImportError:
     pass
 except OSError:
-    # Handle case where litellm is not installed or not available
+    
     pass
 
 def handle_streaming_json(api_params):
@@ -37,6 +38,97 @@ def handle_streaming_json(api_params):
             except json.JSONDecodeError:
                 pass
 
+def get_transformers_response(
+   prompt: str = None,
+   model=None,
+   tokenizer=None, 
+   tools: list = None,
+   tool_map: Dict = None,
+   format: str = None,
+   messages: List[Dict[str, str]] = None,
+   auto_process_tool_calls: bool = False,
+   **kwargs,
+) -> Dict[str, Any]:
+   import torch
+   import json
+   import uuid
+   from transformers import AutoTokenizer, AutoModelForCausalLM
+   
+   result = {
+       "response": None,
+       "messages": messages.copy() if messages else [],
+       "raw_response": None,
+       "tool_calls": [], 
+       "tool_results": []
+   }
+   
+   if model is None or tokenizer is None:
+       model_name = model if isinstance(model, str) else "Qwen/Qwen3-1.7b"
+       tokenizer = AutoTokenizer.from_pretrained(model_name)
+       model = AutoModelForCausalLM.from_pretrained(model_name)
+       
+       if tokenizer.pad_token is None:
+           tokenizer.pad_token = tokenizer.eos_token
+   
+   if prompt:
+       if result['messages'] and result['messages'][-1]["role"] == "user":
+           result['messages'][-1]["content"] = prompt
+       else:
+           result['messages'].append({"role": "user", "content": prompt})
+   
+   if format == "json":
+       json_instruction = """If you are returning a json object, begin directly with the opening {.
+Do not include any additional markdown formatting or leading ```json tags in your response."""
+       if result["messages"] and result["messages"][-1]["role"] == "user":
+           result["messages"][-1]["content"] += "\n" + json_instruction
+
+   chat_text = tokenizer.apply_chat_template(result["messages"], tokenize=False, add_generation_prompt=True)
+   device = next(model.parameters()).device
+   inputs = tokenizer(chat_text, return_tensors="pt", padding=True, truncation=True)
+   inputs = {k: v.to(device) for k, v in inputs.items()}
+   
+       
+   with torch.no_grad():
+       outputs = model.generate(
+           **inputs,
+           max_new_tokens=256,
+           temperature=0.7,
+           do_sample=True,
+           pad_token_id=tokenizer.eos_token_id,
+       )
+   
+   response_content = tokenizer.decode(outputs[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True).strip()
+   result["response"] = response_content
+   result["raw_response"] = response_content
+   result["messages"].append({"role": "assistant", "content": response_content})
+
+   if auto_process_tool_calls and tools and tool_map:
+       detected_tools = []
+       for tool in tools:
+           tool_name = tool.get("function", {}).get("name", "")
+           if tool_name in response_content:
+               detected_tools.append({
+                   "id": str(uuid.uuid4()),
+                   "function": {
+                       "name": tool_name,
+                       "arguments": "{}"
+                   }
+               })
+       
+       if detected_tools:
+           result["tool_calls"] = detected_tools
+           result = process_tool_calls(result, tool_map, "local", "transformers", result["messages"])
+   
+   if format == "json":
+       try:
+           if response_content.startswith("```json"):
+               response_content = response_content.replace("```json", "").replace("```", "").strip()
+           parsed_response = json.loads(response_content)
+           result["response"] = parsed_response
+       except json.JSONDecodeError:
+           result["error"] = f"Invalid JSON response: {response_content}"
+   
+   return result
 
         
 def get_ollama_response(
@@ -58,6 +150,7 @@ def get_ollama_response(
     Generates a response using the Ollama API, supporting both streaming and non-streaming.
     """
 
+    options = {}
 
     image_paths = []
     if images:
@@ -145,6 +238,10 @@ def get_ollama_response(
 
     if tools:
         api_params["tools"] = tools
+        if tool_choice:
+            options["tool_choice"] = tool_choice
+
+
     if think is not None:
         api_params['think'] = think
 
@@ -153,7 +250,6 @@ def get_ollama_response(
     elif isinstance(format, str) and format == "json" and not stream:
         api_params["format"] = "json"
 
-    options = {}
     for key, value in kwargs.items():
         if key in [
             "stop", 
@@ -167,9 +263,6 @@ def get_ollama_response(
             "user",
         ]:
             options[key] = value
-    if tool_choice:
-        options["tool_choice"] = tool_choice
-
 
     result = {
         "response": None,
@@ -179,18 +272,18 @@ def get_ollama_response(
         "tool_results": []
     }
 
-    #print("API params:", api_params)
+    
 
-    # If we want raw tool calls OR no tools, just stream directly
+    
     if not auto_process_tool_calls or not (tools and tool_map):
         res = ollama.chat(**api_params, options=options)
         result["raw_response"] = res
         
         if stream:
-            result["response"] = res  # This is the stream generator
+            result["response"] = res  
             return result
         else:
-            # Non-streaming regular response
+            
             message = res.get("message", {})
             response_content = message.get("content", "")
             result["response"] = response_content
@@ -199,7 +292,7 @@ def get_ollama_response(
             if message.get('tool_calls'):
                 result["tool_calls"] = message['tool_calls']
             
-            # Handle JSON format if specified
+            
             if format == "json":
                 try:
                     if isinstance(response_content, str):
@@ -216,19 +309,19 @@ def get_ollama_response(
             
             return result
 
-    # Only if auto_process_tool_calls=True AND we have tools
-    # Make initial non-streaming call to check for tool calls
+    
+    
     res = ollama.chat(**api_params, options=options)
     result["raw_response"] = res
     
-    #print("Raw Ollama response:", res)
+    
     
     message = res.get("message", {})
     response_content = message.get("content", "")
     
-    # Check for tool calls and process them
+    
     if message.get('tool_calls'):
-        print("Found tool calls, processing automatically:", message['tool_calls'])
+
         
         result["tool_calls"] = message['tool_calls']
         
@@ -239,21 +332,21 @@ def get_ollama_response(
             "tool_calls": message['tool_calls']
         }
         
-        # Process tool calls and get the updated result
+        
         processed_result = process_tool_calls(response_for_processing, 
                                               tool_map, model, 
                                               'ollama', 
                                               messages, 
                                               stream=False)
         
-        # Now if streaming was requested, make a final call with the complete conversation
+        
         if stream:
-            print("Making final streaming call with processed tools")
+
+            
             
-            # Use the updated messages from tool processing
             final_messages = processed_result["messages"]
             
-            # Make the final streaming call
+            
             final_api_params = {
                 "model": model,
                 "messages": final_messages,
@@ -268,13 +361,13 @@ def get_ollama_response(
             
         return processed_result
     
-    # No tool calls found, handle normally
+    
     else:
         result["response"] = response_content
         result["messages"].append({"role": "assistant", "content": response_content})
         
         if stream:
-            # Make streaming call for regular response
+            
             stream_api_params = {
                 "model": model,
                 "messages": messages,
@@ -285,22 +378,53 @@ def get_ollama_response(
             
             result["response"] = ollama.chat(**stream_api_params, options=options)
         else:
-            # Handle JSON format if specified
+                        
             if format == "json":
                 try:
-                    if isinstance(response_content, str):
-                        if response_content.startswith("```json"):
-                            response_content = (
-                                response_content.replace("```json", "")
-                                .replace("```", "")
-                                .strip()
-                            )
-                        parsed_response = json.loads(response_content)
-                        result["response"] = parsed_response
-                except json.JSONDecodeError:
-                    result["error"] = f"Invalid JSON response: {response_content}"
-        
+                    if isinstance(llm_response, str):
+                        llm_response = llm_response.strip()
+                        
+                        if '```json' in llm_response:
+                            start = llm_response.find('```json') + 7
+                            end = llm_response.rfind('```')
+                            if end > start:
+                                llm_response = llm_response[start:end].strip()
+                        
+                        first_brace = llm_response.find('{')
+                        first_bracket = llm_response.find('[')
+                        
+                        if first_brace == -1 and first_bracket == -1:
+                            result["response"] = {}
+                            result["error"] = "No JSON found in response"
+                            return result
+                        
+                        if first_brace != -1 and (first_bracket == -1 or first_brace < first_bracket):
+                            llm_response = llm_response[first_brace:]
+                            last_brace = llm_response.rfind('}')
+                            if last_brace != -1:
+                                llm_response = llm_response[:last_brace+1]
+                        else:
+                            llm_response = llm_response[first_bracket:]
+                            last_bracket = llm_response.rfind(']')
+                            if last_bracket != -1:
+                                llm_response = llm_response[:last_bracket+1]
+                        
+                        parsed_json = json.loads(llm_response, strict=False)
+                        
+                        if "json" in parsed_json:
+                            result["response"] = parsed_json["json"]
+                        else:
+                            result["response"] = parsed_json
+                        
+                except (json.JSONDecodeError, TypeError) as e:
+                    print(f"JSON parsing error: {str(e)}")
+                    print(f"Raw response: {llm_response[:500]}")
+                    result["response"] = {}
+                    result["error"] = "Invalid JSON response"
+
         return result
+    
+import time 
 
 
 def get_litellm_response(
@@ -319,6 +443,7 @@ def get_litellm_response(
     stream: bool = False,
     attachments: List[str] = None,
     auto_process_tool_calls: bool = False, 
+    include_usage: bool = False,
     **kwargs,
 ) -> Dict[str, Any]:
     result = {
@@ -344,6 +469,23 @@ def get_litellm_response(
             auto_process_tool_calls=auto_process_tool_calls, 
             **kwargs
         )
+    elif provider=='transformers':
+        return get_transformers_response(
+            prompt, 
+            model, 
+            images=images, 
+            tools=tools, 
+            tool_choice=tool_choice, 
+            tool_map=tool_map,
+            think=think,
+            format=format, 
+            messages=messages, 
+            stream=stream, 
+            attachments=attachments, 
+            auto_process_tool_calls=auto_process_tool_calls, 
+            **kwargs
+
+        )
     
 
     if attachments:
@@ -362,9 +504,9 @@ def get_litellm_response(
                         pdf_data = load_pdf(attachment)
                         if pdf_data is not None:
                             if prompt:
-                                prompt += f"\n\nContent from PDF: {os.path.basename(attachment)}\n{pdf_data[:5000]}..."
+                                prompt += f"\n\nContent from PDF: {os.path.basename(attachment)}\n{pdf_data}..."
                             else:
-                                prompt = f"Content from PDF: {os.path.basename(attachment)}\n{pdf_data[:5000]}..."
+                                prompt = f"Content from PDF: {os.path.basename(attachment)}\n{pdf_data}..."
 
                     except Exception:
                         pass
@@ -435,12 +577,25 @@ def get_litellm_response(
     
 
     api_params = {"messages": result["messages"]}
-    
-    if api_url is not None and provider == "openai-like":
+
+    if include_usage:
+      litellm.include_cost_in_streaming_usage = True
+      api_params['stream_options'] = {"include_usage": True}
+
+    if api_url is not None and ('openai-like' in provider or provider == "openai-like" or provider == "openai"):
         api_params["api_base"] = api_url
         provider = "openai"
     
     
+    if provider =='enpisi' and api_url is None:
+        api_params['api_base'] = 'https://api.enpisi.com'
+        if api_key is None:
+            api_key = os.environ.get('NPC_STUDIO_LICENSE_KEY')
+            api_params['api_key'] = api_key
+        if '-npc' in model: 
+            model = model.split('-npc')[0]
+        provider = "openai"
+
     if isinstance(format, BaseModel):
         api_params["response_format"] = format
     if model is None:
@@ -465,33 +620,55 @@ def get_litellm_response(
             ]:
                 api_params[key] = value
 
-    # If we want raw tool calls OR no tools, just call directly with requested streaming
     if not auto_process_tool_calls or not (tools and tool_map):
         api_params["stream"] = stream
         resp = completion(**api_params)
         result["raw_response"] = resp
         
         if stream:
-            result["response"] = resp  # This is the stream generator
+            result["response"] = resp  
             return result
         else:
-            # Non-streaming response
+            
             llm_response = resp.choices[0].message.content
             result["response"] = llm_response
             result["messages"].append({"role": "assistant", 
                                        "content": llm_response})
             
-            # Check for tool calls
+            
             if hasattr(resp.choices[0].message, 'tool_calls') and resp.choices[0].message.tool_calls:
                 result["tool_calls"] = resp.choices[0].message.tool_calls
-            
-            # Handle JSON format requests
             if format == "json":
                 try:
                     if isinstance(llm_response, str):
-                        if llm_response.startswith("```json"):
-                            llm_response = llm_response.replace("```json", "").replace("```", "").strip()
-                        parsed_json = json.loads(llm_response)
+                        llm_response = llm_response.strip()
+                        
+                        if '```json' in llm_response:
+                            start = llm_response.find('```json') + 7
+                            end = llm_response.rfind('```')
+                            if end > start:
+                                llm_response = llm_response[start:end].strip()
+                        
+                        first_brace = llm_response.find('{')
+                        first_bracket = llm_response.find('[')
+                        
+                        if first_brace == -1 and first_bracket == -1:
+                            result["response"] = {}
+                            result["error"] = "No JSON found in response"
+                            return result
+                        
+                        if first_brace != -1 and (first_bracket == -1 or first_brace < first_bracket):
+                            llm_response = llm_response[first_brace:]
+                            last_brace = llm_response.rfind('}')
+                            if last_brace != -1:
+                                llm_response = llm_response[:last_brace+1]
+                        else:
+                            llm_response = llm_response[first_bracket:]
+                            last_bracket = llm_response.rfind(']')
+                            if last_bracket != -1:
+                                llm_response = llm_response[:last_bracket+1]
+                        
+                        parsed_json = json.loads(llm_response, strict=False)
                         
                         if "json" in parsed_json:
                             result["response"] = parsed_json["json"]
@@ -500,13 +677,14 @@ def get_litellm_response(
                         
                 except (json.JSONDecodeError, TypeError) as e:
                     print(f"JSON parsing error: {str(e)}")
-                    print(f"Raw response: {llm_response}")
+                    print(f"Raw response: {llm_response[:500]}")
+                    result["response"] = {}
                     result["error"] = "Invalid JSON response"
             
             return result
 
-    # Only if process_tool_calls=True AND we have tools
-    # Make initial non-streaming call to check for tool calls
+    
+    
     initial_api_params = api_params.copy()
     initial_api_params["stream"] = False
     
@@ -514,15 +692,15 @@ def get_litellm_response(
     resp = completion(**initial_api_params)
     result["raw_response"] = resp
     
-    # Check for tool calls
+    
     has_tool_calls = hasattr(resp.choices[0].message, 'tool_calls') and resp.choices[0].message.tool_calls
     
     if has_tool_calls:
-        print("Found tool calls in LiteLLM, processing automatically:", resp.choices[0].message.tool_calls)
+
         
         result["tool_calls"] = resp.choices[0].message.tool_calls
         
-        # Process tool calls
+        
         processed_result = process_tool_calls(result, 
                                               tool_map, 
                                               model, 
@@ -530,52 +708,68 @@ def get_litellm_response(
                                               result["messages"], 
                                               stream=False)
         
-        # If streaming was requested, make final streaming call with processed conversation
+        
         if stream:
-            print("Making final streaming call with processed tools")
+
+            
+
+            clean_messages = []
+            for msg in processed_result["messages"]:
+                if msg.get('role') == 'assistant' and 'tool_calls' in msg:
+                    continue  
+                                
+                else:
+                    clean_messages.append(msg)
             
             final_api_params = api_params.copy()
-            final_api_params["messages"] = processed_result["messages"]
+            final_api_params["messages"] = clean_messages
+            final_api_params["stream"] = True
+
+
+            final_api_params = api_params.copy()
+            final_api_params["messages"] = clean_messages
             final_api_params["stream"] = True
+            if "tools" in final_api_params:
+                del final_api_params["tools"]
+            if "tool_choice" in final_api_params:
+                del final_api_params["tool_choice"]
+
+            final_stream = completion(**final_api_params)
+
             
             final_stream = completion(**final_api_params)
             processed_result["response"] = final_stream
             
         return processed_result
-    
-    # No tool calls found, handle normally
+        
+        
     else:
         llm_response = resp.choices[0].message.content
-        result["response"] = llm_response
         result["messages"].append({"role": "assistant", "content": llm_response})
         
         if stream:
-            # Make streaming call for regular response
-            stream_api_params = api_params.copy()
-            stream_api_params["stream"] = True
-            final_stream = completion(**stream_api_params)
-            result["response"] = final_stream
+            def string_chunk_generator():
+                chunk_size = 1
+                for i, char in enumerate(llm_response):
+                    yield type('MockChunk', (), {
+                        'id': f'mock-chunk-{i}',
+                        'object': 'chat.completion.chunk',
+                        'created': int(time.time()),
+                        'model': model or 'unknown',
+                        'choices': [type('Choice', (), {
+                            'index': 0,
+                            'delta': type('Delta', (), {
+                                'content': char,
+                                'role': 'assistant' if i == 0 else None
+                            })(),
+                            'finish_reason': 'stop' if i == len(llm_response) - 1 else None
+                        })()]
+                    })()
+            
+            result["response"] = string_chunk_generator()
         else:
-            # Handle JSON format requests for non-streaming
-            if format == "json":
-                try:
-                    if isinstance(llm_response, str):
-                        if llm_response.startswith("```json"):
-                            llm_response = llm_response.replace("```json", "").replace("```", "").strip()
-                        parsed_json = json.loads(llm_response)
-                        
-                        if "json" in parsed_json:
-                            result["response"] = parsed_json["json"]
-                        else:
-                            result["response"] = parsed_json
-                        
-                except (json.JSONDecodeError, TypeError) as e:
-                    print(f"JSON parsing error: {str(e)}")
-                    print(f"Raw response: {llm_response}")
-                    result["error"] = "Invalid JSON response"
-
-        return result
-
+            result["response"] = llm_response
+    return result            
 def process_tool_calls(response_dict, tool_map, model, provider, messages, stream=False):
     result = response_dict.copy()
     result["tool_results"] = []
@@ -587,12 +781,12 @@ def process_tool_calls(response_dict, tool_map, model, provider, messages, strea
     
     if not tool_calls:
         return result
-    #print('tm', tool_map)
+    
     for tool_call in tool_calls:
         tool_id = str(uuid.uuid4())
         tool_name = None
         arguments = {}
-        #print('tc', tool_call)
+        
 
         if isinstance(tool_call, dict):
             tool_id = tool_call.get("id", str(uuid.uuid4()))
@@ -611,20 +805,17 @@ def process_tool_calls(response_dict, tool_map, model, provider, messages, strea
             arguments = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
         except json.JSONDecodeError:
             arguments = {"raw_arguments": arguments_str}
-        #print('arg', arguments)
-        #print('tool name in tool map ', tool_name in tool_map)
+        
+        
         if tool_name in tool_map:
             tool_result = None
             tool_result_str = ""
             serializable_result = None
 
             try:
-                print(tool_map[tool_name])
-                print('Executing tool:', tool_name, 'with arguments:', arguments)
                 tool_result = tool_map[tool_name](**arguments)
-                print('Executed Tool Result:', tool_result)
             except Exception as e:
-                tool_result = f"Error executing tool '{tool_name}': {str(e)}"
+                tool_result = f"Error executing tool '{tool_name}': {str(e)}. Tool map is : {tool_map}"
 
             try:
                 tool_result_str = json.dumps(tool_result, default=str)
@@ -643,25 +834,10 @@ def process_tool_calls(response_dict, tool_map, model, provider, messages, strea
                 "result": serializable_result
             })
             
-            result["messages"].append({
-                "role": "assistant",
-                "content": None,
-                "tool_calls": [
-                    {
-                        "id": tool_id,
-                        "type": "function",
-                        "function": {
-                            "name": tool_name,
-                            "arguments": arguments
-                        }
-                    }
-                ]
-            })
             
             result["messages"].append({
-                "role": "tool",
-                "tool_call_id": tool_id,
-                "content": tool_result_str
+                "role": "assistant",
+                "content": f'The results of the tool call for {tool_name} with {arguments} are as follows:' +tool_result_str
             })
     
     return result