lollms-client 1.4.1__py3-none-any.whl → 1.7.10__py3-none-any.whl

lollms_client/llm_bindings/openai/__init__.py

@@ -12,16 +12,73 @@ from typing import List, Dict
 import math
 import httpx
 import pipmaster as pm
-
+import mimetypes
 pm.ensure_packages(["openai","tiktoken"])
 
 import openai
 import tiktoken
 import os
-
+import base64
 BindingName = "OpenAIBinding"
 
 
+def _read_file_as_base64(path):
+    with open(path, "rb") as f:
+        return base64.b64encode(f.read()).decode("utf-8")
+
+def _extract_markdown_path(s):
+    s = s.strip()
+    if s.startswith("[") and s.endswith(")"):
+        lb, rb = s.find("["), s.find("]")
+        if lb != -1 and rb != -1 and rb > lb:
+            return s[lb+1:rb].strip()
+    return s
+
+def _guess_mime_from_name(name, default="image/jpeg"):
+    mime, _ = mimetypes.guess_type(name)
+    return mime or default
+
+def _to_data_url(b64_str, mime):
+    return f"data:{mime};base64,{b64_str}"
+
+def normalize_image_input(img, default_mime="image/jpeg"):
+    """
+    Returns a Responses API-ready content block:
+      { "type": "input_image", "image_url": "data:<mime>;base64,<...>" }
+    Accepts:
+      - dict {'data': '<base64>', 'mime': 'image/png'}
+      - dict {'path': 'E:\\images\\x.png'}
+      - string raw base64
+      - string local path (Windows/POSIX), including markdown-like "[E:\\path\\img.png]()"
+    URLs are intentionally not supported (base64 only).
+    """
+    if isinstance(img, dict):
+        if "data" in img and isinstance(img["data"], str):
+            mime = img.get("mime", default_mime)
+            return {"type": "input_image", "image_url": _to_data_url(img["data"], mime)}
+        if "path" in img and isinstance(img["path"], str):
+            p = _extract_markdown_path(img["path"])
+            b64 = _read_file_as_base64(p)
+            mime = _guess_mime_from_name(p, default_mime)
+            return {"type": "input_image", "image_url": _to_data_url(b64, mime)}
+        if "url" in img:
+            raise ValueError("URL inputs not allowed here; provide base64 or local path")
+        raise ValueError("Unsupported dict format for image input")
+
+    if isinstance(img, str):
+        s = _extract_markdown_path(img)
+        # Accept already-correct data URLs as-is
+        if s.startswith("data:"):
+            return {"type": "input_image", "image_url": s}
+        # Local path heuristics: exists on disk or looks like a path
+        if os.path.exists(s) or (":" in s and "\\" in s) or s.startswith("/") or s.startswith("."):
+            b64 = _read_file_as_base64(s)
+            mime = _guess_mime_from_name(s, default_mime)
+            return {"type": "input_image", "image_url": _to_data_url(b64, mime)}
+        # Otherwise, treat as raw base64 payload
+        return {"type": "input_image", "image_url": _to_data_url(s, default_mime)}
+
+    raise ValueError("Unsupported image input type")
 class OpenAIBinding(LollmsLLMBinding):
     """OpenAI-specific binding implementation"""
     
@@ -123,17 +180,18 @@ class OpenAIBinding(LollmsLLMBinding):
 
         if images:
             if split:
+                # Original call to split message roles
                 messages += self.split_discussion(prompt, user_keyword=user_keyword, ai_keyword=ai_keyword)
-                messages[-1]["content"] = [{"type": "text", "text": messages[-1]["content"]}] + [
-                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encode_image(path)}"}}
-                    for path in images
-                ]
+                # Convert the last message content to the structured content array
+                last = messages[-1]
+                text_block = {"type": "text", "text": last["content"]}
+                image_blocks = [normalize_image_input(img) for img in images]
+                last["content"] = [text_block] + image_blocks
             else:
                 messages.append({
-                    'role': 'user',
-                    'content': [{"type": "text", "text": prompt}] + [
-                        {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encode_image(path)}"}}
-                        for path in images
+                    "role": "user",
+                    "content": [{"type": "text", "text": prompt}] + [
+                        normalize_image_input(img) for img in images
                     ]
                 })
         else:
@@ -242,10 +300,66 @@ class OpenAIBinding(LollmsLLMBinding):
                      streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
                      **kwargs
                      ) -> Union[str, dict]:
-        # Build the request parameters
+        
+        # --- Standardize Messages for OpenAI ---
+        def normalize_message(msg: Dict) -> Dict:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+            text_parts = []
+            images = []
+
+            # 1. Extract Text and Images from input
+            if isinstance(content, str):
+                text_parts.append(content)
+            elif isinstance(content, list):
+                for item in content:
+                    if item.get("type") == "text":
+                        text_parts.append(item.get("text", ""))
+                    elif item.get("type") in ["input_image", "image_url"]:
+                        # Handle various internal representations of images
+                        val = item.get("image_url")
+                        if isinstance(val, dict):
+                            # Handle dicts like {"url": "..."} or {"base64": "..."}
+                            val = val.get("url") or val.get("base64")
+                        
+                        if isinstance(val, str) and val:
+                            images.append(val)
+
+            text_content = "\n".join([p for p in text_parts if p.strip()])
+
+            # 2. Format for OpenAI API
+            if not images:
+                # Simple text-only message
+                return {"role": role, "content": text_content}
+            else:
+                # Multimodal message
+                openai_content = []
+                if text_content:
+                    openai_content.append({"type": "text", "text": text_content})
+                
+                for img in images:
+                    # OpenAI STRICTLY requires the data URI prefix for base64
+                    # or a valid http/https URL.
+                    img_url = img
+                    if not img.startswith("http"):
+                        if not img.startswith("data:"):
+                            # If raw base64 is passed without header, add default jpeg header
+                            img_url = f"data:image/jpeg;base64,{img}"
+                    
+                    openai_content.append({
+                        "type": "image_url",
+                        "image_url": {"url": img_url}
+                    })
+                
+                return {"role": role, "content": openai_content}
+
+        # Process and clean the list
+        openai_messages = [normalize_message(m) for m in messages]
+        
+        # --- Build Request ---
         params = {
             "model": self.model_name,
-            "messages": messages,
+            "messages": openai_messages, # Use the standardized list
             "max_tokens": n_predict,
             "n": 1,
             "temperature": temperature,
@@ -253,34 +367,39 @@ class OpenAIBinding(LollmsLLMBinding):
             "frequency_penalty": repeat_penalty,
             "stream": stream
         }
-        # Add seed if available, as it's supported by newer OpenAI models
+
+        # Add seed if available
         if seed is not None:
             params["seed"] = seed
 
-        # Remove None values, as the API expects them to be absent
+        # Remove None values
         params = {k: v for k, v in params.items() if v is not None}
         
         output = ""
-        # 2. Call the API
+        
+        # --- Call API ---
         try:
             try:
                 completion = self.client.chat.completions.create(**params)
             except Exception as ex:
-                # exception for new openai models
-                params["max_completion_tokens"]=params["max_tokens"]
-                params["temperature"]=1
-                try: del params["max_tokens"] 
-                except Exception: pass
-                try: del params["top_p"]
-                except Exception: pass
-                try: del params["frequency_penalty"]
-                except Exception: pass
-
+                # Fallback/Handling for 'reasoning' models (o1, o3-mini) 
+                # which don't support max_tokens, temperature, etc.
+                if "max_tokens" in params:
+                    params["max_completion_tokens"] = params["max_tokens"]
+                    del params["max_tokens"]
                 
+                # Set temperature to 1 (required for some o-series models) or remove it
+                params["temperature"] = 1
+                
+                keys_to_remove = ["top_p", "frequency_penalty", "presence_penalty"]
+                for k in keys_to_remove:
+                    if k in params:
+                        del params[k]
+
                 completion = self.client.chat.completions.create(**params)
+
             if stream:
                 for chunk in completion:
-                    # The streaming response for chat has a different structure
                     delta = chunk.choices[0].delta
                     if delta.content:
                         word = delta.content
@@ -292,7 +411,6 @@ class OpenAIBinding(LollmsLLMBinding):
                 output = completion.choices[0].message.content
         
         except Exception as e:
-            # Handle API errors gracefully
             error_message = f"An error occurred with the OpenAI API: {e}"
             if streaming_callback:
                 streaming_callback(error_message, MSG_TYPE.MSG_TYPE_EXCEPTION)
@@ -313,7 +431,8 @@ class OpenAIBinding(LollmsLLMBinding):
             seed: Optional[int] = None,
             n_threads: Optional[int] = None,
             ctx_size: Optional[int] = None,
-            streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None
+            streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
+            **kwargs
             ) -> Union[str, dict]:
 
         messages = discussion.export("openai_chat", branch_tip_id)
@@ -590,7 +709,7 @@ class OpenAIBinding(LollmsLLMBinding):
             "model_name": self.model_name
         }
 
-    def listModels(self) -> List[Dict]:
+    def list_models(self) -> List[Dict]:
         # Known context lengths
         known_context_lengths = {
             "gpt-4o": 128000,