dhisana 0.0.1.dev85__py3-none-any.whl → 0.0.1.dev236__py3-none-any.whl

dhisana/utils/openai_assistant_and_file_utils.py

@@ -1,39 +1,43 @@
+"""
+Vector-store and file helpers that work with **either** OpenAI or Azure OpenAI,
+using the shared factory functions defined in `dhisana.utils.fetch_openai_config`.
+
+Only the client initialisation lines changed; all business logic is untouched.
+"""
+
 import json
 import logging
 import re
 import traceback
-from typing import Optional, List, Any, Dict
+from typing import Any, Dict, List, Optional
 
 from fastapi import HTTPException
-from pydantic import BaseModel
-import openai
-from dhisana.utils.openai_helpers import get_openai_access_token
 
+import openai  # still needed for openai.NotFoundError
+from dhisana.utils.fetch_openai_config import (
+    create_openai_client,     # synchronous client
+)
+
+# ---------------------------------------------------------------------------
+# Vector-store helpers
+# ---------------------------------------------------------------------------
 
-# -----------------------------------------------------------------------------
-# Vector Store Helpers
-# -----------------------------------------------------------------------------
 
 async def create_vector_store(
     vector_store_name: str,
-    tool_config: Optional[List[Dict]] = None
+    tool_config: Optional[List[Dict]] = None,
 ) -> Dict[str, Any]:
-    """
-    Create a new vector store with a cleaned name. Returns the vector store details.
-    """
-    normalized_name = vector_store_name.lower()
-    normalized_name = re.sub(r'[^a-z0-9_]+', '_', normalized_name)
-    normalized_name = normalized_name[:64]
-    openai_key = get_openai_access_token(tool_config)
-
-    client = openai.OpenAI(api_key=openai_key)
+    """Create a new vector store and return its metadata."""
+    normalized_name = re.sub(r"[^a-z0-9_]+", "_", vector_store_name.lower())[:64]
+    client = create_openai_client(tool_config)
+
     try:
-        vector_store = client.vector_stores.create(name=normalized_name)
+        vs = client.vector_stores.create(name=normalized_name)
         return {
-            "id": vector_store.id,
-            "name": vector_store.name,
-            "created_at": vector_store.created_at,
-            "file_count": vector_store.file_counts.completed
+            "id": vs.id,
+            "name": vs.name,
+            "created_at": vs.created_at,
+            "file_count": vs.file_counts.completed,
         }
     except Exception as e:
         logging.error(f"Error creating vector store: {e}\n{traceback.format_exc()}")
@@ -42,64 +46,49 @@ async def create_vector_store(
 
 async def delete_vector_store(
     vector_store_id: str,
-    tool_config: Optional[List[Dict]] = None
+    tool_config: Optional[List[Dict]] = None,
 ) -> None:
-    """
-    Delete a vector store by ID.
-    """
-    openai_key = get_openai_access_token(tool_config)
-    client = openai.OpenAI(api_key=openai_key)
+    """Delete a vector store by ID."""
+    client = create_openai_client(tool_config)
     try:
         client.vector_stores.delete(vector_store_id=vector_store_id)
     except Exception as e:
         logging.error(f"Error deleting vector store {vector_store_id}: {e}")
         raise HTTPException(status_code=400, detail=str(e))
 
+# ---------------------------------------------------------------------------
+# File-upload helpers
+# ---------------------------------------------------------------------------
 
-# -----------------------------------------------------------------------------
-# File Upload Helpers
-# -----------------------------------------------------------------------------
 
 async def upload_file_openai_and_vector_store(
     file_path_or_bytes: Any,
     file_name: str,
     mime_type: str,
     vector_store_id: str,
-    tool_config: Optional[List[Dict]] = None
+    tool_config: Optional[List[Dict]] = None,
 ):
-    """
-    Upload a local file or bytes to OpenAI, then attach to a vector store.
-    """
-    openai_key = get_openai_access_token(tool_config)
-    client = openai.OpenAI(api_key=openai_key)
-    purpose = "assistants"
-    if mime_type in ["image/jpeg", "image/png"]:
-        purpose = "vision"
+    """Upload a file and attach it to a vector store (purpose = assistants / vision)."""
+    client = create_openai_client(tool_config)
+    purpose = "vision" if mime_type in {"image/jpeg", "image/png"} else "assistants"
 
     try:
-        # file=(filename, file_content, mime_type) if bytes
-        # or file=open(file_path, "rb") if local path
-        file_upload = None
-        if isinstance(file_path_or_bytes, (str, bytes)):
-            # If string path, open as binary
-            if isinstance(file_path_or_bytes, str):
-                file_upload = client.files.create(
-                    file=open(file_path_or_bytes, "rb"),
-                    purpose=purpose
-                )
-            else:
-                # raw bytes
-                file_upload = client.files.create(
-                    file=(file_name, file_path_or_bytes, mime_type),
-                    purpose=purpose
-                )
+        if isinstance(file_path_or_bytes, str):
+            file_upload = client.files.create(
+                file=open(file_path_or_bytes, "rb"),
+                purpose=purpose,
+            )
+        elif isinstance(file_path_or_bytes, bytes):
+            file_upload = client.files.create(
+                file=(file_name, file_path_or_bytes, mime_type),
+                purpose=purpose,
+            )
         else:
             raise ValueError("Unknown file content type. Must be path or bytes.")
 
         if purpose == "assistants" and vector_store_id:
             client.vector_stores.files.create(
-                vector_store_id=vector_store_id,
-                file_id=file_upload.id
+                vector_store_id=vector_store_id, file_id=file_upload.id
             )
         return file_upload
     except Exception as e:
@@ -111,29 +100,22 @@ async def upload_file_openai(
     file_path_or_bytes: Any,
     file_name: str,
     mime_type: str,
-    tool_config: Optional[List[Dict]] = None
+    tool_config: Optional[List[Dict]] = None,
 ):
-    """
-    Upload a file to OpenAI (not attached to a vector store).
-    """
-    openai_key = get_openai_access_token(tool_config)
-    client = openai.OpenAI(api_key=openai_key)
-
-    purpose = "assistants"
-    if mime_type in ["image/jpeg", "image/png"]:
-        purpose = "vision"
+    """Upload a standalone file (not attached to a vector store)."""
+    client = create_openai_client(tool_config)
+    purpose = "vision" if mime_type in {"image/jpeg", "image/png"} else "assistants"
 
     try:
         if isinstance(file_path_or_bytes, str):
-            # treat as local path
             file_upload = client.files.create(
                 file=open(file_path_or_bytes, "rb"),
-                purpose=purpose
+                purpose=purpose,
             )
         else:
             file_upload = client.files.create(
                 file=(file_name, file_path_or_bytes, mime_type),
-                purpose=purpose
+                purpose=purpose,
             )
         return file_upload
     except Exception as e:
@@ -144,180 +126,142 @@ async def upload_file_openai(
 async def attach_file_to_vector_store(
     file_id: str,
     vector_store_id: str,
-    tool_config: Optional[List[Dict]] = None
+    tool_config: Optional[List[Dict]] = None,
 ):
-    """
-    Attach an already uploaded file to a vector store.
-    """
-    openai_key = get_openai_access_token(tool_config)
-    client = openai.OpenAI(api_key=openai_key)
+    """Attach an already-uploaded file to a vector store."""
+    client = create_openai_client(tool_config)
     try:
-        response = client.vector_stores.files.create(
-            vector_store_id=vector_store_id,
-            file_id=file_id
+        return client.vector_stores.files.create(
+            vector_store_id=vector_store_id, file_id=file_id
         )
-        return response
     except Exception as e:
-        logging.error(f"Error attaching file {file_id} to vector store {vector_store_id}: {e}")
+        logging.error(
+            f"Error attaching file {file_id} to vector store {vector_store_id}: {e}"
+        )
         raise HTTPException(status_code=400, detail=str(e))
 
 
 async def delete_files(
     file_ids: List[str],
     vector_store_id: Optional[str] = None,
-    tool_config: Optional[List[Dict]] = None
+    tool_config: Optional[List[Dict]] = None,
 ):
-    """
-    Delete files from the vector store and from OpenAI's file storage.
-    """
-    openai_key = get_openai_access_token(tool_config)
-    client = openai.OpenAI(api_key=openai_key)
-    for file_id in file_ids:
+    """Delete files from vector store (if given) and OpenAI storage."""
+    client = create_openai_client(tool_config)
+
+    for fid in file_ids:
         try:
             if vector_store_id:
-                client.vector_stores.files.delete(vector_store_id=vector_store_id, file_id=file_id)
-            client.files.delete(file_id=file_id)
+                client.vector_stores.files.delete(
+                    vector_store_id=vector_store_id, file_id=fid
+                )
+            client.files.delete(file_id=fid)
         except openai.NotFoundError:
-            logging.warning(f"File not found: {file_id}")
+            logging.warning(f"File not found: {fid}")
         except Exception as e:
-            logging.error(f"Error deleting file {file_id}: {e}\n{traceback.format_exc()}")
+            logging.error(f"Error deleting file {fid}: {e}\n{traceback.format_exc()}")
 
+# ---------------------------------------------------------------------------
+# RAG / Responses helpers
+# ---------------------------------------------------------------------------
 
-# -----------------------------------------------------------------------------
-# Using File Search (RAG) with the Responses API
-# -----------------------------------------------------------------------------
 
 async def run_file_search(
     query: str,
     vector_store_id: str,
-    model: str = "gpt-4o-mini",
+    model: str = "gpt-5.1-chat",
     max_num_results: int = 5,
     store: bool = True,
-    tool_config: Optional[List[Dict]] = None
+    tool_config: Optional[List[Dict]] = None,
 ) -> Dict[str, Any]:
-    """
-    Execute a single call to the new OpenAI Responses API, using the file_search tool.
-    - 'query': user question
-    - 'vector_store_id': the store where the PDFs are embedded
-    - 'model': which model to use for generating the final answer
-    - 'max_num_results': how many relevant chunks to retrieve
-    - 'store': whether to store the response on the OpenAI side
-    Returns a dict with:
-      {
-        "answer": str,            # the text answer from the LLM
-        "retrieved_files": list,  # the top-k filenames used
-        "annotations": list       # any chunk-level annotations
-      }
-    """
-    openai_key = get_openai_access_token(tool_config)
-    client = openai.OpenAI(api_key=openai_key)
+    """Single-shot file_search + answer with the new Responses API."""
+    client = create_openai_client(tool_config)
 
     try:
-        response = client.responses.create(
+        rsp = client.responses.create(
             input=query,
             model=model,
             store=store,
-            # Provide the file_search tool with vector_store_ids
-            tools=[{
-                "type": "file_search",
-                "vector_store_ids": [vector_store_id],
-                "max_num_results": max_num_results,
-            }]
+            tools=[
+                {
+                    "type": "file_search",
+                    "vector_store_ids": [vector_store_id],
+                    "max_num_results": max_num_results,
+                }
+            ],
         )
 
-        # By default, the file_search call is placed into output[1].
-        # The first chunk in output[1].content can contain both text and annotations.
-        if len(response.output) > 1 and response.output[1].content:
-            # The text & annotations from the file_search step:
-            fs_content = response.output[1].content[0]
-            answer_text = fs_content.text
-            annotations = fs_content.annotations  # chunk-level info (filename, snippet, etc.)
-
-            retrieved_files = []
-            if annotations:
-                retrieved_files = list({result.filename for result in annotations})
-
+        if len(rsp.output) > 1 and rsp.output[1].content:
+            fs_chunk = rsp.output[1].content[0]
+            annotations = fs_chunk.annotations or []
+            retrieved_files = list({ann.filename for ann in annotations})
             return {
-                "answer": answer_text,
+                "answer": fs_chunk.text,
                 "retrieved_files": retrieved_files,
-                "annotations": annotations
-            }
-        else:
-            # If for some reason no file_search step was generated:
-            return {
-                "answer": response.output_text,  # fallback
-                "retrieved_files": [],
-                "annotations": []
+                "annotations": annotations,
             }
+
+        return {
+            "answer": rsp.output_text,
+            "retrieved_files": [],
+            "annotations": [],
+        }
     except Exception as e:
         logging.error(f"Error in run_file_search: {e}\n{traceback.format_exc()}")
         raise HTTPException(status_code=400, detail=str(e))
 
 
-# -----------------------------------------------------------------------------
-# Additional “Responses” Helpers (e.g., run_response_text, run_response_structured)
-# -----------------------------------------------------------------------------
-
 async def run_response_text(
     prompt: str,
-    model: str = "gpt-4o-mini",
+    model: str = "gpt-5.1-chat",
     max_tokens: int = 2048,
     store: bool = True,
-    tool_config: Optional[List[Dict]] = None
+    tool_config: Optional[List[Dict]] = None,
 ) -> (str, str):
-    """
-    Simple text completion with the new Responses API.
-    Returns (answer, status).
-    """
-    openai_key = get_openai_access_token(tool_config)
-    client = openai.OpenAI(api_key=openai_key)
+    """Plain text completion via the Responses API."""
+    client = create_openai_client(tool_config)
 
     try:
-        response = client.responses.create(
+        rsp = client.responses.create(
             input=[{"role": "user", "content": prompt}],
             model=model,
             max_tokens=max_tokens,
-            store=store
+            store=store,
         )
-        return response.output_text, "success"
+        return rsp.output_text, "success"
     except Exception as e:
         logging.error(f"Error in run_response_text: {e}\n{traceback.format_exc()}")
-        return (f"An error occurred: {e}", "error")
+        return f"An error occurred: {e}", "error"
 
 
 async def run_response_structured(
     prompt: str,
     response_format: dict,
-    model: str = "gpt-4o-mini",
+    model: str = "gpt-5.1-chat",
     max_tokens: int = 1024,
     store: bool = True,
-    tool_config: Optional[List[Dict]] = None
+    tool_config: Optional[List[Dict]] = None,
 ) -> (Any, str):
-    """
-    Structured output example. Provide a JSON schema or other format in text={"format": ...}.
-    """
-    openai_key = get_openai_access_token(tool_config)
-    client = openai.OpenAI(api_key=openai_key)
+    """Structured JSON output via Responses API."""
+    client = create_openai_client(tool_config)
 
     try:
-        response = client.responses.create(
+        rsp = client.responses.create(
             input=[{"role": "user", "content": prompt}],
             model=model,
             max_tokens=max_tokens,
             store=store,
-            text={"format": response_format}
+            text={"format": response_format},
         )
-        # If we assume the JSON is in the first output chunk’s .text
-        if response.output and len(response.output) > 0:
-            raw_text = response.output[0].content[0].text
+        if rsp.output:
+            raw = rsp.output[0].content[0].text
             try:
-                parsed = json.loads(raw_text)
-                return parsed, "success"
+                return json.loads(raw), "success"
             except json.JSONDecodeError:
-                # Possibly the model returned partial or invalid JSON
-                return raw_text, "error"
-        else:
-            return "No output returned", "error"
+                return raw, "error"
+        return "No output returned", "error"
     except Exception as e:
-        logging.error(f"Error in run_response_structured: {e}\n{traceback.format_exc()}")
+        logging.error(
+            f"Error in run_response_structured: {e}\n{traceback.format_exc()}"
+        )
         return f"An error occurred: {e}", "error"

dhisana/utils/openai_helpers.py

@@ -12,10 +12,10 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type
 
 from pydantic import BaseModel, Field, create_model
 from fastapi import HTTPException
-from openai import AsyncOpenAI, OpenAIError, LengthFinishReasonError, pydantic_function_tool
+from openai import AsyncOpenAI, OpenAIError, pydantic_function_tool
 
 from dhisana.utils import cache_output_tools
-from dhisana.utils.trasform_json import GLOBAL_GENERATED_PYTHON_CODE, transform_json_with_type
+# from dhisana.utils.trasform_json import GLOBAL_GENERATED_PYTHON_CODE
 
 from .agent_tools import GLOBAL_DATA_MODELS, GLOBAL_TOOLS_FUNCTIONS
 from .google_workspace_tools import get_file_content_from_googledrive_by_name, write_content_to_googledrive
@@ -44,7 +44,7 @@ def get_openai_access_token(tool_config: Optional[List[Dict]] = None) -> str:
         str: The OPENAI_API_KEY access token.
 
     Raises:
-        ValueError: If the access token is not found in the tool configuration or environment variable.
+        ValueError: If the OpenAI integration has not been configured.
     """
     if tool_config:
         openai_config = next(
@@ -64,7 +64,9 @@ def get_openai_access_token(tool_config: Optional[List[Dict]] = None) -> str:
 
     OPENAI_API_KEY = OPENAI_API_KEY or os.getenv("OPENAI_API_KEY")
     if not OPENAI_API_KEY:
-        raise ValueError("OPENAI_API_KEY access token not found in tool_config or environment variable")
+        raise ValueError(
+            "OpenAI integration is not configured. Please configure the connection to OpenAI in Integrations."
+        )
     return OPENAI_API_KEY
 
 async def read_from_google_drive(path):
@@ -328,7 +330,7 @@ async def process_agent_request(row_batch: List[Dict], workflow: Dict, custom_in
             name="AI Assistant",
             instructions=instructions,   
             tools=[],
-            model="o3-mini"
+            model="gpt-5.1-chat"
         )
         thread = await client.beta.threads.create()
         parsed_outputs = []
@@ -480,20 +482,21 @@ async def process_transform_json(task_inputs, response_type, task):
     task_id = task.get('id')
     for input_name, input_info in task_inputs.items():
         data_format = input_info.get('format', 'list')
-        transform_function_name = input_info.get('transform_function_name', f"{task_id}_transform_input_json")
+        input_info.get('transform_function_name', f"{task_id}_transform_input_json")
         items = input_info.get('data')
         if data_format == 'list':
             if items and len(items) > 0:
                 # Generate the transformation function
-                if GLOBAL_GENERATED_PYTHON_CODE.get(transform_function_name, ''):
-                    transformation_function = GLOBAL_GENERATED_PYTHON_CODE[transform_function_name]
-                else:
-                    function_name = await transform_json_with_type(
-                        items[0],
-                        response_type,
-                        transform_function_name
-                    )
-                    transformation_function = GLOBAL_GENERATED_PYTHON_CODE[function_name]
+                # if GLOBAL_GENERATED_PYTHON_CODE.get(transform_function_name, ''):
+                #     transformation_function = GLOBAL_GENERATED_PYTHON_CODE[transform_function_name]
+                # else:
+                #     function_name = await transform_json_with_type(
+                #         items[0],
+                #         response_type,
+                #         transform_function_name
+                #     )
+                #     transformation_function = GLOBAL_GENERATED_PYTHON_CODE[function_name]
+                transformation_function = lambda x: x
                 for item in items:
                     input_json = json.loads(item)
                     output_json = transformation_function(input_json)
@@ -952,7 +955,7 @@ async def get_function_call_arguments(input_text: str, function_name: str) -> Tu
         
         # Make the API call
         response = await client.beta.chat.completions.parse(
-            model="o3-mini",
+            model="gpt-5.1-chat",
             messages=[
                 {"role": "system", "content": "Extract function arguments in JSON format."},
                 {"role": "user", "content": prompt},

dhisana/utils/parse_linkedin_messages_txt.py

@@ -1,7 +1,5 @@
 import uuid
-import hashlib
-from typing import List, Dict, Any
-from pydantic import BaseModel, Field
+from typing import List
 from datetime import datetime, date
 from dhisana.schemas.sales import MessageItem
 
@@ -86,6 +84,7 @@ def parse_conversation(conversation_text: str) -> List[MessageItem]:
         body_text = "\n".join(body_lines).strip()
         message_item = MessageItem(
             message_id=str(uuid.uuid4()),
+            thread_id=str(uuid.uuid4()),
             sender_name=sender,
             sender_email="",      # LinkedIn message => keep empty
             receiver_name="",     # keep empty by default

dhisana/utils/profile.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+from pyinstrument import Profiler
+import logging
+import re
+from typing import Any, Awaitable, Dict, List, Optional, TypeVar, Union
+import mdformat
+
+# --------------------------------------------------------------------------- #
+# Helper: profile any awaited coroutine and log the timing with its call‑site #
+# --------------------------------------------------------------------------- #
+T = TypeVar("T")
+logger = logging.getLogger(__name__)
+
+
+async def profile_async_call(awaitable: Awaitable[T], name: str) -> T:
+    """
+    Run *awaitable*, timing it with an ad‑hoc Profiler instance,
+    and log the profiler output.
+
+    Args:
+        awaitable:   The coroutine to time.
+        name:        Friendly name to show in the log (e.g. function name).
+
+    Returns:
+        The awaited result, typed to whatever the coroutine yields.
+    """
+    profiler = Profiler()            # noqa: F821  (assumes Profiler is already imported)
+    profiler.start()
+    result: T = await awaitable
+    profiler.stop()
+
+    logger.debug(
+        "⏱️  Profiled %s\n%s",
+        name,
+        profiler.output_text(unicode=True, color=True),
+    )
+    return result