dhisana 0.0.1.dev266__py3-none-any.whl → 0.0.1.dev267__py3-none-any.whl

dhisana/utils/generate_structured_output_internal.py

@@ -19,6 +19,126 @@ from dhisana.utils.fetch_openai_config import (
     create_async_openai_client,
 )
 
+# Import search and scrape utilities for web search tools
+try:
+    from dhisana.utils.search_router import search_google_with_tools
+except Exception:
+    async def search_google_with_tools(*a, **k):
+        return []
+
+try:
+    from dhisana.utils.web_download_parse_tools import get_text_content_from_url
+except Exception:
+    async def get_text_content_from_url(url: str) -> str:
+        return ""
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Web search tool definitions for the Responses API
+# ──────────────────────────────────────────────────────────────────────────────
+
+SEARCH_GOOGLE_TOOL = {
+    "type": "function",
+    "name": "search_google",
+    "description": "Search Google for information. Returns a list of search results with titles, links, and snippets.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "query": {
+                "type": "string",
+                "description": "The search query to look up on Google"
+            },
+            "num_results": {
+                "type": "integer",
+                "description": "Number of results to return (default: 5, max: 10)"
+            }
+        },
+        "required": ["query"],
+        "additionalProperties": False
+    }
+}
+
+FETCH_URL_CONTENT_TOOL = {
+    "type": "function",
+    "name": "fetch_url_content",
+    "description": "Fetch and extract text content from a URL. Use this to read the full content of a webpage.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "description": "The URL to fetch content from"
+            }
+        },
+        "required": ["url"],
+        "additionalProperties": False
+    }
+}
+
+
+async def _execute_search_google(
+    query: str, num_results: int, tool_config: Optional[List[Dict]]
+) -> str:
+    """Execute Google search and return results as JSON string."""
+    try:
+        num_results = min(max(num_results, 1), 10)
+        raw = await search_google_with_tools(
+            query, number_of_results=num_results, offset=0, tool_config=tool_config
+        )
+        results = []
+        if isinstance(raw, list):
+            for item in raw:
+                try:
+                    data = json.loads(item) if isinstance(item, str) else item
+                    results.append({
+                        "title": data.get("title", ""),
+                        "link": data.get("link", ""),
+                        "snippet": data.get("snippet", "")
+                    })
+                except Exception:
+                    continue
+        return json.dumps(results, default=str)
+    except Exception as e:
+        logging.warning("search_google tool failed: %s", e)
+        return json.dumps({"error": str(e)})
+
+
+async def _execute_fetch_url_content(url: str) -> str:
+    """Fetch URL content and return as string."""
+    try:
+        content = await get_text_content_from_url(url)
+        if content:
+            max_len = 15000
+            if len(content) > max_len:
+                content = content[:max_len] + "\n... [content truncated]"
+            return content
+        return "Failed to fetch content from URL"
+    except Exception as e:
+        logging.warning("fetch_url_content tool failed for %s: %s", url, e)
+        return f"Error fetching URL: {str(e)}"
+
+
+async def _execute_web_search_tool(
+    tool_name: str, args: dict, tool_config: Optional[List[Dict]]
+) -> str:
+    """Execute a web search tool and return the result as a string."""
+    if tool_name == "search_google":
+        query = args.get("query", "")
+        num_results = args.get("num_results", 5)
+        if not query:
+            return json.dumps({"error": "Missing required parameter: query"})
+        return await _execute_search_google(query, num_results, tool_config)
+
+    elif tool_name == "fetch_url_content":
+        url = args.get("url", "")
+        if not url:
+            return json.dumps({"error": "Missing required parameter: url"})
+        return await _execute_fetch_url_content(url)
+
+    else:
+        logging.warning(f"Unknown tool requested: {tool_name}")
+        return json.dumps({"error": f"Unknown tool: {tool_name}"})
+
 
 # ──────────────────────────────────────────────────────────────────────────────
 # 1.  Helper functions
@@ -71,6 +191,9 @@ async def get_structured_output_internal(
 
     On a 429 (rate-limit) error the call is retried once after
     20 s + random exponential back-off.
+    
+    If use_web_search=True, uses Google search and URL scraping tools
+    to enable web research (works with both OpenAI and Azure OpenAI).
     """
     try:
         # ─── caching bookkeeping ────────────────────────────────────────────
@@ -95,30 +218,24 @@ async def get_structured_output_internal(
             "schema": schema["json_schema"]["schema"],
         }
 
-        # ─── client initialisation (NEW) ────────────────────────────────────
+        # ─── client initialisation ──────────────────────────────────────────
         client_async = create_async_openai_client(tool_config)
 
-        openai_cfg = _extract_config(tool_config, "openai")
-        # TODO: Azure OpenAI does not support web_search yet
-        if not openai_cfg:
-            use_web_search = False
+        # ─── Web search path (uses Google search + URL scraping tools) ──────
+        if use_web_search:
+            return await _get_structured_output_with_web_search(
+                client_async=client_async,
+                prompt=prompt,
+                response_format=response_format,
+                json_schema_format=json_schema_format,
+                model=model,
+                effort=effort,
+                tool_config=tool_config,
+                cache_key=cache_key,
+            )
 
-        # -------------------------------------------------------------------
-        # Internal helper to perform ONE attempt
-        # -------------------------------------------------------------------
+        # ─── Standard path (no web search) ──────────────────────────────────
         async def _make_request():
-            if use_web_search and model.startswith("gpt-"):
-                return await client_async.responses.create(
-                    input=[
-                        {"role": "system", "content": "You are a helpful AI. Output JSON only."},
-                        {"role": "user", "content": prompt},
-                    ],
-                    model=model,
-                    text={"format": json_schema_format},
-                    tool_choice="required",
-                    tools=[{"type": "web_search_preview"}],
-                    store=False,
-                )
             if model.startswith("o"):  # reasoning param only for "o" family
                 return await client_async.responses.create(
                     input=[
@@ -175,45 +292,8 @@ async def get_structured_output_internal(
                 logging.error(f"OpenAI API error: {e}")
                 return f"OpenAI API error: {str(e)}", "API_ERROR"
 
-        # ─── handle model output (unchanged) ────────────────────────────────
-        if completion and completion.output and len(completion.output) > 0:
-            raw_text = None
-            for out in completion.output:
-                if out.type == "message" and out.content:
-                    for content_item in out.content:
-                        if hasattr(content_item, "text"):
-                            raw_text = content_item.text
-                            break
-                        else:
-                            logging.warning("request refused: %s", str(content_item))
-                            return "Request refused.", "FAIL"
-                    if raw_text:
-                        break
-
-            if not raw_text or not raw_text.strip():
-                return "No text returned (possibly refusal or empty response)", "FAIL"
-
-            try:
-                parsed_obj = response_format.parse_raw(raw_text)
-                cache_output_tools.cache_output(
-                    "get_structured_output_internal", cache_key, parsed_obj.json()
-                )
-                return parsed_obj, "SUCCESS"
-
-            except Exception:
-                logging.warning("ERROR: Could not parse JSON from model output.")
-                try:
-                    fixed_json = repair_json(raw_text)
-                    parsed_obj = response_format.parse_raw(fixed_json)
-                    cache_output_tools.cache_output(
-                        "get_structured_output_internal", cache_key, parsed_obj.json()
-                    )
-                    return parsed_obj, "SUCCESS"
-                except Exception as e2:
-                    logging.warning("JSON repair failed: %s", str(e2))
-                    return raw_text, "FAIL"
-        else:
-            return "No output returned", "FAIL"
+        # ─── handle model output ────────────────────────────────────────────
+        return _parse_completion_response(completion, response_format, cache_key)
 
     # Safety fallback: catch any OpenAI errors not caught by inner retry loop
     except OpenAIError as e:
@@ -226,6 +306,173 @@ async def get_structured_output_internal(
         return f"Unexpected error: {str(e)}", "ERROR"
 
 
+async def _get_structured_output_with_web_search(
+    client_async,
+    prompt: str,
+    response_format: BaseModel,
+    json_schema_format: Dict,
+    model: str,
+    effort: str,
+    tool_config: Optional[List[Dict]],
+    cache_key: str,
+):
+    """
+    Handles structured output with web search using Google search and URL scraping tools.
+    Works with both OpenAI and Azure OpenAI.
+    """
+    tools = [SEARCH_GOOGLE_TOOL, FETCH_URL_CONTENT_TOOL]
+    
+    system_content = (
+        "You are a helpful AI. Output JSON only.\n\n"
+        "Web Search Instructions:\n"
+        "- Use search_google to find relevant information on the web.\n"
+        "- Use fetch_url_content to read the full content of relevant URLs.\n"
+        "- After gathering information, provide your response in the required JSON format."
+    )
+    
+    input_messages = [
+        {"role": "system", "content": system_content},
+        {"role": "user", "content": prompt},
+    ]
+    
+    base_request = {
+        "input": input_messages,
+        "model": model,
+        "text": {"format": json_schema_format},
+        "tools": tools,
+        "store": False,
+    }
+    
+    if model.startswith("o"):
+        base_request["reasoning"] = {"effort": effort}
+    
+    max_tool_iterations = 10
+    tool_iteration = 0
+    completion = None
+    
+    while tool_iteration < max_tool_iterations:
+        tool_iteration += 1
+        
+        # Retry logic for rate limits
+        for attempt in range(2):
+            try:
+                completion = await client_async.responses.create(**base_request)
+                break
+            except (RateLimitError, OpenAIError) as e:
+                is_rl = (
+                    isinstance(e, RateLimitError)
+                    or getattr(e, "status_code", None) == 429
+                    or "rate_limit" in str(e).lower()
+                )
+                if attempt == 0 and is_rl:
+                    wait_time = 20 + random.uniform(0, 2.0)
+                    logging.warning(f"Rate-limit hit (429). Waiting {wait_time:.2f}s then retrying.")
+                    await asyncio.sleep(wait_time)
+                    continue
+                logging.error(f"OpenAI API error: {e}")
+                raise HTTPException(status_code=502, detail="Error communicating with the OpenAI API.")
+        
+        if not completion:
+            raise HTTPException(status_code=502, detail="OpenAI request failed.")
+        
+        # Check for function tool calls in the response
+        tool_calls = []
+        for item in (completion.output or []):
+            item_type = getattr(item, "type", None)
+            if item_type == "function_call":
+                tool_calls.append(item)
+        
+        if not tool_calls:
+            # No tool calls, we have the final response
+            break
+        
+        # Execute tool calls and prepare for next iteration
+        logging.info(f"Processing {len(tool_calls)} web search tool call(s) in iteration {tool_iteration}")
+        
+        # Use previous_response_id for conversation threading
+        if hasattr(completion, "id") and completion.id:
+            base_request = {
+                "model": model,
+                "text": {"format": json_schema_format},
+                "tools": tools,
+                "store": False,
+                "previous_response_id": completion.id,
+                "input": [],
+            }
+            if model.startswith("o"):
+                base_request["reasoning"] = {"effort": effort}
+            
+            for tc in tool_calls:
+                func_name = getattr(tc, "name", "")
+                call_id = getattr(tc, "call_id", "")
+                args_str = getattr(tc, "arguments", "{}")
+                
+                try:
+                    args = json.loads(args_str) if args_str else {}
+                except json.JSONDecodeError:
+                    args = {}
+                
+                # Execute the tool
+                tool_result = await _execute_web_search_tool(func_name, args, tool_config)
+                
+                # Add tool result to input
+                base_request["input"].append({
+                    "type": "function_call_output",
+                    "call_id": call_id,
+                    "output": tool_result
+                })
+                
+                logging.info(f"Executed web search tool {func_name}, result length: {len(tool_result)}")
+        else:
+            logging.warning("No response ID available, breaking tool call loop")
+            break
+    
+    # Parse and return the final response
+    return _parse_completion_response(completion, response_format, cache_key)
+
+
+def _parse_completion_response(completion, response_format: BaseModel, cache_key: str):
+    """Parse completion response and return structured output."""
+    if completion and completion.output and len(completion.output) > 0:
+        raw_text = None
+        for out in completion.output:
+            if out.type == "message" and out.content:
+                for content_item in out.content:
+                    if hasattr(content_item, "text"):
+                        raw_text = content_item.text
+                        break
+                    else:
+                        logging.warning("request refused: %s", str(content_item))
+                        return "Request refused.", "FAIL"
+                if raw_text:
+                    break
+
+        if not raw_text or not raw_text.strip():
+            return "No text returned (possibly refusal or empty response)", "FAIL"
+
+        try:
+            parsed_obj = response_format.parse_raw(raw_text)
+            cache_output_tools.cache_output(
+                "get_structured_output_internal", cache_key, parsed_obj.json()
+            )
+            return parsed_obj, "SUCCESS"
+
+        except Exception:
+            logging.warning("ERROR: Could not parse JSON from model output.")
+            try:
+                fixed_json = repair_json(raw_text)
+                parsed_obj = response_format.parse_raw(fixed_json)
+                cache_output_tools.cache_output(
+                    "get_structured_output_internal", cache_key, parsed_obj.json()
+                )
+                return parsed_obj, "SUCCESS"
+            except Exception as e2:
+                logging.warning("JSON repair failed: %s", str(e2))
+                return raw_text, "FAIL"
+    else:
+        return "No output returned", "FAIL"
+
+
 
 async def get_structured_output_with_mcp(
     prompt: str,

{dhisana-0.0.1.dev266.dist-info → dhisana-0.0.1.dev267.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dhisana
-Version: 0.0.1.dev266
+Version: 0.0.1.dev267
 Summary: A Python SDK for Dhisana AI Platform
 Home-page: https://github.com/dhisana-ai/dhisana-python-sdk
 Author: Admin

{dhisana-0.0.1.dev266.dist-info → dhisana-0.0.1.dev267.dist-info}/RECORD

@@ -45,7 +45,7 @@ dhisana/utils/generate_flow.py,sha256=QMn6bWo0nH0fBvy2Ebub1XfH5udnVAqsPsbIqCtQPX
 dhisana/utils/generate_leads_salesnav.py,sha256=FG7q6GSm9IywZ9TgQnn5_N3QNfiI-Qk2gaO_3GS99nY,12236
 dhisana/utils/generate_linkedin_connect_message.py,sha256=QxsxDiT-3eQOqAAbW13d0HGJXV36WYPvC-7Zsw_2VTI,10208
 dhisana/utils/generate_linkedin_response_message.py,sha256=mWoSs5p2JSTIoFZFGm86x1kgs67J7dHPvGKZPzcdGdU,14569
-dhisana/utils/generate_structured_output_internal.py,sha256=k6w5zaaMigp7cUgFALr-TRBsoEQTlzyGfH4R8HIIfGU,22116
+dhisana/utils/generate_structured_output_internal.py,sha256=N1C-LhczMiJhNaDECeFJIY1vc9ta6cwkLWKxvdK4v7M,31266
 dhisana/utils/google_custom_search.py,sha256=5rQ4uAF-hjFpd9ooJkd6CjRvSmhZHhqM0jfHItsbpzk,10071
 dhisana/utils/google_oauth_tools.py,sha256=ReG5lCpXL3_e_s0yn6ai4U7B4-feOWHJVtbv_c0g0rE,28525
 dhisana/utils/google_workspace_tools.py,sha256=fuV0UcvAqF9drLzj7-p6D5zh7d5jMXl1jNJTICk4XOo,50224
@@ -95,8 +95,8 @@ dhisana/workflow/agent.py,sha256=esv7_i_XuMkV2j1nz_UlsHov_m6X5WZZiZm_tG4OBHU,565
 dhisana/workflow/flow.py,sha256=xWE3qQbM7j2B3FH8XnY3zOL_QXX4LbTW4ArndnEYJE0,1638
 dhisana/workflow/task.py,sha256=HlWz9mtrwLYByoSnePOemBUBrMEcj7KbgNjEE1oF5wo,1830
 dhisana/workflow/test.py,sha256=E7lRnXK0PguTNzyasHytLzTJdkqIPxG5_4qk4hMEeKc,3399
-dhisana-0.0.1.dev266.dist-info/METADATA,sha256=KEVAlP8-K5O412xaBb3WeJWM6S1thO8dRYiEu3PQKvw,1190
-dhisana-0.0.1.dev266.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dhisana-0.0.1.dev266.dist-info/entry_points.txt,sha256=jujxteZmNI9EkEaK-pOCoWuBujU8TCevdkfl9ZcKHek,49
-dhisana-0.0.1.dev266.dist-info/top_level.txt,sha256=NETTHt6YifG_P7XtRHbQiXZlgSFk9Qh9aR-ng1XTf4s,8
-dhisana-0.0.1.dev266.dist-info/RECORD,,
+dhisana-0.0.1.dev267.dist-info/METADATA,sha256=zAT6piCTBLFYagzP8-Vuogwo1s1meUG3ch_nk8yLiwg,1190
+dhisana-0.0.1.dev267.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dhisana-0.0.1.dev267.dist-info/entry_points.txt,sha256=jujxteZmNI9EkEaK-pOCoWuBujU8TCevdkfl9ZcKHek,49
+dhisana-0.0.1.dev267.dist-info/top_level.txt,sha256=NETTHt6YifG_P7XtRHbQiXZlgSFk9Qh9aR-ng1XTf4s,8
+dhisana-0.0.1.dev267.dist-info/RECORD,,