selectools 0.2.0__py3-none-any.whl

toolcalling/examples/bbox.py

@@ -0,0 +1,272 @@
+"""
+Bounding-box detection tool example using OpenAI Vision.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import re
+from pathlib import Path
+from typing import Dict, Optional
+
+from PIL import Image, ImageDraw, ImageFont
+
+from ..tools import Tool, ToolParameter
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[3]
+ASSETS_DIR = PROJECT_ROOT / "assets"
+BBOX_MOCK_ENV = "TOOLCALLING_BBOX_MOCK_JSON"
+
+
+def _resolve_image_path(image_path: str) -> Path:
+    candidate = Path(image_path)
+    if not candidate.is_absolute():
+        asset_candidate = ASSETS_DIR / candidate
+        if asset_candidate.exists():
+            candidate = asset_candidate
+    return candidate.resolve()
+
+
+def _load_openai_client():
+    try:
+        from openai import OpenAI
+    except ImportError as exc:  # noqa: BLE001
+        raise RuntimeError("openai package is required for bounding-box detection.") from exc
+
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError("Set OPENAI_API_KEY to run bounding-box detection.")
+    return OpenAI(api_key=api_key)
+
+
+def detect_bounding_box_impl(target_object: str, image_path: str) -> str:
+    """
+    Detect a target object in an image and draw a bounding box.
+
+    Returns a JSON string containing success status, coordinates, and output path.
+    """
+    resolved_path = _resolve_image_path(image_path)
+    if not resolved_path.exists():
+        return json.dumps(
+            {
+                "success": False,
+                "message": f"Image file not found: {resolved_path}",
+                "coordinates": None,
+                "output_path": None,
+            }
+        )
+
+    mock_json = os.getenv(BBOX_MOCK_ENV)
+    if mock_json:
+        detection_data = _load_mock_detection(Path(mock_json))
+    else:
+        detection_data = _call_openai_vision(target_object=target_object, image_path=resolved_path)
+
+    if not detection_data:
+        return json.dumps(
+            {
+                "success": False,
+                "message": "No detection data returned.",
+                "coordinates": None,
+                "output_path": None,
+            }
+        )
+
+    if not detection_data.get("found"):
+        return json.dumps(
+            {
+                "success": False,
+                "message": f"Could not find {target_object}: {detection_data.get('description', '')}",
+                "coordinates": None,
+                "output_path": None,
+            }
+        )
+
+    x_min = float(detection_data["x_min"])
+    y_min = float(detection_data["y_min"])
+    x_max = float(detection_data["x_max"])
+    y_max = float(detection_data["y_max"])
+
+    if not _coordinates_valid(x_min, y_min, x_max, y_max):
+        return json.dumps(
+            {
+                "success": False,
+                "message": f"Invalid coordinates returned (must be between 0 and 1): {detection_data}",
+                "coordinates": None,
+                "output_path": None,
+            }
+        )
+
+    output_path, pixel_coordinates = _draw_box(resolved_path, target_object, x_min, y_min, x_max, y_max)
+
+    return json.dumps(
+        {
+            "success": True,
+            "message": f"Detected {target_object}; output saved to {output_path}",
+            "coordinates": {
+                "normalized": {
+                    "x_min": x_min,
+                    "y_min": y_min,
+                    "x_max": x_max,
+                    "y_max": y_max,
+                },
+                "pixels": pixel_coordinates,
+            },
+            "output_path": str(output_path),
+            "confidence": detection_data.get("confidence", "unknown"),
+            "description": detection_data.get("description", ""),
+        },
+        indent=2,
+    )
+
+
+def _parse_detection_response(response_text: str) -> Dict[str, str]:
+    json_match = re.search(r"\{.*\}", response_text, re.DOTALL)
+    if json_match:
+        return json.loads(json_match.group())
+    return json.loads(response_text)
+
+
+def _load_mock_detection(mock_path: Path) -> Dict[str, str]:
+    """Load a deterministic mock response for offline testing."""
+    if not mock_path.exists():
+        return {"found": False, "description": f"Mock file not found at {mock_path}"}
+    try:
+        return json.loads(mock_path.read_text())
+    except Exception as exc:  # noqa: BLE001
+        return {"found": False, "description": f"Failed to read mock file: {exc}"}
+
+
+def _call_openai_vision(target_object: str, image_path: Path) -> Optional[Dict[str, str]]:
+    client = _load_openai_client()
+    image_base64 = base64.b64encode(image_path.read_bytes()).decode("utf-8")
+
+    prompt = f"""Analyze this image and locate the {target_object}.
+
+Return ONLY a JSON object with the bounding box coordinates in normalized format (0.0 to 1.0):
+
+{{
+    "found": true/false,
+    "x_min": 0.0-1.0,
+    "y_min": 0.0-1.0,
+    "x_max": 0.0-1.0,
+    "y_max": 0.0-1.0,
+    "confidence": "high/medium/low",
+    "description": "brief description of what you found"
+}}
+
+If the {target_object} is not found, set "found" to false and explain why in the description.
+Coordinates should be normalized (0.0 = left/top edge, 1.0 = right/bottom edge).
+Return ONLY the JSON object, no other text."""
+
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4o",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                        {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}},
+                    ],
+                }
+            ],
+            max_tokens=500,
+        )
+    except Exception as exc:  # noqa: BLE001
+        return {"found": False, "description": f"Vision API error: {exc}"}
+
+    response_text = response.choices[0].message.content
+    return _parse_detection_response(response_text)
+
+
+def _coordinates_valid(x_min: float, y_min: float, x_max: float, y_max: float) -> bool:
+    return 0 <= x_min <= 1 and 0 <= y_min <= 1 and 0 <= x_max <= 1 and 0 <= y_max <= 1
+
+
+def _draw_box(image_path: Path, target_object: str, x_min: float, y_min: float, x_max: float, y_max: float):
+    image = Image.open(image_path)
+    width, height = image.size
+
+    pixel_coordinates = {
+        "x_min": int(x_min * width),
+        "y_min": int(y_min * height),
+        "x_max": int(x_max * width),
+        "y_max": int(y_max * height),
+    }
+
+    draw = ImageDraw.Draw(image)
+    thickness = max(3, int(min(width, height) * 0.005))
+    for offset in range(thickness):
+        draw.rectangle(
+            [
+                pixel_coordinates["x_min"] - offset,
+                pixel_coordinates["y_min"] - offset,
+                pixel_coordinates["x_max"] + offset,
+                pixel_coordinates["y_max"] + offset,
+            ],
+            outline="red",
+            width=1,
+        )
+
+    label = target_object.upper()
+    try:
+        font = ImageFont.truetype("arial.ttf", size=max(20, int(height * 0.03)))
+    except Exception:
+        font = ImageFont.load_default()
+
+    bbox = draw.textbbox((0, 0), label, font=font)
+    text_width = bbox[2] - bbox[0]
+    text_height = bbox[3] - bbox[1]
+    label_x = pixel_coordinates["x_min"]
+    label_y = pixel_coordinates["y_min"] - text_height - 10
+    if label_y < 0:
+        label_y = pixel_coordinates["y_min"] + 5
+
+    padding = 5
+    draw.rectangle(
+        [
+            label_x - padding,
+            label_y - padding,
+            label_x + text_width + padding,
+            label_y + text_height + padding,
+        ],
+        fill="red",
+    )
+    draw.text((label_x, label_y), label, fill="white", font=font)
+
+    output_path = image_path.parent / f"{image_path.stem}_with_bbox.png"
+    image.save(output_path)
+    return output_path, pixel_coordinates
+
+
+def create_bounding_box_tool() -> Tool:
+    """Factory for the bounding-box detection tool."""
+    return Tool(
+        name="detect_bounding_box",
+        description=(
+            "Detects and draws a bounding box around a specific object in an image. "
+            "Returns normalized and pixel coordinates plus the output image path."
+        ),
+        parameters=[
+            ToolParameter(
+                name="target_object",
+                param_type=str,
+                description="The object to locate in the image.",
+                required=True,
+            ),
+            ToolParameter(
+                name="image_path",
+                param_type=str,
+                description="Path to the image file (absolute or relative to assets/).",
+                required=True,
+            ),
+        ],
+        function=detect_bounding_box_impl,
+    )
+
+
+__all__ = ["create_bounding_box_tool", "detect_bounding_box_impl"]

toolcalling/parser.py

@@ -0,0 +1,114 @@
+"""
+Parser for TOOL_CALL directives emitted by language models.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from .types import ToolCall
+
+
+@dataclass
+class ParseResult:
+    """Result of attempting to parse a tool call."""
+
+    tool_call: Optional[ToolCall]
+    raw_text: str
+
+
+class ToolCallParser:
+    """Robustly extract TOOL_CALL directives from model output."""
+
+    def __init__(self, marker: str = "TOOL_CALL", max_payload_chars: int = 8000):
+        self.marker = marker
+        self.max_payload_chars = max_payload_chars
+
+    def parse(self, text: str) -> ParseResult:
+        """
+        Attempt to parse a tool call from the provided text.
+
+        Supports fenced code blocks, inline JSON, and newline-heavy outputs.
+        """
+        candidates = self._extract_candidate_blocks(text)
+        for candidate in candidates:
+            if self.max_payload_chars and len(candidate) > self.max_payload_chars:
+                continue
+            tool_data = self._load_json(candidate)
+            if not tool_data:
+                continue
+            tool_name = tool_data.get("tool_name") or tool_data.get("tool") or tool_data.get("name")
+            parameters: Dict[str, Any] = tool_data.get("parameters") or tool_data.get("params") or {}
+            if tool_name:
+                return ParseResult(tool_call=ToolCall(tool_name=tool_name, parameters=parameters), raw_text=text)
+        return ParseResult(tool_call=None, raw_text=text)
+
+    def _extract_candidate_blocks(self, text: str) -> List[str]:
+        """Pull out all JSON substrings that might contain the TOOL_CALL payload."""
+        blocks: List[str] = []
+
+        marker_positions = [m.start() for m in re.finditer(self.marker, text)]
+        for pos in marker_positions:
+            subset = text[pos:]
+            blocks.extend(self._find_balanced_json(subset))
+
+        fenced_blocks = re.findall(r"```.*?```", text, re.DOTALL)
+        for block in fenced_blocks:
+            if self.marker in block or "tool_name" in block or "parameters" in block:
+                cleaned = block.strip("` \n")
+                blocks.extend(self._find_balanced_json(cleaned))
+
+        if not blocks:
+            blocks.extend(self._find_balanced_json(text))
+
+        # Deduplicate while preserving order
+        deduped = []
+        seen = set()
+        for block in blocks:
+            if block in seen:
+                continue
+            deduped.append(block)
+            seen.add(block)
+        return deduped
+
+    def _load_json(self, candidate: str) -> Optional[Dict[str, Any]]:
+        """Attempt JSON parsing with lenient fallbacks."""
+        normalized = candidate
+        if self.marker in normalized:
+            normalized = normalized.split(self.marker, maxsplit=1)[-1]
+        normalized = normalized.strip("` \n:")
+
+        attempts = [
+            normalized,
+            normalized.replace("'", '"'),
+            normalized.replace("\n", "\\n"),
+        ]
+        for attempt in attempts:
+            try:
+                return json.loads(attempt)
+            except json.JSONDecodeError:
+                continue
+        return None
+
+    def _find_balanced_json(self, text: str) -> List[str]:
+        """Collect balanced JSON-like substrings from text."""
+        candidates: List[str] = []
+        starts = [m.start() for m in re.finditer(r"\{", text)]
+        for start in starts:
+            depth = 0
+            for idx in range(start, len(text)):
+                char = text[idx]
+                if char == "{":
+                    depth += 1
+                elif char == "}":
+                    depth -= 1
+                    if depth == 0:
+                        candidates.append(text[start : idx + 1])
+                        break
+        return candidates
+
+
+__all__ = ["ToolCallParser", "ParseResult"]

toolcalling/prompt.py

@@ -0,0 +1,44 @@
+"""
+Prompt templating for the TOOL_CALL contract.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import List
+
+from .tools import Tool
+
+
+DEFAULT_SYSTEM_INSTRUCTIONS = """You are an assistant that can call tools when helpful.
+
+Tool call contract:
+- Emit TOOL_CALL with JSON: {"tool_name": "<name>", "parameters": {...}}
+- Include every required parameter. Ask for missing details instead of guessing.
+- Wait for tool results before giving a final answer.
+- Do not invent tool outputs; only report what was returned.
+- Keep tool payloads compact (<=8k chars) and emit one tool call at a time.
+"""
+
+
+class PromptBuilder:
+    """Render a system prompt that includes tool schemas."""
+
+    def __init__(self, base_instructions: str = DEFAULT_SYSTEM_INSTRUCTIONS):
+        self.base_instructions = base_instructions
+
+    def build(self, tools: List[Tool]) -> str:
+        tool_blocks = []
+        for tool in tools:
+            tool_blocks.append(json.dumps(tool.schema(), indent=2))
+        tools_text = "\n\n".join(tool_blocks)
+
+        return (
+            f"{self.base_instructions.strip()}\n\n"
+            f"Available tools (JSON schema):\n\n{tools_text}\n\n"
+            "If a relevant tool exists, respond with a TOOL_CALL first. "
+            "When no tool is useful, answer directly."
+        )
+
+
+__all__ = ["PromptBuilder", "DEFAULT_SYSTEM_INSTRUCTIONS"]

toolcalling/providers/base.py

@@ -0,0 +1,55 @@
+"""
+Provider abstraction for model-agnostic tool calling.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from ..types import Message
+
+
+class ProviderError(RuntimeError):
+    """Raised when an adapter cannot complete a request."""
+
+
+@runtime_checkable
+class Provider(Protocol):
+    """Interface every provider adapter must satisfy."""
+
+    name: str
+    supports_streaming: bool
+
+    def complete(
+        self,
+        *,
+        model: str,
+        system_prompt: str,
+        messages: list[Message],
+        temperature: float = 0.0,
+        max_tokens: int = 1000,
+        timeout: float | None = None,
+    ) -> str:
+        """Return assistant text given conversation state."""
+        ...
+
+    def stream(
+        self,
+        *,
+        model: str,
+        system_prompt: str,
+        messages: list[Message],
+        temperature: float = 0.0,
+        max_tokens: int = 1000,
+        timeout: float | None = None,
+    ):
+        """
+        Yield assistant text chunks for providers that support streaming.
+
+        Implementations should raise ProviderError if streaming is not supported
+        or fails.
+        """
+        ...
+
+
+__all__ = ["Provider", "ProviderError"]

toolcalling/providers/openai_provider.py

@@ -0,0 +1,122 @@
+"""
+OpenAI provider adapter for the tool-calling library.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import List
+
+from ..env import load_default_env
+from ..types import Message, Role
+from .base import Provider, ProviderError
+
+
+class OpenAIProvider(Provider):
+    """Adapter that speaks to OpenAI's Chat Completions API."""
+
+    name = "openai"
+    supports_streaming = True
+
+    def __init__(self, api_key: str | None = None, default_model: str = "gpt-4o"):
+        load_default_env()
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY")
+        if not self.api_key:
+            raise ProviderError("OPENAI_API_KEY is not set. Set it in env or pass api_key.")
+
+        try:
+            from openai import OpenAI
+        except ImportError as exc:
+            raise ProviderError("openai package not installed. Install with `pip install openai`.") from exc
+
+        self._client = OpenAI(api_key=self.api_key)
+        self.default_model = default_model
+
+    def complete(
+        self,
+        *,
+        model: str,
+        system_prompt: str,
+        messages: List[Message],
+        temperature: float = 0.0,
+        max_tokens: int = 1000,
+        timeout: float | None = None,
+    ) -> str:
+        formatted = self._format_messages(system_prompt=system_prompt, messages=messages)
+
+        try:
+            response = self._client.chat.completions.create(
+                model=model or self.default_model,
+                messages=formatted,
+                temperature=temperature,
+                max_tokens=max_tokens,
+                timeout=timeout,
+            )
+        except Exception as exc:  # noqa: BLE001
+            raise ProviderError(f"OpenAI completion failed: {exc}") from exc
+
+        content = response.choices[0].message.content
+        return content or ""
+
+    def stream(
+        self,
+        *,
+        model: str,
+        system_prompt: str,
+        messages: List[Message],
+        temperature: float = 0.0,
+        max_tokens: int = 1000,
+        timeout: float | None = None,
+    ):
+        formatted = self._format_messages(system_prompt=system_prompt, messages=messages)
+        try:
+            response = self._client.chat.completions.create(
+                model=model or self.default_model,
+                messages=formatted,
+                temperature=temperature,
+                max_tokens=max_tokens,
+                stream=True,
+                timeout=timeout,
+            )
+        except Exception as exc:  # noqa: BLE001
+            raise ProviderError(f"OpenAI streaming failed: {exc}") from exc
+
+        for chunk in response:
+            try:
+                delta = chunk.choices[0].delta
+                if not delta or not delta.content:
+                    continue
+                content = delta.content
+                if isinstance(content, list):
+                    content = "".join([part.text for part in content if getattr(part, "text", None)])
+                yield content
+            except Exception as exc:  # noqa: BLE001
+                raise ProviderError(f"OpenAI stream parsing failed: {exc}") from exc
+
+    def _format_messages(self, system_prompt: str, messages: List[Message]):
+        payload = [{"role": "system", "content": system_prompt}]
+        for message in messages:
+            role = message.role.value
+            if role == Role.TOOL.value:
+                role = Role.ASSISTANT.value
+            payload.append(
+                {
+                    "role": role,
+                    "content": self._format_content(message),
+                }
+            )
+        return payload
+
+    def _format_content(self, message: Message):
+        if message.image_base64:
+            return [
+                {"type": "text", "text": message.content},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{message.image_base64}"},
+                },
+            ]
+        return message.content
+
+
+__all__ = ["OpenAIProvider"]