aip-agents-binary 0.0.0__py3-none-any.whl

aip_agents/middleware/vision.py

@@ -0,0 +1,322 @@
+"""Vision middleware and read_image tool for image context engineering.
+
+Authors:
+    Raymond Christopher (raymond.christopher@gdplabs.id)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import os
+from typing import Any, Literal
+
+from langchain_core.runnables import RunnableConfig
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, Field
+
+from aip_agents.schema.attachments import MAX_IMAGE_ATTACHMENT_BYTES
+from aip_agents.utils.attachment_helpers import (
+    _artifact_attachment,
+    _matches_attachment,
+    resolve_attachment_reference,
+)
+
+VisionDetail = Literal["auto", "low", "high"]
+
+
+VISION_SYSTEM_PROMPT = """## Vision Tool `read_image`
+
+You can inspect registered image attachments or generated image artifacts by calling `read_image`.
+Use it when a user's request depends on visual details. Do not assume a file path alone means you can see the image.
+"""
+
+
+class ReadImageInput(BaseModel):
+    """Input schema for read_image."""
+
+    image: str = Field(description="Image attachment/artifact id, registered filename, or allowed local path")
+    query: str | None = Field(default=None, description="Optional targeted question about the image")
+
+
+class VisionToolConfig(BaseModel):
+    """Configuration for the vision model used by read_image."""
+
+    model: str | None = None
+    api_key: str | None = None
+    base_url: str | None = None
+    detail: VisionDetail = "auto"  # internal/reserved - not wired through gllm-inference yet
+
+
+class ReadImageTool(BaseTool):
+    """Analyze a registered image and return text context."""
+
+    name: str = "read_image"
+    description: str = (
+        "Inspect an image attachment or generated image artifact and return text context. "
+        "Use this for visual questions about images passed by the user or created by tools."
+    )
+    args_schema: type[BaseModel] = ReadImageInput
+    vision_config: VisionToolConfig = Field(default_factory=VisionToolConfig)
+    vision_analyzer: Any | None = None
+
+    def invoke(self, input: Any, config: RunnableConfig | None = None, **kwargs: Any) -> str:
+        """Invoke while preserving RunnableConfig metadata for attachment lookup."""
+        tool_args = self._normalize_tool_input(input, kwargs)
+        return self._run(**tool_args, config=config)
+
+    async def ainvoke(self, input: Any, config: RunnableConfig | None = None, **kwargs: Any) -> str:
+        """Async invoke while preserving RunnableConfig metadata for attachment lookup."""
+        tool_args = self._normalize_tool_input(input, kwargs)
+        return await self._arun(**tool_args, config=config)
+
+    @staticmethod
+    def _normalize_tool_input(input: Any, kwargs: dict[str, Any]) -> dict[str, Any]:
+        if isinstance(input, dict):
+            tool_args = dict(input)
+        else:
+            tool_args = {"image": input}
+        tool_args.update(kwargs)
+        return tool_args
+
+    def _run(
+        self,
+        image: str,
+        query: str | None = None,
+        *,
+        detail: VisionDetail | None = None,
+        config: RunnableConfig | None = None,
+    ) -> str:
+        """Analyze an image using configured vision settings."""
+        metadata = self._extract_metadata(config)
+        resolved = self._resolve_image(image, metadata)
+        if resolved.get("error"):
+            return f"Error: {resolved['error']}"
+
+        prompt = query or "Describe this image in detail. Include visible objects, text, and important context."
+        data_url = resolved["data_url"]
+        try:
+            return self._analyze_image(data_url=data_url, prompt=prompt)
+        except Exception as exc:
+            return f"Error: {self._describe_exception(exc)}"
+
+    async def _arun(
+        self,
+        image: str,
+        query: str | None = None,
+        *,
+        detail: VisionDetail | None = None,
+        config: RunnableConfig | None = None,
+    ) -> str:
+        """Async entry point for LangChain tool execution."""
+        metadata = self._extract_metadata(config)
+        resolved = self._resolve_image(image, metadata)
+        if resolved.get("error"):
+            return f"Error: {resolved['error']}"  # pragma: no cover
+
+        prompt = query or "Describe this image in detail. Include visible objects, text, and important context."
+        data_url = resolved["data_url"]
+
+        if self.vision_analyzer is not None:
+            return str(self.vision_analyzer(data_url=data_url, prompt=prompt, detail=self.vision_config.detail))
+
+        try:
+            return await self._analyze_image_live_async(data_url=data_url, prompt=prompt)
+        except Exception as exc:  # pragma: no cover
+            return f"Error: {self._describe_exception(exc)}"  # pragma: no cover
+
+    @staticmethod
+    def _extract_metadata(config: RunnableConfig | None) -> dict[str, Any]:
+        if not isinstance(config, dict):
+            return {}
+        metadata = config.get("metadata")
+        return metadata if isinstance(metadata, dict) else {}
+
+    def _resolve_image(self, image: str, metadata: dict[str, Any]) -> dict[str, str]:
+        attachment = resolve_attachment_reference(image, metadata)
+        if attachment is None:
+            return {"error": f"Image not found: {image}"}
+
+        mime_type = str(attachment.get("mime_type") or "")
+        if not mime_type.startswith("image/"):
+            return {"error": f"Unsupported image MIME type: {mime_type or 'unknown'}"}
+
+        data_ref = str(attachment.get("data_ref") or "")
+        if data_ref.startswith("artifact://"):
+            return self._resolve_artifact_data_url(image, attachment, metadata, mime_type)
+
+        if os.path.isfile(data_ref):
+            unsafe_ref_error = self._unsafe_local_data_ref_error(image, attachment, data_ref)
+            if unsafe_ref_error:
+                return {"error": unsafe_ref_error}
+            oversized_error = self._oversized_error(image, attachment, data_ref)
+            if oversized_error:
+                return {"error": oversized_error}
+            with open(data_ref, "rb") as handle:
+                encoded = base64.b64encode(handle.read()).decode("ascii")
+            return {"data_url": f"data:{mime_type};base64,{encoded}"}
+
+        return {"error": f"Image data is not locally readable: {image}"}
+
+    @staticmethod
+    def _unsafe_local_data_ref_error(image: str, attachment: dict[str, Any], data_ref: str) -> str | None:
+        """Reject local refs that do not match the registered filename."""
+        if attachment.get("source") != "user_input":
+            return None
+        filename = str(attachment.get("filename") or "")
+        if filename and os.path.basename(data_ref) != filename:
+            return f"Image data reference does not match registered image: {image}"
+        return None
+
+    @staticmethod
+    def _oversized_error(image: str, attachment: dict[str, Any], data_ref: str) -> str | None:
+        """Return an error message if an image exceeds the runtime size limit."""
+        candidates: list[int] = []
+        size_value = attachment.get("size_bytes")
+        if isinstance(size_value, int):
+            candidates.append(size_value)
+        try:
+            candidates.append(os.path.getsize(data_ref))
+        except OSError:
+            pass
+        if candidates and max(candidates) > MAX_IMAGE_ATTACHMENT_BYTES:
+            return f"Image is too large: {image} exceeds {MAX_IMAGE_ATTACHMENT_BYTES} bytes"
+        return None
+
+    def _resolve_artifact_data_url(
+        self, image: str, attachment: dict[str, Any], metadata: dict[str, Any], mime_type: str
+    ) -> dict[str, str]:
+        artifacts = metadata.get("artifacts")
+        if not isinstance(artifacts, list):
+            return {"error": f"Image artifact data is unavailable: {image}"}
+
+        for artifact in artifacts:
+            if not isinstance(artifact, dict):
+                continue
+            artifact_attachment = _artifact_attachment(artifact)
+            if artifact_attachment == attachment or _matches_attachment(image, artifact_attachment):
+                encoded = artifact.get("data")
+                if isinstance(encoded, str) and encoded:
+                    oversized_error = self._oversized_artifact_error(image, artifact_attachment, encoded)
+                    if oversized_error:
+                        return {"error": oversized_error}
+                    return {"data_url": f"data:{mime_type};base64,{encoded}"}
+        return {"error": f"Image artifact data is unavailable: {image}"}
+
+    @staticmethod
+    def _oversized_artifact_error(image: str, attachment: dict[str, Any] | None, encoded: str) -> str | None:
+        """Return an error if an artifact image exceeds the runtime size limit."""
+        size_value = attachment.get("size_bytes") if isinstance(attachment, dict) else None
+        size_bytes = size_value if isinstance(size_value, int) else None
+        if size_bytes is None:
+            try:
+                size_bytes = len(base64.b64decode(encoded, validate=True))
+            except ValueError:
+                size_bytes = None
+        if size_bytes is not None and size_bytes > MAX_IMAGE_ATTACHMENT_BYTES:
+            return f"Image is too large: {image} exceeds {MAX_IMAGE_ATTACHMENT_BYTES} bytes"
+        return None
+
+    @staticmethod
+    def _infer_image_mime(filename: str) -> str | None:
+        lower = filename.lower()
+        if lower.endswith(".png"):
+            return "image/png"
+        if lower.endswith((".jpg", ".jpeg")):
+            return "image/jpeg"
+        if lower.endswith(".webp"):
+            return "image/webp"
+        return None
+
+    def _analyze_image(self, *, data_url: str, prompt: str) -> str:
+        if self.vision_analyzer is not None:
+            return str(self.vision_analyzer(data_url=data_url, prompt=prompt, detail=self.vision_config.detail))
+
+        if not self.vision_config.model:
+            return (
+                "Error: read_image could not determine a vision-capable model. "
+                "By default it uses the agent model; set vision.model when the agent model cannot be resolved."
+            )
+
+        return asyncio.run(self._analyze_image_live_async(data_url=data_url, prompt=prompt))
+
+    async def _analyze_image_live_async(self, *, data_url: str, prompt: str) -> str:
+        """Async live model path using gllm-inference."""
+        from gllm_inference.lm_invoker.build_lm_invoker import build_lm_invoker
+        from gllm_inference.schema.attachment import Attachment
+        from gllm_inference.schema.lm_output import LMOutput
+        from gllm_inference.schema.message import Message
+
+        model_str = self.vision_config.model
+        if self.vision_config.base_url:
+            model_str = f"openai-chat-completions/{self.vision_config.base_url}:{self.vision_config.model}"
+
+        invoker = build_lm_invoker(
+            model_id=model_str,
+            credentials=self.vision_config.api_key,
+        )
+
+        attachment = Attachment.from_data_url(data_url, filename="image.png")
+        message = Message.user([prompt, attachment])
+
+        result = await invoker.invoke([message])
+        if isinstance(result, str):
+            return result
+        if isinstance(result, LMOutput):  # pragma: no cover
+            texts = result.texts  # pragma: no cover
+            if texts:  # pragma: no cover
+                return texts[0]  # pragma: no cover
+        return str(result)  # pragma: no cover
+
+    @staticmethod
+    def _describe_exception(exc: Exception) -> str:
+        message = str(exc).strip()
+        return message or exc.__class__.__name__
+
+
+__all__ = ["ReadImageTool", "VisionMiddleware", "VisionToolConfig"]
+
+
+class VisionMiddleware:
+    """Middleware that contributes the read_image tool."""
+
+    def __init__(
+        self, vision_config: dict[str, Any] | VisionToolConfig | None = None, vision_analyzer: Any | None = None
+    ):
+        """Initialize the middleware with optional model config or analyzer hook."""
+        if isinstance(vision_config, VisionToolConfig):
+            config = vision_config
+        elif isinstance(vision_config, dict):
+            config = VisionToolConfig(**vision_config)
+        else:
+            config = VisionToolConfig()
+
+        self.tools: list[BaseTool] = [ReadImageTool(vision_config=config, vision_analyzer=vision_analyzer)]
+        self.system_prompt_additions = VISION_SYSTEM_PROMPT
+
+    def before_model(self, state: dict[str, Any]) -> dict[str, Any]:
+        """No-op model hook for middleware manager compatibility."""
+        del state
+        return {}
+
+    async def abefore_model(self, state: dict[str, Any]) -> dict[str, Any]:
+        """Async no-op model hook for middleware manager compatibility."""
+        del state
+        await asyncio.sleep(0)
+        return {}
+
+    def modify_model_request(self, request: dict[str, Any], state: dict[str, Any]) -> dict[str, Any]:
+        """Return model requests unchanged."""
+        del state
+        return request
+
+    def after_model(self, state: dict[str, Any]) -> dict[str, Any]:
+        """No-op after-model hook for middleware manager compatibility."""
+        del state
+        return {}
+
+    async def aafter_model(self, state: dict[str, Any]) -> dict[str, Any]:
+        """Async no-op after-model hook for middleware manager compatibility."""
+        del state
+        await asyncio.sleep(0)
+        return {}

aip_agents/ptc/__init__.py

@@ -0,0 +1,87 @@
+"""PTC (Programmatic Tool Calling) core module.
+
+This module provides core PTC functionality, including executor, prompt builder,
+sandbox bridge, and custom tool configuration validation.
+
+Authors:
+    Putu Ravindra Wiguna (putu.r.wiguna@gdplabs.id)
+"""
+
+from aip_agents.ptc.custom_tools import (
+    PTCCustomToolConfig,
+    PTCCustomToolValidationError,
+    PTCFileToolDef,
+    PTCPackageToolDef,
+    PTCToolDef,
+    enrich_tool_def_with_metadata,
+    extract_tool_metadata,
+    validate_custom_tool_config,
+)
+from aip_agents.ptc.custom_tools_payload import (
+    CustomToolPayloadResult,
+    build_custom_tools_payload,
+)
+from aip_agents.ptc.exceptions import PTCError, PTCToolError
+from aip_agents.ptc.prompt_builder import PromptConfig, build_ptc_prompt, compute_ptc_prompt_hash
+from aip_agents.ptc.tool_def_helpers import file_tool, package_tool
+from aip_agents.ptc.tool_enrichment import (
+    build_tool_lookup,
+    enrich_custom_tools_from_agent,
+    match_tool_by_name,
+)
+
+__all__ = [
+    # Exceptions
+    "PTCError",
+    "PTCToolError",
+    "PTCCustomToolValidationError",
+    # Executor
+    "PTCSandboxConfig",
+    "PTCSandboxExecutor",
+    # Custom tools
+    "PTCCustomToolConfig",
+    "PTCToolDef",
+    "PTCPackageToolDef",
+    "PTCFileToolDef",
+    "validate_custom_tool_config",
+    "extract_tool_metadata",
+    "enrich_tool_def_with_metadata",
+    # Tool enrichment
+    "build_tool_lookup",
+    "match_tool_by_name",
+    "enrich_custom_tools_from_agent",
+    # Tool definition helpers
+    "package_tool",
+    "file_tool",
+    # Custom tools payload
+    "CustomToolPayloadResult",
+    "build_custom_tools_payload",
+    # Prompt builder
+    "PromptConfig",
+    "build_ptc_prompt",
+    "compute_ptc_prompt_hash",
+    # Sandbox bridge
+    "build_sandbox_payload",
+    "wrap_ptc_code",
+]
+
+
+def __getattr__(name: str) -> object:
+    """Lazy import to avoid circular dependencies."""
+    if name == "PTCSandboxConfig":
+        from aip_agents.ptc.executor import PTCSandboxConfig
+
+        return PTCSandboxConfig
+    elif name == "PTCSandboxExecutor":
+        from aip_agents.ptc.executor import PTCSandboxExecutor
+
+        return PTCSandboxExecutor
+    elif name == "build_sandbox_payload":
+        from aip_agents.ptc.sandbox_bridge import build_sandbox_payload
+
+        return build_sandbox_payload
+    elif name == "wrap_ptc_code":
+        from aip_agents.ptc.sandbox_bridge import wrap_ptc_code
+
+        return wrap_ptc_code
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")