intentkit 1.2.5__tar.gz → 1.2.7__tar.gz

{intentkit-1.2.5 → intentkit-1.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: intentkit
-Version: 1.2.5
+Version: 1.2.7
 Summary: Intent-based AI Agent Platform - Core Package
 Project-URL: Homepage, https://github.com/crestalnetwork/intentkit
 Project-URL: Repository, https://github.com/crestalnetwork/intentkit

{intentkit-1.2.5 → intentkit-1.2.7}/intentkit/__init__.py

@@ -3,7 +3,7 @@
 A powerful platform for building AI agents with blockchain and cryptocurrency capabilities.
 """
 
-__version__ = "1.2.5"
+__version__ = "1.2.7"
 __author__ = "Ruihua"
 __email__ = "ruihua@crestal.network"
 

{intentkit-1.2.5 → intentkit-1.2.7}/intentkit/core/engine.py

@@ -13,6 +13,7 @@ The module uses a global cache to store initialized agents for better performanc
 # pyright: reportImportCycles=false
 
 import asyncio
+import base64
 import logging
 import mimetypes
 import re
@@ -73,11 +74,10 @@ _FORWARDABLE_TYPES = frozenset(
 )
 
 # Maps each forwardable attachment type to (model capability flag,
-# unsupported-error system message). Image keeps the legacy OpenAI-style
-# `image_url` block, accepted by every LangChain provider adapter we use.
-# The other three emit LangChain v0.3 standard multimodal blocks
-# `{"type": ..., "url": ...}` — currently only langchain-google-genai 3.2+
-# converts these to provider Parts. See _MEDIA_PROVIDER_SUPPORT.
+# unsupported-error system message). The CSV's `supports_<type>_input`
+# flags are the source of truth; if a model advertises a capability that
+# its provider's LangChain adapter cannot actually deliver, the call
+# surfaces an internal error and the operator should correct the CSV.
 _MEDIA_INPUT_SPECS: list[tuple[ChatMessageAttachmentType, str, SystemMessageType]] = [
     (
         ChatMessageAttachmentType.IMAGE,
@@ -101,31 +101,9 @@ _MEDIA_INPUT_SPECS: list[tuple[ChatMessageAttachmentType, str, SystemMessageType
     ),
 ]
 
-# Providers whose LangChain adapter accepts our content-block shape for each
-# media type. Non-image types use the v0.3 standard block which currently
-# only langchain-google-genai handles; OpenAI/Anthropic require base64 data
-# blocks (`input_audio` / `document`) and are not yet implemented. A model
-# advertising capability=true on an unsupported provider is gated the same
-# way as one that doesn't advertise the capability — better than emitting
-# blocks that crash mid-stream with AGENT_INTERNAL_ERROR.
-_MEDIA_PROVIDER_SUPPORT: dict[ChatMessageAttachmentType, set[LLMProvider] | None] = {
-    ChatMessageAttachmentType.IMAGE: None,  # all providers
-    ChatMessageAttachmentType.AUDIO: {LLMProvider.GOOGLE},
-    ChatMessageAttachmentType.VIDEO: {LLMProvider.GOOGLE},
-    ChatMessageAttachmentType.FILE: {LLMProvider.GOOGLE},
-}
-
-
-def _model_can_deliver_media(
-    atype: ChatMessageAttachmentType, provider: LLMProvider
-) -> bool:
-    allowed = _MEDIA_PROVIDER_SUPPORT.get(atype)
-    return allowed is None or provider in allowed
-
-
 # Per-type fallback when both the HTTP Content-Type and the URL extension
-# fail to yield a usable mime — Gemini rejects inlineData with an empty
-# mimeType, so we always need to supply something.
+# fail to yield a usable mime. Gemini's inlineData and OpenAI's input_audio
+# both reject empty mime/format values, so we always need a concrete value.
 _MEDIA_DEFAULT_MIME: dict[ChatMessageAttachmentType, str] = {
     ChatMessageAttachmentType.IMAGE: "image/jpeg",
     ChatMessageAttachmentType.AUDIO: "audio/mpeg",
@@ -150,8 +128,7 @@ async def _fetch_media_bytes(
 
     Order of preference for mime: HTTP Content-Type response header
     (explicit, set by S3 at upload time) → URL path extension via
-    `mimetypes` → per-type default. Gemini's `inlineData` rejects an empty
-    mimeType, so we always end up with a non-empty value.
+    `mimetypes` → per-type default.
     """
     async with httpx.AsyncClient(
         timeout=_MEDIA_FETCH_TIMEOUT, follow_redirects=True
@@ -172,20 +149,31 @@ async def _fetch_media_bytes(
 
 
 def _build_image_url_block(url: str) -> dict[str, Any]:
+    """OpenAI Chat Completions image block — accepted by every adapter."""
     return {"type": "image_url", "image_url": {"url": url}}
 
 
-def _build_inline_media_block(fetched: _FetchedMedia) -> dict[str, Any]:
-    """Build langchain-google-genai's `media` content block.
+def _build_v1_data_block(
+    atype: ChatMessageAttachmentType, fetched: _FetchedMedia
+) -> dict[str, Any]:
+    """LangChain v1 standard multimodal data block with embedded base64.
+
+    This shape is recognized by every provider adapter we use:
+      - langchain-google-genai decodes base64 and forwards as Gemini
+        ``inlineData(data, mimeType)``
+      - langchain-openai converts to ``input_audio`` / ``file`` Chat
+        Completions blocks (audio + PDFs only)
+      - langchain-anthropic converts ``file`` to ``document`` blocks
+        (PDFs only — Anthropic has no audio/video input)
+      - langchain-openrouter / langchain-xai inherit OpenAI conversion
 
-    Carries raw bytes + an explicit mime_type, which the adapter forwards
-    to Gemini as `inlineData(data, mimeType)`. Doing the fetch ourselves
-    sidesteps LangChain's URL-fetch + mime-guess path, which has produced
-    empty-mimeType errors against Gemini in the field.
+    Fetching ourselves and embedding the bytes avoids LangChain's URL fetch
+    + mime guessing path, which has produced empty-mimeType errors against
+    Gemini in the field.
     """
     return {
-        "type": "media",
-        "data": fetched.data,
+        "type": atype.value,
+        "base64": base64.b64encode(fetched.data).decode("ascii"),
         "mime_type": fetched.mime_type,
     }
 
@@ -945,9 +933,7 @@ async def stream_agent_raw(
     for atype, capability, error_type in _MEDIA_INPUT_SPECS:
         if not media_urls_by_type[atype]:
             continue
-        if not getattr(model, capability) or not _model_can_deliver_media(
-            atype, model.provider
-        ):
+        if not getattr(model, capability):
             yield await _create_system_error_response(
                 error_type,
                 user_message,
@@ -1031,8 +1017,8 @@ async def stream_agent_raw(
             )
             return
         messages.extend(
-            HumanMessage(content=[_build_inline_media_block(fetched)])
-            for fetched in fetched_media
+            HumanMessage(content=[_build_v1_data_block(atype, fetched)])
+            for atype, fetched in zip(fetch_meta, fetched_media, strict=True)
         )
 
     # stream config

{intentkit-1.2.5 → intentkit-1.2.7}/intentkit/core/executor.py

@@ -92,6 +92,7 @@ async def build_executor(
     from intentkit.core.middleware import (
         DynamicPromptMiddleware,
         EmptyContentSafetyMiddleware,
+        MediaBlockSanitizerMiddleware,
         SafeLLMToolSelectorMiddleware,
         StepTrackingMiddleware,
         SummarizationMiddleware,
@@ -250,6 +251,7 @@ async def build_executor(
         ToolBindingMiddleware(llm_model, tools, private_tools, extra_llm_params),
         DynamicPromptMiddleware(agent, agent_data),
         EmptyContentSafetyMiddleware(),
+        MediaBlockSanitizerMiddleware(),
         StepTrackingMiddleware(),
         ToolRetryMiddleware(retry_on=_should_retry_tool_failure),
         ModelRetryMiddleware(),

{intentkit-1.2.5 → intentkit-1.2.7}/intentkit/core/middleware.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
 import logging
+import mimetypes
 from collections.abc import Awaitable, Callable
 from typing import TYPE_CHECKING, Any, override
 
 from langchain.agents.middleware import AgentMiddleware, LLMToolSelectorMiddleware
 from langchain.agents.middleware.summarization import SummarizationMiddleware
-from langchain_core.messages import AIMessage
+from langchain_core.messages import AIMessage, HumanMessage
 from langchain_core.tools import BaseTool
 from langgraph.runtime import Runtime
 
@@ -141,6 +142,123 @@ class EmptyContentSafetyMiddleware(AgentMiddleware[AgentState, AgentContext]):
         return await handler(request)
 
 
+_MEDIA_BLOCK_TYPES = frozenset({"image", "audio", "video", "file"})
+_MEDIA_DEFAULT_MIME: dict[str, str] = {
+    "image": "image/jpeg",
+    "audio": "audio/mpeg",
+    "video": "video/mp4",
+    "file": "application/octet-stream",
+}
+
+
+def _as_media_block(block: Any) -> dict[str, Any] | None:
+    if not isinstance(block, dict):
+        return None
+    if block.get("type") not in _MEDIA_BLOCK_TYPES:
+        return None
+    if not (block.get("url") or block.get("base64") or block.get("data")):
+        return None
+    return block
+
+
+def _resolve_mime_type(block: dict[str, Any]) -> str | None:
+    btype = block.get("type")
+    url = block.get("url")
+    guessed: str | None = None
+    if isinstance(url, str) and url:
+        path_only = url.split("?", 1)[0].split("#", 1)[0]
+        guessed, _ = mimetypes.guess_type(path_only)
+    if btype == "file":
+        # Downstream model APIs (e.g. Gemini) reject application/octet-stream
+        # outright, so a `.bin` extension or a per-type default doesn't help.
+        # Only return a guess we'd actually consider sendable.
+        if guessed and guessed != "application/octet-stream":
+            return guessed
+        return None
+    if guessed:
+        return guessed
+    return _MEDIA_DEFAULT_MIME.get(btype) if isinstance(btype, str) else None
+
+
+class MediaBlockSanitizerMiddleware(AgentMiddleware[AgentState, AgentContext]):
+    """Repair or strip historical media content blocks lacking `mime_type`.
+
+    Background: an early version of the engine stored `HumanMessage`s with
+    blocks like ``{"type": "audio", "url": "..."}`` — no ``mime_type``. The
+    LangGraph checkpointer faithfully replays these on every turn, and
+    Gemini's adapter falls back to ``mimetypes.guess_type`` against the raw
+    URL (query string included), which silently fails on signed URLs or
+    extensions it doesn't know. The result is an ``inlineData`` payload
+    with empty ``mimeType`` and the request is rejected with INVALID_ARGUMENT.
+
+    For each historical block we either:
+      - attach a guessed (or per-type default) ``mime_type``, so the model
+        sees a complete payload, or
+      - drop the block entirely when no reasonable mime can be inferred,
+        which is safer than sending something the API will refuse.
+
+    The textual ``[Attachments]`` URL list embedded in the original message
+    body is preserved, so the model still has the URL reference for
+    delegation/recall even when a binary block is dropped.
+    """
+
+    @override
+    async def awrap_model_call(  # type: ignore[override]
+        self,
+        request: ModelRequest[AgentContext],
+        handler: Callable[[ModelRequest[AgentContext]], Awaitable[ModelResponse]],
+    ) -> ModelResponse:
+        messages = request.messages
+        patched = False
+        for i, msg in enumerate(messages):
+            if not isinstance(msg, HumanMessage):
+                continue
+            if not isinstance(msg.content, list):
+                continue
+            cleaned: list[Any] = []
+            modified = False
+            for block in msg.content:
+                media_block = _as_media_block(block)
+                if media_block is None:
+                    cleaned.append(block)
+                    continue
+                if media_block.get("mime_type"):
+                    cleaned.append(block)
+                    continue
+                resolved = _resolve_mime_type(media_block)
+                btype = media_block.get("type")
+                if resolved is None:
+                    modified = True
+                    logger.warning(
+                        "Dropping historical %s block at msg[%d] — no resolvable mime_type",
+                        btype,
+                        i,
+                    )
+                    continue
+                modified = True
+                cleaned.append({**media_block, "mime_type": resolved})
+                logger.info(
+                    "Repaired historical %s block at msg[%d] with mime_type=%s",
+                    btype,
+                    i,
+                    resolved,
+                )
+            if not modified:
+                continue
+            patched = True
+            new_content: Any
+            if cleaned:
+                new_content = cleaned
+            else:
+                # All blocks stripped — replace with empty text so the
+                # message stays valid for providers that reject empty content.
+                new_content = ""
+            messages[i] = msg.model_copy(update={"content": new_content})
+        if patched:
+            request = request.override(messages=messages)
+        return await handler(request)
+
+
 class SafeLLMToolSelectorMiddleware(LLMToolSelectorMiddleware):
     """`LLMToolSelectorMiddleware` that silently drops `always_include` names
     missing from the current request's tool list.
@@ -175,6 +293,7 @@ class SafeLLMToolSelectorMiddleware(LLMToolSelectorMiddleware):
 __all__ = [
     "DynamicPromptMiddleware",
     "EmptyContentSafetyMiddleware",
+    "MediaBlockSanitizerMiddleware",
     "SafeLLMToolSelectorMiddleware",
     "StepTrackingMiddleware",
     "SummarizationMiddleware",

{intentkit-1.2.5 → intentkit-1.2.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "intentkit"
-version = "1.2.5"
+version = "1.2.7"
 description = "Intent-based AI Agent Platform - Core Package"
 authors = [
     { name = "Ruihua", email = "ruihua@crestal.network" },