aixtools 0.3.10__tar.gz → 0.4.0__tar.gz

{aixtools-0.3.10 → aixtools-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aixtools
-Version: 0.3.10
+Version: 0.4.0
 Summary: Tools for AI exploration and debugging
 Requires-Python: >=3.11.2
 Description-Content-Type: text/markdown
@@ -26,6 +26,7 @@ Requires-Dist: rich>=14.0.0
 Requires-Dist: ruff>=0.11.6
 Requires-Dist: streamlit>=1.44.1
 Requires-Dist: watchdog>=6.0.0
+Requires-Dist: markitdown[docx,pdf,pptx,xls,xlsx]>=0.1.3
 Provides-Extra: test
 Requires-Dist: pyyaml; extra == "test"
 Provides-Extra: feature

{aixtools-0.3.10 → aixtools-0.4.0}/aixtools/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.3.10'
-__version_tuple__ = version_tuple = (0, 3, 10)
+__version__ = version = '0.4.0'
+__version_tuple__ = version_tuple = (0, 4, 0)
 
-__commit_id__ = commit_id = 'g5b197998a'
+__commit_id__ = commit_id = 'gc209901eb'

{aixtools-0.3.10 → aixtools-0.4.0}/aixtools/a2a/app.py

@@ -32,6 +32,7 @@ from starlette.responses import RedirectResponse
 from aixtools.a2a.auth_middleware import AuthMiddleware
 from aixtools.auth.auth import AccessTokenAuthProvider
 from aixtools.context import session_id_var, user_id_var
+from aixtools.utils import config
 
 
 class AgentWorkerWithMetadataParser(AgentWorker):
@@ -140,6 +141,11 @@ def build_a2a_starlette_app(
         http_handler=request_handler,
     )
 
+    if config.SKIP_MCP_AUTHORIZATION:
+        auth_provider = None
+    elif auth_provider is None:
+        auth_provider = AccessTokenAuthProvider()
+
     app = server.build()
     if auth_provider:
         app.add_middleware(AuthMiddleware, provider=auth_provider)

aixtools-0.4.0/aixtools/agents/prompt.py

@@ -0,0 +1,175 @@
+"""Prompt building utilities for Pydantic AI agent, including file handling and context management."""
+
+import mimetypes
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Optional
+
+from markitdown import MarkItDown
+from pydantic_ai import BinaryContent
+
+from aixtools.context import SessionIdTuple
+from aixtools.logging.logging_config import get_logger
+from aixtools.server import container_to_host_path
+from aixtools.utils.config import (
+    EXTRACTABLE_DOCUMENT_TYPES,
+    IMAGE_ATTACHMENT_TYPES,
+    MAX_EXTRACTED_TEXT_SIZE,
+    MAX_IMAGE_ATTACHMENT_SIZE,
+)
+from aixtools.utils.files import is_text_content
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class FileExtractionResult:
+    """Result of file content extraction.
+
+    Attributes:
+        content: Extracted file content (str for text/documents, BinaryContent for images, None on failure)
+        success: True if file was successfully read or extracted, False on any failure
+        error_message: Error description if extraction failed, None otherwise
+        was_extracted: True if document extraction via markitdown was used successfully
+    """
+
+    content: str | BinaryContent | None
+    success: bool
+    error_message: str | None = None
+    was_extracted: bool = False
+
+
+def should_be_included_into_context(
+    file_content: BinaryContent | str | None,
+    *,
+    max_image_size_bytes: int = MAX_IMAGE_ATTACHMENT_SIZE,
+    max_extracted_text_size_bytes: int = MAX_EXTRACTED_TEXT_SIZE,
+) -> bool:
+    """Check if file content should be included in model context based on type and size limits."""
+    if file_content is None:
+        return False
+
+    # Handle extracted text (strings)
+    if isinstance(file_content, str):
+        text_size = len(file_content.encode("utf-8"))
+        return text_size < max_extracted_text_size_bytes
+
+    # Handle binary content (images only)
+    if isinstance(file_content, BinaryContent):
+        if file_content.media_type not in IMAGE_ATTACHMENT_TYPES:
+            return False
+        image_size = len(file_content.data)
+        return image_size < max_image_size_bytes
+
+    return False
+
+
+def file_to_binary_content(file_path: str | Path, mime_type: Optional[str] = None) -> FileExtractionResult:
+    """Read file and extract text from documents (PDF, DOCX, XLSX, PPTX) using markitdown."""
+    if not mime_type:
+        mime_type, _ = mimetypes.guess_type(file_path)
+        mime_type = mime_type or "application/octet-stream"
+
+    # Extract text from supported document types using markitdown
+    if mime_type in EXTRACTABLE_DOCUMENT_TYPES:
+        try:
+            markitdown = MarkItDown()
+            result = markitdown.convert(str(file_path))
+            return FileExtractionResult(
+                content=result.text_content, success=True, error_message=None, was_extracted=True
+            )
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            error_msg = f"Extraction failed: {type(e).__name__}: {str(e)}"
+            logger.error("Document extraction failed for %s: %s", file_path, error_msg)
+            return FileExtractionResult(content=None, success=False, error_message=error_msg)
+
+    # Read the file data for non-document types
+    try:
+        with open(file_path, "rb") as f:
+            data = f.read()
+
+        # Return as string if it's text content
+        if is_text_content(data, mime_type):
+            return FileExtractionResult(content=data.decode("utf-8"), success=True)
+
+        # Return as binary content for images and other binary files
+        return FileExtractionResult(content=BinaryContent(data=data, media_type=mime_type), success=True)
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        error_msg = f"Failed to read file: {type(e).__name__}: {str(e)}"
+        logger.error("File reading failed for %s: %s", file_path, error_msg)
+        return FileExtractionResult(content=None, success=False, error_message=error_msg)
+
+
+def truncate_extracted_text(text: str, max_bytes: int = MAX_EXTRACTED_TEXT_SIZE) -> str:
+    """Truncate text to max_bytes with warning prefix."""
+    truncated_bytes = text.encode("utf-8")[:max_bytes]
+    truncated_text = truncated_bytes.decode("utf-8", errors="ignore")
+
+    total_chars = len(text)
+    truncated_chars = len(truncated_text)
+
+    return f"[TRUNCATED - showing first {truncated_chars} of {total_chars} characters]\n\n{truncated_text}"
+
+
+def build_user_input(
+    session_tuple: SessionIdTuple,
+    user_text: str,
+    file_paths: list[Path],
+) -> str | list[str | BinaryContent]:
+    """Build user input for the Pydantic AI agent, including file attachments if provided."""
+    if not file_paths:
+        return user_text
+
+    attachment_info_lines = []
+    binary_attachments: list[str | BinaryContent] = []
+
+    for workspace_path in file_paths:
+        # Convert Path to PurePosixPath for container_to_host_path
+        workspace_posix_path = PurePosixPath(workspace_path)
+        host_path = container_to_host_path(workspace_posix_path, ctx=session_tuple)
+
+        # Handle None return from container_to_host_path
+        if host_path is None:
+            attachment_info = (
+                f"* {workspace_path.name} (path in workspace: {workspace_path}) -- conversion failed: invalid path"
+            )
+            attachment_info_lines.append(attachment_info)
+            continue
+
+        file_size = host_path.stat().st_size
+        mime_type, _ = mimetypes.guess_type(host_path)
+        mime_type = mime_type or "application/octet-stream"
+
+        attachment_info = f"* {workspace_path.name} (file_size={file_size} bytes) (path in workspace: {workspace_path})"
+        extraction_result = file_to_binary_content(host_path, mime_type)
+
+        # Handle extraction failure - exclude from attachments
+        if not extraction_result.success:
+            attachment_info += f" -- extraction failed: {extraction_result.error_message}"
+            attachment_info_lines.append(attachment_info)
+            continue
+
+        # Handle successful extraction
+        if extraction_result.was_extracted:
+            attachment_info += " -- extracted as text"
+
+        # Check if content should be included in context
+        if should_be_included_into_context(extraction_result.content) and extraction_result.content is not None:
+            binary_attachments.append(extraction_result.content)
+            attachment_info += f" -- provided to model context at index {len(binary_attachments) - 1}"
+        elif (
+            isinstance(extraction_result.content, str) and extraction_result.content and extraction_result.was_extracted
+        ):
+            # Truncate large extracted text and include with warning (only for extracted documents)
+            truncated_content = truncate_extracted_text(extraction_result.content)
+            binary_attachments.append(truncated_content)
+            attachment_info += f" -- truncated and provided to model context at index {len(binary_attachments) - 1}"
+        elif extraction_result.content is not None:
+            # Content exists but excluded from context (e.g., images too large, non-extracted text)
+            attachment_info += " -- too large for context"
+
+        attachment_info_lines.append(attachment_info)
+
+    full_prompt = user_text + "\nAttachments:\n" + "\n".join(attachment_info_lines)
+
+    return [full_prompt] + binary_attachments

{aixtools-0.3.10 → aixtools-0.4.0}/aixtools/utils/config.py

@@ -146,3 +146,26 @@ APP_DEFAULT_SCOPE = get_variable_env("APP_DEFAULT_SCOPE", allow_empty=True)
 AUTH_TEST_TOKEN = get_variable_env("AUTH_TEST_TOKEN", allow_empty=True)
 
 MCP_TOOLS_MAX_RETRIES = int(get_variable_env("MCP_TOOLS_MAX_RETRIES", default=10))
+
+
+# File attachment limits and supported types for model context
+# Maximum extracted document text size (5MB default, planned for future use)
+MAX_EXTRACTED_TEXT_SIZE = int(get_variable_env("MAX_EXTRACTED_TEXT_SIZE", default=str(5 * 1024 * 1024)))
+# Maximum image attachment size (2MB default)
+MAX_IMAGE_ATTACHMENT_SIZE = int(get_variable_env("MAX_IMAGE_ATTACHMENT_SIZE", default=str(2 * 1024 * 1024)))
+# Image MIME types that can be attached to model context
+IMAGE_ATTACHMENT_TYPES = {
+    "image/png",
+    "image/jpeg",
+    "image/jpg",
+    "image/gif",
+    "image/webp",
+}
+# Document MIME types that can be extracted as text
+EXTRACTABLE_DOCUMENT_TYPES = {
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation",  # .pptx
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",  # .docx
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",  # .xlsx
+    "application/vnd.ms-excel",  # .xls
+    "application/pdf",  # .pdf
+}

{aixtools-0.3.10 → aixtools-0.4.0}/pyproject.toml

@@ -33,7 +33,8 @@ dependencies = [
     "rich>=14.0.0",
     "ruff>=0.11.6",
     "streamlit>=1.44.1",
-    "watchdog>=6.0.0"
+    "watchdog>=6.0.0",
+    "markitdown[docx,pdf,pptx,xls,xlsx]>=0.1.3",
 ]
 
 [project.scripts]

aixtools-0.3.10/aixtools/agents/prompt.py

@@ -1,97 +0,0 @@
-"""Prompt building utilities for Pydantic AI agent, including file handling and context management."""
-
-import mimetypes
-from pathlib import Path
-
-from pydantic_ai import BinaryContent
-
-from aixtools.context import SessionIdTuple
-from aixtools.server import container_to_host_path
-from aixtools.utils.files import is_text_content
-
-CLAUDE_MAX_FILE_SIZE_IN_CONTEXT = 4 * 1024 * 1024  # Claude limit 4.5 MB for PDF files
-CLAUDE_IMAGE_MAX_FILE_SIZE_IN_CONTEXT = (
-    5 * 1024 * 1024
-)  # Claude limit 5 MB for images, to avoid large image files in context
-
-
-def should_be_included_into_context(
-    file_content: BinaryContent | str | None,
-    file_size: int,
-    *,
-    max_img_size_bytes: int = CLAUDE_IMAGE_MAX_FILE_SIZE_IN_CONTEXT,
-    max_file_size_bytes: int = CLAUDE_MAX_FILE_SIZE_IN_CONTEXT,
-) -> bool:
-    """Decide whether a file content should be included into the model context based on its type and size."""
-    if not isinstance(file_content, BinaryContent):
-        return False
-
-    if file_content.media_type.startswith("text/"):
-        return False
-
-    # Exclude archive files as they're not supported by OpenAI models
-    archive_types = {
-        "application/zip",
-        "application/x-tar",
-        "application/gzip",
-        "application/x-gzip",
-        "application/x-rar-compressed",
-        "application/x-7z-compressed",
-    }
-    if file_content.media_type in archive_types:
-        return False
-
-    if file_content.is_image and file_size < max_img_size_bytes:
-        return True
-
-    return file_size < max_file_size_bytes
-
-
-def file_to_binary_content(file_path: str | Path, mime_type: str = "") -> str | BinaryContent:
-    """
-    Read a file and return its content as either a UTF-8 string (for text files)
-    or BinaryContent (for binary files).
-    """
-    with open(file_path, "rb") as f:
-        data = f.read()
-
-    if not mime_type:
-        mime_type, _ = mimetypes.guess_type(file_path)
-        mime_type = mime_type or "application/octet-stream"
-
-    if is_text_content(data, mime_type):
-        return data.decode("utf-8")
-
-    return BinaryContent(data=data, media_type=mime_type)
-
-
-def build_user_input(
-    session_tuple: SessionIdTuple,
-    user_text: str,
-    file_paths: list[Path],
-) -> str | list[str | BinaryContent]:
-    """Build user input for the Pydantic AI agent, including file attachments if provided."""
-    if not file_paths:
-        return user_text
-
-    attachment_info_lines = []
-    binary_attachments = []
-
-    for workspace_path in file_paths:
-        host_path = container_to_host_path(workspace_path, ctx=session_tuple)
-        file_size = host_path.stat().st_size
-        mime_type, _ = mimetypes.guess_type(host_path)
-        mime_type = mime_type or "application/octet-stream"
-
-        attachment_info = f"* {workspace_path.name} (file_size={file_size} bytes) (path in workspace: {workspace_path})"
-        binary_content = file_to_binary_content(host_path, mime_type)
-
-        if should_be_included_into_context(binary_content, file_size):
-            binary_attachments.append(binary_content)
-            attachment_info += f" -- provided to model context at index {len(binary_attachments) - 1}"
-
-        attachment_info_lines.append(attachment_info)
-
-    full_prompt = user_text + "\nAttachments:\n" + "\n".join(attachment_info_lines)
-
-    return [full_prompt] + binary_attachments