synth-ai 0.2.14__py3-none-any.whl → 0.2.17__py3-none-any.whl

synth_ai/learning/sft/data.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
 import json
+import logging
 from collections.abc import Iterable, Iterator, Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
+logger = logging.getLogger(__name__)
+
 SFTMessageContent = str | dict[str, Any] | list[Any] | None
 
 
@@ -37,6 +40,8 @@ class SFTMessage:
     tool_calls: list[SFTToolCall] = field(default_factory=list)
     tool_call_id: str | None = None
     name: str | None = None
+    reasoning: str | None = None  # NEW: Explicit reasoning/thinking content
+    raw_content: str | None = None  # NEW: Original unparsed content (before reasoning extraction)
     extra: dict[str, Any] = field(default_factory=dict)
 
 
@@ -86,9 +91,11 @@ def _coerce_tool_call(raw: Any, *, index: int) -> SFTToolCall:
     name: str | None = None
     arguments: Any = None
 
-    if isinstance(raw.get("function"), dict):
-        fn_payload = raw["function"]
-        name = fn_payload.get("name") if isinstance(fn_payload.get("name"), str) else None
+    fn_obj = raw.get("function")
+    if isinstance(fn_obj, dict):
+        fn_payload = fn_obj
+        name_val = fn_payload.get("name")
+        name = name_val if isinstance(name_val, str) else None
         arguments = fn_payload.get("arguments")
     if name is None:
         maybe_name = raw.get("name")
@@ -143,11 +150,20 @@ def _coerce_message(raw: Any, *, index: int) -> SFTMessage:
     name = raw.get("name")
     if name is not None and not isinstance(name, str):
         raise SFTDataError(f"message {index} name must be a string if present")
+    
+    # NEW: Extract reasoning and raw_content if present
+    reasoning = raw.get("reasoning")
+    if reasoning is not None and not isinstance(reasoning, str):
+        raise SFTDataError(f"message {index} reasoning must be a string if present")
+    
+    raw_content = raw.get("raw_content")
+    if raw_content is not None and not isinstance(raw_content, str):
+        raise SFTDataError(f"message {index} raw_content must be a string if present")
 
     extra = {
         key: value
         for key, value in raw.items()
-        if key not in {"role", "content", "tool_calls", "tool_call_id", "name"}
+        if key not in {"role", "content", "tool_calls", "tool_call_id", "name", "reasoning", "raw_content"}
     }
 
     return SFTMessage(
@@ -156,6 +172,8 @@ def _coerce_message(raw: Any, *, index: int) -> SFTMessage:
         tool_calls=tool_calls,
         tool_call_id=tool_call_id,
         name=name,
+        reasoning=reasoning,
+        raw_content=raw_content,
         extra=extra,
     )
 
@@ -280,6 +298,378 @@ def load_jsonl(path: Path, *, min_messages: int = 1) -> list[SFTExample]:
         return list(iter_sft_examples(fh, min_messages=min_messages))
 
 
+# Reasoning/Thinking Utilities
+# ============================================================================
+
+
+def extract_reasoning(content: str, *, tag: str = "think") -> tuple[str | None, str]:
+    """Extract reasoning from content with <think> tags.
+    
+    Args:
+        content: Raw content string
+        tag: Tag name to extract (default: "think")
+    
+    Returns:
+        Tuple of (reasoning, clean_content)
+        - reasoning: Content inside tags, or None if no tags found
+        - clean_content: Content with tags removed
+    
+    Examples:
+        >>> extract_reasoning("<think>Let me analyze...</think>The answer is 42")
+        ('Let me analyze...', 'The answer is 42')
+        >>> extract_reasoning("Just plain text")
+        (None, 'Just plain text')
+    """
+    import re
+    
+    pattern = rf"<{tag}>(.*?)</{tag}>"
+    matches = re.findall(pattern, content, re.DOTALL)
+    
+    if not matches:
+        return None, content
+    
+    # Combine all reasoning blocks
+    reasoning = "\n\n".join(m.strip() for m in matches)
+    
+    # Remove all reasoning blocks from content
+    clean_content = re.sub(pattern, "", content, flags=re.DOTALL).strip()
+    
+    return reasoning, clean_content
+
+
+def strip_reasoning(content: str, *, tag: str = "think") -> str:
+    """Remove reasoning tags from content.
+    
+    Args:
+        content: Content with potential reasoning tags
+        tag: Tag name to strip (default: "think")
+    
+    Returns:
+        Content with reasoning tags removed
+    """
+    _, clean = extract_reasoning(content, tag=tag)
+    return clean
+
+
+def message_has_reasoning(message: SFTMessage) -> bool:
+    """Check if a message has explicit reasoning.
+    
+    Args:
+        message: SFTMessage to check
+    
+    Returns:
+        True if message has reasoning field or <think> tags in content
+    """
+    # Check explicit reasoning field
+    if message.reasoning:
+        return True
+    
+    # Check for reasoning tags in content
+    if isinstance(message.content, str):
+        reasoning, _ = extract_reasoning(message.content)
+        return reasoning is not None
+    
+    return False
+
+
+def validate_message_content(
+    message: SFTMessage, *, require_content: bool = True
+) -> tuple[bool, str | None]:
+    """Validate that message has valid content combinations.
+    
+    Rules:
+    - Must have at least one of: reasoning + tool_calls, reasoning + content, 
+      content, raw_content, or tool_calls
+    - If raw_content present with reasoning + content, they should be consistent
+    - Cannot have neither reasoning, content, raw_content, nor tool_calls
+    
+    Args:
+        message: SFTMessage to validate
+        require_content: If True, require some form of content (default: True)
+    
+    Returns:
+        Tuple of (is_valid, error_message)
+    """
+    has_reasoning = bool(message.reasoning)
+    has_content = message.content is not None and message.content != ""
+    has_raw = bool(message.raw_content)
+    has_tools = len(message.tool_calls) > 0
+    
+    # Check for completely empty message
+    if require_content and not (has_reasoning or has_content or has_raw or has_tools):
+        return False, "Message has no reasoning, content, raw_content, or tool_calls"
+    
+    # Valid combinations:
+    # 1. reasoning + tool_calls (reasoning-based action)
+    if has_reasoning and has_tools:
+        return True, None
+    
+    # 2. reasoning + content (reasoning then output)
+    if has_reasoning and has_content:
+        # If raw_content present, validate consistency
+        if has_raw and message.raw_content:
+            # Raw should contain both reasoning and content
+            reasoning_in_raw, content_in_raw = extract_reasoning(message.raw_content)
+            if message.reasoning and reasoning_in_raw != message.reasoning.strip():
+                logger.warning(
+                    "raw_content reasoning doesn't match reasoning field"
+                )
+            # This is okay - just a warning, not an error
+        return True, None
+    
+    # 3. content only (standard message)
+    if has_content and not has_reasoning:
+        return True, None
+    
+    # 4. raw_content only (unparsed content)
+    if has_raw and not (has_reasoning and has_content):
+        return True, None
+    
+    # 5. tool_calls only (action without reasoning/content - like OpenAI format)
+    if has_tools and not has_content:
+        return True, None
+    
+    # 6. reasoning only (pure thinking turn)
+    if has_reasoning and not has_content and not has_tools:
+        return True, None
+    
+    return True, None
+
+
+# Vision/Multimodal Utilities
+# ============================================================================
+
+
+def has_image_content(content: SFTMessageContent) -> bool:
+    """Check if message content contains image data (OpenAI multimodal format).
+    
+    Supports:
+    - List of content parts: [{"type": "text", ...}, {"type": "image_url", ...}]
+    - Single dict with type field: {"type": "image_url", "image_url": {...}}
+    
+    Args:
+        content: Message content (can be str, list, dict, or None)
+    
+    Returns:
+        True if content contains an image segment
+    
+    Examples:
+        >>> has_image_content([{"type": "text", "text": "What's this?"}, 
+        ...                    {"type": "image_url", "image_url": {"url": "..."}}])
+        True
+        >>> has_image_content("Just text")
+        False
+    """
+    if isinstance(content, list):
+        return any(
+            isinstance(part, dict) and part.get("type") in {"image", "image_url"}
+            for part in content
+        )
+    elif isinstance(content, dict):
+        return content.get("type") in {"image", "image_url"}
+    return False
+
+
+def message_has_image(message: SFTMessage) -> bool:
+    """Check if an SFTMessage contains image content.
+    
+    Args:
+        message: SFTMessage to check
+    
+    Returns:
+        True if the message contains image content
+    """
+    return has_image_content(message.content)
+
+
+def example_has_image(example: SFTExample) -> bool:
+    """Check if an SFTExample contains any image content.
+    
+    Args:
+        example: SFTExample to check
+    
+    Returns:
+        True if any message in the example contains image content
+    """
+    return any(message_has_image(msg) for msg in example.messages)
+
+
+def count_images_in_content(content: SFTMessageContent) -> int:
+    """Count the number of images in message content.
+    
+    Args:
+        content: Message content to analyze
+    
+    Returns:
+        Number of image segments found
+    """
+    if isinstance(content, list):
+        return sum(
+            1 for part in content
+            if isinstance(part, dict) and part.get("type") in {"image", "image_url"}
+        )
+    elif isinstance(content, dict) and content.get("type") in {"image", "image_url"}:
+        return 1
+    return 0
+
+
+def extract_image_urls(content: SFTMessageContent) -> list[str]:
+    """Extract all image URLs from message content.
+    
+    Filters out invalid entries:
+    - Non-string URLs
+    - Empty strings
+    - Whitespace-only strings
+    
+    Args:
+        content: Message content to extract from
+    
+    Returns:
+        List of valid image URL strings (may be http(s):// URLs or data:image/... base64)
+    """
+    urls: list[str] = []
+    
+    if isinstance(content, list):
+        for part in content:
+            if isinstance(part, dict) and part.get("type") in {"image", "image_url"}:
+                # Handle both formats:
+                # {"type": "image_url", "image_url": {"url": "..."}}
+                # {"type": "image", "image": "..."}
+                if "image_url" in part and isinstance(part["image_url"], dict):
+                    url = part["image_url"].get("url")
+                    if isinstance(url, str) and url.strip():  # Filter empty/whitespace
+                        urls.append(url)
+                elif "image" in part and isinstance(part["image"], str):
+                    if part["image"].strip():  # Filter empty/whitespace
+                        urls.append(part["image"])
+    elif isinstance(content, dict) and content.get("type") in {"image", "image_url"}:
+        image_url_data = content.get("image_url")
+        if isinstance(image_url_data, dict):
+            url = image_url_data.get("url")
+            if isinstance(url, str) and url.strip():  # Filter empty/whitespace
+                urls.append(url)
+        else:
+            image_value = content.get("image")
+            if isinstance(image_value, str) and image_value.strip():  # Filter empty/whitespace
+                urls.append(image_value)
+    
+    return urls
+
+
+def validate_vision_example(
+    example: SFTExample, *, require_images: bool = True
+) -> tuple[bool, str | None]:
+    """Validate a vision SFT example.
+    
+    Checks:
+    - If require_images is True, at least one message must contain an image
+    - All image URLs must be non-empty, non-whitespace strings
+    - Image entries must have valid URL data
+    - Messages must follow valid structure
+    
+    Args:
+        example: SFTExample to validate
+        require_images: If True, fail if no images are present
+    
+    Returns:
+        Tuple of (is_valid, error_message)
+        If valid, error_message is None
+    """
+    # Count actual valid URLs and detect any invalid entries
+    total_valid_urls = 0
+    
+    # Validate image URLs in each message
+    for i, msg in enumerate(example.messages):
+        # Check if this message has image_url type entries
+        if not isinstance(msg.content, list | dict):
+            continue
+        
+        # Count image_url type entries vs valid URLs
+        content_list = msg.content if isinstance(msg.content, list) else [msg.content]
+        image_type_count = sum(
+            1 for item in content_list
+            if isinstance(item, dict) and item.get("type") in {"image", "image_url"}
+        )
+        
+        if image_type_count > 0:
+            # Extract valid URLs (after filtering)
+            urls = extract_image_urls(msg.content)
+            
+            # If we have image_url type entries but fewer valid URLs, some are invalid
+            if len(urls) < image_type_count:
+                return False, f"Message {i}: Has {image_type_count} image_url entries but only {len(urls)} valid URLs (some are empty, null, or missing)"
+            
+            # Validate each URL (double-check, though extract_image_urls should have filtered)
+            for url in urls:
+                # extract_image_urls already filters for isinstance(url, str) and url.strip()
+                # but let's be defensive
+                if not isinstance(url, str):
+                    return False, f"Message {i}: Image URL is not a string: {type(url)}"
+                
+                if not url.strip():
+                    return False, f"Message {i}: Invalid or empty image URL"
+                
+                # Basic URL format check
+                if not url.startswith(("http://", "https://", "data:image/")):
+                    logger.warning(
+                        f"Message {i}: Image URL doesn't start with http://, https://, or data:image/ - "
+                        f"this may cause issues during training. URL: {url[:100]}"
+                    )
+                
+                total_valid_urls += 1
+    
+    # Final check: if images are required, ensure we found at least one valid URL
+    if require_images and total_valid_urls == 0:
+        return False, "No image content found in any message"
+    
+    return True, None
+
+
+def iter_vision_examples(
+    source: Iterable[str],
+    *,
+    min_messages: int = 1,
+    skip_empty: bool = True,
+    require_images: bool = True,
+    log_validation_errors: bool = False,
+) -> Iterator[SFTExample]:
+    """Iterate over vision SFT examples from JSONL source.
+    
+    Similar to iter_sft_examples but with vision-specific validation.
+    
+    Args:
+        source: Iterable of JSONL lines
+        min_messages: Minimum number of messages required
+        skip_empty: Skip empty lines
+        require_images: If True, skip examples without images
+        log_validation_errors: If True, log validation failures
+    
+    Yields:
+        Valid vision SFTExample objects
+    """
+    for line in source:
+        if skip_empty and not line.strip():
+            continue
+        
+        try:
+            example = parse_jsonl_line(line, min_messages=min_messages)
+            
+            # Validate vision content if required
+            if require_images:
+                is_valid, error = validate_vision_example(example, require_images=True)
+                if not is_valid:
+                    if log_validation_errors:
+                        logger.warning(f"Skipping invalid vision example: {error}")
+                    continue
+            
+            yield example
+            
+        except (json.JSONDecodeError, SFTDataError) as exc:
+            if log_validation_errors:
+                logger.warning(f"Failed to parse vision example: {exc}")
+            continue
+
+
 __all__ = [
     "SFTDataError",
     "SFTExample",
@@ -292,4 +682,17 @@ __all__ = [
     "load_jsonl",
     "parse_jsonl_line",
     "validate_jsonl_or_raise",
+    # Reasoning utilities
+    "extract_reasoning",
+    "strip_reasoning",
+    "message_has_reasoning",
+    "validate_message_content",
+    # Vision utilities
+    "has_image_content",
+    "message_has_image",
+    "example_has_image",
+    "count_images_in_content",
+    "extract_image_urls",
+    "validate_vision_example",
+    "iter_vision_examples",
 ]

synth_ai/learning/validators.py

@@ -37,7 +37,10 @@ def validate_training_jsonl(path: str | Path, *, sample_lines: int = 50) -> None
 def validate_task_app_url(url: str, *, name: str = "TASK_APP_BASE_URL") -> None:
     from synth_ai.task.validators import validate_task_app_url as _vt
 
-    _vt(url, name=name)
+    try:
+        _vt(url)
+    except ValueError as exc:
+        raise ValueError(f"{name}: {exc}") from exc
 
 
 def validate_trainer_cfg_rl(trainer: dict[str, Any]) -> None:

synth_ai/streaming/__init__.py

@@ -0,0 +1,29 @@
+from .config import StreamConfig
+from .handlers import (
+    BufferedHandler,
+    CallbackHandler,
+    CLIHandler,
+    IntegrationTestHandler,
+    JSONHandler,
+    LossCurveHandler,
+    RichHandler,
+    StreamHandler,
+)
+from .streamer import JobStreamer, StreamEndpoints
+from .types import StreamMessage, StreamType
+
+__all__ = [
+    "BufferedHandler",
+    "CallbackHandler",
+    "CLIHandler",
+    "IntegrationTestHandler",
+    "JSONHandler",
+    "LossCurveHandler",
+    "JobStreamer",
+    "RichHandler",
+    "StreamEndpoints",
+    "StreamConfig",
+    "StreamHandler",
+    "StreamMessage",
+    "StreamType",
+]

synth_ai/streaming/config.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .types import StreamType
+
+
+@dataclass(slots=True)
+class StreamConfig:
+    """Configuration describing which streams to consume and how to filter them."""
+
+    enabled_streams: set[StreamType] = field(default_factory=lambda: set(StreamType))
+    event_types: set[str] | None = None  # Whitelist: only include these event types
+    event_types_exclude: set[str] | None = None  # Blacklist: exclude these event types
+    event_levels: set[str] | None = None
+    metric_names: set[str] | None = None
+    metric_phases: set[str] | None = None
+    timeline_phases: set[str] | None = None
+    sample_rate: float = 1.0
+    max_events_per_poll: int | None = None
+    deduplicate: bool = True
+
+    @classmethod
+    def default(cls) -> StreamConfig:
+        """Return a configuration representing the default (all streams) view."""
+        return cls(
+            event_types_exclude={
+                # Filter out noisy events that just announce what metrics already show
+                "sft.progress",  # Generic "Training progress" with no data
+                "sft.loss",      # Generic "Loss update" with no data
+                "sft.upstream.status",  # Very verbose status echo events
+            }
+        )
+
+    @classmethod
+    def minimal(cls) -> StreamConfig:
+        """Return a configuration streaming status updates only."""
+        return cls(enabled_streams={StreamType.STATUS})
+
+    @classmethod
+    def verbose(cls) -> StreamConfig:
+        """Return a configuration with all streams and events (no filters)."""
+        return cls()
+
+    @classmethod
+    def progress_only(cls) -> StreamConfig:
+        """Return a configuration tailored to show training progress."""
+        return cls(
+            enabled_streams={StreamType.STATUS, StreamType.EVENTS, StreamType.METRICS},
+            event_types={"sft.progress", "rl.train.step", "sft.validation.summary"},
+            metric_names={"train.loss", "eval.reward_mean"},
+        )
+
+    @classmethod
+    def errors_only(cls) -> StreamConfig:
+        """Return a configuration that focuses on heightened severity signals."""
+        return cls(
+            enabled_streams={StreamType.STATUS, StreamType.EVENTS},
+            event_levels={"error", "warning"},
+        )
+
+    def should_include_event(self, event: dict[str, Any]) -> bool:
+        """Determine whether an event message should be included."""
+        event_type = event.get("type")
+        
+        # Apply blacklist first (takes precedence)
+        if self.event_types_exclude and event_type in self.event_types_exclude:
+            return False
+        
+        # Then apply whitelist
+        if self.event_types and event_type not in self.event_types:
+            return False
+        
+        if self.event_levels:
+            return event.get("level") in self.event_levels
+        return True
+
+    def should_include_metric(self, metric: dict[str, Any]) -> bool:
+        """Determine whether a metric point should be included."""
+        if self.metric_names and metric.get("name") not in self.metric_names:
+            return False
+        if self.metric_phases:
+            return metric.get("phase") in self.metric_phases
+        return True
+
+    def should_include_timeline(self, timeline_entry: dict[str, Any]) -> bool:
+        """Determine whether a timeline entry should be included."""
+        if self.timeline_phases:
+            return timeline_entry.get("phase") in self.timeline_phases
+        return True
+
+
+__all__ = ["StreamConfig"]