synth-ai 0.2.14__py3-none-any.whl → 0.2.16__py3-none-any.whl

synth_ai/learning/sft/data.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
 import json
+import logging
 from collections.abc import Iterable, Iterator, Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
 
+logger = logging.getLogger(__name__)
+
 SFTMessageContent = str | dict[str, Any] | list[Any] | None
 
 
@@ -37,6 +40,8 @@ class SFTMessage:
     tool_calls: list[SFTToolCall] = field(default_factory=list)
     tool_call_id: str | None = None
     name: str | None = None
+    reasoning: str | None = None  # NEW: Explicit reasoning/thinking content
+    raw_content: str | None = None  # NEW: Original unparsed content (before reasoning extraction)
     extra: dict[str, Any] = field(default_factory=dict)
 
 
@@ -86,9 +91,11 @@ def _coerce_tool_call(raw: Any, *, index: int) -> SFTToolCall:
     name: str | None = None
     arguments: Any = None
 
-    if isinstance(raw.get("function"), dict):
-        fn_payload = raw["function"]
-        name = fn_payload.get("name") if isinstance(fn_payload.get("name"), str) else None
+    fn_obj = raw.get("function")
+    if isinstance(fn_obj, dict):
+        fn_payload = fn_obj
+        name_val = fn_payload.get("name")
+        name = name_val if isinstance(name_val, str) else None
         arguments = fn_payload.get("arguments")
     if name is None:
         maybe_name = raw.get("name")
@@ -143,11 +150,20 @@ def _coerce_message(raw: Any, *, index: int) -> SFTMessage:
     name = raw.get("name")
     if name is not None and not isinstance(name, str):
         raise SFTDataError(f"message {index} name must be a string if present")
+    
+    # NEW: Extract reasoning and raw_content if present
+    reasoning = raw.get("reasoning")
+    if reasoning is not None and not isinstance(reasoning, str):
+        raise SFTDataError(f"message {index} reasoning must be a string if present")
+    
+    raw_content = raw.get("raw_content")
+    if raw_content is not None and not isinstance(raw_content, str):
+        raise SFTDataError(f"message {index} raw_content must be a string if present")
 
     extra = {
         key: value
         for key, value in raw.items()
-        if key not in {"role", "content", "tool_calls", "tool_call_id", "name"}
+        if key not in {"role", "content", "tool_calls", "tool_call_id", "name", "reasoning", "raw_content"}
     }
 
     return SFTMessage(
@@ -156,6 +172,8 @@ def _coerce_message(raw: Any, *, index: int) -> SFTMessage:
         tool_calls=tool_calls,
         tool_call_id=tool_call_id,
         name=name,
+        reasoning=reasoning,
+        raw_content=raw_content,
         extra=extra,
     )
 
@@ -280,6 +298,378 @@ def load_jsonl(path: Path, *, min_messages: int = 1) -> list[SFTExample]:
         return list(iter_sft_examples(fh, min_messages=min_messages))
 
 
+# Reasoning/Thinking Utilities
+# ============================================================================
+
+
+def extract_reasoning(content: str, *, tag: str = "think") -> tuple[str | None, str]:
+    """Extract reasoning from content with <think> tags.
+    
+    Args:
+        content: Raw content string
+        tag: Tag name to extract (default: "think")
+    
+    Returns:
+        Tuple of (reasoning, clean_content)
+        - reasoning: Content inside tags, or None if no tags found
+        - clean_content: Content with tags removed
+    
+    Examples:
+        >>> extract_reasoning("<think>Let me analyze...</think>The answer is 42")
+        ('Let me analyze...', 'The answer is 42')
+        >>> extract_reasoning("Just plain text")
+        (None, 'Just plain text')
+    """
+    import re
+    
+    pattern = rf"<{tag}>(.*?)</{tag}>"
+    matches = re.findall(pattern, content, re.DOTALL)
+    
+    if not matches:
+        return None, content
+    
+    # Combine all reasoning blocks
+    reasoning = "\n\n".join(m.strip() for m in matches)
+    
+    # Remove all reasoning blocks from content
+    clean_content = re.sub(pattern, "", content, flags=re.DOTALL).strip()
+    
+    return reasoning, clean_content
+
+
+def strip_reasoning(content: str, *, tag: str = "think") -> str:
+    """Remove reasoning tags from content.
+    
+    Args:
+        content: Content with potential reasoning tags
+        tag: Tag name to strip (default: "think")
+    
+    Returns:
+        Content with reasoning tags removed
+    """
+    _, clean = extract_reasoning(content, tag=tag)
+    return clean
+
+
+def message_has_reasoning(message: SFTMessage) -> bool:
+    """Check if a message has explicit reasoning.
+    
+    Args:
+        message: SFTMessage to check
+    
+    Returns:
+        True if message has reasoning field or <think> tags in content
+    """
+    # Check explicit reasoning field
+    if message.reasoning:
+        return True
+    
+    # Check for reasoning tags in content
+    if isinstance(message.content, str):
+        reasoning, _ = extract_reasoning(message.content)
+        return reasoning is not None
+    
+    return False
+
+
+def validate_message_content(
+    message: SFTMessage, *, require_content: bool = True
+) -> tuple[bool, str | None]:
+    """Validate that message has valid content combinations.
+    
+    Rules:
+    - Must have at least one of: reasoning + tool_calls, reasoning + content, 
+      content, raw_content, or tool_calls
+    - If raw_content present with reasoning + content, they should be consistent
+    - Cannot have neither reasoning, content, raw_content, nor tool_calls
+    
+    Args:
+        message: SFTMessage to validate
+        require_content: If True, require some form of content (default: True)
+    
+    Returns:
+        Tuple of (is_valid, error_message)
+    """
+    has_reasoning = bool(message.reasoning)
+    has_content = message.content is not None and message.content != ""
+    has_raw = bool(message.raw_content)
+    has_tools = len(message.tool_calls) > 0
+    
+    # Check for completely empty message
+    if require_content and not (has_reasoning or has_content or has_raw or has_tools):
+        return False, "Message has no reasoning, content, raw_content, or tool_calls"
+    
+    # Valid combinations:
+    # 1. reasoning + tool_calls (reasoning-based action)
+    if has_reasoning and has_tools:
+        return True, None
+    
+    # 2. reasoning + content (reasoning then output)
+    if has_reasoning and has_content:
+        # If raw_content present, validate consistency
+        if has_raw and message.raw_content:
+            # Raw should contain both reasoning and content
+            reasoning_in_raw, content_in_raw = extract_reasoning(message.raw_content)
+            if message.reasoning and reasoning_in_raw != message.reasoning.strip():
+                logger.warning(
+                    "raw_content reasoning doesn't match reasoning field"
+                )
+            # This is okay - just a warning, not an error
+        return True, None
+    
+    # 3. content only (standard message)
+    if has_content and not has_reasoning:
+        return True, None
+    
+    # 4. raw_content only (unparsed content)
+    if has_raw and not (has_reasoning and has_content):
+        return True, None
+    
+    # 5. tool_calls only (action without reasoning/content - like OpenAI format)
+    if has_tools and not has_content:
+        return True, None
+    
+    # 6. reasoning only (pure thinking turn)
+    if has_reasoning and not has_content and not has_tools:
+        return True, None
+    
+    return True, None
+
+
+# Vision/Multimodal Utilities
+# ============================================================================
+
+
+def has_image_content(content: SFTMessageContent) -> bool:
+    """Check if message content contains image data (OpenAI multimodal format).
+    
+    Supports:
+    - List of content parts: [{"type": "text", ...}, {"type": "image_url", ...}]
+    - Single dict with type field: {"type": "image_url", "image_url": {...}}
+    
+    Args:
+        content: Message content (can be str, list, dict, or None)
+    
+    Returns:
+        True if content contains an image segment
+    
+    Examples:
+        >>> has_image_content([{"type": "text", "text": "What's this?"}, 
+        ...                    {"type": "image_url", "image_url": {"url": "..."}}])
+        True
+        >>> has_image_content("Just text")
+        False
+    """
+    if isinstance(content, list):
+        return any(
+            isinstance(part, dict) and part.get("type") in {"image", "image_url"}
+            for part in content
+        )
+    elif isinstance(content, dict):
+        return content.get("type") in {"image", "image_url"}
+    return False
+
+
+def message_has_image(message: SFTMessage) -> bool:
+    """Check if an SFTMessage contains image content.
+    
+    Args:
+        message: SFTMessage to check
+    
+    Returns:
+        True if the message contains image content
+    """
+    return has_image_content(message.content)
+
+
+def example_has_image(example: SFTExample) -> bool:
+    """Check if an SFTExample contains any image content.
+    
+    Args:
+        example: SFTExample to check
+    
+    Returns:
+        True if any message in the example contains image content
+    """
+    return any(message_has_image(msg) for msg in example.messages)
+
+
+def count_images_in_content(content: SFTMessageContent) -> int:
+    """Count the number of images in message content.
+    
+    Args:
+        content: Message content to analyze
+    
+    Returns:
+        Number of image segments found
+    """
+    if isinstance(content, list):
+        return sum(
+            1 for part in content
+            if isinstance(part, dict) and part.get("type") in {"image", "image_url"}
+        )
+    elif isinstance(content, dict) and content.get("type") in {"image", "image_url"}:
+        return 1
+    return 0
+
+
+def extract_image_urls(content: SFTMessageContent) -> list[str]:
+    """Extract all image URLs from message content.
+    
+    Filters out invalid entries:
+    - Non-string URLs
+    - Empty strings
+    - Whitespace-only strings
+    
+    Args:
+        content: Message content to extract from
+    
+    Returns:
+        List of valid image URL strings (may be http(s):// URLs or data:image/... base64)
+    """
+    urls: list[str] = []
+    
+    if isinstance(content, list):
+        for part in content:
+            if isinstance(part, dict) and part.get("type") in {"image", "image_url"}:
+                # Handle both formats:
+                # {"type": "image_url", "image_url": {"url": "..."}}
+                # {"type": "image", "image": "..."}
+                if "image_url" in part and isinstance(part["image_url"], dict):
+                    url = part["image_url"].get("url")
+                    if isinstance(url, str) and url.strip():  # Filter empty/whitespace
+                        urls.append(url)
+                elif "image" in part and isinstance(part["image"], str):
+                    if part["image"].strip():  # Filter empty/whitespace
+                        urls.append(part["image"])
+    elif isinstance(content, dict) and content.get("type") in {"image", "image_url"}:
+        image_url_data = content.get("image_url")
+        if isinstance(image_url_data, dict):
+            url = image_url_data.get("url")
+            if isinstance(url, str) and url.strip():  # Filter empty/whitespace
+                urls.append(url)
+        else:
+            image_value = content.get("image")
+            if isinstance(image_value, str) and image_value.strip():  # Filter empty/whitespace
+                urls.append(image_value)
+    
+    return urls
+
+
+def validate_vision_example(
+    example: SFTExample, *, require_images: bool = True
+) -> tuple[bool, str | None]:
+    """Validate a vision SFT example.
+    
+    Checks:
+    - If require_images is True, at least one message must contain an image
+    - All image URLs must be non-empty, non-whitespace strings
+    - Image entries must have valid URL data
+    - Messages must follow valid structure
+    
+    Args:
+        example: SFTExample to validate
+        require_images: If True, fail if no images are present
+    
+    Returns:
+        Tuple of (is_valid, error_message)
+        If valid, error_message is None
+    """
+    # Count actual valid URLs and detect any invalid entries
+    total_valid_urls = 0
+    
+    # Validate image URLs in each message
+    for i, msg in enumerate(example.messages):
+        # Check if this message has image_url type entries
+        if not isinstance(msg.content, list | dict):
+            continue
+        
+        # Count image_url type entries vs valid URLs
+        content_list = msg.content if isinstance(msg.content, list) else [msg.content]
+        image_type_count = sum(
+            1 for item in content_list
+            if isinstance(item, dict) and item.get("type") in {"image", "image_url"}
+        )
+        
+        if image_type_count > 0:
+            # Extract valid URLs (after filtering)
+            urls = extract_image_urls(msg.content)
+            
+            # If we have image_url type entries but fewer valid URLs, some are invalid
+            if len(urls) < image_type_count:
+                return False, f"Message {i}: Has {image_type_count} image_url entries but only {len(urls)} valid URLs (some are empty, null, or missing)"
+            
+            # Validate each URL (double-check, though extract_image_urls should have filtered)
+            for url in urls:
+                # extract_image_urls already filters for isinstance(url, str) and url.strip()
+                # but let's be defensive
+                if not isinstance(url, str):
+                    return False, f"Message {i}: Image URL is not a string: {type(url)}"
+                
+                if not url.strip():
+                    return False, f"Message {i}: Invalid or empty image URL"
+                
+                # Basic URL format check
+                if not url.startswith(("http://", "https://", "data:image/")):
+                    logger.warning(
+                        f"Message {i}: Image URL doesn't start with http://, https://, or data:image/ - "
+                        f"this may cause issues during training. URL: {url[:100]}"
+                    )
+                
+                total_valid_urls += 1
+    
+    # Final check: if images are required, ensure we found at least one valid URL
+    if require_images and total_valid_urls == 0:
+        return False, "No image content found in any message"
+    
+    return True, None
+
+
+def iter_vision_examples(
+    source: Iterable[str],
+    *,
+    min_messages: int = 1,
+    skip_empty: bool = True,
+    require_images: bool = True,
+    log_validation_errors: bool = False,
+) -> Iterator[SFTExample]:
+    """Iterate over vision SFT examples from JSONL source.
+    
+    Similar to iter_sft_examples but with vision-specific validation.
+    
+    Args:
+        source: Iterable of JSONL lines
+        min_messages: Minimum number of messages required
+        skip_empty: Skip empty lines
+        require_images: If True, skip examples without images
+        log_validation_errors: If True, log validation failures
+    
+    Yields:
+        Valid vision SFTExample objects
+    """
+    for line in source:
+        if skip_empty and not line.strip():
+            continue
+        
+        try:
+            example = parse_jsonl_line(line, min_messages=min_messages)
+            
+            # Validate vision content if required
+            if require_images:
+                is_valid, error = validate_vision_example(example, require_images=True)
+                if not is_valid:
+                    if log_validation_errors:
+                        logger.warning(f"Skipping invalid vision example: {error}")
+                    continue
+            
+            yield example
+            
+        except (json.JSONDecodeError, SFTDataError) as exc:
+            if log_validation_errors:
+                logger.warning(f"Failed to parse vision example: {exc}")
+            continue
+
+
 __all__ = [
     "SFTDataError",
     "SFTExample",
@@ -292,4 +682,17 @@ __all__ = [
     "load_jsonl",
     "parse_jsonl_line",
     "validate_jsonl_or_raise",
+    # Reasoning utilities
+    "extract_reasoning",
+    "strip_reasoning",
+    "message_has_reasoning",
+    "validate_message_content",
+    # Vision utilities
+    "has_image_content",
+    "message_has_image",
+    "example_has_image",
+    "count_images_in_content",
+    "extract_image_urls",
+    "validate_vision_example",
+    "iter_vision_examples",
 ]

synth_ai/learning/validators.py

@@ -37,7 +37,10 @@ def validate_training_jsonl(path: str | Path, *, sample_lines: int = 50) -> None
 def validate_task_app_url(url: str, *, name: str = "TASK_APP_BASE_URL") -> None:
     from synth_ai.task.validators import validate_task_app_url as _vt
 
-    _vt(url, name=name)
+    try:
+        _vt(url)
+    except ValueError as exc:
+        raise ValueError(f"{name}: {exc}") from exc
 
 
 def validate_trainer_cfg_rl(trainer: dict[str, Any]) -> None:

synth_ai/task/apps/__init__.py

@@ -50,10 +50,12 @@ class TaskAppRegistry:
 
     def register(self, entry: TaskAppEntry) -> None:
         if entry.app_id in self._entries:
-            raise ValueError(f"Task app already registered: {entry.app_id}")
+            # Allow idempotent registration when modules are imported multiple times.
+            return
         self._entries[entry.app_id] = entry
         for alias in entry.aliases:
-            if alias in self._alias_to_id:
+            existing = self._alias_to_id.get(alias)
+            if existing and existing != entry.app_id:
                 raise ValueError(f"Alias already registered: {alias}")
             self._alias_to_id[alias] = entry.app_id
 

synth_ai/task/config.py

@@ -185,9 +185,12 @@ class FilterConfig:
             raise ValueError(f"output must be a .jsonl or .json file, got: {self.output}")
         
         # Validate score thresholds
-        if self.min_official_score is not None and self.max_official_score is not None:
-            if self.min_official_score > self.max_official_score:
-                raise ValueError("min_official_score cannot be greater than max_official_score")
+        if (
+            self.min_official_score is not None
+            and self.max_official_score is not None
+            and self.min_official_score > self.max_official_score
+        ):
+            raise ValueError("min_official_score cannot be greater than max_official_score")
         
         # Validate limit/offset
         if self.limit is not None and self.limit < 1:
@@ -254,4 +257,3 @@ class FilterConfig:
         output_path.parent.mkdir(parents=True, exist_ok=True)
         return output_path
 
-

synth_ai/task/rubrics/__init__.py

@@ -9,10 +9,9 @@ This module provides:
 """
 
 # Core models (flexible validation)
-from .models import Criterion, Rubric
-
 # Loading and blending
 from .loaders import blend_rubrics, load_rubric
+from .models import Criterion, Rubric
 
 # Scoring
 from .scoring import score_events_against_rubric, score_outcome_against_rubric

synth_ai/task/rubrics/loaders.py

@@ -78,15 +78,20 @@ def load_rubric(source: str | dict[str, Any] | Rubric | None) -> Rubric | None:
         data = _parse_structured(text, suffix)
     
     # Check if this looks like a backend judge rubric (wrong format)
-    if isinstance(data, dict) and "event" in data and "outcome" in data:
-        # Missing required task app rubric fields
-        if "version" not in data and "goal_text" not in data and "criteria" not in data:
-            source_hint = f" ({source})" if isinstance(source, str) else ""
-            raise ValueError(
-                f"Rubric appears to be in backend judge format (has 'event'/'outcome' keys){source_hint}. "
-                f"Task apps require rubrics with 'version', 'goal_text', and 'criteria' fields. "
-                f"Backend judge rubrics should be named '*_backend_judge.json' and loaded by judge functions."
-            )
+    if (
+        isinstance(data, dict)
+        and "event" in data
+        and "outcome" in data
+        and "version" not in data
+        and "goal_text" not in data
+        and "criteria" not in data
+    ):
+        source_hint = f" ({source})" if isinstance(source, str) else ""
+        raise ValueError(
+            f"Rubric appears to be in backend judge format (has 'event'/'outcome' keys){source_hint}. "
+            f"Task apps require rubrics with 'version', 'goal_text', and 'criteria' fields. "
+            f"Backend judge rubrics should be named '*_backend_judge.json' and loaded by judge functions."
+        )
     
     return Rubric.model_validate(data)
 
@@ -149,4 +154,3 @@ def blend_rubrics(base: Rubric | None, override: Rubric | None) -> Rubric | None
         criteria=merged,
         aggregation=aggregation,
     )
-