synth-ai 0.2.8.dev4__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

synth_ai/spec/serializer.py

@@ -0,0 +1,199 @@
+"""Serializers for converting specs to prompt-friendly formats."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from synth_ai.spec.dataclasses import Spec
+
+
+def spec_to_prompt_context(
+    spec: Spec,
+    include_examples: bool = True,
+    include_tests: bool = False,
+    include_glossary: bool = True,
+    max_rules: Optional[int] = None,
+    priority_threshold: Optional[int] = None,
+) -> str:
+    """Convert a Spec to a prompt-friendly markdown format.
+    
+    Args:
+        spec: The specification to serialize
+        include_examples: Whether to include rule examples
+        include_tests: Whether to include test cases
+        include_glossary: Whether to include glossary terms
+        max_rules: Maximum number of rules to include (None = all)
+        priority_threshold: Only include rules with priority >= threshold
+        
+    Returns:
+        Markdown-formatted string suitable for inclusion in prompts
+    """
+    lines = []
+    
+    # Header
+    lines.append(f"# {spec.metadata.title}")
+    if spec.metadata.description:
+        lines.append(f"\n{spec.metadata.description}")
+    lines.append(f"\n**Version:** {spec.metadata.version}")
+    if spec.metadata.scope:
+        lines.append(f"**Scope:** {spec.metadata.scope}")
+    lines.append("")
+    
+    # Principles
+    if spec.principles:
+        lines.append("## Guiding Principles\n")
+        for principle in spec.principles:
+            lines.append(f"**{principle.id}**: {principle.text}")
+            if principle.rationale:
+                lines.append(f"  - *Rationale:* {principle.rationale}")
+            lines.append("")
+    
+    # Rules
+    if spec.rules:
+        # Filter rules by priority if specified
+        rules_to_include = spec.rules
+        if priority_threshold is not None:
+            rules_to_include = [
+                r for r in rules_to_include
+                if r.priority is not None and r.priority >= priority_threshold
+            ]
+        
+        # Sort by priority (highest first)
+        rules_to_include = sorted(
+            rules_to_include,
+            key=lambda r: r.priority if r.priority is not None else 0,
+            reverse=True,
+        )
+        
+        # Limit number of rules
+        if max_rules is not None:
+            rules_to_include = rules_to_include[:max_rules]
+        
+        if rules_to_include:
+            lines.append("## Rules and Policies\n")
+            
+            for rule in rules_to_include:
+                # Rule header
+                priority_str = f" [Priority: {rule.priority}]" if rule.priority else ""
+                lines.append(f"### {rule.id}: {rule.title}{priority_str}\n")
+                
+                if rule.rationale:
+                    lines.append(f"*Rationale:* {rule.rationale}\n")
+                
+                # Constraints
+                if rule.constraints.must or rule.constraints.must_not:
+                    lines.append("**Constraints:**")
+                    
+                    if rule.constraints.must:
+                        lines.append("- **MUST:**")
+                        for constraint in rule.constraints.must:
+                            lines.append(f"  - {constraint}")
+                    
+                    if rule.constraints.must_not:
+                        lines.append("- **MUST NOT:**")
+                        for constraint in rule.constraints.must_not:
+                            lines.append(f"  - {constraint}")
+                    
+                    if rule.constraints.should:
+                        lines.append("- **SHOULD:**")
+                        for constraint in rule.constraints.should:
+                            lines.append(f"  - {constraint}")
+                    
+                    if rule.constraints.should_not:
+                        lines.append("- **SHOULD NOT:**")
+                        for constraint in rule.constraints.should_not:
+                            lines.append(f"  - {constraint}")
+                    
+                    lines.append("")
+                
+                # Examples
+                if include_examples and rule.examples:
+                    lines.append("**Examples:**\n")
+                    
+                    good_examples = [e for e in rule.examples if e.kind == "good"]
+                    bad_examples = [e for e in rule.examples if e.kind == "bad"]
+                    
+                    if good_examples:
+                        lines.append("✅ **Good:**")
+                        for ex in good_examples:
+                            lines.append(f"- Prompt: \"{ex.prompt}\"")
+                            lines.append(f"  Response: \"{ex.response}\"")
+                            if ex.description:
+                                lines.append(f"  *{ex.description}*")
+                        lines.append("")
+                    
+                    if bad_examples:
+                        lines.append("❌ **Bad:**")
+                        for ex in bad_examples:
+                            lines.append(f"- Prompt: \"{ex.prompt}\"")
+                            lines.append(f"  Response: \"{ex.response}\"")
+                            if ex.description:
+                                lines.append(f"  *{ex.description}*")
+                        lines.append("")
+                
+                # Tests
+                if include_tests and rule.tests:
+                    lines.append("**Test Cases:**\n")
+                    for test in rule.tests:
+                        lines.append(f"- {test.id}: {test.challenge}")
+                        if test.asserts:
+                            lines.append(f"  Asserts: {', '.join(test.asserts)}")
+                        if test.expected_behavior:
+                            lines.append(f"  Expected: {test.expected_behavior}")
+                    lines.append("")
+    
+    # Glossary
+    if include_glossary and spec.glossary:
+        lines.append("## Glossary\n")
+        for item in spec.glossary:
+            aliases_str = f" (aliases: {', '.join(item.aliases)})" if item.aliases else ""
+            lines.append(f"**{item.term}**{aliases_str}: {item.definition}")
+        lines.append("")
+    
+    return "\n".join(lines)
+
+
+def spec_to_compact_context(spec: Spec, max_tokens: int = 5000) -> str:
+    """Convert a Spec to a compact prompt context within token limit.
+    
+    Prioritizes high-priority rules and essential information.
+    
+    Args:
+        spec: The specification to serialize
+        max_tokens: Approximate maximum tokens (uses char estimation: 4 chars ≈ 1 token)
+        
+    Returns:
+        Compact markdown-formatted string
+    """
+    # Start with high-priority rules only
+    context = spec_to_prompt_context(
+        spec,
+        include_examples=True,
+        include_tests=False,
+        include_glossary=True,
+        priority_threshold=7,
+    )
+    
+    # If still too long, try with fewer examples
+    max_chars = max_tokens * 4
+    if len(context) > max_chars:
+        context = spec_to_prompt_context(
+            spec,
+            include_examples=False,
+            include_tests=False,
+            include_glossary=True,
+            priority_threshold=7,
+        )
+    
+    # If still too long, reduce glossary
+    if len(context) > max_chars:
+        context = spec_to_prompt_context(
+            spec,
+            include_examples=False,
+            include_tests=False,
+            include_glossary=False,
+            priority_threshold=8,
+        )
+    
+    return context
+

synth_ai/spec/validation.py

@@ -0,0 +1,250 @@
+"""Validation utilities for system specifications."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+from synth_ai.spec.dataclasses import Spec
+
+
+class SpecValidationError(Exception):
+    """Raised when spec validation fails."""
+    pass
+
+
+class SpecValidator:
+    """Validator for system specifications."""
+    
+    def __init__(self):
+        self.errors: List[str] = []
+        self.warnings: List[str] = []
+    
+    def validate(self, spec: Spec, strict: bool = False) -> bool:
+        """Validate a spec and collect errors/warnings.
+        
+        Args:
+            spec: The spec to validate
+            strict: If True, warnings are treated as errors
+            
+        Returns:
+            True if validation passes (no errors, or warnings if not strict)
+            
+        Raises:
+            SpecValidationError: If validation fails with errors
+        """
+        self.errors = []
+        self.warnings = []
+        
+        self._validate_metadata(spec)
+        self._validate_principles(spec)
+        self._validate_rules(spec)
+        self._validate_glossary(spec)
+        self._validate_consistency(spec)
+        
+        if strict and self.warnings:
+            self.errors.extend(self.warnings)
+            self.warnings = []
+        
+        if self.errors:
+            error_msg = "\n".join([f"  - {err}" for err in self.errors])
+            raise SpecValidationError(f"Spec validation failed:\n{error_msg}")
+        
+        return True
+    
+    def _validate_metadata(self, spec: Spec) -> None:
+        """Validate metadata fields."""
+        md = spec.metadata
+        
+        if not md.id:
+            self.errors.append("Metadata: 'id' is required")
+        elif not md.id.startswith("spec."):
+            self.warnings.append("Metadata: 'id' should start with 'spec.' prefix")
+        
+        if not md.title:
+            self.errors.append("Metadata: 'title' is required")
+        
+        if not md.version:
+            self.errors.append("Metadata: 'version' is required")
+        elif not self._is_valid_semver(md.version):
+            self.warnings.append(f"Metadata: version '{md.version}' is not valid semver (X.Y.Z)")
+        
+        if not md.scope:
+            self.warnings.append("Metadata: 'scope' should be specified")
+    
+    def _validate_principles(self, spec: Spec) -> None:
+        """Validate principles."""
+        seen_ids = set()
+        
+        for i, principle in enumerate(spec.principles):
+            if not principle.id:
+                self.errors.append(f"Principle {i}: 'id' is required")
+            elif principle.id in seen_ids:
+                self.errors.append(f"Principle {i}: duplicate id '{principle.id}'")
+            else:
+                seen_ids.add(principle.id)
+            
+            if not principle.text:
+                self.errors.append(f"Principle {principle.id}: 'text' is required")
+            
+            if not principle.id.startswith("P-"):
+                self.warnings.append(f"Principle {principle.id}: id should start with 'P-' prefix")
+    
+    def _validate_rules(self, spec: Spec) -> None:
+        """Validate rules."""
+        seen_ids = set()
+        
+        for i, rule in enumerate(spec.rules):
+            if not rule.id:
+                self.errors.append(f"Rule {i}: 'id' is required")
+            elif rule.id in seen_ids:
+                self.errors.append(f"Rule {i}: duplicate id '{rule.id}'")
+            else:
+                seen_ids.add(rule.id)
+            
+            if not rule.title:
+                self.errors.append(f"Rule {rule.id}: 'title' is required")
+            
+            if not rule.id.startswith("R-"):
+                self.warnings.append(f"Rule {rule.id}: id should start with 'R-' prefix")
+            
+            # Validate constraints
+            if not rule.constraints.must and not rule.constraints.must_not:
+                self.warnings.append(
+                    f"Rule {rule.id}: no constraints defined (must/must_not are empty)"
+                )
+            
+            # Validate examples
+            for j, example in enumerate(rule.examples):
+                if example.kind not in ("good", "bad"):
+                    self.errors.append(
+                        f"Rule {rule.id}, Example {j}: kind must be 'good' or 'bad', got '{example.kind}'"
+                    )
+                
+                if not example.prompt or not example.response:
+                    self.errors.append(
+                        f"Rule {rule.id}, Example {j}: both 'prompt' and 'response' are required"
+                    )
+            
+            # Validate priority
+            if rule.priority is not None and (not isinstance(rule.priority, int) or rule.priority < 1 or rule.priority > 10):
+                self.errors.append(
+                    f"Rule {rule.id}: priority must be an integer between 1 and 10, got {rule.priority}"
+                )
+    
+    def _validate_glossary(self, spec: Spec) -> None:
+        """Validate glossary."""
+        seen_terms = set()
+        
+        for item in spec.glossary:
+            term_lower = item.term.lower()
+            
+            if term_lower in seen_terms:
+                self.errors.append(f"Glossary: duplicate term '{item.term}'")
+            else:
+                seen_terms.add(term_lower)
+            
+            if not item.definition:
+                self.errors.append(f"Glossary: term '{item.term}' missing definition")
+            
+            # Check for duplicate aliases
+            for alias in item.aliases:
+                alias_lower = alias.lower()
+                if alias_lower in seen_terms:
+                    self.warnings.append(
+                        f"Glossary: alias '{alias}' for term '{item.term}' conflicts with existing term"
+                    )
+                seen_terms.add(alias_lower)
+    
+    def _validate_consistency(self, spec: Spec) -> None:
+        """Validate cross-references and consistency."""
+        # Check for orphaned imports
+        if spec.metadata.imports:
+            self.warnings.append(
+                f"Metadata: imports specified but not validated ({len(spec.metadata.imports)} imports)"
+            )
+        
+        # Warn if no rules or principles
+        if not spec.rules and not spec.principles:
+            self.warnings.append("Spec has no rules or principles defined")
+        
+        # Check for rules without examples
+        rules_without_examples = [r.id for r in spec.rules if not r.examples]
+        if rules_without_examples:
+            self.warnings.append(
+                f"Rules without examples: {', '.join(rules_without_examples)}"
+            )
+    
+    @staticmethod
+    def _is_valid_semver(version: str) -> bool:
+        """Check if version follows semver format (X.Y.Z)."""
+        parts = version.split(".")
+        if len(parts) != 3:
+            return False
+        try:
+            for part in parts:
+                int(part)
+            return True
+        except ValueError:
+            return False
+
+
+def validate_spec_dict(data: Dict[str, Any], strict: bool = False) -> List[str]:
+    """Validate spec dictionary before loading.
+    
+    Args:
+        data: Dictionary representation of spec
+        strict: If True, treat warnings as errors
+        
+    Returns:
+        List of validation errors (empty if valid)
+    """
+    errors = []
+    
+    # Check required top-level keys
+    if "metadata" not in data:
+        errors.append("Missing required key: 'metadata'")
+    elif not isinstance(data["metadata"], dict):
+        errors.append("'metadata' must be a dictionary")
+    else:
+        # Check required metadata fields
+        for field in ["id", "title", "version"]:
+            if field not in data["metadata"]:
+                errors.append(f"Missing required metadata field: '{field}'")
+    
+    # Check optional top-level keys are correct type
+    type_checks = {
+        "principles": list,
+        "rules": list,
+        "interfaces": dict,
+        "glossary": list,
+        "changelog": list,
+    }
+    
+    for key, expected_type in type_checks.items():
+        if key in data and not isinstance(data[key], expected_type):
+            errors.append(f"'{key}' must be a {expected_type.__name__}")
+    
+    return errors
+
+
+def validate_spec_file(path: str, strict: bool = False) -> bool:
+    """Validate a spec file.
+    
+    Args:
+        path: Path to spec JSON file
+        strict: If True, treat warnings as errors
+        
+    Returns:
+        True if validation passes
+        
+    Raises:
+        SpecValidationError: If validation fails
+        FileNotFoundError: If file doesn't exist
+        json.JSONDecodeError: If file is not valid JSON
+    """
+    from synth_ai.spec.loader import load_spec_from_file
+    
+    spec = load_spec_from_file(path)
+    validator = SpecValidator()
+    return validator.validate(spec, strict=strict)
+

synth_ai/streaming/__init__.py

@@ -0,0 +1,29 @@
+from .config import StreamConfig
+from .handlers import (
+    BufferedHandler,
+    CallbackHandler,
+    CLIHandler,
+    IntegrationTestHandler,
+    JSONHandler,
+    LossCurveHandler,
+    RichHandler,
+    StreamHandler,
+)
+from .streamer import JobStreamer, StreamEndpoints
+from .types import StreamMessage, StreamType
+
+__all__ = [
+    "BufferedHandler",
+    "CallbackHandler",
+    "CLIHandler",
+    "IntegrationTestHandler",
+    "JSONHandler",
+    "LossCurveHandler",
+    "JobStreamer",
+    "RichHandler",
+    "StreamEndpoints",
+    "StreamConfig",
+    "StreamHandler",
+    "StreamMessage",
+    "StreamType",
+]

synth_ai/streaming/config.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .types import StreamType
+
+
+@dataclass(slots=True)
+class StreamConfig:
+    """Configuration describing which streams to consume and how to filter them."""
+
+    enabled_streams: set[StreamType] = field(default_factory=lambda: set(StreamType))
+    event_types: set[str] | None = None  # Whitelist: only include these event types
+    event_types_exclude: set[str] | None = None  # Blacklist: exclude these event types
+    event_levels: set[str] | None = None
+    metric_names: set[str] | None = None
+    metric_phases: set[str] | None = None
+    timeline_phases: set[str] | None = None
+    sample_rate: float = 1.0
+    max_events_per_poll: int | None = None
+    deduplicate: bool = True
+
+    @classmethod
+    def default(cls) -> StreamConfig:
+        """Return a configuration representing the default (all streams) view."""
+        return cls(
+            event_types_exclude={
+                # Filter out noisy events that just announce what metrics already show
+                "sft.progress",  # Generic "Training progress" with no data
+                "sft.loss",      # Generic "Loss update" with no data
+                "sft.upstream.status",  # Very verbose status echo events
+            }
+        )
+
+    @classmethod
+    def minimal(cls) -> StreamConfig:
+        """Return a configuration streaming status updates only."""
+        return cls(enabled_streams={StreamType.STATUS})
+
+    @classmethod
+    def verbose(cls) -> StreamConfig:
+        """Return a configuration with all streams and events (no filters)."""
+        return cls()
+
+    @classmethod
+    def progress_only(cls) -> StreamConfig:
+        """Return a configuration tailored to show training progress."""
+        return cls(
+            enabled_streams={StreamType.STATUS, StreamType.EVENTS, StreamType.METRICS},
+            event_types={"sft.progress", "rl.train.step", "sft.validation.summary"},
+            metric_names={"train.loss", "eval.reward_mean"},
+        )
+
+    @classmethod
+    def errors_only(cls) -> StreamConfig:
+        """Return a configuration that focuses on heightened severity signals."""
+        return cls(
+            enabled_streams={StreamType.STATUS, StreamType.EVENTS},
+            event_levels={"error", "warning"},
+        )
+
+    def should_include_event(self, event: dict[str, Any]) -> bool:
+        """Determine whether an event message should be included."""
+        event_type = event.get("type")
+        
+        # Apply blacklist first (takes precedence)
+        if self.event_types_exclude and event_type in self.event_types_exclude:
+            return False
+        
+        # Then apply whitelist
+        if self.event_types and event_type not in self.event_types:
+            return False
+        
+        if self.event_levels:
+            return event.get("level") in self.event_levels
+        return True
+
+    def should_include_metric(self, metric: dict[str, Any]) -> bool:
+        """Determine whether a metric point should be included."""
+        if self.metric_names and metric.get("name") not in self.metric_names:
+            return False
+        if self.metric_phases:
+            return metric.get("phase") in self.metric_phases
+        return True
+
+    def should_include_timeline(self, timeline_entry: dict[str, Any]) -> bool:
+        """Determine whether a timeline entry should be included."""
+        if self.timeline_phases:
+            return timeline_entry.get("phase") in self.timeline_phases
+        return True
+
+
+__all__ = ["StreamConfig"]