synth-ai 0.2.3__tar.gz → 0.2.4.dev2__tar.gz

{synth_ai-0.2.3/synth_ai.egg-info → synth_ai-0.2.4.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-ai
-Version: 0.2.3
+Version: 0.2.4.dev2
 Summary: Software for aiding the best and multiplying the will - Core AI functionality and tracing
 Author-email: Synth AI <josh@usesynth.ai>
 License-Expression: MIT
@@ -85,7 +85,7 @@ Modern Compound AI System Development
 
 [![Python](https://img.shields.io/badge/python-3.11+-blue)](https://www.python.org/)
 [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
-[![PyPI](https://img.shields.io/badge/PyPI-0.2.1.dev0-orange)](https://pypi.org/project/synth-ai/)
+[![PyPI](https://img.shields.io/badge/PyPI-0.2.3.dev0-orange)](https://pypi.org/project/synth-ai/)
 ![Coverage](https://img.shields.io/badge/coverage-0.0%25-red)
 ![Tests](https://img.shields.io/badge/tests-17%2F17%20passing-brightgreen)
 
@@ -189,6 +189,7 @@ Notes:
   - `export LEARNING_V2_BASE_URL="http://localhost:8000/api"` (or your proxy)
   - `export SYNTH_API_KEY="sk_live_..."`
 - v3 trace data is stored under `traces/v3/synth_ai.db/` by default. Inspect with `uvx synth-ai traces`.
+ - LM tracing: all model calls (prompts, outputs, tool calls, token usage, latency, cost) are automatically captured via v3 tracing and stored locally; inspect with `uvx synth-ai traces`.
 
 ### One-Command Demos
 

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/README.md

@@ -6,7 +6,7 @@ Modern Compound AI System Development
 
 [![Python](https://img.shields.io/badge/python-3.11+-blue)](https://www.python.org/)
 [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
-[![PyPI](https://img.shields.io/badge/PyPI-0.2.1.dev0-orange)](https://pypi.org/project/synth-ai/)
+[![PyPI](https://img.shields.io/badge/PyPI-0.2.3.dev0-orange)](https://pypi.org/project/synth-ai/)
 ![Coverage](https://img.shields.io/badge/coverage-0.0%25-red)
 ![Tests](https://img.shields.io/badge/tests-17%2F17%20passing-brightgreen)
 
@@ -110,6 +110,7 @@ Notes:
   - `export LEARNING_V2_BASE_URL="http://localhost:8000/api"` (or your proxy)
   - `export SYNTH_API_KEY="sk_live_..."`
 - v3 trace data is stored under `traces/v3/synth_ai.db/` by default. Inspect with `uvx synth-ai traces`.
+ - LM tracing: all model calls (prompts, outputs, tool calls, token usage, latency, cost) are automatically captured via v3 tracing and stored locally; inspect with `uvx synth-ai traces`.
 
 ### One-Command Demos
 

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "synth-ai"
-version = "0.2.3"
+version = "0.2.4.dev2"
 description = "Software for aiding the best and multiplying the will - Core AI functionality and tracing"
 authors = [{name = "Synth AI", email = "josh@usesynth.ai"}]
 license = "MIT"

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/filter_traces_sft_turso.py

@@ -251,6 +251,111 @@ class FinetuningDataExtractorV3:
         
         return qualifying_sessions
     
+    async def extract_openai_format_from_call_records(self, session_ids: List[str], min_reward: float = 0.0) -> List[Dict[str, Any]]:
+        """Extract training data in OpenAI format from call_records in LMCAISEvents.
+        
+        This is the new method that uses the detailed LLM interaction data stored
+        in call_records instead of relying on separate message records.
+        """
+        training_data = []
+        
+        for session_id in session_ids:
+            # Get LM CAIS events with call_records from the proper column
+            events_query = """
+            SELECT e.call_records, st.turn_number
+            FROM events e
+            LEFT JOIN session_timesteps st ON e.timestep_id = st.id
+            WHERE e.session_id = :session_id
+            AND e.event_type = 'cais'
+            AND e.call_records IS NOT NULL
+            ORDER BY COALESCE(st.turn_number, e.message_time), e.id
+            """
+            
+            events_df = await self.db_manager.query_traces(events_query, {"session_id": session_id})
+            
+            if len(events_df) == 0:
+                # Fall back to old method if no call_records
+                continue
+            
+            # Extract messages from call_records
+            all_messages = []
+            
+            for _, row in events_df.iterrows():
+                call_records_json = row['call_records']
+                if not call_records_json:
+                    continue
+                
+                # Parse the call_records JSON directly from the column
+                try:
+                    import json
+                    if isinstance(call_records_json, str):
+                        call_records = json.loads(call_records_json)
+                    else:
+                        call_records = call_records_json
+                    
+                    # Process each call record
+                    for record in call_records:
+                        # Extract input messages
+                        for msg in record.get('input_messages', []):
+                            role = msg.get('role', 'user')
+                            parts = msg.get('parts', [])
+                            
+                            # Combine text parts
+                            text_content = []
+                            for part in parts:
+                                if part.get('type') == 'text' and part.get('text'):
+                                    text_content.append(part['text'])
+                            
+                            if text_content:
+                                content = ' '.join(text_content)
+                                if role == 'system' and not any(m['role'] == 'system' for m in all_messages):
+                                    all_messages.insert(0, {"role": "system", "content": content})
+                                elif role != 'system':
+                                    all_messages.append({"role": role, "content": content})
+                        
+                        # Extract output messages
+                        for msg in record.get('output_messages', []):
+                            role = msg.get('role', 'assistant')
+                            parts = msg.get('parts', [])
+                            
+                            # Combine text parts
+                            text_content = []
+                            for part in parts:
+                                if part.get('type') == 'text' and part.get('text'):
+                                    text_content.append(part['text'])
+                            
+                            if text_content:
+                                content = ' '.join(text_content)
+                                all_messages.append({"role": role, "content": content})
+                
+                except Exception as e:
+                    print(f"Error parsing call_records for session {session_id}: {e}")
+                    continue
+            
+            # Only include if we have a complete conversation
+            if len(all_messages) > 1:
+                # Get total reward for this session
+                reward_query = """
+                SELECT COALESCE(SUM(reward), 0) as total_reward
+                FROM events
+                WHERE session_id = :session_id
+                AND event_type = 'environment'
+                AND reward IS NOT NULL
+                """
+                reward_df = await self.db_manager.query_traces(reward_query, {"session_id": session_id})
+                total_reward = reward_df.iloc[0]['total_reward'] if len(reward_df) > 0 else 0
+                
+                training_data.append({
+                    "messages": all_messages,
+                    "metadata": {
+                        "session_id": session_id,
+                        "total_reward": float(total_reward),
+                        "source": "call_records"  # Mark that this came from call_records
+                    }
+                })
+        
+        return training_data
+    
     async def extract_openai_format(self, session_ids: List[str], min_reward: float = 0.0) -> List[Dict[str, Any]]:
         """Extract training data in OpenAI format from filtered sessions."""
         training_data = []
@@ -440,10 +545,19 @@ async def filter_traces_from_turso(
         
         # Extract training data
         if mode == "trajectory":
-            training_data = await extractor.extract_openai_format(
+            # Try new method first (using call_records)
+            training_data = await extractor.extract_openai_format_from_call_records(
                 session_ids=filtered_sessions,
                 min_reward=min_reward
             )
+            
+            # If no data from call_records, fall back to old method
+            if not training_data:
+                print("No call_records found, falling back to message-based extraction...")
+                training_data = await extractor.extract_openai_format(
+                    session_ids=filtered_sessions,
+                    min_reward=min_reward
+                )
         else:  # window mode
             # For window mode, we need to implement window extraction
             # For now, use trajectory mode

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/test_crafter_react_agent_lm_synth.py

@@ -70,7 +70,7 @@ from synth_ai.lm.config import SynthConfig
 # Import session tracer for v3 tracing
 from synth_ai.tracing_v3 import SessionTracer
 from synth_ai.tracing_v3.abstractions import (
-    SessionEventMessage, TimeRecord,
+    SessionEventMarkovBlanketMessage, TimeRecord,
     RuntimeEvent, EnvironmentEvent, LMCAISEvent
 )
 # create_experiment_context will be defined as a helper function below
@@ -255,7 +255,7 @@ async def retry_http_request(client: AsyncClient, method: str, url: str, **kwarg
     raise last_exception
 
 
-def create_message(content: Any, message_type: str, origin_system_id: Any, turn: int) -> SessionEventMessage:
+def create_message(content: Any, message_type: str, origin_system_id: Any, turn: int) -> SessionEventMarkovBlanketMessage:
     """Create a message with origin system ID embedded in content."""
     # Map custom message types to valid v3 message types
     type_mapping = {
@@ -267,7 +267,7 @@ def create_message(content: Any, message_type: str, origin_system_id: Any, turn:
         "tool_result": "tool_result"
     }
     
-    return SessionEventMessage(
+    return SessionEventMarkovBlanketMessage(
         content=json.dumps({
             "origin_system_id": str(origin_system_id),
             "payload": content

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/test_crafter_react_agent_lm_synth_v2_backup.py

@@ -67,7 +67,7 @@ from synth_ai.lm.config import SynthConfig
 
 # Import session tracer for v2 tracing
 from synth_ai.tracing_v2.session_tracer import (
-    SessionTracer, SessionEventMessage, TimeRecord,
+    SessionTracer, SessionEventMarkovBlanketMessage, TimeRecord,
     RuntimeEvent, EnvironmentEvent, LMCAISEvent
 )
 from synth_ai.tracing_v2.utils import create_experiment_context
@@ -175,9 +175,9 @@ async def retry_http_request(client: AsyncClient, method: str, url: str, **kwarg
     raise last_exception
 
 
-def create_message(content: Any, message_type: str, origin_system_id: Any, turn: int) -> SessionEventMessage:
+def create_message(content: Any, message_type: str, origin_system_id: Any, turn: int) -> SessionEventMarkovBlanketMessage:
     """Create a message with origin system ID embedded in content."""
-    return SessionEventMessage(
+    return SessionEventMarkovBlanketMessage(
         content={
             "origin_system_id": str(origin_system_id),
             "payload": content

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/run_rollouts_for_models_and_compare_v3.py

@@ -52,7 +52,7 @@ from synth_ai.lm.core.main_v3 import LM
 # Import session tracer for v3 tracing
 from synth_ai.tracing_v3.session_tracer import SessionTracer
 from synth_ai.tracing_v3.abstractions import (
-    SessionEventMessage, TimeRecord,
+    SessionEventMarkovBlanketMessage, TimeRecord,
     RuntimeEvent, EnvironmentEvent
 )
 # from synth_ai.tracing_v3.utils import create_experiment_context  # Not needed
@@ -176,9 +176,9 @@ def compress_observation_for_trace(obs: dict[str, Any]) -> str:
         return f"{{\"error\": \"{str(e)}\"}}"
 
 
-def create_message(content: str, message_type: str, system_id: str, turn: int) -> SessionEventMessage:
-    """Create a SessionEventMessage with metadata."""
-    return SessionEventMessage(
+def create_message(content: str, message_type: str, system_id: str, turn: int) -> SessionEventMarkovBlanketMessage:
+    """Create a SessionEventMarkovBlanketMessage with metadata."""
+    return SessionEventMarkovBlanketMessage(
         content=content,
         message_type=message_type,
         metadata={"system_id": system_id, "turn": turn},

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/test_crafter_react_agent_openai_v2_backup.py

@@ -71,7 +71,7 @@ import numpy as np
 
 # Import session tracer for CAIS event capture
 from synth_ai.tracing_v2.session_tracer import (
-    SessionTracer, SessionEventMessage, TimeRecord,
+    SessionTracer, SessionEventMarkovBlanketMessage, TimeRecord,
     RuntimeEvent, EnvironmentEvent
 )
 from synth_ai.tracing_v2.abstractions import CAISEvent
@@ -150,9 +150,9 @@ except ImportError:
 
 
 # Create a proper message structure with origin_system_id
-def create_message(content: Any, message_type: str, origin_system_id: Any, turn: int) -> SessionEventMessage:
+def create_message(content: Any, message_type: str, origin_system_id: Any, turn: int) -> SessionEventMarkovBlanketMessage:
     """Create a message with origin system ID embedded in content."""
-    return SessionEventMessage(
+    return SessionEventMarkovBlanketMessage(
         content={
             "origin_system_id": str(origin_system_id),
             "payload": content

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/synth_ai/environments/examples/crafter_classic/agent_demos/example_v3_usage.py

@@ -18,7 +18,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent.parent.parent)
 from synth_ai.tracing_v3.session_tracer import SessionTracer
 from synth_ai.tracing_v3.abstractions import (
     RuntimeEvent, EnvironmentEvent, LMCAISEvent,
-    TimeRecord, SessionEventMessage
+    TimeRecord, SessionEventMarkovBlanketMessage
 )
 from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
 from synth_ai.tracing_v3.decorators import set_session_id, set_turn_number

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/synth_ai/environments/examples/crafter_classic/environment.py

@@ -13,7 +13,7 @@ from .config_logging import safe_compare
 # Import tracing abstractions
 from synth_ai.tracing_v3.abstractions import (
     RuntimeEvent,
-    SessionEventMessage,
+    SessionEventMarkovBlanketMessage,
     TimeRecord,
 )
 

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/synth_ai/environments/examples/crafter_custom/environment.py

@@ -13,7 +13,7 @@ from synth_ai.environments.examples.crafter_classic.config_logging import safe_c
 # Import tracing abstractions
 from synth_ai.tracing_v3.abstractions import (
     RuntimeEvent,
-    SessionEventMessage,
+    SessionEventMarkovBlanketMessage,
     TimeRecord,
 )
 

{synth_ai-0.2.3 → synth_ai-0.2.4.dev2}/synth_ai/environments/service/core_routes.py

@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
 # Import tracing abstractions from v3
 from synth_ai.tracing_v3.abstractions import (
     RuntimeEvent,
-    SessionEventMessage,
+    SessionEventMarkovBlanketMessage,
     TimeRecord,
 )
 

synth_ai-0.2.4.dev2/synth_ai/learning/prompts/mipro.py

@@ -0,0 +1,8 @@
+
+
+class ExampleTwoStepDag:
+    pass
+
+"""
+A -> B
+"""