synth-ai 0.2.4.dev3__py3-none-any.whl → 0.2.4.dev5__py3-none-any.whl

synth_ai/environments/examples/verilog/taskset.py

@@ -0,0 +1,418 @@
+from synth_ai.environments.tasks.core import (
+    Task,
+    TaskInstance,
+    TaskInstanceMetadata,
+    TaskInstanceSet,
+    SplitInfo,
+    Impetus,
+    Intent,
+)
+from uuid import uuid4, UUID
+from dataclasses import dataclass, asdict, fields
+from typing import Optional
+from pathlib import Path
+import tempfile
+import os
+import shutil
+import atexit
+from datasets import load_dataset
+
+# Global list to track temp directories for cleanup
+_temp_dirs = []
+
+
+def _cleanup_temp_dirs():
+    """Clean up all temporary directories created during task instances."""
+    for temp_dir in _temp_dirs:
+        try:
+            if os.path.exists(temp_dir):
+                shutil.rmtree(temp_dir)
+        except Exception:
+            pass  # Ignore cleanup errors
+    _temp_dirs.clear()
+
+
+# Register cleanup function to run at exit
+atexit.register(_cleanup_temp_dirs)
+
+verilog_task = Task(
+    global_premises="Implement and verify Verilog hardware designs",
+    global_constraints="Must pass testbench verification",
+    global_objectives="Write correct Verilog code that passes all tests",
+    shared_env_params={},
+)
+
+
+@dataclass
+class VerilogTaskInstanceMetadata(TaskInstanceMetadata):
+    problem_name: str
+    difficulty: str
+    description: str
+    files_provided: list[str]
+
+
+@dataclass
+class VerilogTaskInstance(TaskInstance):
+    pristine_dir: Optional[str] = None
+    snapshot_dir: Optional[str] = None
+
+    async def serialize(self) -> dict:
+        data = asdict(self)
+        if "id" in data and isinstance(data["id"], UUID):
+            data["id"] = str(data["id"])
+        if "intent" in data and data["intent"] is not None:
+            if "deterministic_eval_functions" in data["intent"]:
+                data["intent"]["deterministic_eval_functions"] = []
+        return data
+
+    @classmethod
+    async def deserialize(cls, data: dict) -> "VerilogTaskInstance":
+        """Gracefully accept non-UUID ids and rebuild required objects."""
+        if "id" in data:
+            try:
+                data["id"] = UUID(str(data["id"]))
+            except (ValueError, TypeError, AttributeError):
+                pass  # keep original string
+
+        if "impetus" in data and isinstance(data["impetus"], dict):
+            impetus_data = data["impetus"]
+            # Ensure instructions field exists with default if missing
+            if "instructions" not in impetus_data:
+                impetus_data["instructions"] = "Implement the Verilog module"
+            data["impetus"] = Impetus(**impetus_data)
+
+        if "intent" in data and isinstance(data["intent"], dict):
+            intent_data = data["intent"]
+            if "deterministic_eval_functions" not in intent_data:
+                intent_data["deterministic_eval_functions"] = []
+            # Provide default values for required fields if missing
+            if "rubric" not in intent_data:
+                intent_data["rubric"] = {"goal": "Pass all testbench tests"}
+            if "gold_trajectories" not in intent_data:
+                intent_data["gold_trajectories"] = None
+            if "gold_state_diff" not in intent_data:
+                intent_data["gold_state_diff"] = {}
+            data["intent"] = Intent(**intent_data)
+
+        if "metadata" in data and isinstance(data["metadata"], dict):
+            metadata_data = data["metadata"]
+            # Ensure required fields exist with defaults if missing
+            if "problem_name" not in metadata_data:
+                metadata_data["problem_name"] = "unknown"
+            if "difficulty" not in metadata_data:
+                metadata_data["difficulty"] = "medium"
+            if "description" not in metadata_data:
+                metadata_data["description"] = "Verilog implementation task"
+            if "files_provided" not in metadata_data:
+                metadata_data["files_provided"] = []
+            data["metadata"] = VerilogTaskInstanceMetadata(**metadata_data)
+
+        constructor_field_names = {f.name for f in fields(cls)}
+        filtered_data = {k: v for k, v in data.items() if k in constructor_field_names}
+
+        # Add default values for required fields if missing
+        if "is_reproducible" not in filtered_data:
+            filtered_data["is_reproducible"] = True
+        if "initial_engine_snapshot" not in filtered_data:
+            filtered_data["initial_engine_snapshot"] = None
+
+        return cls(**filtered_data)
+
+
+async def create_verilog_taskset(max_instances: int = 10) -> TaskInstanceSet:
+    """Create a Verilog task set from HuggingFace VerilogEval v2 dataset."""
+    # Load VerilogEval v2 dataset from HuggingFace
+    ds = load_dataset("dakies/nvlabs-verilogeval-v2-spec-to-rtl", split="test")
+
+    instances = []
+
+    # Limit the number of instances for faster testing
+    dataset_size = min(max_instances, len(ds))  # type: ignore[arg-type]
+
+    # Convert each dataset item to VerilogTaskInstance
+    for i in range(dataset_size):
+        item = ds[i]
+        instance = _create_hf_task_instance(item, i)
+        instances.append(instance)
+
+    # Create split info - use first 80% for validation, last 20% for test
+    total_instances = len(instances)
+    val_split = int(0.8 * total_instances)
+
+    val_ids = {inst.id for inst in instances[:val_split]}
+    test_ids = {inst.id for inst in instances[val_split:]}
+
+    split_info = SplitInfo(
+        val_instance_ids=val_ids,
+        test_instance_ids=test_ids,
+        _is_split_defined=True,
+    )
+
+    return TaskInstanceSet(
+        name="VerilogEval v2 TaskSet",
+        description="VerilogEval v2 spec-to-RTL tasks from HuggingFace",
+        instances=instances,
+        split_info=split_info,
+    )
+
+
+def _create_hf_task_instance(item, index: int) -> VerilogTaskInstance:
+    """Create a VerilogTaskInstance from a HuggingFace dataset item."""
+    instance_id = uuid4()
+
+    # Create temporary directory for this task
+    temp_dir = tempfile.mkdtemp(prefix=f"verilog_hf_{index}_{instance_id}_")
+    _temp_dirs.append(temp_dir)  # Track for cleanup
+    pristine_dir = Path(temp_dir)
+    pristine_dir.mkdir(exist_ok=True)
+
+    # Extract information from dataset item
+    problem_id = item["problem_id"]
+    prompt = item["prompt"]
+    testbench = item["test"]
+    ref_solution = item["ref"]
+
+    # Create incomplete module template (TopModule is the expected name in tests)
+    module_content = (
+        """module TopModule();
+    // TODO: Implement the module based on the specification below
+    /*
+    Specification:
+    """
+        + prompt.strip()
+        + """
+    */
+endmodule"""
+    )
+
+    # Write files to pristine directory
+    module_file = "TopModule.v"
+    testbench_file = f"{problem_id}_tb.v"
+    ref_file = "RefModule.v"
+
+    (pristine_dir / module_file).write_text(module_content)
+    (pristine_dir / testbench_file).write_text(testbench)
+    (pristine_dir / ref_file).write_text(ref_solution)  # Include reference module
+
+    files_provided = [module_file, testbench_file, ref_file]
+
+    # Create task components
+    impetus = Impetus(
+        instructions=f"Problem: {problem_id}\n\n{prompt.strip()}\n\nImplement the TopModule according to the specification. The testbench will verify your implementation."
+    )
+
+    intent = Intent(
+        rubric={
+            "goal": f"Implement correct TopModule for {problem_id} that passes testbench verification"
+        },
+        gold_trajectories=None,
+        gold_state_diff={},
+    )
+
+    metadata = VerilogTaskInstanceMetadata(
+        problem_name=problem_id,
+        difficulty="medium",  # VerilogEval doesn't specify difficulty levels
+        description=prompt.strip(),  # Full description
+        files_provided=files_provided,
+    )
+
+    # Create snapshot directory and track for cleanup
+    snapshot_dir = tempfile.mkdtemp(prefix=f"verilog_snapshot_{instance_id}_")
+    _temp_dirs.append(snapshot_dir)
+
+    return VerilogTaskInstance(
+        id=instance_id,
+        impetus=impetus,
+        intent=intent,
+        metadata=metadata,
+        is_reproducible=True,
+        initial_engine_snapshot=None,
+        pristine_dir=str(pristine_dir),
+        snapshot_dir=snapshot_dir,
+    )
+
+
+def _create_adder_task() -> VerilogTaskInstance:
+    """Create a simple 4-bit adder task."""
+    instance_id = uuid4()
+
+    # Create temporary directory for this task
+    temp_dir = tempfile.mkdtemp(prefix=f"verilog_adder_{instance_id}_")
+    _temp_dirs.append(temp_dir)  # Track for cleanup
+
+    # Write adder testbench
+    adder_tb_content = """`timescale 1ns/1ps
+module adder4_tb;
+    reg [3:0] a, b;
+    wire [4:0] sum;
+    
+    adder4 dut(.a(a), .b(b), .sum(sum));
+    
+    initial begin
+        a = 4'b0000; b = 4'b0000; #10;
+        if (sum != 5'b00000) $fatal(1, "Test failed: 0 + 0 != 0");
+        
+        a = 4'b0001; b = 4'b0001; #10;
+        if (sum != 5'b00010) $fatal(1, "Test failed: 1 + 1 != 2");
+        
+        a = 4'b1111; b = 4'b0001; #10;
+        if (sum != 5'b10000) $fatal(1, "Test failed: 15 + 1 != 16");
+        
+        $display("ALL_TESTS_PASSED");
+        $finish;
+    end
+endmodule"""
+
+    # Write incomplete adder module (for student to complete)
+    adder_content = """module adder4(
+    input [3:0] a,
+    input [3:0] b,
+    output [4:0] sum
+);
+    // TODO: Implement 4-bit adder
+    // assign sum = ?;
+endmodule"""
+
+    pristine_dir = Path(temp_dir)
+    pristine_dir.mkdir(exist_ok=True)
+
+    (pristine_dir / "adder4_tb.v").write_text(adder_tb_content)
+    (pristine_dir / "adder4.v").write_text(adder_content)
+
+    impetus = Impetus(
+        instructions="Implement a 4-bit adder module that takes two 4-bit inputs 'a' and 'b' and produces a 5-bit output 'sum'."
+    )
+
+    intent = Intent(
+        rubric="Implement correct 4-bit adder that passes testbench",
+        gold_trajectories=None,
+        gold_state_diff={},
+    )
+
+    metadata = VerilogTaskInstanceMetadata(
+        problem_name="adder4",
+        difficulty="easy",
+        description="4-bit adder implementation",
+        files_provided=["adder4.v", "adder4_tb.v"],
+    )
+
+    return VerilogTaskInstance(
+        id=instance_id,
+        impetus=impetus,
+        intent=intent,
+        metadata=metadata,
+        is_reproducible=True,
+        initial_engine_snapshot=None,
+        pristine_dir=str(pristine_dir),
+        snapshot_dir=(
+            lambda: (
+                _temp_dirs.append(d := tempfile.mkdtemp(prefix=f"verilog_snapshot_{instance_id}_")),
+                d,
+            )[1]
+        )(),
+    )
+
+
+def _create_and_gate_task() -> VerilogTaskInstance:
+    """Create a simple AND gate task."""
+    instance_id = uuid4()
+
+    # Create temporary directory for this task
+    temp_dir = tempfile.mkdtemp(prefix=f"verilog_and_{instance_id}_")
+    _temp_dirs.append(temp_dir)  # Track for cleanup
+
+    # Write AND gate testbench
+    and_tb_content = """`timescale 1ns/1ps
+module and_gate_tb;
+    reg a, b;
+    wire y;
+    
+    and_gate dut(.a(a), .b(b), .y(y));
+    
+    initial begin
+        a = 0; b = 0; #10;
+        if (y != 0) $fatal(1, "Test failed: 0 AND 0 != 0");
+        
+        a = 0; b = 1; #10;
+        if (y != 0) $fatal(1, "Test failed: 0 AND 1 != 0");
+        
+        a = 1; b = 0; #10;
+        if (y != 0) $fatal(1, "Test failed: 1 AND 0 != 0");
+        
+        a = 1; b = 1; #10;
+        if (y != 1) $fatal(1, "Test failed: 1 AND 1 != 1");
+        
+        $display("ALL_TESTS_PASSED");
+        $finish;
+    end
+endmodule"""
+
+    # Write incomplete AND gate module
+    and_content = """module and_gate(
+    input a,
+    input b,
+    output y
+);
+    // TODO: Implement AND gate
+    // assign y = ?;
+endmodule"""
+
+    pristine_dir = Path(temp_dir)
+    pristine_dir.mkdir(exist_ok=True)
+
+    (pristine_dir / "and_gate_tb.v").write_text(and_tb_content)
+    (pristine_dir / "and_gate.v").write_text(and_content)
+
+    impetus = Impetus(
+        instructions="Implement an AND gate module that takes two inputs 'a' and 'b' and produces output 'y'."
+    )
+
+    intent = Intent(
+        rubric="Implement correct AND gate that passes testbench",
+        gold_trajectories=None,
+        gold_state_diff={},
+    )
+
+    metadata = VerilogTaskInstanceMetadata(
+        problem_name="and_gate",
+        difficulty="easy",
+        description="Basic AND gate implementation",
+        files_provided=["and_gate.v", "and_gate_tb.v"],
+    )
+
+    return VerilogTaskInstance(
+        id=instance_id,
+        impetus=impetus,
+        intent=intent,
+        metadata=metadata,
+        is_reproducible=True,
+        initial_engine_snapshot=None,
+        pristine_dir=str(pristine_dir),
+        snapshot_dir=(
+            lambda: (
+                _temp_dirs.append(d := tempfile.mkdtemp(prefix=f"verilog_snapshot_{instance_id}_")),
+                d,
+            )[1]
+        )(),
+    )
+
+
+# Example usage
+if __name__ == "__main__":
+    import asyncio
+
+    async def main():
+        taskset = await create_verilog_taskset()
+
+        serialized = await asyncio.gather(*(inst.serialize() for inst in taskset.instances))
+
+        print(f"Created {len(serialized)} Verilog task instances")
+
+        # Print summary
+        for i, inst in enumerate(taskset.instances):
+            print(f"Task {i + 1}: {inst.metadata.problem_name} ({inst.metadata.difficulty})")
+            print(f"  Description: {inst.metadata.description}")
+            print(f"  Files: {inst.metadata.files_provided}")
+            print()
+
+    asyncio.run(main())

synth_ai/tracing_v3/examples/basic_usage.py

@@ -0,0 +1,188 @@
+"""Basic usage example for tracing v3."""
+
+import asyncio
+import time
+from datetime import datetime
+
+from synth_ai.tracing_v3 import SessionTracer
+from synth_ai.tracing_v3.abstractions import LMCAISEvent, EnvironmentEvent, RuntimeEvent, TimeRecord
+from synth_ai.tracing_v3.turso.daemon import SqldDaemon
+
+
+async def simulate_llm_call(model: str, prompt: str) -> dict:
+    """Simulate an LLM API call."""
+    await asyncio.sleep(0.1)  # Simulate network latency
+
+    # Simulate response
+    tokens = len(prompt.split()) * 3
+    return {
+        "model": model,
+        "response": f"Response to: {prompt[:50]}...",
+        "usage": {
+            "prompt_tokens": len(prompt.split()) * 2,
+            "completion_tokens": tokens,
+            "total_tokens": len(prompt.split()) * 2 + tokens,
+        },
+    }
+
+
+async def main():
+    """Demonstrate basic tracing v3 usage."""
+    print("Starting tracing v3 example...")
+
+    # Option 1: Start sqld daemon programmatically
+    with SqldDaemon() as daemon:
+        print("✓ Started sqld daemon")
+
+        # Wait for daemon to be ready
+        await asyncio.sleep(1)
+
+        # Create tracer
+        tracer = SessionTracer()
+        await tracer.initialize()
+        print("✓ Initialized tracer")
+
+        # Example 1: Basic session with events
+        print("\n--- Example 1: Basic Session ---")
+        async with tracer.session(metadata={"example": "basic"}) as session_id:
+            print(f"Started session: {session_id}")
+
+            # Timestep 1: LLM interaction
+            async with tracer.timestep("llm_step", turn_number=1):
+                # Simulate LLM call
+                result = await simulate_llm_call("gpt-4", "What is the capital of France?")
+
+                # Record LLM event
+                event = LMCAISEvent(
+                    system_instance_id="llm_system",
+                    time_record=TimeRecord(event_time=time.time()),
+                    model_name=result["model"],
+                    input_tokens=result["usage"]["prompt_tokens"],
+                    output_tokens=result["usage"]["completion_tokens"],
+                    total_tokens=result["usage"]["total_tokens"],
+                    cost_usd=0.003,  # $0.003
+                    latency_ms=100,
+                    metadata={"prompt": "What is the capital of France?"},
+                )
+                await tracer.record_event(event)
+
+                # Record messages
+                await tracer.record_message(
+                    content="What is the capital of France?", message_type="user"
+                )
+                await tracer.record_message(content=result["response"], message_type="assistant")
+                print("✓ Recorded LLM interaction")
+
+            # Timestep 2: Environment interaction
+            async with tracer.timestep("env_step", turn_number=2):
+                # Record environment event
+                env_event = EnvironmentEvent(
+                    system_instance_id="environment",
+                    time_record=TimeRecord(event_time=time.time()),
+                    reward=0.8,
+                    terminated=False,
+                    system_state_before={"position": [0, 0]},
+                    system_state_after={"position": [1, 0]},
+                )
+                await tracer.record_event(env_event)
+                print("✓ Recorded environment event")
+
+            # Timestep 3: Runtime action
+            async with tracer.timestep("runtime_step", turn_number=3):
+                # Record runtime event
+                runtime_event = RuntimeEvent(
+                    system_instance_id="agent",
+                    time_record=TimeRecord(event_time=time.time()),
+                    actions=[1, 0, 0, 1],  # Example action vector
+                    metadata={"action_type": "move_right"},
+                )
+                await tracer.record_event(runtime_event)
+                print("✓ Recorded runtime event")
+
+        print(f"✓ Session {session_id} saved\n")
+
+        # Example 2: Concurrent sessions
+        print("--- Example 2: Concurrent Sessions ---")
+
+        async def run_concurrent_session(session_num: int):
+            """Run a session concurrently."""
+            async with tracer.session(
+                metadata={"example": "concurrent", "session_num": session_num}
+            ) as sid:
+                for i in range(3):
+                    async with tracer.timestep(f"step_{i}", turn_number=i):
+                        # Simulate some work
+                        await asyncio.sleep(0.05)
+
+                        # Record event
+                        event = RuntimeEvent(
+                            system_instance_id=f"worker_{session_num}",
+                            time_record=TimeRecord(event_time=time.time()),
+                            actions=[i],
+                            metadata={"iteration": i},
+                        )
+                        await tracer.record_event(event)
+
+                return sid
+
+        # Run 5 concurrent sessions
+        tasks = [run_concurrent_session(i) for i in range(5)]
+        session_ids = await asyncio.gather(*tasks)
+        print(f"✓ Completed {len(session_ids)} concurrent sessions")
+
+        # Example 3: Query stored data
+        print("\n--- Example 3: Querying Data ---")
+
+        # Get model usage statistics
+        model_usage = await tracer.db.get_model_usage()
+        print("\nModel Usage:")
+        print(model_usage)
+
+        # Query recent sessions
+        recent_sessions = await tracer.get_session_history(limit=5)
+        print(f"\nRecent Sessions: {len(recent_sessions)} found")
+        for session in recent_sessions:
+            print(
+                f"  - {session['session_id']}: "
+                f"{session['num_events']} events, "
+                f"{session['num_messages']} messages"
+            )
+
+        # Get specific session details
+        if recent_sessions:
+            session_detail = await tracer.db.get_session_trace(recent_sessions[0]["session_id"])
+            print(f"\nSession Detail for {session_detail['session_id']}:")
+            print(f"  Created: {session_detail['created_at']}")
+            print(f"  Timesteps: {len(session_detail['timesteps'])}")
+
+        # Example 4: Using hooks
+        print("\n--- Example 4: Hooks ---")
+
+        # Add a custom hook
+        call_count = {"count": 0}
+
+        async def count_events(event, **kwargs):
+            call_count["count"] += 1
+            print(f"  Hook: Event #{call_count['count']} recorded")
+
+        tracer.hooks.register("event_recorded", count_events, name="event_counter")
+
+        async with tracer.session(metadata={"example": "hooks"}) as session_id:
+            async with tracer.timestep("hook_test"):
+                for i in range(3):
+                    event = RuntimeEvent(
+                        system_instance_id="hook_test",
+                        time_record=TimeRecord(event_time=time.time()),
+                        actions=[i],
+                    )
+                    await tracer.record_event(event)
+
+        print(f"✓ Hook called {call_count['count']} times")
+
+        # Cleanup
+        await tracer.close()
+        print("\n✓ Example completed successfully!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

{synth_ai-0.2.4.dev3.dist-info → synth_ai-0.2.4.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-ai
-Version: 0.2.4.dev3
+Version: 0.2.4.dev5
 Summary: Software for aiding the best and multiplying the will - Core AI functionality and tracing
 Author-email: Synth AI <josh@usesynth.ai>
 License-Expression: MIT