synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/v0/tracing/upload.py

@@ -1,409 +0,0 @@
-# ruff: noqa
-import asyncio
-import json
-import logging
-import os
-import ssl
-import time
-from typing import Any, Dict, List, Tuple, TypedDict
-
-import requests
-from dotenv import load_dotenv
-from pydantic import BaseModel, ConfigDict, field_validator
-from requests.adapters import HTTPAdapter
-from urllib3.poolmanager import PoolManager
-
-from synth_ai.config.base_url import PROD_BASE_URL_DEFAULT
-
-from .abstractions import Dataset, SystemTrace
-from .events.store import event_store
-
-load_dotenv()
-
-
-# NOTE: This may cause memory issues in the future
-def validate_json(data: Dict[str, Any]) -> None:
-    """Validate that a dictionary contains only JSON-serializable values.
-
-    Args:
-        data: Dictionary to validate for JSON serialization
-
-    Raises:
-        ValueError: If the dictionary contains non-serializable values
-    """
-
-    try:
-        json.dumps(data)
-    except (TypeError, OverflowError) as e:
-        raise ValueError(f"Contains non-JSON-serializable values: {e}. {data}")
-
-
-def createPayload(dataset: Dataset, traces: List[SystemTrace]) -> Dict[str, Any]:
-    payload = {
-        "traces": [trace.to_dict() for trace in traces],  # Convert SystemTrace objects to dicts
-        "dataset": dataset.to_dict(),
-    }
-    return payload
-
-
-class TLSAdapter(HTTPAdapter):
-    def init_poolmanager(self, connections, maxsize, block=False):
-        """Create and initialize the urllib3 PoolManager."""
-        ctx = ssl.create_default_context()
-        ctx.set_ciphers("DEFAULT@SECLEVEL=1")
-        self.poolmanager = PoolManager(
-            num_pools=connections,
-            maxsize=maxsize,
-            block=block,
-            ssl_version=ssl.PROTOCOL_TLSv1_2,
-            ssl_context=ctx,
-        )
-
-
-def load_signed_url(signed_url: str, dataset: Dataset, traces: List[SystemTrace]) -> None:
-    payload = createPayload(dataset, traces)
-    validate_json(payload)
-
-    session = requests.Session()
-    adapter = TLSAdapter()
-    session.mount("https://", adapter)
-
-    try:
-        response = session.put(
-            signed_url, json=payload, headers={"Content-Type": "application/json"}
-        )
-        response.raise_for_status()
-    except requests.exceptions.RequestException as e:
-        print(f"Error making request: {str(e)}")
-        print(f"Request payload: {payload}")  # Debugging info
-        raise
-
-    if response.status_code != 200:
-        raise ValueError(
-            f"Failed to load signed URL Status Code: {response.status_code} "
-            f"Response: {response.text}, Signed URL: {signed_url}"
-        )
-    else:
-        pass
-
-
-def send_system_traces_s3(
-    dataset: Dataset,
-    traces: List[SystemTrace],
-    base_url: str,
-    api_key: str,
-    system_id: str,
-    system_name: str,
-    verbose: bool = False,
-):
-    upload_id, signed_url = get_upload_id(base_url, api_key, system_id, system_name, verbose)
-    load_signed_url(signed_url, dataset, traces)
-
-    token_url = f"{base_url}/v1/auth/token"
-    try:
-        token_response = requests.get(token_url, headers={"customer_specific_api_key": api_key})
-        token_response.raise_for_status()
-        access_token = token_response.json()["access_token"]
-    except requests.exceptions.RequestException as e:
-        logging.error(f"Error obtaining access token: {e}")
-        raise
-
-    api_url = f"{base_url}/v1/uploads/process-upload/{upload_id}"
-    data = {"signed_url": signed_url}
-    headers = {
-        "Content-Type": "application/json",
-        "Authorization": f"Bearer {access_token}",
-    }
-
-    try:
-        response = requests.post(api_url, headers=headers, json=data)
-        response.raise_for_status()
-
-        response_data = response.json()
-        upload_id = response_data.get("upload_id")
-        signed_url = response_data.get("signed_url")
-        return upload_id, signed_url
-    except requests.exceptions.HTTPError as e:
-        logging.error(f"HTTP error occurred: {e}")
-        raise
-    except Exception as e:
-        logging.error(f"An error occurred: {e}")
-        raise
-
-
-def get_upload_id(
-    base_url: str, api_key: str, system_id: str, system_name: str, verbose: bool = False
-) -> Tuple[str, str]:
-    """
-    Modified client-side function to send both system_id and system_name.
-    """
-    token_url = f"{base_url}/v1/auth/token"
-    token_response = requests.get(token_url, headers={"customer_specific_api_key": api_key})
-    token_response.raise_for_status()
-    access_token = token_response.json()["access_token"]
-
-    # Include system_name in the query parameters
-    api_url = (
-        f"{base_url}/v1/uploads/get-upload-id-signed-url?"
-        f"system_id={system_id}&system_name={system_name}"
-    )
-    headers = {
-        "Content-Type": "application/json",
-        "Authorization": f"Bearer {access_token}",
-    }
-
-    try:
-        response = requests.get(api_url, headers=headers)
-        response.raise_for_status()
-        upload_id = response.json()["upload_id"]
-        signed_url = response.json()["signed_url"]
-        return upload_id, signed_url
-    except requests.exceptions.HTTPError as e:
-        logging.error(f"HTTP error occurred: {e}")
-        raise
-    except Exception as e:
-        logging.error(f"An error occurred: {e}")
-        raise
-
-
-class UploadValidator(BaseModel):
-    """Validator for upload data format."""
-
-    model_config = ConfigDict(
-        from_attributes=True,  # Replaces the deprecated orm_mode
-        validate_assignment=True,
-        extra="forbid",  # Prevent additional fields
-    )
-
-    traces: List[Dict[str, Any]]
-    dataset: Dict[str, Any]
-
-    @field_validator("traces")
-    @classmethod
-    def validate_traces(cls, traces: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        if not traces:
-            raise ValueError("Traces list cannot be empty")
-
-        for trace in traces:
-            # Validate required fields in each trace
-            if "system_instance_id" not in trace:
-                raise ValueError("Each trace must have a system_instance_id")
-            if "partition" not in trace:
-                raise ValueError("Each trace must have a partition")
-
-            # Validate metadata if present
-            if "metadata" in trace and trace["metadata"] is not None:
-                if not isinstance(trace["metadata"], dict):
-                    raise ValueError("Metadata must be a dictionary")
-
-            # Validate partition structure
-            partition = trace["partition"]
-            if not isinstance(partition, list):
-                raise ValueError("Partition must be a list")
-
-            for part in partition:
-                if "partition_index" not in part:
-                    raise ValueError("Each partition element must have a partition_index")
-                if "events" not in part:
-                    raise ValueError("Each partition element must have an events list")
-
-                # Validate events
-                events = part["events"]
-                if not isinstance(events, list):
-                    raise ValueError("Events must be a list")
-
-                for event in events:
-                    required_fields = [
-                        "event_type",
-                        "opened",
-                        "closed",
-                        "partition_index",
-                    ]
-                    missing_fields = [f for f in required_fields if f not in event]
-                    if missing_fields:
-                        raise ValueError(f"Event missing required fields: {missing_fields}")
-
-        return traces
-
-    @field_validator("dataset")
-    @classmethod
-    def validate_dataset(cls, dataset: Dict[str, Any]) -> Dict[str, Any]:
-        required_fields = ["questions", "reward_signals"]
-        missing_fields = [f for f in required_fields if f not in dataset]
-        if missing_fields:
-            raise ValueError(f"Dataset missing required fields: {missing_fields}")
-
-        # Validate questions
-        questions = dataset["questions"]
-        if not isinstance(questions, list):
-            raise ValueError("Questions must be a list")
-
-        for question in questions:
-            if "intent" not in question or "criteria" not in question:
-                raise ValueError("Each question must have intent and criteria")
-
-        # Validate reward signals
-        reward_signals = dataset["reward_signals"]
-        if not isinstance(reward_signals, list):
-            raise ValueError("Reward signals must be a list")
-
-        return dataset
-
-
-def validate_upload(traces: List[Dict[str, Any]], dataset: Dict[str, Any]):
-    # Validate the upload format before sending to server.
-    # Raises ValueError if validation fails.
-    try:
-        UploadValidator(traces=traces, dataset=dataset)
-        return True
-    except ValueError as e:
-        raise ValueError(f"Upload validation failed: {str(e)}")
-
-
-def is_event_loop_running():
-    try:
-        asyncio.get_running_loop()  # Check if there's a running event loop
-        return True
-    except RuntimeError:
-        # This exception is raised if no event loop is running
-        return False
-
-
-def format_upload_output(
-    dataset: Dataset, traces: List[SystemTrace]
-) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]]:
-    # Format questions array
-    questions_data = [
-        {"intent": q.intent, "criteria": q.criteria, "id": q.id} for q in dataset.questions
-    ]
-
-    # Format reward signals array with error handling
-    reward_signals_data = [
-        {
-            "system_instance_id": rs.system_instance_id,
-            "reward": rs.reward,
-            "question_id": rs.question_id,
-            "annotation": rs.annotation if hasattr(rs, "annotation") else None,
-        }
-        for rs in dataset.reward_signals
-    ]
-
-    # Format traces array
-    traces_data = [
-        {
-            "system_instance_id": t.system_instance_id,
-            "metadata": t.metadata if t.metadata else None,
-            "partition": [
-                {
-                    "partition_index": p.partition_index,
-                    "events": [e.to_dict() for e in p.events],
-                }
-                for p in t.partition
-            ],
-        }
-        for t in traces
-    ]
-
-    return questions_data, reward_signals_data, traces_data
-
-
-class UploadIdResponse(TypedDict):
-    message: str
-    upload_id: str
-    signed_url: str
-
-
-class ProcessUploadResponse(TypedDict):
-    status: str
-    upload_id: str
-    signed_url: str
-
-
-def upload(
-    dataset: Dataset,
-    traces: List[SystemTrace] = [],
-    verbose: bool = False,
-    show_payload: bool = False,
-) -> Tuple[
-    ProcessUploadResponse,
-    List[Dict[str, Any]],
-    List[Dict[str, Any]],
-    List[Dict[str, Any]],
-]:
-    """Upload all system traces and dataset to the server.
-
-    Args:
-        dataset: Dataset containing questions and reward signals
-        traces: List of system traces to upload
-        verbose: Whether to print verbose output
-        show_payload: Whether to show the payload being sent
-
-    Returns:
-        Tuple containing:
-        - response: Server response with status, upload_id, and signed_url
-        - questions_json: List of formatted questions
-        - reward_signals_json: List of formatted reward signals
-        - traces_json: List of formatted traces
-
-    Raises:
-        ValueError: If no system traces found or validation fails
-        requests.exceptions.HTTPError: If server request fails
-        RuntimeError: If SYNTH_API_KEY environment variable not set
-    """
-    return upload_helper(dataset, traces, verbose, show_payload)
-
-
-def upload_helper(
-    dataset: Dataset,
-    traces: List[SystemTrace] = [],
-    verbose: bool = False,
-    show_payload: bool = False,
-) -> Tuple[
-    ProcessUploadResponse,
-    List[Dict[str, Any]],
-    List[Dict[str, Any]],
-    List[Dict[str, Any]],
-]:
-    """Helper function to handle the upload process.
-
-    Returns same type as upload() function.
-    """
-    api_key = os.getenv("SYNTH_API_KEY")
-    if not api_key:
-        raise ValueError("SYNTH_API_KEY environment variable not set")
-    base_url = os.getenv("SYNTH_ENDPOINT_OVERRIDE", PROD_BASE_URL_DEFAULT)
-
-    logged_traces = event_store.get_system_traces()
-    combined_traces: List[SystemTrace] = list(logged_traces) + list(traces)
-
-    if not combined_traces:
-        raise ValueError("No system traces found")
-
-    traces_dict = [trace.to_dict() for trace in combined_traces]
-    dataset_dict = dataset.to_dict()
-
-    if verbose:
-        print("Validating upload format...")
-    validate_upload(traces_dict, dataset_dict)
-
-    upload_id, signed_url = send_system_traces_s3(
-        dataset=dataset,
-        traces=combined_traces,
-        base_url=base_url,
-        api_key=api_key,
-        system_id=combined_traces[0].system_id,
-        system_name=combined_traces[0].system_name,
-        verbose=verbose,
-    )
-
-    questions_json, reward_signals_json, traces_json = format_upload_output(dataset, combined_traces)
-    return (
-        {
-            "status": "success",
-            "upload_id": upload_id,
-            "signed_url": signed_url,
-        },
-        questions_json,
-        reward_signals_json,
-        traces_json,
-    )

synth_ai/v0/tracing/utils.py

@@ -1,9 +0,0 @@
-import uuid
-
-
-def get_system_id(system_name: str) -> str:
-    """Create a deterministic system_instance_id from system_name using UUID5."""
-    if not system_name:
-        raise ValueError("system_name cannot be empty")
-    system_id = uuid.uuid5(uuid.NAMESPACE_DNS, system_name)
-    return str(system_id)

synth_ai/v0/tracing_v1/__init__.py

@@ -1,16 +0,0 @@
-import warnings
-
-# Issue deprecation warning when this module is imported
-warnings.warn(
-    "synth_ai.tracing (now tracing_v1) is deprecated. "
-    "Please use synth_ai.tracing_v2 instead. "
-    "Backend upload functionality is no longer supported in v1.",
-    DeprecationWarning,
-    stacklevel=2,
-)
-
-# Re-export the main components with deprecation warnings
-from .abstractions import *
-from .config import *
-from .decorators import *
-from .trackers import *

synth_ai/v0/tracing_v1/abstractions.py

@@ -1,224 +0,0 @@
-import logging
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Any
-
-from pydantic import BaseModel
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class MessageInputs:
-    messages: list[dict[str, str]]  # {"role": "", "content": ""}
-
-
-@dataclass
-class ArbitraryInputs:
-    inputs: dict[str, Any]
-
-
-@dataclass
-class MessageOutputs:
-    messages: list[dict[str, str]]
-
-
-@dataclass
-class ArbitraryOutputs:
-    outputs: dict[str, Any]
-
-
-@dataclass
-class ComputeStep:
-    event_order: int
-    compute_ended: datetime  # time step
-    compute_began: datetime  # time step
-    compute_input: dict[str, Any]  # {variable_name: value}
-    compute_output: dict[str, Any]  # {variable_name: value}
-
-    def to_dict(self):
-        # Serialize compute_input
-        serializable_input = [
-            input_item.__dict__
-            for input_item in self.compute_input
-            if isinstance(input_item, (MessageInputs, ArbitraryInputs))
-        ]
-
-        # Serialize compute_output
-        serializable_output = [
-            output_item.__dict__
-            for output_item in self.compute_output
-            if isinstance(output_item, (MessageOutputs, ArbitraryOutputs))
-        ]
-
-        # Warn about non-serializable inputs/outputs
-        for item in self.compute_input:
-            if not isinstance(item, (MessageInputs, ArbitraryInputs)):
-                logger.warning(f"Skipping non-serializable input: {item}")
-        for item in self.compute_output:
-            if not isinstance(item, (MessageOutputs, ArbitraryOutputs)):
-                logger.warning(f"Skipping non-serializable output: {item}")
-
-        return {
-            "event_order": self.event_order,
-            "compute_ended": self.compute_ended.isoformat()
-            if isinstance(self.compute_ended, datetime)
-            else self.compute_ended,
-            "compute_began": self.compute_began.isoformat()
-            if isinstance(self.compute_began, datetime)
-            else self.compute_began,
-            "compute_input": serializable_input,
-            "compute_output": serializable_output,
-        }
-
-
-@dataclass
-class AgentComputeStep(ComputeStep):
-    model_name: str | None = None
-    model_params: dict[str, Any] | None = None
-    should_learn: bool | None = None
-    compute_input: list[MessageInputs | ArbitraryInputs]
-    compute_output: list[MessageOutputs | ArbitraryOutputs]
-
-    def to_dict(self):
-        base_dict = super().to_dict()  # Get the parent class serialization
-        base_dict["model_name"] = self.model_name  # Add model_name
-        return base_dict
-
-
-@dataclass
-class EnvironmentComputeStep(ComputeStep):
-    compute_input: list[ArbitraryInputs]
-    compute_output: list[ArbitraryOutputs]
-
-
-@dataclass
-class Event:
-    system_instance_id: str
-    event_type: str
-    opened: float
-    closed: float | None
-    partition_index: int
-    agent_compute_step: AgentComputeStep
-    environment_compute_steps: list["EnvironmentComputeStep"]
-    system_name: str | None = None
-    system_id: str | None = None
-    event_metadata: dict[str, Any] = None  # JSON-serializable metadata for this specific event
-
-    def __post_init__(self):
-        if self.event_metadata is None:
-            self.event_metadata = {}
-
-    def to_dict(self):
-        return {
-            "event_type": self.event_type,
-            "opened": self.opened.isoformat() if isinstance(self.opened, datetime) else self.opened,
-            "closed": self.closed.isoformat() if isinstance(self.closed, datetime) else self.closed,
-            "partition_index": self.partition_index,
-            "agent_compute_step": self.agent_compute_step.to_dict()
-            if self.agent_compute_step
-            else None,
-            "environment_compute_steps": [
-                step.to_dict() for step in self.environment_compute_steps
-            ],
-            "event_metadata": self.event_metadata,
-        }
-
-    # backwards compatibility
-    @property
-    def agent_compute_steps(self) -> list[AgentComputeStep]:
-        """Backwards compatibility method that returns a list containing the agent_compute_step."""
-        return [self.agent_compute_step] if self.agent_compute_step is not None else []
-
-
-@dataclass
-class EventPartitionElement:
-    partition_index: int
-    events: list[Event]
-
-    def to_dict(self):
-        return {
-            "partition_index": self.partition_index,
-            "events": [event.to_dict() for event in self.events],
-        }
-
-
-@dataclass
-class SystemTrace:
-    system_name: str
-    system_id: str
-    system_instance_id: str
-    partition: list[EventPartitionElement]
-    metadata: dict[str, Any] | None = None  # System-level metadata
-    instance_metadata: dict[str, Any] = (
-        None  # JSON-serializable metadata for this specific instance
-    )
-    current_partition_index: int = 0  # Track current partition
-
-    def __post_init__(self):
-        if self.instance_metadata is None:
-            self.instance_metadata = {}
-
-    def to_dict(self):
-        return {
-            "system_name": self.system_name,
-            "system_id": self.system_id,
-            "system_instance_id": self.system_instance_id,
-            "partition": [element.to_dict() for element in self.partition],
-            "current_partition_index": self.current_partition_index,
-            "metadata": self.metadata if self.metadata else None,
-            "instance_metadata": self.instance_metadata,
-        }
-
-
-class TrainingQuestion(BaseModel):
-    """
-    A training question is a question that an agent (system_instance_id) is trying to answer.
-    It contains an intent and criteria that the agent is trying to meet.
-    """
-
-    id: str
-    intent: str
-    criteria: str
-
-    def to_dict(self):
-        return {
-            "id": self.id,
-            "intent": self.intent,
-            "criteria": self.criteria,
-        }
-
-
-class RewardSignal(BaseModel):
-    """
-    A reward signal tells us how well an agent (system_instance_id) is doing on a particular question (question_id).
-    """
-
-    question_id: str
-    system_instance_id: str
-    reward: float | int | bool
-    annotation: str | None = None
-
-    def to_dict(self):
-        return {
-            "question_id": self.question_id,
-            "system_instance_id": self.system_instance_id,
-            "reward": self.reward,
-            "annotation": self.annotation,
-        }
-
-
-class Dataset(BaseModel):
-    """
-    A dataset is a collection of training questions and reward signals.
-    This better represents the data that is used to train a model, and gives us more information about the data.
-    """
-
-    questions: list[TrainingQuestion]
-    reward_signals: list[RewardSignal]
-
-    def to_dict(self):
-        return {
-            "questions": [question.to_dict() for question in self.questions],
-            "reward_signals": [signal.to_dict() for signal in self.reward_signals],
-        }