dslighting 1.3.9__py3-none-any.whl

dsat/benchmark/sciencebench.py

@@ -0,0 +1,304 @@
+# dsat/benchmark/sciencebench.py
+
+import logging
+import time
+import uuid
+import yaml
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from dsat.benchmark.benchmark import BaseBenchmark
+from dsat.models.task import TaskDefinition
+
+from sciencebench.data import is_dataset_prepared
+from sciencebench.grade import grade_submission
+from sciencebench.grade_helpers import CompetitionReport
+from sciencebench.registry import Competition, Registry
+from sciencebench.registry import registry as DEFAULT_SCIENCEBENCH_REGISTRY
+
+logger = logging.getLogger(__name__)
+
+
+class ScienceBenchBenchmark(BaseBenchmark):
+    """
+    Benchmark class to integrate ScienceBench competitions into the DSAT framework.
+
+    Expected `data_dir` layout (same as MLEBench):
+      <data_dir>/<competition_id>/prepared/public/...
+      <data_dir>/<competition_id>/prepared/private/...
+    """
+
+    def __init__(
+        self,
+        name: str,
+        file_path: Optional[str],
+        log_path: str,
+        data_dir: Optional[str] = None,
+        competitions: Optional[List[str]] = None,
+    ):
+        self.data_dir = Path(data_dir) if data_dir else DEFAULT_SCIENCEBENCH_REGISTRY.get_data_dir()
+        self.registry: Registry = DEFAULT_SCIENCEBENCH_REGISTRY.set_data_dir(self.data_dir)
+
+        self.config = self._load_config()
+        if competitions:
+            self.config["competitions"] = list(competitions)
+
+        super().__init__(name, file_path, log_path)
+        Path(self.log_path).mkdir(parents=True, exist_ok=True)
+
+        self.problems = self._load_problems()
+        logger.info("ScienceBenchBenchmark initialized with data_dir: %s", self.data_dir)
+
+    def _load_config(self) -> Dict[str, Any]:
+        try:
+            framework_dir = Path(__file__).parent.parent.parent
+            config_path = framework_dir / "config.yaml"
+            if not config_path.exists():
+                logger.warning("Config file not found at %s, using default configuration", config_path)
+                return {"competitions": []}
+
+            with open(config_path, "r", encoding="utf-8") as f:
+                config = yaml.safe_load(f) or {}
+
+            competitions = config.get("sciencebench_competitions", [])
+            if competitions:
+                return {"competitions": list(competitions)}
+            return {"competitions": []}
+        except Exception as exc:
+            logger.error("Error loading config file for ScienceBench: %s", exc)
+            return {"competitions": []}
+
+    def _load_problems(self) -> List[Dict[str, Any]]:
+        logger.info("Discovering prepared ScienceBench competitions in %s...", self.data_dir)
+
+        if not self.data_dir.exists():
+            raise FileNotFoundError(
+                f"ScienceBench data directory not found: {self.data_dir}. "
+                "Please provide a valid path via --data-dir."
+            )
+
+        competition_ids = self.config.get("competitions", [])
+        if not competition_ids:
+            logger.warning("No sciencebench competitions provided; pass --task-id <competition_id>.")
+            return []
+
+        problems: List[Dict[str, Any]] = []
+        for competition_id in competition_ids:
+            try:
+                competition = self.registry.get_competition(competition_id)
+                if is_dataset_prepared(competition, grading_only=False):
+                    problems.append({"competition_id": competition_id})
+                else:
+                    logger.warning(
+                        "Skipping competition '%s' as its dataset is not fully prepared under '%s'.",
+                        competition_id,
+                        self.data_dir,
+                    )
+            except Exception as exc:
+                logger.warning("Error loading competition '%s': %s", competition_id, exc)
+
+        if not problems:
+            logger.error(
+                "No prepared ScienceBench competitions found in %s. "
+                "Please prepare the dataset or point --data-dir to a prepared competitions folder.",
+                self.data_dir,
+            )
+        return problems
+
+    def set_mode(self, mode: str):
+        logger.info("Setting ScienceBenchBenchmark mode to '%s'", mode)
+        self.registry.set_mode(mode)
+
+    async def grade(self, submission_path: Path) -> float:
+        """Grades a submission and returns a numerical fitness score for AFlow."""
+        if not self.problems:
+            return 0.0
+
+        competition_id = self.problems[0].get("competition_id")
+        if not competition_id:
+            return 0.0
+
+        try:
+            competition = self.registry.get_competition(competition_id)
+            if not submission_path.exists():
+                logger.warning("Grading failed: submission file not found at %s", submission_path)
+                return 0.0
+
+            report = grade_submission(submission_path, competition)
+            score = report.score if report.score is not None else 0.0
+
+            is_lower_better = False
+            try:
+                if competition.leaderboard.exists():
+                    import pandas as pd
+
+                    leaderboard = pd.read_csv(competition.leaderboard)
+                    if "score" in leaderboard.columns and len(leaderboard.index) > 1:
+                        is_lower_better = competition.grader.is_lower_better(leaderboard)
+            except Exception as exc:
+                logger.warning(
+                    "Could not determine score direction for %s: %s",
+                    competition_id,
+                    exc,
+                )
+
+            if is_lower_better:
+                return 1.0 / (1.0 + score) if score > 0 else 1.0
+            return float(score)
+
+        except Exception as exc:
+            logger.error("Error during grading for %s: %s", competition_id, exc)
+            return 0.0
+
+    def get_result_columns(self) -> List[str]:
+        return [
+            "competition_id",
+            "submission_path",
+            "answers_path",
+            "score",
+            "cost",
+            "running_time",
+            "input_tokens",
+            "output_tokens",
+            "total_tokens",
+            "gold_medal",
+            "silver_medal",
+            "bronze_medal",
+            "above_median",
+            "submission_exists",
+            "valid_submission",
+            "error_message",
+        ]
+
+    def _create_error_report(self, competition_id: str, submission_path: Path, error_msg: str) -> CompetitionReport:
+        base_data = {
+            "competition_id": competition_id,
+            "score": None,
+            "gold_threshold": float("nan"),
+            "silver_threshold": float("nan"),
+            "bronze_threshold": float("nan"),
+            "median_threshold": float("nan"),
+            "any_medal": False,
+            "gold_medal": False,
+            "silver_medal": False,
+            "bronze_medal": False,
+            "above_median": False,
+            "submission_exists": submission_path.exists(),
+            "valid_submission": False,
+            "is_lower_better": False,
+            "created_at": datetime.now().isoformat(),
+            "submission_path": str(submission_path),
+        }
+        report = CompetitionReport.from_dict(base_data)
+        logger.error("Error for %s: %s", competition_id, error_msg)
+        return report
+
+    async def evaluate_problem(
+        self, problem: dict, eval_fn: Callable
+    ) -> Tuple[Tuple, CompetitionReport, Optional[str]]:
+        competition_id = problem.get("competition_id")
+        if not competition_id:
+            raise ValueError("Problem data must contain 'competition_id'")
+
+        unique_id = uuid.uuid4().hex[:6]
+        competition: Optional[Competition] = None
+
+        # Default output: a unique CSV name under log_path/.
+        # If the competition declares a specific submission format in config (e.g. `.npy`),
+        # keep the same naming scheme but switch the file extension accordingly (resolved after
+        # loading the competition config).
+        output_filename = f"submission_{competition_id}_{unique_id}.csv"
+        output_submission_path = (Path(self.log_path) / output_filename).absolute()
+
+        start_time = time.perf_counter()
+
+        cost = 0.0
+        input_tokens = 0
+        output_tokens = 0
+        total_tokens = 0
+        report: Optional[CompetitionReport] = None
+        error_message: Optional[str] = None
+
+        try:
+            competition = self.registry.get_competition(competition_id)
+
+            submission_filename = getattr(competition, "submission_filename", None)
+            submission_suffix = Path(submission_filename).suffix if submission_filename else ".csv"
+            if submission_suffix and not submission_suffix.startswith("."):
+                submission_suffix = f".{submission_suffix}"
+            output_filename = f"submission_{competition_id}_{unique_id}{submission_suffix}"
+            output_submission_path = (Path(self.log_path) / output_filename).absolute()
+
+            if not is_dataset_prepared(competition, grading_only=False):
+                raise ValueError(f"Dataset for '{competition_id}' not prepared in '{self.data_dir}'.")
+
+            task = TaskDefinition(
+                task_id=competition_id,
+                task_type="kaggle",
+                payload={
+                    "description": competition.description,
+                    "public_data_dir": str(competition.public_dir.absolute()),
+                    "output_submission_path": str(output_submission_path.absolute()),
+                },
+            )
+
+            result, cost, usage_summary = await eval_fn(task)
+
+            input_tokens = usage_summary.get("prompt_tokens", 0)
+            output_tokens = usage_summary.get("completion_tokens", 0)
+            total_tokens = usage_summary.get("total_tokens", 0)
+
+            if isinstance(result, Path):
+                report = grade_submission(result, competition)
+            elif isinstance(result, str) and result.startswith("[ERROR]"):
+                error_message = f"DSAT workflow failed: {result}"
+                report = self._create_error_report(competition_id, output_submission_path, error_message)
+            else:
+                error_message = f"Unexpected result type from eval_fn: {type(result).__name__}"
+                report = self._create_error_report(competition_id, output_submission_path, error_message)
+
+        except Exception as exc:
+            error_message = f"Error during ScienceBench evaluation of {competition_id}: {exc}"
+            logger.error(error_message, exc_info=True)
+            report = self._create_error_report(competition_id, output_submission_path, error_message)
+
+        if report is None:
+            final_error = error_message or "Unknown error: report is None"
+            report = self._create_error_report(competition_id, output_submission_path, final_error)
+            error_message = final_error
+
+        if not report.valid_submission:
+            answers_path_str = str(getattr(competition, "answers", "N/A")) if competition else "N/A"
+            self.log_mismatch(
+                problem=competition_id,
+                expected_output=answers_path_str,
+                prediction=f"File: {output_submission_path}, Exists: {report.submission_exists}, Valid: {report.valid_submission}",
+                extracted_output=report.score,
+                extract_answer_code=error_message or "Grading function failed or file invalid/missing",
+            )
+            if not error_message:
+                error_message = "Submission invalid or missing."
+
+        running_time = round(time.perf_counter() - start_time, 4)
+        answers_path_str = str(getattr(competition, "answers", "N/A")) if competition else "N/A"
+
+        csv_tuple = (
+            report.competition_id,
+            str(report.submission_path),
+            answers_path_str,
+            report.score,
+            cost,
+            running_time,
+            input_tokens,
+            output_tokens,
+            total_tokens,
+            bool(getattr(report, "gold_medal", False)),
+            bool(getattr(report, "silver_medal", False)),
+            bool(getattr(report, "bronze_medal", False)),
+            bool(getattr(report, "above_median", False)),
+            report.submission_exists,
+            report.valid_submission,
+            error_message,
+        )
+        return csv_tuple, report, error_message

dsat/common/constants.py

@@ -0,0 +1,11 @@
+# Default timeout values (in seconds)
+DEFAULT_SANDBOX_TIMEOUT = 24 * 3600
+DEFAULT_LLM_TIMEOUT = 300
+
+# Default configuration values
+DEFAULT_LLM_MODEL = "gpt-4o-mini"
+DEFAULT_TEMPERATURE = 0.7
+
+# Path constants
+DEFAULT_WORKSPACE_DIR = "runs"
+DEFAULT_BENCHMARK_DIR = "benchmarks"

dsat/common/exceptions.py

@@ -0,0 +1,48 @@
+class DSATError(Exception):
+    """Base exception for all DSAT-related errors."""
+    def __init__(self, message=None, *args, **kwargs):
+        if message is None:
+            message = "An error occurred in the DSAT framework."
+        super().__init__(message, *args)
+        self.message = message
+        self.details = kwargs.get("details", None)
+
+    def __str__(self):
+        if self.details:
+            return f"{self.message} Details: {self.details}"
+        return self.message
+
+class WorkspaceError(DSATError):
+    """Exception raised for workspace-related errors."""
+    def __init__(self, message=None, *args, **kwargs):
+        if message is None:
+            message = "A workspace-related error occurred."
+        super().__init__(message, *args, **kwargs)
+
+class SandboxError(DSATError):
+    """Exception raised for sandbox execution errors."""
+    def __init__(self, message=None, *args, **kwargs):
+        if message is None:
+            message = "A sandbox execution error occurred."
+        super().__init__(message, *args, **kwargs)
+
+class LLMError(DSATError):
+    """Exception raised for LLM service errors."""
+    def __init__(self, message=None, *args, **kwargs):
+        if message is None:
+            message = "An LLM service error occurred."
+        super().__init__(message, *args, **kwargs)
+
+class DynamicImportError(DSATError):
+    """Exception raised when dynamically importing code fails (e.g., syntax errors in generated workflows)."""
+    def __init__(self, message=None, *args, **kwargs):
+        if message is None:
+            message = "A dynamic code import error occurred."
+        super().__init__(message, *args, **kwargs)
+
+class WorkflowError(DSATError):
+    """Exception raised for workflow execution errors."""
+    def __init__(self, message=None, *args, **kwargs):
+        if message is None:
+            message = "A workflow execution error occurred."
+        super().__init__(message, *args, **kwargs)

dsat/common/typing.py

@@ -0,0 +1,19 @@
+# dsat/common/typing.py
+from pydantic import BaseModel, Field
+from typing import List, Optional, Dict, Any
+
+class ExecutionResult(BaseModel):
+    """
+    Standardized result of executing a code snippet in any sandbox mode.
+    """
+    success: bool = Field(description="True if the execution completed without errors, False otherwise.")
+    stdout: str = Field(default="", description="The captured standard output stream.")
+    stderr: str = Field(default="", description="The captured standard error stream.")
+    exc_type: Optional[str] = Field(default=None, description="The type of exception if one was raised (e.g., 'TimeoutError', 'ValueError').")
+    # For notebook mode, this can hold base64 encoded images or other rich outputs.
+    artifacts: List[str] = Field(default_factory=list, description="A list of generated artifacts, like image filenames.")
+    metadata: Dict[str, Any] = Field(default_factory=dict, description="Arbitrary execution metadata (timestamps, paths, etc.).")
+
+    class Config:
+        """Pydantic configuration."""
+        extra = 'forbid'

dsat/config.py

@@ -0,0 +1,79 @@
+# dsat/config.py
+
+from typing import Dict, Any, Optional, List
+from pydantic import BaseModel, Field
+
+class LLMConfig(BaseModel):
+    """LLM service settings."""
+    model: str = "gpt-4o-mini"
+    temperature: float = 0.7
+    api_key: Optional[str] = Field(None, description="API key, defaults to API_KEY env var if not set.")
+    api_base: Optional[str] = "https://api.openai.com/v1"
+    provider: Optional[str] = Field(None, description="Optional LiteLLM provider alias, e.g. 'siliconflow'.")
+    max_retries: int = 3
+
+class SandboxConfig(BaseModel):
+    """Code execution sandbox settings."""
+    timeout: int = 6 * 3600
+
+class TaskConfig(BaseModel):
+    """Defines the problem to be solved."""
+    goal: str = "Solve the given data science task."
+    eval_metric: Optional[str] = None
+    data_dir: Optional[str] = None
+
+class RunConfig(BaseModel):
+    """Settings for a specific execution run."""
+    name: str = "dsat_run"
+    total_steps: int = 4
+    keep_all_workspaces: bool = Field(False, description="If True, do not delete any workspace after execution.")
+    keep_workspace_on_failure: bool = Field(True, description="If True, keep the workspace only if the task execution fails.")
+    parameters: Dict[str, Any] = Field(default_factory=dict, description="Arbitrary runtime parameters saved for telemetry.")
+
+class AgentSearchConfig(BaseModel):
+    """Parameters for Paradigm 2 (AIDE/AutoMind) search."""
+    num_drafts: int = 5
+    debug_prob: float = 0.8
+    max_iterations: int = 5
+    max_debug_depth: int = 10
+
+class AutoKaggleConfig(BaseModel):
+    """Parameters for the AutoKaggle SOP workflow."""
+    max_attempts_per_phase: int = 10
+    success_threshold: float = 3.0
+
+class AgentConfig(BaseModel):
+    """Configuration for a specific agent's behavior."""
+    search: AgentSearchConfig = Field(default_factory=AgentSearchConfig)
+    max_retries: int = 10
+    autokaggle: AutoKaggleConfig = Field(default_factory=AutoKaggleConfig)
+
+class OptimizerConfig(BaseModel):
+    """Parameters for Paradigm 3 (AFlow) meta-optimization."""
+    max_rounds: int = 10
+    validation_runs_per_candidate: int = 1
+    top_k_selection: int = 2
+
+
+class WorkflowConfig(BaseModel):
+    """Specifies which workflow to run and its parameters."""
+    name: str
+    params: Dict[str, Any] = Field(default_factory=dict)
+    # This field is populated at runtime by main.py, not from the YAML file.
+    class_ref: Optional[Any] = Field(None, exclude=True)
+
+class DSATConfig(BaseModel):
+    """The root configuration model for the entire DSAT application."""
+    run: RunConfig = Field(default_factory=RunConfig)
+    task: TaskConfig = Field(default_factory=TaskConfig)
+    llm: LLMConfig = Field(default_factory=LLMConfig)
+    sandbox: SandboxConfig = Field(default_factory=SandboxConfig)
+    
+    # Paradigm-specific configurations
+    workflow: Optional[WorkflowConfig] = None
+    agent: AgentConfig = Field(default_factory=AgentConfig)
+    optimizer: Optional[OptimizerConfig] = None
+
+    class Config:
+        """Pydantic configuration."""
+        extra = 'forbid'

dsat/models/__init__.py

@@ -0,0 +1,3 @@
+from .task import TaskDefinition, TaskType
+from .candidates import *
+from .formats import *

dsat/models/candidates.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel, Field
+from typing import Optional, List
+
+class WorkflowCandidate(BaseModel):
+    """
+    Represents a proposed orchestration workflow being evaluated by a
+    Paradigm 3 (AFlow-style) meta-optimization driver.
+    """
+    workflow_code: str = Field(description="The code for the proposed orchestration workflow.")
+    fitness: Optional[float] = Field(default=None, description="The fitness score of this workflow.")
+    lineage: List[str] = Field(default_factory=list, description="IDs of parent workflow candidates.")
+    round_num: Optional[int] = Field(default=None, description="The optimization round this candidate was generated in.")
+
+    class Config:
+        """Pydantic configuration."""
+        extra = 'forbid'

dsat/models/formats.py

@@ -0,0 +1,52 @@
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field
+
+# --- Pydantic Models for Operator Outputs ---
+# These models are now used with LLMService.call_with_json
+
+class ReviewResult(BaseModel):
+    """Structured result from the ReviewOperator."""
+    is_buggy: bool = Field(..., description="True if the execution failed or has a clear bug, otherwise False.")
+    summary: str = Field(..., description="If buggy, a proposal to fix the bug. If successful, a summary of empirical findings.")
+    metric_value: Optional[float] = Field(..., description="A quantitative measure of success based on the task requirements. Null if the task does not define a quantitative metric or if it cannot be determined.")
+    lower_is_better: bool = Field(default=True, description="True if a lower metric is better (e.g., RMSE), False if higher is better (e.g., Accuracy).")
+
+class Task(BaseModel):
+    """A single task within a larger plan."""
+    task_id: str = Field(..., description="Unique identifier for a task, e.g., '1', '2.1'.")
+    instruction: str = Field(..., description="Clear, concise instruction for what to do in this task.")
+    dependent_task_ids: List[str] = Field(default_factory=list, description="List of task_ids this task depends on.")
+
+class Plan(BaseModel):
+    """A structured plan consisting of multiple tasks."""
+    tasks: List[Task] = Field(..., description="A list of tasks to achieve the overall goal.")
+
+
+class ComplexityScore(BaseModel):
+    """Structured result from the ComplexityScorerOperator."""
+    complexity: int = Field(..., description="An integer score from 1 to 5 representing the plan's complexity.", ge=1, le=5)
+    justification: str = Field(..., description="A brief justification for the assigned score.")
+
+class DecomposedPlan(BaseModel):
+    """A structured plan decomposed into sequential tasks, used for stepwise execution."""
+    tasks: List[Task] = Field(..., description="A list of sequential tasks to achieve the overall goal.")
+
+
+
+class FileArtifact(BaseModel):
+    filename: str = Field(description="The name of the file, e.g., 'input_data.dat', 'image_001.png', or 'results.json'.")
+    description: str = Field(description="A brief description of the file's purpose and content.")
+
+
+class TaskContract(BaseModel):
+    task_goal: str = Field(description="A clear, one-sentence summary of the main objective.")
+    task_type: str = Field(description="A high-level categorization derived from the task goal (e.g., 'Classification', 'Generation', 'Simulation', 'Data Processing').")
+    input_files: List[FileArtifact] = Field(description="A list of all input files required to complete the task.")
+    output_files: List[FileArtifact] = Field(description="A list of all final output files that must be generated.")
+    evaluation_metric: str = Field(description="The primary metric for evaluating the success of the output, e.g., 'Accuracy Score', 'ROUGE Score', 'Visual Appeal'.")
+
+class StepPlan(BaseModel):
+    """A detailed plan for a single phase and the artifacts it's expected to create."""
+    plan: str = Field(description="A detailed, step-by-step natural language plan for the developer.")
+    input_artifacts: List[str] = Field(default_factory=list, description="A list of artifact filenames from the global state that this plan will use as input (e.g., ['train_preprocessed.csv', 'scaler.joblib']).")
+    output_files: List[str] = Field(description="A list of filenames that the plan is expected to generate (e.g., ['model.pkl', 'submission.csv']).")

dsat/models/task.py

@@ -0,0 +1,64 @@
+from pydantic import BaseModel, Field
+from typing import Literal, Dict, Any
+
+# Define a controlled vocabulary for task types.
+# The framework will choose appropriate TaskHandler based on this type.
+TaskType = Literal["kaggle", "qa", "code", "datasci", "open_ended"]
+
+# Define execution modes.
+# 'standard_ml': Strict evaluation against ground truth (e.g., RMSE, Accuracy).
+# 'open_ended': Exploratory tasks evaluated by LLM judge or artifact existence.
+TaskMode = Literal["standard_ml", "open_ended"]
+
+
+class TaskDefinition(BaseModel):
+    """
+    Standardized, serializable representation of any task in the DSAT framework.
+
+    This is the "logical contract" for the framework to interact with external benchmarks.
+    It encapsulates all information about a task so that DSATRunner and TaskHandler
+    can understand and process it.
+    """
+    task_id: str = Field(
+        description="Unique identifier for the task instance, e.g., 'house-prices-advanced-regression-techniques', 'gsm8k_train_001'."
+    )
+    task_type: TaskType = Field(
+        description="General category of the task, used to select the correct TaskHandler for processing."
+    )
+    mode: TaskMode = Field(
+        default="standard_ml",
+        description="The execution mode for the task. Defaults to 'standard_ml' for backward compatibility."
+    )
+    payload: Dict[str, Any] = Field(
+        description="A dictionary containing task-specific data."
+    )
+
+    class Config:
+        """
+        Pydantic configuration with documentation examples.
+
+        Payload examples:
+        - task_type='kaggle':
+          {
+              "description": "Predict sales prices for houses in Ames, Iowa.",
+              "public_data_dir": "/path/to/benchmark/data",
+              "output_submission_path": "/path/to/run/artifacts/submission.csv"
+          }
+        - task_type='qa':
+          {
+              "question": "What is the result of 9*8-2?"
+          }
+        - task_type='code':
+          {
+              "prompt": "Write a Python function that computes the nth Fibonacci number.",
+              "entry_point": "fibonacci",
+              "test_cases": "[...]"
+          }
+        - task_type='open_ended':
+          {
+              "description": "Design a mathematical model to optimize traffic flow in a city.",
+              "rubric": "Evaluation criteria: model accuracy (40%), creativity (30%), clarity (30%)",
+              "output_submission_path": "/path/to/run/artifacts"
+          }
+        """
+        frozen = True  # Task definitions should be immutable after creation.