dslighting 1.3.9__py3-none-any.whl

dsat/templates/open_ended/grade_template.py

@@ -0,0 +1,107 @@
+import os
+from pathlib import Path
+from typing import Any
+import pandas as pd
+
+# This is a generic LLM-based grader for open-ended tasks.
+# It reads 'rubric.md' from the task directory and evaluates the submission.
+
+try:
+    from dsat.services.llm import LLMService
+    from dsat.config import LLMConfig
+except ImportError:
+    # Fallback for when running outside of dsat package context
+    import sys
+    sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent))
+    from dsat.services.llm import LLMService
+    from dsat.config import LLMConfig
+
+class Report:
+    def __init__(self, score, feedback):
+        self.score = score
+        self.feedback = feedback
+        # Standard fields expected by the framework
+        self.is_lower_better = False
+        self.submission_exists = True
+        self.valid_submission = True
+        self.gold_medal = score >= 0.9
+        self.silver_medal = score >= 0.7
+        self.bronze_medal = score >= 0.5
+        self.above_median = score >= 0.5
+        self.submission_path = ""
+        self.competition_id = "open_ended_task"
+
+def grade(submission_path: Path, competition: Any) -> Report:
+    """
+    Grades the submission using an LLM Judge against rubric.md.
+    """
+    # 1. Load the Rubric
+    task_dir = competition.raw_dir.parent
+    rubric_path = task_dir / "rubric.md"
+    
+    if not rubric_path.exists():
+        # Fallback if no rubric exists
+        print(f"Warning: Rubric not found at {rubric_path}. Returning default score.")
+        return Report(0.5, "No grading rubric defined.")
+        
+    rubric_content = rubric_path.read_text(encoding="utf-8")
+    
+    # 2. Load the Submission Content (Preview)
+    # Since it's open-ended, the 'submission_path' might be a CSV, code, or just a marker.
+    # We'll try to peek at the output artifacts if possible, or assume the agent's recent work 
+    # is what we are grading. Ideally, AIDE produces a submission file.
+    
+    submission_content = "No submission content readable."
+    if submission_path.exists():
+        try:
+            if submission_path.suffix == '.csv':
+                df = pd.read_csv(submission_path)
+                submission_content = f"CSV Submission Preview:\n{df.head().to_markdown()}"
+            else:
+                submission_content = submission_path.read_text(encoding="utf-8")[:2000]
+        except Exception as e:
+            submission_content = f"Error reading submission: {e}"
+
+    # 3. Setup LLM for Judging
+    # Note: In a real run, we might want to inject the API key securely.
+    # Here we assume environment variables are set (which they are in DSATRunner).
+    try:
+        api_key = os.getenv("API_KEY", "EMPTY")
+        base_url = os.getenv("API_BASE", "https://api.openai.com/v1")
+        model = os.getenv("LLM_MODEL", "gpt-4o")
+        
+        llm = LLMService(LLMConfig(api_key=api_key, api_base=base_url, model=model))
+        
+        prompt = f"""You are an impartial Judge. Evaluate the following submission against the provided Rubric.
+
+# RUBRIC
+{rubric_content}
+
+# SUBMISSION CONTENT
+{submission_content}
+
+# INSTRUCTION
+Assess the submission. 
+Output ONLY a float number between 0.0 and 1.0 on the first line.
+On subsequent lines, provide brief feedback.
+"""
+        # Synchronous call wrapper or direct call if possible. 
+        # Since grade() is synchronous in standard mlebench, we need a way to run async code.
+        import asyncio
+        response = asyncio.run(llm.achat([{"role": "user", "content": prompt}]))
+        
+        lines = response.strip().split('\n')
+        try:
+            score = float(lines[0].strip())
+        except ValueError:
+            # Fallback if LLM is chatty
+            import re
+            match = re.search(r"(\d+(\.\d+)?)", lines[0])
+            score = float(match.group(1)) if match else 0.5
+            
+        feedback = "\n".join(lines[1:])
+        return Report(score, feedback)
+
+    except Exception as e:
+        print(f"LLM Judging failed: {e}")
+        return Report(0.0, f"Judging failed: {e}")

dsat/tools/__init__.py

@@ -0,0 +1,4 @@
+"""dsat.tools
+
+Tools and utilities for DSAT workflows.
+"""

dsat/utils/context.py

@@ -0,0 +1,172 @@
+from itertools import groupby
+import logging
+from typing import List, Dict, Optional, Any
+
+from dsat.services.llm import LLMService
+
+logger = logging.getLogger(__name__)
+
+# Use character count as a simple, fast proxy for token count.
+# In a production system, you would use a real tokenizer.
+MAX_HISTORY_CHARS = 32000
+MAX_ERROR_CHARS = 8000
+MAX_KNOWLEDGE_CHARS = 8000
+MAX_OUTPUT_CHARS = 16000  # Maximum characters for execution output
+
+class ContextManager:
+    """
+    A utility for intelligently building prompt context to prevent overflow.
+    It uses summarization, windowing, and truncation to manage context size.
+    """
+    def __init__(self, llm_service: Optional[LLMService] = None):
+        self.llm_service = llm_service
+
+    def build_history_context(self, history: List[Dict[str, Any]], key_order: List[str]) -> str:
+        """
+        Builds a context string from a list of historical events using a windowing strategy.
+
+        Args:
+            history: A list of dictionary-like objects representing historical steps.
+            key_order: The keys to extract from each history object and their order.
+
+        Returns:
+            A formatted string of the most recent history that fits within the budget.
+        """
+        if not history:
+            return "No history available."
+
+        context_parts = []
+        total_chars = 0
+
+        # Iterate backwards from the most recent history
+        for item in reversed(history):
+            part = "\n".join([f"{key.capitalize()}: {item.get(key, 'N/A')}" for key in key_order])
+            part_len = len(part)
+
+            if total_chars + part_len > MAX_HISTORY_CHARS:
+                logger.warning("History context truncated to fit budget.")
+                break
+
+            context_parts.append(part)
+            total_chars += part_len
+
+        # Reverse again to restore chronological order
+        final_context = "\n---\n".join(reversed(context_parts))
+        
+        if len(history) > len(context_parts):
+            final_context = f"[... {len(history) - len(context_parts)} older steps summarized ...]\n\n{final_context}"
+
+        return final_context
+
+    def summarize_error(self, stderr: str, exc_type: Optional[str] = None) -> str:
+        """
+        Extracts the most relevant parts of a long error message.
+        """
+        if not stderr:
+            return "No error output."
+        
+        # Prioritize the exception type if available
+        summary = f"Exception Type: {exc_type}\n\n" if exc_type else ""
+        
+        if len(stderr) > MAX_ERROR_CHARS:
+            # Keep the beginning and the end of the traceback
+            head = stderr[:MAX_ERROR_CHARS // 2]
+            tail = stderr[-MAX_ERROR_CHARS // 2:]
+            summary += f"Traceback (truncated):\n{head}\n[...]\n{tail}"
+            logger.warning("Error context truncated to fit budget.")
+        else:
+            summary += f"Traceback:\n{stderr}"
+            
+        return summary
+    
+    async def summarize_knowledge(self, knowledge_docs: List[str], task_goal: str) -> str:
+        """
+        Uses an LLM to summarize a list of retrieved documents into a concise
+        knowledge block relevant to the task.
+        """
+        if not knowledge_docs:
+            return "No relevant knowledge was retrieved for this task."
+        if not self.llm_service:
+            logger.warning("LLMService not provided to ContextManager; returning raw knowledge.")
+            return "\n\n".join(knowledge_docs)[:MAX_KNOWLEDGE_CHARS]
+
+        full_knowledge = "\n\n---\n\n".join(knowledge_docs)
+        
+        if len(full_knowledge) < MAX_KNOWLEDGE_CHARS:
+            return full_knowledge
+
+        logger.info("Retrieved knowledge is too long; summarizing with LLM...")
+        prompt = (
+            f"The user is trying to achieve the following goal: '{task_goal}'.\n\n"
+            "The following are retrieved documents that might be helpful. Summarize the most critical "
+            "patterns, code snippets, and strategies from these documents that are directly relevant "
+            "to the user's goal. Be concise."
+            f"\n\n# DOCUMENTS\n{full_knowledge}"
+        )
+        
+        summary = await self.llm_service.call(prompt)
+        return summary
+
+
+def summarize_repetitive_logs(log_text: str, min_repeats: int = 3) -> str:
+    """
+    Summarizes consecutive, identical lines in a log string.
+
+    Args:
+        log_text: The raw log output.
+        min_repeats: The minimum number of consecutive repeats to summarize.
+
+    Returns:
+        A cleaned log string with repeated lines summarized.
+    """
+    if not log_text:
+        return ""
+
+    lines = log_text.strip().split('\n')
+    summarized_lines = []
+
+    # Use groupby to find consecutive identical elements
+    for line, group in groupby(lines):
+        count = len(list(group))
+        if count >= min_repeats:
+            # Add a summary line for the repeated block
+            summarized_lines.append(f"<{line.strip()} (repeated {count} times)>")
+        else:
+            # If not repeated enough, add the lines back as they were
+            summarized_lines.extend([line] * count)
+
+    return '\n'.join(summarized_lines)
+
+
+def truncate_output(output: str, max_chars: int = MAX_OUTPUT_CHARS) -> str:
+    """
+    Truncates long output by keeping the beginning and end portions.
+    This is useful when execution output exceeds max_seq_len limits.
+
+    Args:
+        output: The raw execution output.
+        max_chars: Maximum allowed characters (default: MAX_OUTPUT_CHARS).
+
+    Returns:
+        Truncated output with truncation notice if needed.
+    """
+    if not output:
+        return ""
+
+    if len(output) <= max_chars:
+        return output
+
+    # Keep the first and last portions
+    head_size = max_chars // 2
+    tail_size = max_chars - head_size
+
+    head = output[:head_size]
+    tail = output[-tail_size:]
+
+    truncation_notice = (
+        f"\n\n... [TRUNCATED: {len(output) - max_chars} characters omitted "
+        f"to prevent context overflow] ...\n\n"
+    )
+
+    logger.warning(f"Output truncated from {len(output)} to {max_chars} characters.")
+    return head + truncation_notice + tail

dsat/utils/dynamic_import.py

@@ -0,0 +1,71 @@
+"""
+Utility for dynamically importing and instantiating classes from code strings.
+Ported from the AFlow project for use in the meta-optimization evaluation step.
+"""
+import importlib.util
+import sys
+from typing import Any, Optional, Dict
+import logging
+from dsat.common.exceptions import DynamicImportError # Import the new exception
+
+logger = logging.getLogger(__name__)
+
+def import_workflow_from_string(code_string: str, class_name: str = "Workflow") -> Any:
+    """
+    Dynamically imports a workflow class from a code string.
+    
+    Args:
+        code_string: The string containing the Python code.
+        class_name: The name of the class to import (default: "Workflow").
+    
+    Returns:
+        The workflow class.
+        
+    Raises:
+        DynamicImportError: If the import fails for any reason.
+    """
+    try:
+        # Create a temporary, unique module name to avoid conflicts
+        module_name = f"dynamic_workflow_module_{hash(code_string)}"
+        if module_name in sys.modules:
+            del sys.modules[module_name]
+
+        spec = importlib.util.spec_from_loader(module_name, loader=None)
+        module = importlib.util.module_from_spec(spec)
+
+        # Inject necessary base classes and types into the module's scope
+        # to prevent NameError during exec.
+        module.__dict__['DSATWorkflow'] = __import__('dsat.workflows.base', fromlist=['DSATWorkflow']).DSATWorkflow
+        module.__dict__['Path'] = __import__('pathlib').Path
+        module.__dict__['asyncio'] = __import__('asyncio')
+        module.__dict__['shutil'] = __import__('shutil')
+        module.__dict__['Dict'] = __import__('typing').Dict
+        module.__dict__['Any'] = __import__('typing').Any
+        module.__dict__['List'] = __import__('typing').List
+        module.__dict__['LLMService'] = __import__('dsat.services.llm', fromlist=['LLMService']).LLMService
+        module.__dict__['SandboxService'] = __import__('dsat.services.sandbox', fromlist=['SandboxService']).SandboxService
+        module.__dict__['parse_plan_and_code'] = __import__('dsat.utils.parsing', fromlist=['parse_plan_and_code']).parse_plan_and_code
+        
+        # Execute the code within the new module's namespace
+        exec(code_string, module.__dict__)
+
+        # Get the class from the module
+        WorkflowClass = getattr(module, class_name, None)
+
+        if WorkflowClass:
+            return WorkflowClass
+        else:
+            error_msg = f"Class '{class_name}' not found in the provided dynamic code."
+            logger.error(error_msg)
+            raise DynamicImportError(error_msg)
+            
+    except DynamicImportError:
+        raise # Re-raise if already caught
+    except Exception as e:
+        error_msg = f"Error during dynamic class import (e.g., syntax error): {e}"
+        logger.error(error_msg, exc_info=True)
+        raise DynamicImportError(error_msg) from e
+    finally:
+        if module_name in sys.modules:
+            del sys.modules[module_name]
+

dsat/utils/parsing.py

@@ -0,0 +1,33 @@
+"""
+Helper functions for parsing structured content from raw LLM text responses.
+"""
+import re
+
+def parse_plan_and_code(response: str) -> tuple[str, str]:
+    """
+    Extracts a natural language plan and a Python code block from an LLM's response.
+    Assumes a format where the plan precedes the code block.
+
+    Args:
+        response: The raw text response from the LLM.
+
+    Returns:
+        A tuple containing the extracted plan and code.
+        Returns a default error message for code if not found.
+    """
+    # Use a non-greedy match for the plan to capture everything before the first code block.
+    plan_match = re.search(r"(.*?)```(?:python|py)?", response, re.DOTALL)
+    if plan_match:
+        plan = plan_match.group(1).strip()
+    else:
+        # If no code block is found, assume the entire response is the plan.
+        plan = response.strip()
+
+    code_match = re.search(r"```(?:python|py)?\n(.*?)\n```", response, re.DOTALL)
+    if code_match:
+        code = code_match.group(1).strip()
+    else:
+        # Fallback if the code block is malformed or missing
+        code = "# ERROR: Could not parse code block from LLM response."
+
+    return plan, code

dsat/workflows/__init__.py

@@ -0,0 +1,12 @@
+# dsat/workflows/__init__.py
+
+# This file makes the 'workflows' directory a Python package.
+from .base import DSATWorkflow
+from .factory import (
+    WorkflowFactory,
+    AIDEWorkflowFactory,
+    AutoMindWorkflowFactory, 
+    DSAgentWorkflowFactory,
+    DataInterpreterWorkflowFactory,
+    AutoKaggleWorkflowFactory
+)

dsat/workflows/base.py

@@ -0,0 +1,53 @@
+# dsat/workflows/base.py
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Dict, Any
+import logging
+
+
+# --- New, standardized workflow interface (core) ---
+class DSATWorkflow(ABC):
+    """
+    New standardized workflow abstract base class defining the "physical interface contract".
+
+    Any workflow implementing this interface becomes a generic problem solver
+    that is completely decoupled from the specific form of the task (QA, Kaggle, etc.).
+    It only understands files and directories.
+    """
+    def __init__(self, operators: Dict[str, Any], services: Dict[str, Any], agent_config: Dict[str, Any]):
+        """
+        Initialize through dependency injection.
+
+        Args:
+            operators: A dictionary containing all operator instances needed by this workflow.
+            services: A dictionary containing required service instances (e.g., LLMService, SandboxService).
+            agent_config: A dictionary containing agent behavior-specific configuration.
+        """
+        self.operators = operators
+        self.services = services
+        self.agent_config = agent_config
+
+    @abstractmethod
+    async def solve(
+        self,
+        description: str,
+        io_instructions: str, # NEW ARGUMENT
+        data_dir: Path,
+        output_path: Path
+    ) -> None:
+        """
+        Solve a given task based on the physical file interface.
+
+        This is the core method for all standardized workflows. Workflows implementing this method need to:
+        1.  Treat `data_dir` as their only input source.
+        2.  Execute their internal logic (e.g., call LLM, run code) to solve the task described in `description`.
+        3.  Write their final, evaluable answer as a single file to the complete path specified by `output_path`.
+
+        Args:
+            description: Natural language goal description and data analysis report.
+            io_instructions: Explicit, standardized instructions for reading input and writing output.
+            data_dir: A directory containing all input files needed to solve the task (e.g., `problem.txt`, `train.csv`).
+            output_path: The complete path where the final output file (e.g., `answer.txt`, `submission.csv`) must be saved.
+        """
+        raise NotImplementedError