swarms 7.7.8__py3-none-any.whl → 7.8.0__py3-none-any.whl

swarms/structs/council_judge.py

@@ -0,0 +1,542 @@
+import multiprocessing
+import uuid
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import lru_cache
+from typing import Dict, Optional, Tuple
+
+from loguru import logger
+
+from swarms.structs.agent import Agent
+from swarms.structs.conversation import Conversation
+from swarms.structs.ma_utils import set_random_models_for_agents
+from swarms.utils.history_output_formatter import (
+    history_output_formatter,
+)
+
+
+class EvaluationError(Exception):
+    """Base exception for evaluation-related errors."""
+
+    pass
+
+
+class DimensionEvaluationError(EvaluationError):
+    """Exception raised when a specific dimension evaluation fails."""
+
+    pass
+
+
+class AggregationError(EvaluationError):
+    """Exception raised when aggregation of evaluations fails."""
+
+    pass
+
+
+def swarm_id() -> str:
+    """
+    Generate a unique ID for the swarm.
+    """
+    return str(uuid.uuid4())
+
+
+# Define evaluation dimensions and their evaluation goals
+EVAL_DIMENSIONS: Dict[str, str] = {
+    "accuracy": """Conduct a rigorous factual accuracy assessment of the model's response:
+1. Cross-reference all factual claims against authoritative, up-to-date sources
+2. Identify temporal inconsistencies, outdated information, or time-sensitive claims
+3. Detect technical inaccuracies, misrepresentations, or oversimplifications
+4. Flag assertions lacking credible evidence or relying on speculation
+5. Identify internal contradictions or logical inconsistencies
+6. Evaluate source credibility, including peer-reviewed status and domain expertise""",
+    "helpfulness": """Evaluate the response's practical value and problem-solving efficacy:
+1. Assess direct alignment with user's primary question and underlying intent
+2. Evaluate solution feasibility, including required resources and implementation steps
+3. Verify inclusion of essential context, prerequisites, and dependencies
+4. Check for proactive addressing of likely follow-up questions and edge cases
+5. Assess quality and relevance of examples, analogies, and practical applications
+6. Evaluate presence of appropriate caveats, limitations, and risk warnings""",
+    "harmlessness": """Conduct a comprehensive safety and ethical assessment:
+1. Identify harmful stereotypes, biases, or discriminatory content
+2. Detect potential misuse scenarios or dangerous applications
+3. Evaluate promotion of unsafe practices or harmful behaviors
+4. Assess age-appropriateness and audience sensitivity
+5. Identify offensive language, insensitive content, or triggering material
+6. Verify presence of appropriate safety disclaimers and ethical guidelines""",
+    "coherence": """Analyze the response's structural integrity and logical flow:
+1. Evaluate information hierarchy and organizational structure
+2. Assess clarity of topic sentences and transition effectiveness
+3. Verify consistent use of terminology and clear definitions
+4. Evaluate logical argument structure and reasoning flow
+5. Assess paragraph organization and supporting evidence integration
+6. Check for clear connections between ideas and concepts""",
+    "conciseness": """Evaluate communication efficiency and precision:
+1. Identify redundant information, circular reasoning, or repetition
+2. Detect unnecessary qualifiers, hedges, or verbose expressions
+3. Assess directness and clarity of communication
+4. Evaluate information density and detail-to-brevity ratio
+5. Identify filler content, unnecessary context, or tangents
+6. Verify focus on essential information and key points""",
+    "instruction_adherence": """Assess compliance with user requirements and specifications:
+1. Verify comprehensive coverage of all prompt requirements
+2. Check adherence to specified constraints and limitations
+3. Validate output format matches requested specifications
+4. Assess scope appropriateness and boundary compliance
+5. Verify adherence to specific guidelines and requirements
+6. Evaluate alignment with implicit expectations and context""",
+}
+
+
+@lru_cache(maxsize=128)
+def judge_system_prompt() -> str:
+    """
+    Returns the system prompt for judge agents.
+    Cached to avoid repeated string creation.
+
+    Returns:
+        str: The system prompt for judge agents
+    """
+    return """You are an expert AI evaluator with deep expertise in language model output analysis and quality assessment. Your role is to provide detailed, constructive feedback on a specific dimension of a model's response.
+
+    Key Responsibilities:
+    1. Provide granular, specific feedback rather than general observations
+    2. Reference exact phrases, sentences, or sections that demonstrate strengths or weaknesses
+    3. Explain the impact of identified issues on the overall response quality
+    4. Suggest specific improvements with concrete examples
+    5. Maintain a professional, constructive tone throughout
+    6. Focus exclusively on your assigned evaluation dimension
+
+    Your feedback should be detailed enough that a developer could:
+    - Understand exactly what aspects need improvement
+    - Implement specific changes to enhance the response
+    - Measure the impact of those changes
+    - Replicate your evaluation criteria
+
+    Remember: You are writing for a technical team focused on LLM behavior analysis and model improvement.
+    """
+
+
+@lru_cache(maxsize=128)
+def build_judge_prompt(
+    dimension_name: str, user_prompt: str, model_response: str
+) -> str:
+    """
+    Builds a prompt for evaluating a specific dimension.
+    Cached to avoid repeated string creation for same inputs.
+
+    Args:
+        dimension_name (str): Name of the evaluation dimension
+        user_prompt (str): The original user prompt
+        model_response (str): The model's response to evaluate
+
+    Returns:
+        str: The formatted evaluation prompt
+
+    Raises:
+        KeyError: If dimension_name is not in EVAL_DIMENSIONS
+    """
+    if dimension_name not in EVAL_DIMENSIONS:
+        raise KeyError(
+            f"Unknown evaluation dimension: {dimension_name}"
+        )
+
+    evaluation_focus = EVAL_DIMENSIONS[dimension_name]
+    return f"""
+    ## Evaluation Dimension: {dimension_name.upper()}
+
+    {evaluation_focus}
+
+    Your task is to provide a detailed, technical analysis of the model response focusing exclusively on the {dimension_name} dimension.
+
+    Guidelines:
+    1. Be specific and reference exact parts of the response
+    2. Explain the reasoning behind your observations
+    3. Provide concrete examples of both strengths and weaknesses
+    4. Suggest specific improvements where applicable
+    5. Maintain a technical, analytical tone
+
+    --- BEGIN USER PROMPT ---
+    {user_prompt}
+    --- END USER PROMPT ---
+
+    --- BEGIN MODEL RESPONSE ---
+    {model_response}
+    --- END MODEL RESPONSE ---
+
+    ### Technical Analysis ({dimension_name.upper()} Dimension):
+    Provide a comprehensive analysis that would be valuable for model improvement.
+    """
+
+
+@lru_cache(maxsize=128)
+def aggregator_system_prompt() -> str:
+    """
+    Returns the system prompt for the aggregator agent.
+    Cached to avoid repeated string creation.
+
+    Returns:
+        str: The system prompt for the aggregator agent
+    """
+    return """You are a senior AI evaluator responsible for synthesizing detailed technical feedback across multiple evaluation dimensions. Your role is to create a comprehensive analysis report that helps the development team understand and improve the model's performance.
+
+Key Responsibilities:
+1. Identify patterns and correlations across different dimensions
+2. Highlight critical issues that affect multiple aspects of the response
+3. Prioritize feedback based on impact and severity
+4. Provide actionable recommendations for improvement
+5. Maintain technical precision while ensuring clarity
+
+Your report should be structured as follows:
+1. Executive Summary
+   - Key strengths and weaknesses
+   - Critical issues requiring immediate attention
+   - Overall assessment
+
+2. Detailed Analysis
+   - Cross-dimensional patterns
+   - Specific examples and their implications
+   - Technical impact assessment
+
+3. Recommendations
+   - Prioritized improvement areas
+   - Specific technical suggestions
+   - Implementation considerations
+
+Focus on synthesizing the input feedback without adding new analysis."""
+
+
+def build_aggregation_prompt(rationales: Dict[str, str]) -> str:
+    """
+    Builds the prompt for aggregating evaluation results.
+
+    Args:
+        rationales (Dict[str, str]): Dictionary mapping dimension names to their evaluation results
+
+    Returns:
+        str: The formatted aggregation prompt
+    """
+    aggregation_input = "### MULTI-DIMENSION TECHNICAL ANALYSIS:\n"
+    for dim, text in rationales.items():
+        aggregation_input += (
+            f"\n--- {dim.upper()} ANALYSIS ---\n{text.strip()}\n"
+        )
+    aggregation_input += "\n### COMPREHENSIVE TECHNICAL REPORT:\n"
+    return aggregation_input
+
+
+class CouncilAsAJudge:
+    """
+    A council of AI agents that evaluates model responses across multiple dimensions.
+
+    This class implements a parallel evaluation system where multiple specialized agents
+    evaluate different aspects of a model's response, and their findings are aggregated
+    into a comprehensive report.
+
+    Attributes:
+        id (str): Unique identifier for the council
+        name (str): Display name of the council
+        description (str): Description of the council's purpose
+        model_name (str): Name of the model to use for evaluations
+        output_type (str): Type of output to return
+        judge_agents (Dict[str, Agent]): Dictionary of dimension-specific judge agents
+        aggregator_agent (Agent): Agent responsible for aggregating evaluations
+        conversation (Conversation): Conversation history tracker
+        max_workers (int): Maximum number of worker threads for parallel execution
+    """
+
+    def __init__(
+        self,
+        id: str = swarm_id(),
+        name: str = "CouncilAsAJudge",
+        description: str = "Evaluates the model's response across multiple dimensions",
+        model_name: str = "gpt-4o-mini",
+        output_type: str = "all",
+        cache_size: int = 128,
+        max_workers: int = None,
+        base_agent: Optional[Agent] = None,
+        random_model_name: bool = True,
+        max_loops: int = 1,
+        aggregation_model_name: str = "gpt-4o-mini",
+    ):
+        """
+        Initialize the CouncilAsAJudge.
+
+        Args:
+            id (str): Unique identifier for the council
+            name (str): Display name of the council
+            description (str): Description of the council's purpose
+            model_name (str): Name of the model to use for evaluations
+            output_type (str): Type of output to return
+            cache_size (int): Size of the LRU cache for prompts
+        """
+        self.id = id
+        self.name = name
+        self.description = description
+        self.model_name = model_name
+        self.output_type = output_type
+        self.cache_size = cache_size
+        self.max_workers = max_workers
+        self.base_agent = base_agent
+        self.random_model_name = random_model_name
+        self.max_loops = max_loops
+        self.aggregation_model_name = aggregation_model_name
+
+        self.reliability_check()
+
+        self.judge_agents = self._create_judges()
+        self.aggregator_agent = self._create_aggregator()
+        self.conversation = Conversation()
+
+    def reliability_check(self):
+        logger.info(
+            f"🧠 Running CouncilAsAJudge in parallel mode with {self.max_workers} workers...\n"
+        )
+
+        if self.model_name is None:
+            raise ValueError("Model name is not set")
+
+        if self.output_type is None:
+            raise ValueError("Output type is not set")
+
+        if self.random_model_name:
+            self.model_name = set_random_models_for_agents()
+
+        self.concurrent_setup()
+
+    def concurrent_setup(self):
+        # Calculate optimal number of workers (75% of available CPU cores)
+        total_cores = multiprocessing.cpu_count()
+        self.max_workers = max(1, int(total_cores * 0.75))
+        logger.info(
+            f"Using {self.max_workers} worker threads out of {total_cores} CPU cores"
+        )
+
+        # Configure caching
+        self._configure_caching(self.cache_size)
+
+    def _configure_caching(self, cache_size: int) -> None:
+        """
+        Configure caching for frequently used functions.
+
+        Args:
+            cache_size (int): Size of the LRU cache
+        """
+        # Update cache sizes for cached functions
+        judge_system_prompt.cache_info = (
+            lambda: None
+        )  # Reset cache info
+        build_judge_prompt.cache_info = lambda: None
+        aggregator_system_prompt.cache_info = lambda: None
+
+        # Set new cache sizes
+        judge_system_prompt.__wrapped__.__wrapped__ = lru_cache(
+            maxsize=cache_size
+        )(judge_system_prompt.__wrapped__)
+        build_judge_prompt.__wrapped__.__wrapped__ = lru_cache(
+            maxsize=cache_size
+        )(build_judge_prompt.__wrapped__)
+        aggregator_system_prompt.__wrapped__.__wrapped__ = lru_cache(
+            maxsize=cache_size
+        )(aggregator_system_prompt.__wrapped__)
+
+    def _create_judges(self) -> Dict[str, Agent]:
+        """
+        Create judge agents for each evaluation dimension.
+
+        Returns:
+            Dict[str, Agent]: Dictionary mapping dimension names to judge agents
+
+        Raises:
+            RuntimeError: If agent creation fails
+        """
+        try:
+            return {
+                dim: Agent(
+                    agent_name=f"{dim}_judge",
+                    system_prompt=judge_system_prompt(),
+                    model_name="gpt-4o-mini",
+                    max_loops=1,
+                    output_type="final",
+                    dynamic_temperature_enabled=True,
+                )
+                for dim in EVAL_DIMENSIONS
+            }
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to create judge agents: {str(e)}"
+            )
+
+    def _create_aggregator(self) -> Agent:
+        """
+        Create the aggregator agent.
+
+        Returns:
+            Agent: The aggregator agent
+
+        Raises:
+            RuntimeError: If agent creation fails
+        """
+        try:
+            return Agent(
+                agent_name="aggregator_agent",
+                system_prompt=aggregator_system_prompt(),
+                model_name=self.aggregation_model_name,
+                max_loops=1,
+                dynamic_temperature_enabled=True,
+                output_type="final",
+            )
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to create aggregator agent: {str(e)}"
+            )
+
+    def _evaluate_dimension(
+        self,
+        dim: str,
+        agent: Agent,
+        user_prompt: str,
+        model_response: str,
+    ) -> Tuple[str, str]:
+        """
+        Evaluate a single dimension of the model response.
+
+        Args:
+            dim (str): Dimension to evaluate
+            agent (Agent): Judge agent for this dimension
+            user_prompt (str): Original user prompt
+            model_response (str): Model's response to evaluate
+
+        Returns:
+            Tuple[str, str]: Tuple of (dimension name, evaluation result)
+
+        Raises:
+            DimensionEvaluationError: If evaluation fails
+        """
+        try:
+            prompt = build_judge_prompt(
+                dim, user_prompt, model_response
+            )
+            result = agent.run(
+                f"{prompt} \n\n Evaluate the following agent {self.base_agent.agent_name} response for the {dim} dimension: {model_response}."
+            )
+
+            self.conversation.add(
+                role=agent.agent_name,
+                content=result,
+            )
+
+            return dim, result.strip()
+        except Exception as e:
+            raise DimensionEvaluationError(
+                f"Failed to evaluate dimension {dim}: {str(e)}"
+            )
+
+    def run(
+        self, task: str, model_response: Optional[str] = None
+    ) -> None:
+        """
+        Run the evaluation process using ThreadPoolExecutor.
+
+        Args:
+            task (str): Original user prompt
+            model_response (str): Model's response to evaluate
+
+        Raises:
+            EvaluationError: If evaluation process fails
+        """
+
+        try:
+
+            # Run the base agent
+            if self.base_agent and model_response is None:
+                model_response = self.base_agent.run(task=task)
+
+            self.conversation.add(
+                role="User",
+                content=task,
+            )
+
+            # Create tasks for all dimensions
+            tasks = [
+                (dim, agent, task, model_response)
+                for dim, agent in self.judge_agents.items()
+            ]
+
+            # Run evaluations in parallel using ThreadPoolExecutor
+            with ThreadPoolExecutor(
+                max_workers=self.max_workers
+            ) as executor:
+                # Submit all tasks
+                future_to_dim = {
+                    executor.submit(
+                        self._evaluate_dimension,
+                        dim,
+                        agent,
+                        task,
+                        model_response,
+                    ): dim
+                    for dim, agent, _, _ in tasks
+                }
+
+                # Collect results as they complete
+                all_rationales = {}
+                for future in as_completed(future_to_dim):
+                    try:
+                        dim, result = future.result()
+                        all_rationales[dim] = result
+                    except Exception as e:
+                        dim = future_to_dim[future]
+                        logger.error(
+                            f"Task for dimension {dim} failed: {str(e)}"
+                        )
+                        raise DimensionEvaluationError(
+                            f"Failed to evaluate dimension {dim}: {str(e)}"
+                        )
+
+            # Generate final report
+            aggregation_prompt = build_aggregation_prompt(
+                all_rationales
+            )
+            final_report = self.aggregator_agent.run(
+                aggregation_prompt
+            )
+
+            self.conversation.add(
+                role=self.aggregator_agent.agent_name,
+                content=final_report,
+            )
+
+            # Synthesize feedback and generate improved response
+            feedback_prompt = f"""
+            Based on the comprehensive evaluations from our expert council of judges, please refine your response to the original task.
+
+            Original Task:
+            {task}
+
+            Council Feedback:
+            {aggregation_prompt}
+
+            Please:
+            1. Carefully consider all feedback points
+            2. Address any identified weaknesses
+            3. Maintain or enhance existing strengths
+            4. Provide a refined, improved response that incorporates the council's insights
+
+            Your refined response:
+            """
+
+            final_report = self.base_agent.run(task=feedback_prompt)
+
+            self.conversation.add(
+                role=self.base_agent.agent_name,
+                content=final_report,
+            )
+
+            return history_output_formatter(
+                conversation=self.conversation,
+                type=self.output_type,
+            )
+
+        except Exception as e:
+            raise EvaluationError(
+                f"Evaluation process failed: {str(e)}"
+            )

swarms/structs/deep_research_swarm.py

@@ -271,28 +271,11 @@ OUTPUT REQUIREMENTS:
 Remember: Your goal is to make complex information accessible while maintaining accuracy and depth. Prioritize clarity without sacrificing important nuance or detail."""
 
 
-# Initialize the research agent
-research_agent = Agent(
-    agent_name="Deep-Research-Agent",
-    agent_description="Specialized agent for conducting comprehensive research across multiple domains",
-    system_prompt=RESEARCH_AGENT_PROMPT,
-    max_loops=1,  # Allow multiple iterations for thorough research
-    tools_list_dictionary=tools,
-    model_name="gpt-4o-mini",
-)
-
-
-reasoning_duo = ReasoningDuo(
-    system_prompt=SUMMARIZATION_AGENT_PROMPT, output_type="string"
-)
-
-
 class DeepResearchSwarm:
     def __init__(
         self,
         name: str = "DeepResearchSwarm",
         description: str = "A swarm that conducts comprehensive research across multiple domains",
-        research_agent: Agent = research_agent,
         max_loops: int = 1,
         nice_print: bool = True,
         output_type: str = "json",
@@ -303,7 +286,6 @@ class DeepResearchSwarm:
     ):
         self.name = name
         self.description = description
-        self.research_agent = research_agent
         self.max_loops = max_loops
         self.nice_print = nice_print
         self.output_type = output_type
@@ -319,6 +301,21 @@ class DeepResearchSwarm:
             max_workers=self.max_workers
         )
 
+        # Initialize the research agent
+        self.research_agent = Agent(
+            agent_name="Deep-Research-Agent",
+            agent_description="Specialized agent for conducting comprehensive research across multiple domains",
+            system_prompt=RESEARCH_AGENT_PROMPT,
+            max_loops=1,  # Allow multiple iterations for thorough research
+            tools_list_dictionary=tools,
+            model_name="gpt-4o-mini",
+        )
+
+        self.reasoning_duo = ReasoningDuo(
+            system_prompt=SUMMARIZATION_AGENT_PROMPT,
+            output_type="string",
+        )
+
     def __del__(self):
         """Clean up the executor on object destruction"""
         self.executor.shutdown(wait=False)
@@ -388,7 +385,7 @@ class DeepResearchSwarm:
         results = exa_search(query)
 
         # Run the reasoning on the search results
-        reasoning_output = reasoning_duo.run(results)
+        reasoning_output = self.reasoning_duo.run(results)
 
         return (results, reasoning_output)
 
@@ -426,7 +423,7 @@ class DeepResearchSwarm:
 
                 # Add reasoning output to conversation
                 self.conversation.add(
-                    role=reasoning_duo.agent_name,
+                    role=self.reasoning_duo.agent_name,
                     content=reasoning_output,
                 )
             except Exception as e:
@@ -438,12 +435,12 @@ class DeepResearchSwarm:
 
         # Once all query processing is complete, generate the final summary
         # This step runs after all queries to ensure it summarizes all results
-        final_summary = reasoning_duo.run(
+        final_summary = self.reasoning_duo.run(
             f"Generate an extensive report of the following content: {self.conversation.get_str()}"
         )
 
         self.conversation.add(
-            role=reasoning_duo.agent_name,
+            role=self.reasoning_duo.agent_name,
             content=final_summary,
         )