flock-core 0.4.0b3__py3-none-any.whl → 0.4.0b5__py3-none-any.whl

flock/core/evaluation/utils.py

@@ -0,0 +1,312 @@
+# src/flock/core/util/evaluation_helpers.py
+import inspect
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Union
+
+import pandas as pd
+from box import Box
+from datasets import get_dataset_config_names, load_dataset
+
+from flock.core.flock_agent import FlockAgent
+from flock.core.flock_evaluator import FlockEvaluator
+from flock.core.logging.logging import get_logger
+
+# Potentially import metrics libraries like rouge_score, nltk, sentence_transformers
+
+logger_helpers = get_logger("util.evaluation")
+
+
+def load_and_merge_all_configs(dataset_name: str) -> pd.DataFrame:
+    all_configs = get_dataset_config_names(dataset_name)
+    all_dfs = []
+
+    for config in all_configs:
+        dataset_dict = load_dataset(dataset_name, config)
+        for split_name, split_dataset in dataset_dict.items():
+            df = split_dataset.to_pandas()
+            df["config"] = config
+            df["split"] = split_name
+            all_dfs.append(df)
+
+    merged_df = pd.concat(all_dfs, ignore_index=True)
+    return merged_df
+
+
+def normalize_dataset(dataset: Any) -> pd.DataFrame:
+    """Converts various dataset formats into a pandas DataFrame."""
+    if isinstance(dataset, pd.DataFrame):
+        return dataset.copy()
+    elif isinstance(dataset, str | Path):
+        path = Path(dataset)
+        if not path.exists():
+            try:
+                return load_and_merge_all_configs(dataset)
+            except Exception as e:
+                raise FileNotFoundError(
+                    f"Dataset file not found: {path}"
+                ) from e
+        if path.suffix.lower() == ".csv":
+            return pd.read_csv(path)
+        # Add support for json, jsonl etc. if needed
+        else:
+            raise ValueError(
+                f"Unsupported file type for dataset: {path.suffix}"
+            )
+    elif isinstance(dataset, list):
+        if not dataset or not isinstance(dataset[0], dict):
+            raise ValueError("Dataset list must contain dictionaries.")
+        return pd.DataFrame(dataset)
+    elif "datasets" in sys.modules and isinstance(
+        dataset, sys.modules["datasets"].Dataset
+    ):
+        # Requires 'datasets' library to be installed
+        return dataset.to_pandas()
+    else:
+        raise TypeError(f"Unsupported dataset type: {type(dataset)}")
+
+
+def extract_value_by_dot_notation(data: dict | Box, key: str) -> Any:
+    """Retrieves a value from a nested dictionary or Box object using dot notation."""
+    if not key:
+        return None
+    keys = key.split(".")
+    value = data
+    try:
+        for k in keys:
+            if isinstance(value, (dict, Box)):
+                value = value.get(k)
+            # Add list index handling if needed: e.g., 'results[0].field'
+            # elif isinstance(value, list) and k.isdigit():
+            #     value = value[int(k)]
+            else:
+                return None  # Cannot traverse further
+            if value is None:
+                return None  # Key not found at this level
+        return value
+    except (KeyError, IndexError, AttributeError):
+        return None
+
+
+def calculate_evaluation_metrics(
+    metrics: list[Union[str, Callable, "FlockAgent", "FlockEvaluator"]],
+    metric_configs: dict[str, dict[str, Any]],
+    predicted_answers: dict[str, Any],
+    expected_answers: dict[str, Any],
+    agent_inputs: dict[str, Any],  # For context
+    agent_output: Any,  # For context
+) -> dict[str, Any]:
+    """Calculates all specified metrics for a single evaluation item."""
+    results = {}
+    for metric in metrics:
+        metric_name = ""
+        metric_result = None
+        try:
+            if isinstance(metric, str):
+                metric_name = metric
+                # Find predicted/expected values relevant to this metric string
+                # Simple case: metric name matches an answer_mapping key
+                if (
+                    metric_name in predicted_answers
+                    and metric_name in expected_answers
+                ):
+                    predicted = predicted_answers[metric_name]
+                    expected = expected_answers[metric_name]
+                    metric_func = _get_metric_function(metric_name)
+                    config = metric_configs.get(metric_name, {})
+                    metric_result = metric_func(predicted, expected, **config)
+                else:
+                    logger_helpers.warning(
+                        f"Could not find matching predicted/expected values for metric '{metric_name}' based on answer_mapping keys."
+                    )
+                    metric_result = None  # Or some error indicator
+
+            elif isinstance(metric, Callable):
+                metric_name = getattr(metric, "__name__", "custom_function")
+                # Custom functions might need specific predicted/expected pairs, or all of them
+                # Let's pass all for flexibility, user function needs to handle it
+                config = metric_configs.get(metric_name, {})
+                # Allow passing context if function signature supports it
+                sig = inspect.signature(metric)
+                call_kwargs = config.copy()
+                if "agent_inputs" in sig.parameters:
+                    call_kwargs["agent_inputs"] = agent_inputs
+                if "agent_output" in sig.parameters:
+                    call_kwargs["agent_output"] = agent_output
+
+                metric_result = metric(
+                    predicted_answers, expected_answers, **call_kwargs
+                )
+
+            # --- Placeholder for Agent/Evaluator based metrics ---
+            elif "FlockAgent" in str(
+                type(metric)
+            ):  # Avoid hard import if possible
+                metric_name = getattr(metric, "name", "judge_agent")
+                config = metric_configs.get(metric_name, {})
+                # Requires running the judge agent - needs async context
+                # metric_result = asyncio.run(_run_judge_agent(metric, predicted_answers, expected_answers, config))
+                logger_helpers.warning(
+                    f"Agent-based metric '{metric_name}' execution not implemented in this sketch."
+                )
+                metric_result = "[Agent Judge Not Implemented]"
+
+            elif "FlockEvaluator" in str(
+                type(metric)
+            ):  # Avoid hard import if possible
+                metric_name = getattr(metric, "name", "judge_evaluator")
+                config = metric_configs.get(metric_name, {})
+                # Requires running the evaluator - needs async context
+                # metric_result = asyncio.run(_run_judge_evaluator(metric, predicted_answers, expected_answers, config))
+                logger_helpers.warning(
+                    f"Evaluator-based metric '{metric_name}' execution not implemented in this sketch."
+                )
+                metric_result = "[Evaluator Judge Not Implemented]"
+            # --- End Placeholder ---
+
+            else:
+                logger_helpers.warning(
+                    f"Unsupported metric type: {type(metric)}"
+                )
+                continue
+
+            # Store result - handle dict results from metrics
+            if isinstance(metric_result, dict):
+                for sub_key, sub_value in metric_result.items():
+                    results[f"{metric_name}_{sub_key}"] = sub_value
+            else:
+                results[metric_name] = metric_result
+
+        except Exception as e:
+            logger_helpers.error(
+                f"Error calculating metric '{metric_name}': {e}"
+            )
+            results[metric_name] = f"[Error: {e}]"
+
+    return results
+
+
+def _get_metric_function(metric_name: str) -> Callable:
+    """Maps metric names to their implementation functions."""
+    # Lazy load metric libraries
+    if metric_name == "exact_match":
+        return lambda pred, act, **kw: str(pred).strip() == str(act).strip()
+    elif metric_name == "fuzzy_match":
+        try:
+            from thefuzz import fuzz
+
+            return (
+                lambda pred, act, threshold=85, **kw: fuzz.ratio(
+                    str(pred), str(act)
+                )
+                >= threshold
+            )
+        except ImportError:
+            logger_helpers.warning(
+                "fuzzy_match requires 'thefuzz': pip install thefuzz[speedup]"
+            )
+            return lambda p, a, **kw: None
+    elif metric_name.startswith("rouge"):  # rouge_1, rouge_2, rouge_l
+        try:
+            from rouge_score import rouge_scorer
+
+            scorer = rouge_scorer.RougeScorer(
+                [metric_name.replace("_", "")], use_stemmer=True
+            )
+
+            def calculate_rouge(pred, act, score_type="fmeasure", **kw):
+                scores = scorer.score(str(act), str(pred))
+                return (
+                    scores[metric_name.replace("_", "")]
+                    ._asdict()
+                    .get(score_type, 0.0)
+                )
+
+            return calculate_rouge
+        except ImportError:
+            logger_helpers.warning(
+                "rouge requires 'rouge-score': pip install rouge-score"
+            )
+            return lambda p, a, **kw: None
+    elif metric_name == "semantic_similarity":
+        try:
+            from sentence_transformers import SentenceTransformer, util
+
+            # Cache the model? Maybe pass it in via config?
+            model = SentenceTransformer("all-MiniLM-L6-v2")
+
+            def calculate_similarity(pred, act, **kw):
+                emb1 = model.encode(str(pred), convert_to_tensor=True)
+                emb2 = model.encode(str(act), convert_to_tensor=True)
+                return util.pytorch_cos_sim(emb1, emb2).item()
+
+            return calculate_similarity
+        except ImportError:
+            logger_helpers.warning(
+                "semantic_similarity requires 'sentence-transformers': pip install sentence-transformers"
+            )
+            return lambda p, a, **kw: None
+    # Add bleu, f1 etc.
+    elif metric_name == "llm_judge":
+        # This is handled by checking type in calculate_evaluation_metrics
+        # but we need a placeholder callable here if we map by string first
+        return lambda p, a, **kw: "[LLM Judge Not Implemented Directly]"
+    else:
+        raise ValueError(f"Unknown built-in metric: {metric_name}")
+
+
+def aggregate_results(results_list: list[dict[str, Any]]) -> dict[str, Any]:
+    """Aggregates evaluation results across all items."""
+    summary = {"total_items": len(results_list), "errors": 0}
+    metric_values: dict[str, list[float | bool]] = {}
+
+    for item in results_list:
+        if item.get("error"):
+            summary["errors"] += 1
+        metrics = item.get("metrics", {})
+        for name, value in metrics.items():
+            if isinstance(
+                value, (float, int, bool)
+            ):  # Only aggregate numerics/bools
+                if name not in metric_values:
+                    metric_values[name] = []
+                metric_values[name].append(value)
+
+    summary["metrics_summary"] = {}
+    for name, values in metric_values.items():
+        if not values:
+            continue
+        # Calculate different stats based on value type
+        if all(isinstance(v, bool) for v in values):
+            summary["metrics_summary"][name] = {
+                "accuracy": sum(values) / len(values)
+            }
+        elif all(isinstance(v, (int, float)) for v in values):
+            numeric_values = [v for v in values if isinstance(v, (int, float))]
+            if numeric_values:
+                summary["metrics_summary"][name] = {
+                    "mean": sum(numeric_values) / len(numeric_values),
+                    "count": len(numeric_values),
+                    # Add min, max, stddev if needed
+                }
+
+    return summary
+
+
+# --- Placeholder for async judge execution ---
+# Need to run these within the main async context or manage loops carefully
+async def _run_judge_agent(judge_agent, predicted, expected, config):
+    # Prepare input for the judge agent based on its signature
+    # E.g., judge_input = {"prediction": predicted_value, "reference": expected_value, "criteria": ...}
+    # judge_result = await judge_agent.run_async(judge_input)
+    # return judge_result # Or extract specific score/judgement
+    return "[Agent Judge Not Implemented]"
+
+
+async def _run_judge_evaluator(judge_evaluator, predicted, expected, config):
+    # Prepare input for the judge evaluator based on its signature
+    # judge_input = {"prediction": predicted_value, "reference": expected_value, **config}
+    # judge_result = await judge_evaluator.evaluate(None, judge_input, []) # Agent might not be needed
+    # return judge_result # Or extract specific score/judgement
+    return "[Evaluator Judge Not Implemented]"

flock/core/execution/batch_executor.py

@@ -0,0 +1,325 @@
+import asyncio
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from box import Box
+from opentelemetry import trace
+from pandas import DataFrame
+from rich.progress import (  # Import Rich Progress
+    BarColumn,
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+
+from flock.config import TELEMETRY
+from flock.core.context.context import FlockContext
+from flock.core.context.context_vars import FLOCK_BATCH_SILENT_MODE
+from flock.core.flock_agent import FlockAgent
+from flock.core.logging.logging import get_logger
+
+try:
+    import pandas as pd
+
+    PANDAS_AVAILABLE = True
+except ImportError:
+    pd = None
+    PANDAS_AVAILABLE = False
+
+if TYPE_CHECKING:
+    from flock.core.flock import Flock
+
+logger = get_logger("flock")
+TELEMETRY.setup_tracing()  # Setup OpenTelemetry
+tracer = trace.get_tracer(__name__)
+
+
+class BatchProcessor:
+    def __init__(self, flock_instance: "Flock"):
+        self.flock = flock_instance
+
+    async def run_batch_async(
+        self,
+        start_agent: FlockAgent | str,
+        batch_inputs: list[dict[str, Any]] | DataFrame | str,
+        input_mapping: dict[str, str] | None = None,
+        static_inputs: dict[str, Any] | None = None,
+        parallel: bool = True,
+        max_workers: int = 5,
+        use_temporal: bool | None = None,
+        box_results: bool = True,
+        return_errors: bool = False,
+        silent_mode: bool = False,
+        write_to_csv: str | None = None,
+    ) -> list[Box | dict | None | Exception]:
+        """Runs the specified agent/workflow for each item in a batch asynchronously.
+
+        Args:
+            start_agent: Agent instance or name to start each run.
+            batch_inputs: Input data in one of these forms:
+                - List of dictionaries, each representing inputs for one run
+                - Pandas DataFrame where each row is inputs for one run
+                - String path to a CSV file to load as DataFrame
+            input_mapping: Maps DataFrame/CSV column names to agent input keys (required for DataFrame/CSV).
+            static_inputs: Dictionary of inputs constant across all batch runs.
+            parallel: Whether to run local jobs in parallel (ignored if use_temporal=True).
+            max_workers: Max concurrent local workers (used if parallel=True and use_temporal=False).
+            use_temporal: Override Flock's 'enable_temporal' setting for this batch.
+            box_results: Wrap successful dictionary results in Box objects.
+            return_errors: If True, return Exception objects for failed runs instead of raising.
+            silent_mode: If True, suppress output and show progress bar instead.
+            write_to_csv: Path to save results as CSV file.
+
+        Returns:
+            List containing results (Box/dict), None (if error and not return_errors),
+            or Exception objects (if error and return_errors). Order matches input.
+
+        Raises:
+            ValueError: For invalid input combinations.
+            ImportError: If DataFrame/CSV used without pandas.
+            Exception: First exception from a run if return_errors is False.
+        """
+        effective_use_temporal = (
+            use_temporal if use_temporal is not None else self.enable_temporal
+        )
+        exec_mode = (
+            "Temporal"
+            if effective_use_temporal
+            else ("Parallel Local" if parallel else "Sequential Local")
+        )
+        logger.info(
+            f"Starting batch run for agent '{start_agent}'. Execution: {exec_mode}, Silent: {silent_mode}"
+        )
+
+        # --- Input Preparation ---
+        prepared_batch_inputs: list[dict[str, Any]] = []
+
+        if input_mapping == {}:
+            input_mapping = None
+        if static_inputs == {}:
+            static_inputs = None
+
+        if isinstance(batch_inputs, str):
+            # Handle CSV file input
+            try:
+                df = pd.read_csv(batch_inputs)
+                logger.debug(
+                    f"Loaded CSV file with {len(df)} rows: {batch_inputs}"
+                )
+                batch_inputs = df  # Convert to DataFrame for unified handling
+            except Exception as e:
+                raise ValueError(
+                    f"Failed to load CSV file '{batch_inputs}': {e}"
+                )
+
+        if isinstance(batch_inputs, DataFrame):
+            # Handle DataFrame input
+            logger.debug(
+                f"Converting DataFrame ({len(batch_inputs)} rows) to batch inputs."
+            )
+            for _, row in batch_inputs.iterrows():
+                if input_mapping:
+                    item_input = {
+                        agent_key: row[df_col]
+                        for df_col, agent_key in input_mapping.items()
+                        if df_col in row
+                    }
+                else:
+                    item_input = row.to_dict()
+                prepared_batch_inputs.append(item_input)
+        else:
+            # Handle list of dictionaries
+            if not isinstance(batch_inputs, list):
+                raise ValueError(
+                    "batch_inputs must be a list of dictionaries, DataFrame, or CSV file path"
+                )
+
+            if input_mapping:
+                # Apply mapping to dictionary inputs
+                logger.debug("Applying input mapping to dictionary inputs")
+                for item in batch_inputs:
+                    mapped_input = {}
+                    for df_col, agent_key in input_mapping.items():
+                        if df_col in item:
+                            mapped_input[agent_key] = item[df_col]
+                        else:
+                            logger.warning(
+                                f"Input mapping key '{df_col}' not found in input dictionary"
+                            )
+                    prepared_batch_inputs.append(mapped_input)
+            else:
+                # Use dictionaries as-is if no mapping provided
+                prepared_batch_inputs = batch_inputs
+
+            logger.debug(
+                f"Using provided list of {len(prepared_batch_inputs)} batch inputs."
+            )
+
+        if not prepared_batch_inputs:
+            return []
+
+        # --- Setup Progress Bar if Silent ---
+        progress_context = None
+        progress_task_id = None
+        if silent_mode:
+            progress = Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+                TextColumn("({task.completed}/{task.total})"),
+                TimeElapsedColumn(),
+                # transient=True # Optionally remove progress bar when done
+            )
+            progress_context = progress  # Use as context manager
+            progress_task_id = progress.add_task(
+                f"Processing Batch ({exec_mode})",
+                total=len(prepared_batch_inputs),
+            )
+            progress.start()
+
+        results = [None] * len(
+            prepared_batch_inputs
+        )  # Pre-allocate results list
+        tasks = []
+        semaphore = asyncio.Semaphore(
+            max_workers if parallel and not effective_use_temporal else 1
+        )  # Semaphore for parallel local
+
+        async def worker(index, item_inputs):
+            async with semaphore:
+                full_input = {**(static_inputs or {}), **item_inputs}
+                context = FlockContext()
+                context.set_variable(FLOCK_BATCH_SILENT_MODE, silent_mode)
+
+                run_desc = f"Batch item {index + 1}"
+                logger.debug(f"{run_desc} started.")
+                try:
+                    result = await self.run_async(
+                        start_agent,
+                        full_input,
+                        box_result=box_results,
+                        context=context,
+                    )
+                    results[index] = result
+                    logger.debug(f"{run_desc} finished successfully.")
+                except Exception as e:
+                    logger.error(
+                        f"{run_desc} failed: {e}", exc_info=not return_errors
+                    )
+                    if return_errors:
+                        results[index] = e
+                    else:
+                        # If not returning errors, ensure the exception propagates
+                        # to stop asyncio.gather if running in parallel.
+                        if parallel and not effective_use_temporal:
+                            raise  # Re-raise to stop gather
+                        else:
+                            # For sequential, we just store None or the exception if return_errors=True
+                            # For Temporal, error handling happens within the workflow/activity usually
+                            results[index] = e if return_errors else None
+                finally:
+                    if progress_context:
+                        progress.update(
+                            progress_task_id, advance=1
+                        )  # Update progress
+
+        try:
+            if effective_use_temporal:
+                # Temporal Batching (Simplified: sequential execution for this example)
+                # A real implementation might use start_workflow or signals
+                logger.info(
+                    "Running batch using Temporal (executing sequentially for now)..."
+                )
+                for i, item_data in enumerate(prepared_batch_inputs):
+                    await worker(i, item_data)  # Run sequentially for demo
+                # TODO: Implement true parallel Temporal workflow execution if needed
+
+            elif parallel:
+                logger.info(
+                    f"Running batch in parallel with max_workers={max_workers}..."
+                )
+                for i, item_data in enumerate(prepared_batch_inputs):
+                    tasks.append(asyncio.create_task(worker(i, item_data)))
+                await asyncio.gather(
+                    *tasks
+                )  # gather handles exceptions based on return_errors logic in worker
+
+            else:  # Sequential Local
+                logger.info("Running batch sequentially...")
+                for i, item_data in enumerate(prepared_batch_inputs):
+                    await worker(
+                        i, item_data
+                    )  # Already handles errors internally based on return_errors
+
+            logger.info("Batch execution finished.")
+
+        except Exception as batch_error:
+            # This catch handles errors re-raised from workers when return_errors=False
+            logger.error(f"Batch execution stopped due to error: {batch_error}")
+            # No need to cancel tasks here as gather would have stopped
+            if not return_errors:
+                raise  # Re-raise the first error encountered if not returning errors
+        finally:
+            if progress_context:
+                progress.stop()
+
+        if write_to_csv:
+            try:
+                df = pd.DataFrame(results)
+                # create write_to_csv directory if it doesn't exist
+                Path(write_to_csv).parent.mkdir(parents=True, exist_ok=True)
+                df.to_csv(write_to_csv, index=False)
+                logger.info(f"Results written to CSV file: {write_to_csv}")
+            except Exception as e:
+                logger.error(f"Failed to write results to CSV: {e}")
+
+        return results
+
+    def run_batch(  # Synchronous wrapper
+        self,
+        start_agent: FlockAgent | str,
+        batch_inputs: list[dict[str, Any]] | DataFrame | str,
+        input_mapping: dict[str, str] | None = None,
+        static_inputs: dict[str, Any] | None = None,
+        parallel: bool = True,
+        max_workers: int = 5,
+        use_temporal: bool | None = None,
+        box_results: bool = True,
+        return_errors: bool = False,
+        silent_mode: bool = False,
+        write_to_csv: str | None = None,
+    ) -> list[Box | dict | None | Exception]:
+        """Synchronous wrapper for run_batch_async."""
+        # (Standard asyncio run wrapper - same as in previous suggestion)
+        try:
+            loop = asyncio.get_running_loop()
+            if loop.is_closed():
+                raise RuntimeError("Event loop is closed")
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+        coro = self.run_batch_async(
+            start_agent=start_agent,
+            batch_inputs=batch_inputs,
+            input_mapping=input_mapping,
+            static_inputs=static_inputs,
+            parallel=parallel,
+            max_workers=max_workers,
+            use_temporal=use_temporal,
+            box_results=box_results,
+            return_errors=return_errors,
+            silent_mode=silent_mode,
+            write_to_csv=write_to_csv,
+        )
+
+        if asyncio.get_event_loop() is loop and not loop.is_running():
+            results = loop.run_until_complete(coro)
+            # loop.close() # Avoid closing potentially shared loop
+            return results
+        else:
+            # Run within an existing loop
+            future = asyncio.ensure_future(coro)
+            return loop.run_until_complete(future)