sdg-hub 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

sdg_hub/__init__.py

@@ -8,7 +8,6 @@ from .core import (
     BlockRegistry,
     Flow,
     FlowMetadata,
-    FlowParameter,
     FlowRegistry,
     FlowValidator,
     GenerateError,
@@ -23,7 +22,6 @@ __all__ = [
     "FlowRegistry",
     # Metadata and utilities
     "FlowMetadata",
-    "FlowParameter",
     "FlowValidator",
     "GenerateError",
     "resolve_path",

sdg_hub/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.4.0'
-__version_tuple__ = version_tuple = (0, 4, 0)
+__version__ = version = '0.4.2'
+__version_tuple__ = version_tuple = (0, 4, 2)
 
 __commit_id__ = commit_id = None

sdg_hub/core/__init__.py

@@ -3,7 +3,7 @@
 
 # Local
 from .blocks import BaseBlock, BlockRegistry
-from .flow import Flow, FlowMetadata, FlowParameter, FlowRegistry, FlowValidator
+from .flow import Flow, FlowMetadata, FlowRegistry, FlowValidator
 from .utils import GenerateError, resolve_path
 
 __all__ = [
@@ -14,7 +14,6 @@ __all__ = [
     "Flow",
     "FlowRegistry",
     "FlowMetadata",
-    "FlowParameter",
     "FlowValidator",
     # Utils
     "GenerateError",

sdg_hub/core/flow/__init__.py

@@ -1,20 +1,19 @@
 # SPDX-License-Identifier: Apache-2.0
 """New flow implementation for SDG Hub.
 
-This module provides a redesigned Flow class with metadata support,
-dual initialization modes, and runtime parameter overrides.
+This module provides a redesigned Flow class with metadata support
+and dual initialization modes.
 """
 
 # Local
 from .base import Flow
-from .metadata import FlowMetadata, FlowParameter
+from .metadata import FlowMetadata
 from .registry import FlowRegistry
 from .validation import FlowValidator
 
 __all__ = [
     "Flow",
     "FlowMetadata",
-    "FlowParameter",
     "FlowRegistry",
     "FlowValidator",
 ]

sdg_hub/core/flow/base.py

@@ -30,12 +30,17 @@ from ..blocks.base import BaseBlock
 from ..blocks.registry import BlockRegistry
 from ..utils.datautils import safe_concatenate_with_validation, validate_no_duplicates
 from ..utils.error_handling import EmptyDatasetError, FlowValidationError
-from ..utils.flow_metrics import display_metrics_summary, save_metrics_to_json
+from ..utils.flow_metrics import (
+    display_metrics_summary,
+    display_time_estimation_summary,
+    save_metrics_to_json,
+)
 from ..utils.logger_config import setup_logger
 from ..utils.path_resolution import resolve_path
+from ..utils.time_estimator import estimate_execution_time
 from ..utils.yaml_utils import save_flow_yaml
 from .checkpointer import FlowCheckpointer
-from .metadata import DatasetRequirements, FlowMetadata, FlowParameter
+from .metadata import DatasetRequirements, FlowMetadata
 from .migration import FlowMigration
 from .validation import FlowValidator
 
@@ -55,8 +60,6 @@ class Flow(BaseModel):
         Ordered list of blocks to execute in the flow.
     metadata : FlowMetadata
         Flow metadata including name, version, author, etc.
-    parameters : Dict[str, FlowParameter]
-        Runtime parameters that can be overridden during execution.
     """
 
     blocks: list[BaseBlock] = Field(
@@ -66,10 +69,6 @@ class Flow(BaseModel):
     metadata: FlowMetadata = Field(
         description="Flow metadata including name, version, author, etc."
     )
-    parameters: dict[str, FlowParameter] = Field(
-        default_factory=dict,
-        description="Runtime parameters that can be overridden during execution",
-    )
 
     model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
 
@@ -96,32 +95,6 @@ class Flow(BaseModel):
 
         return v
 
-    @field_validator("parameters")
-    @classmethod
-    def validate_parameters(
-        cls, v: dict[str, FlowParameter]
-    ) -> dict[str, FlowParameter]:
-        """Validate parameter names and ensure they are FlowParameter instances."""
-        if not v:
-            return v
-
-        validated = {}
-        for param_name, param_value in v.items():
-            if not isinstance(param_name, str) or not param_name.strip():
-                raise ValueError(
-                    f"Parameter name must be a non-empty string: {param_name}"
-                )
-
-            if not isinstance(param_value, FlowParameter):
-                raise ValueError(
-                    f"Parameter '{param_name}' must be a FlowParameter instance, "
-                    f"got: {type(param_value)}"
-                )
-
-            validated[param_name.strip()] = param_value
-
-        return validated
-
     @model_validator(mode="after")
     def validate_block_names_unique(self) -> "Flow":
         """Ensure all block names are unique within the flow."""
@@ -215,17 +188,6 @@ class Flow(BaseModel):
         except Exception as exc:
             raise FlowValidationError(f"Invalid metadata configuration: {exc}") from exc
 
-        # Extract and validate parameters
-        parameters = {}
-        params_dict = flow_config.get("parameters", {})
-        for param_name, param_config in params_dict.items():
-            try:
-                parameters[param_name] = FlowParameter(**param_config)
-            except Exception as exc:
-                raise FlowValidationError(
-                    f"Invalid parameter '{param_name}': {exc}"
-                ) from exc
-
         # Create blocks with validation
         blocks = []
         block_configs = flow_config.get("blocks", [])
@@ -254,7 +216,7 @@ class Flow(BaseModel):
 
         # Create and validate the flow
         try:
-            flow = cls(blocks=blocks, metadata=metadata, parameters=parameters)
+            flow = cls(blocks=blocks, metadata=metadata)
             # Persist generated id back to the YAML file (only on initial load)
             # If the file had no metadata.id originally, update and rewrite
             if not flow_config.get("metadata", {}).get("id"):
@@ -1049,6 +1011,8 @@ class Flow(BaseModel):
         dataset: Dataset,
         sample_size: int = 2,
         runtime_params: Optional[dict[str, dict[str, Any]]] = None,
+        max_concurrency: Optional[int] = None,
+        enable_time_estimation: bool = False,
     ) -> dict[str, Any]:
         """Perform a dry run of the flow with a subset of data.
 
@@ -1060,11 +1024,18 @@ class Flow(BaseModel):
             Number of samples to use for dry run testing.
         runtime_params : Optional[Dict[str, Dict[str, Any]]], optional
             Runtime parameters organized by block name.
+        max_concurrency : Optional[int], optional
+            Maximum concurrent requests for LLM blocks. If None, no limit is applied.
+        enable_time_estimation : bool, default=False
+            If True, estimates execution time for the full dataset and displays it
+            in a Rich table. Automatically runs a second dry run if needed for
+            accurate scaling analysis.
 
         Returns
         -------
         Dict[str, Any]
             Dry run results with execution info and sample outputs.
+            Time estimation is displayed in a table but not included in return value.
 
         Raises
         ------
@@ -1082,6 +1053,19 @@ class Flow(BaseModel):
 
         validate_no_duplicates(dataset)
 
+        # Validate max_concurrency parameter
+        if max_concurrency is not None:
+            if isinstance(max_concurrency, bool) or not isinstance(
+                max_concurrency, int
+            ):
+                raise FlowValidationError(
+                    f"max_concurrency must be an int, got {type(max_concurrency).__name__}"
+                )
+            if max_concurrency <= 0:
+                raise FlowValidationError(
+                    f"max_concurrency must be greater than 0, got {max_concurrency}"
+                )
+
         # Use smaller sample size if dataset is smaller
         actual_sample_size = min(sample_size, len(dataset))
 
@@ -1099,6 +1083,7 @@ class Flow(BaseModel):
             "flow_version": self.metadata.version,
             "sample_size": actual_sample_size,
             "original_dataset_size": len(dataset),
+            "max_concurrency": max_concurrency,
             "input_columns": dataset.column_names,
             "blocks_executed": [],
             "final_dataset": None,
@@ -1125,6 +1110,10 @@ class Flow(BaseModel):
                 # Prepare block execution parameters
                 block_kwargs = self._prepare_block_kwargs(block, runtime_params)
 
+                # Add max_concurrency to block kwargs if provided
+                if max_concurrency is not None:
+                    block_kwargs["_flow_max_concurrency"] = max_concurrency
+
                 # Check if this is a deprecated block and skip validations
                 is_deprecated_block = (
                     hasattr(block, "__class__")
@@ -1142,7 +1131,9 @@ class Flow(BaseModel):
                     # Execute block with validation and logging
                     current_dataset = block(current_dataset, **block_kwargs)
 
-                block_execution_time = time.time() - block_start_time
+                block_execution_time = (
+                    time.perf_counter() - block_start_time
+                )  # Fixed: use perf_counter consistently
 
                 # Record block execution info
                 block_info = {
@@ -1181,6 +1172,12 @@ class Flow(BaseModel):
                 f"in {execution_time:.2f}s"
             )
 
+            # Perform time estimation if requested (displays table but doesn't store in results)
+            if enable_time_estimation:
+                self._estimate_total_time(
+                    dry_run_results, dataset, runtime_params, max_concurrency
+                )
+
             return dry_run_results
 
         except Exception as exc:
@@ -1193,6 +1190,103 @@ class Flow(BaseModel):
 
             raise FlowValidationError(f"Dry run failed: {exc}") from exc
 
+    def _estimate_total_time(
+        self,
+        first_run_results: dict[str, Any],
+        dataset: Dataset,
+        runtime_params: Optional[dict[str, dict[str, Any]]],
+        max_concurrency: Optional[int],
+    ) -> dict[str, Any]:
+        """Estimate execution time using 2 dry runs (private method).
+
+        This method contains all the estimation logic. It determines if a second
+        dry run is needed, executes it, and calls estimate_execution_time.
+
+        Parameters
+        ----------
+        first_run_results : dict
+            Results from the first dry run.
+        dataset : Dataset
+            Full dataset for estimation.
+        runtime_params : Optional[dict]
+            Runtime parameters.
+        max_concurrency : Optional[int]
+            Maximum concurrency.
+
+        Returns
+        -------
+        dict
+            Estimation results with estimated_time_seconds, total_estimated_requests, etc.
+        """
+        first_sample_size = first_run_results["sample_size"]
+
+        # Check if we need a second dry run
+        has_async_blocks = any(
+            getattr(block, "async_mode", False) for block in self.blocks
+        )
+
+        # For sequential or no async blocks, single run is sufficient
+        if max_concurrency == 1 or not has_async_blocks:
+            estimation = estimate_execution_time(
+                dry_run_1=first_run_results,
+                dry_run_2=None,
+                total_dataset_size=len(dataset),
+                max_concurrency=max_concurrency,
+            )
+        else:
+            # Need second measurement - always use canonical (1, 5) pair
+            if first_sample_size == 1:
+                # Already have 1, need 5
+                logger.info("Running second dry run with 5 samples for time estimation")
+                second_run = self.dry_run(
+                    dataset,
+                    5,
+                    runtime_params,
+                    max_concurrency,
+                    enable_time_estimation=False,
+                )
+                dry_run_1, dry_run_2 = first_run_results, second_run
+            elif first_sample_size == 5:
+                # Already have 5, need 1
+                logger.info("Running second dry run with 1 sample for time estimation")
+                second_run = self.dry_run(
+                    dataset,
+                    1,
+                    runtime_params,
+                    max_concurrency,
+                    enable_time_estimation=False,
+                )
+                dry_run_1, dry_run_2 = second_run, first_run_results
+            else:
+                # For other sizes: run both 1 and 5 for canonical pair
+                logger.info("Running dry runs with 1 and 5 samples for time estimation")
+                dry_run_1 = self.dry_run(
+                    dataset,
+                    1,
+                    runtime_params,
+                    max_concurrency,
+                    enable_time_estimation=False,
+                )
+                dry_run_2 = self.dry_run(
+                    dataset,
+                    5,
+                    runtime_params,
+                    max_concurrency,
+                    enable_time_estimation=False,
+                )
+
+            estimation = estimate_execution_time(
+                dry_run_1=dry_run_1,
+                dry_run_2=dry_run_2,
+                total_dataset_size=len(dataset),
+                max_concurrency=max_concurrency,
+            )
+
+        # Display estimation summary
+        display_time_estimation_summary(estimation, len(dataset), max_concurrency)
+
+        return estimation
+
     def add_block(self, block: BaseBlock) -> "Flow":
         """Add a block to the flow, returning a new Flow instance.
 
@@ -1225,17 +1319,12 @@ class Flow(BaseModel):
         # Create new flow with added block
         new_blocks = self.blocks + [block]
 
-        return Flow(
-            blocks=new_blocks, metadata=self.metadata, parameters=self.parameters
-        )
+        return Flow(blocks=new_blocks, metadata=self.metadata)
 
     def get_info(self) -> dict[str, Any]:
         """Get information about the flow."""
         return {
             "metadata": self.metadata.model_dump(),
-            "parameters": {
-                name: param.model_dump() for name, param in self.parameters.items()
-            },
             "blocks": [
                 {
                     "block_type": block.__class__.__name__,
@@ -1339,8 +1428,7 @@ class Flow(BaseModel):
 
         The summary contains:
         1. Flow metadata (name, version, author, description)
-        2. Defined runtime parameters with type hints and defaults
-        3. A table of all blocks with their input and output columns
+        2. A table of all blocks with their input and output columns
 
         Notes
         -----
@@ -1374,17 +1462,6 @@ class Flow(BaseModel):
                 f"Description: [white]{self.metadata.description}[/white]"
             )
 
-        # Parameters section
-        if self.parameters:
-            params_branch = flow_tree.add(
-                "[bold bright_yellow]Parameters[/bold bright_yellow]"
-            )
-            for name, param in self.parameters.items():
-                param_info = f"[bright_cyan]{name}[/bright_cyan]: [white]{param.type_hint}[/white]"
-                if param.default is not None:
-                    param_info += f" = [bright_white]{param.default}[/bright_white]"
-                params_branch.add(param_info)
-
         # Blocks overview
         flow_tree.add(
             f"[bold bright_magenta]Blocks[/bold bright_magenta] ({len(self.blocks)} total)"
@@ -1446,11 +1523,6 @@ class Flow(BaseModel):
             ],
         }
 
-        if self.parameters:
-            config["parameters"] = {
-                name: param.model_dump() for name, param in self.parameters.items()
-            }
-
         save_flow_yaml(output_path, config)
 
     def __len__(self) -> int:

sdg_hub/core/flow/metadata.py

@@ -2,9 +2,8 @@
 """Flow metadata and parameter definitions."""
 
 # Standard
-from datetime import datetime
 from enum import Enum
-from typing import Any, Optional
+from typing import Optional
 
 # Third Party
 from pydantic import BaseModel, Field, field_validator, model_validator
@@ -118,39 +117,6 @@ class RecommendedModels(BaseModel):
         return None
 
 
-class FlowParameter(BaseModel):
-    """Represents a runtime parameter for a flow.
-
-    Attributes
-    ----------
-    default : Any
-        Default value for the parameter.
-    description : str
-        Human-readable description of the parameter.
-    type_hint : str
-        Type hint as string (e.g., "float", "str").
-    required : bool
-        Whether this parameter is required at runtime.
-    constraints : Dict[str, Any]
-        Additional constraints for the parameter.
-    """
-
-    default: Any = Field(..., description="Default value for the parameter")
-    description: str = Field(default="", description="Human-readable description")
-    type_hint: str = Field(default="Any", description="Type hint as string")
-    required: bool = Field(default=False, description="Whether parameter is required")
-    constraints: dict[str, Any] = Field(
-        default_factory=dict, description="Additional constraints for the parameter"
-    )
-
-    @model_validator(mode="after")
-    def validate_required_default(self) -> "FlowParameter":
-        """Validate that required parameters have appropriate defaults."""
-        if self.required and self.default is None:
-            raise ValueError("Required parameters cannot have None as default")
-        return self
-
-
 class DatasetRequirements(BaseModel):
     """Dataset requirements for flow execution.
 
@@ -255,20 +221,10 @@ class FlowMetadata(BaseModel):
         Simplified recommended models structure with default, compatible, and experimental lists.
     tags : List[str]
         Tags for categorization and search.
-    created_at : str
-        Creation timestamp.
-    updated_at : str
-        Last update timestamp.
     license : str
         License identifier.
-    min_sdg_hub_version : str
-        Minimum required SDG Hub version.
     dataset_requirements : Optional[DatasetRequirements]
         Requirements for input datasets.
-    estimated_cost : str
-        Estimated cost tier for running the flow.
-    estimated_duration : str
-        Estimated duration for flow execution.
     """
 
     name: str = Field(..., min_length=1, description="Human-readable name")
@@ -288,29 +244,10 @@ class FlowMetadata(BaseModel):
     tags: list[str] = Field(
         default_factory=list, description="Tags for categorization and search"
     )
-    created_at: str = Field(
-        default_factory=lambda: datetime.now().isoformat(),
-        description="Creation timestamp",
-    )
-    updated_at: str = Field(
-        default_factory=lambda: datetime.now().isoformat(),
-        description="Last update timestamp",
-    )
     license: str = Field(default="Apache-2.0", description="License identifier")
-    min_sdg_hub_version: str = Field(
-        default="", description="Minimum required SDG Hub version"
-    )
     dataset_requirements: Optional[DatasetRequirements] = Field(
         default=None, description="Requirements for input datasets"
     )
-    estimated_cost: str = Field(
-        default="medium",
-        pattern="^(low|medium|high)$",
-        description="Estimated cost tier for running the flow",
-    )
-    estimated_duration: str = Field(
-        default="", description="Estimated duration for flow execution"
-    )
 
     @field_validator("id")
     @classmethod
@@ -352,10 +289,6 @@ class FlowMetadata(BaseModel):
         # Validation is handled within RecommendedModels class
         return v
 
-    def update_timestamp(self) -> None:
-        """Update the updated_at timestamp."""
-        self.updated_at = datetime.now().isoformat()
-
     @model_validator(mode="after")
     def ensure_id(self) -> "FlowMetadata":
         """Ensure id is set.

sdg_hub/core/flow/registry.py

@@ -360,7 +360,6 @@ class FlowRegistry:
                     "tags": ", ".join(metadata.tags) if metadata.tags else "-",
                     "description": metadata.description or "No description",
                     "version": metadata.version,
-                    "cost": metadata.estimated_cost,
                 }
             )
 

sdg_hub/core/utils/__init__.py

@@ -1,8 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Local
-from .flow_identifier import get_flow_identifier
-from .path_resolution import resolve_path
+from .flow_identifier import get_flow_identifier as get_flow_identifier
+from .path_resolution import resolve_path as resolve_path
+from .time_estimator import estimate_execution_time as estimate_execution_time
+from .time_estimator import is_llm_using_block as is_llm_using_block
 
 
 # This is part of the public API, and used by instructlab
@@ -10,4 +12,10 @@ class GenerateError(Exception):
     """An exception raised during generate step."""
 
 
-__all__ = ["GenerateError", "resolve_path", "get_flow_identifier"]
+__all__ = [
+    "GenerateError",
+    "resolve_path",
+    "get_flow_identifier",
+    "estimate_execution_time",
+    "is_llm_using_block",
+]

sdg_hub/core/utils/flow_metrics.py

@@ -188,6 +188,122 @@ def display_metrics_summary(
     console.print()
 
 
+def display_time_estimation_summary(
+    time_estimation: dict[str, Any],
+    dataset_size: int,
+    max_concurrency: Optional[int] = None,
+) -> None:
+    """Display a rich table summarizing time estimation results.
+
+    Parameters
+    ----------
+    time_estimation : dict[str, Any]
+        Time estimation results from estimate_total_time().
+    dataset_size : int
+        Total number of samples in the dataset.
+    max_concurrency : Optional[int], optional
+        Maximum concurrency used for estimation.
+    """
+    console = Console()
+
+    # Create main summary table
+    summary_table = Table(
+        show_header=False,
+        box=None,
+        padding=(0, 1),
+    )
+    summary_table.add_column("Metric", style="bright_cyan")
+    summary_table.add_column("Value", style="bright_white")
+
+    # Format time
+    est_seconds = time_estimation["estimated_time_seconds"]
+    if est_seconds < 60:
+        time_str = f"{est_seconds:.1f} seconds"
+    elif est_seconds < 3600:
+        time_str = f"{est_seconds / 60:.1f} minutes ({est_seconds / 3600:.2f} hours)"
+    else:
+        time_str = f"{est_seconds / 3600:.2f} hours ({est_seconds / 60:.0f} minutes)"
+
+    summary_table.add_row("Estimated Time:", time_str)
+    summary_table.add_row(
+        "Total LLM Requests:", f"{time_estimation.get('total_estimated_requests', 0):,}"
+    )
+
+    if time_estimation.get("total_estimated_requests", 0) > 0:
+        requests_per_sample = time_estimation["total_estimated_requests"] / dataset_size
+        summary_table.add_row("Requests per Sample:", f"{requests_per_sample:.1f}")
+
+    if max_concurrency is not None:
+        summary_table.add_row("Max Concurrency:", str(max_concurrency))
+
+    # Display summary panel
+    console.print()
+    console.print(
+        Panel(
+            summary_table,
+            title=f"[bold bright_white]Time Estimation for {dataset_size:,} Samples[/bold bright_white]",
+            border_style="bright_blue",
+        )
+    )
+
+    # Display per-block breakdown if available
+    block_estimates = time_estimation.get("block_estimates", [])
+    if block_estimates:
+        console.print()
+
+        # Create per-block table
+        block_table = Table(
+            show_header=True,
+            header_style="bold bright_white",
+        )
+        block_table.add_column("Block Name", style="bright_cyan", width=20)
+        block_table.add_column("Time", justify="right", style="bright_yellow", width=10)
+        block_table.add_column(
+            "Requests", justify="right", style="bright_green", width=10
+        )
+        block_table.add_column(
+            "Throughput", justify="right", style="bright_blue", width=12
+        )
+        block_table.add_column(
+            "Amplif.", justify="right", style="bright_magenta", width=10
+        )
+
+        for block in block_estimates:
+            # Format time
+            block_seconds = block["estimated_time"]
+            if block_seconds < 60:
+                time_str = f"{block_seconds:.1f}s"
+            else:
+                time_str = f"{block_seconds / 60:.1f}min"
+
+            # Format requests
+            requests_str = f"{block['estimated_requests']:,.0f}"
+
+            # Format throughput
+            throughput_str = f"{block['throughput']:.2f}/s"
+
+            # Format amplification
+            amplif_str = f"{block['amplification']:.1f}x"
+
+            block_table.add_row(
+                block["block"],
+                time_str,
+                requests_str,
+                throughput_str,
+                amplif_str,
+            )
+
+        console.print(
+            Panel(
+                block_table,
+                title="[bold bright_white]Per-Block Breakdown[/bold bright_white]",
+                border_style="bright_blue",
+            )
+        )
+
+    console.print()
+
+
 def save_metrics_to_json(
     block_metrics: list[dict[str, Any]],
     flow_name: str,

sdg_hub/core/utils/time_estimator.py

@@ -0,0 +1,344 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Time estimation utility for predicting full dataset execution time from dry_run results."""
+
+# Standard
+from typing import Dict, Optional
+import math
+
+# Default max concurrent requests used during dry runs
+DRY_RUN_MAX_CONCURRENT = 100
+
+# Conservative estimation factor (20% buffer for API variability, network latency, etc.)
+ESTIMATION_BUFFER_FACTOR = 1.2
+
+
+def is_llm_using_block(block_info: Dict) -> bool:
+    """Detect if a block uses LLMs.
+
+    Identifies blocks that make LLM API calls based on their type or parameters.
+    This is used to calculate request amplification for LLM blocks.
+
+    Parameters
+    ----------
+    block_info : Dict
+        Block information from dry_run results containing block_type and parameters_used.
+
+    Returns
+    -------
+    bool
+        True if the block uses LLMs, False otherwise.
+
+    Examples
+    --------
+    >>> block = {"block_type": "LLMChatBlock", "parameters_used": {"model": "gpt-4"}}
+    >>> is_llm_using_block(block)
+    True
+    """
+    block_type = block_info.get("block_type", "")
+
+    # Direct LLM blocks or evaluation/verification blocks
+    if any(kw in block_type for kw in ["LLMChatBlock", "Evaluate", "Verify"]):
+        return True
+
+    # Check for model parameters
+    params = block_info.get("parameters_used", {})
+    if any(key in params for key in ["model", "api_base", "api_key"]):
+        return True
+
+    return False
+
+
+def calculate_block_throughput(
+    block_1: Dict, block_2: Dict, samples_1: int, samples_2: int
+) -> Dict:
+    """Calculate throughput and amplification from two dry runs.
+
+    Analyzes performance metrics from two dry runs with different sample sizes
+    to estimate throughput (requests/second), amplification factor, and startup overhead.
+
+    Parameters
+    ----------
+    block_1 : Dict
+        Block execution info from first dry run.
+    block_2 : Dict
+        Block execution info from second dry run.
+    samples_1 : int
+        Number of samples in first dry run.
+    samples_2 : int
+        Number of samples in second dry run.
+
+    Returns
+    -------
+    Dict
+        Dictionary containing:
+        - throughput: float, requests per second
+        - amplification: float, average requests per input sample
+        - startup_overhead: float, fixed startup time in seconds
+
+    Raises
+    ------
+    ValueError
+        If throughput cannot be calculated due to invalid measurements.
+
+    Examples
+    --------
+    >>> block1 = {"execution_time_seconds": 1.0, "input_rows": 1, "block_name": "test"}
+    >>> block2 = {"execution_time_seconds": 2.0, "input_rows": 5, "block_name": "test"}
+    >>> result = calculate_block_throughput(block1, block2, 1, 5)
+    >>> assert result["throughput"] > 0
+    """
+    time_1 = block_1.get("execution_time_seconds", 0)
+    time_2 = block_2.get("execution_time_seconds", 0)
+    requests_1 = block_1.get("input_rows", 0)
+    requests_2 = block_2.get("input_rows", 0)
+
+    # Calculate amplification (requests per sample)
+    amp_1 = requests_1 / samples_1 if samples_1 > 0 else 1
+    amp_2 = requests_2 / samples_2 if samples_2 > 0 else 1
+    avg_amplification = (amp_1 + amp_2) / 2
+
+    # Use linear scaling to extract throughput and overhead from two data points
+    # Model: time = startup_overhead + (requests / throughput)
+
+    if requests_2 > requests_1 and time_2 > time_1:
+        # Calculate marginal time per request (slope of the line)
+        marginal_time = (time_2 - time_1) / (requests_2 - requests_1)
+
+        # Throughput is the inverse of marginal time
+        measured_throughput = 1.0 / marginal_time if marginal_time > 0 else 0
+
+        # Y-intercept is the startup overhead
+        startup_overhead = max(0, time_1 - (requests_1 * marginal_time))
+    else:
+        # Fallback to simple calculation if we don't have good data for scaling
+        throughput_1 = requests_1 / time_1 if time_1 > 0 else 0
+        throughput_2 = requests_2 / time_2 if time_2 > 0 else 0
+        measured_throughput = max(throughput_1, throughput_2)
+
+        # Estimate overhead as a small fraction of time
+        startup_overhead = min(2.0, time_1 * 0.1)  # Assume 10% overhead, max 2 seconds
+
+    # If we have no valid measurements, raise an error
+    if measured_throughput == 0:
+        raise ValueError(
+            f"Cannot calculate throughput for block '{block_1.get('block_name', 'unknown')}': "
+            f"No valid measurements from dry runs (time_1={time_1}, time_2={time_2}, "
+            f"requests_1={requests_1}, requests_2={requests_2})"
+        )
+
+    return {
+        "throughput": measured_throughput,
+        "amplification": avg_amplification,
+        "startup_overhead": startup_overhead,
+    }
+
+
+def calculate_time_with_pipeline(
+    num_requests: float,
+    throughput: float,
+    startup_overhead: float,
+    max_concurrent: int = DRY_RUN_MAX_CONCURRENT,
+) -> float:
+    """Calculate time considering pipeline behavior and max concurrent limit.
+
+    Models the execution time for a given number of requests based on throughput,
+    startup overhead, and concurrency constraints. Applies non-linear scaling
+    for diminishing returns at high concurrency levels.
+
+    Parameters
+    ----------
+    num_requests : float
+        Total number of requests to process.
+    throughput : float
+        Base throughput in requests per second.
+    startup_overhead : float
+        Fixed startup time overhead in seconds.
+    max_concurrent : int, optional
+        Maximum number of concurrent requests, by default 100.
+
+    Returns
+    -------
+    float
+        Estimated total execution time in seconds.
+
+    Examples
+    --------
+    >>> time = calculate_time_with_pipeline(1000, 10.0, 0.5, 50)
+    >>> assert time > 0
+    """
+    if num_requests <= 0:
+        return 0
+
+    # Validate and clamp max_concurrent to avoid division by zero
+    if max_concurrent is None or max_concurrent <= 0:
+        max_concurrent = 1
+
+    # The throughput is what we measured - it represents the server's processing capability
+    if max_concurrent == 1:
+        # Sequential execution - no pipelining benefit
+        effective_throughput = throughput
+    else:
+        # Concurrent execution - small pipelining benefit
+        # At most 10% improvement from perfect pipelining (conservative estimate)
+        # Logarithmic growth to model diminishing returns
+        pipelining_factor = 1.0 + (0.1 * math.log(max_concurrent) / math.log(100))
+        pipelining_factor = min(pipelining_factor, 1.1)  # Cap at 10% improvement
+        effective_throughput = throughput * pipelining_factor
+
+    # Calculate total time
+    base_time = startup_overhead + (num_requests / effective_throughput)
+
+    return base_time
+
+
+def estimate_execution_time(
+    dry_run_1: Dict,
+    dry_run_2: Optional[Dict] = None,
+    total_dataset_size: Optional[int] = None,
+    max_concurrency: Optional[int] = None,
+) -> Dict:
+    """Estimate execution time based on dry run results.
+
+    Estimates the total execution time for a full dataset based on one or two
+    dry runs with smaller sample sizes. For async blocks (with two dry runs),
+    calculates throughput and concurrency benefits. For sync blocks (single dry run),
+    performs simple linear scaling.
+
+    The estimates include a conservative buffer (20%) to account for API variability,
+    network latency, and other real-world factors.
+
+    Parameters
+    ----------
+    dry_run_1 : Dict
+        Results from first dry run, must contain 'sample_size' and 'execution_time_seconds'.
+    dry_run_2 : Optional[Dict], optional
+        Results from second dry run for async estimation, by default None.
+    total_dataset_size : Optional[int], optional
+        Size of full dataset to estimate for. If None, uses original_dataset_size from dry_run_1.
+    max_concurrency : Optional[int], optional
+        Maximum concurrent requests allowed, by default 100.
+
+    Returns
+    -------
+    Dict
+        Estimation results containing:
+        - estimated_time_seconds: float, estimated time with current configuration (includes buffer)
+        - total_estimated_requests: int, total LLM requests (0 for sync blocks)
+        - block_estimates: list, per-block estimates (for async blocks)
+        - note: str, additional information about the estimation
+
+    Examples
+    --------
+    >>> dry_run = {"sample_size": 2, "execution_time_seconds": 10.0}
+    >>> result = estimate_execution_time(dry_run, total_dataset_size=100)
+    >>> assert result["estimated_time_seconds"] > 0
+    >>>
+    >>> # With two dry runs for async estimation
+    >>> dry_run_1 = {"sample_size": 1, "execution_time_seconds": 5.0, "blocks_executed": [...]}
+    >>> dry_run_2 = {"sample_size": 5, "execution_time_seconds": 20.0, "blocks_executed": [...]}
+    >>> result = estimate_execution_time(dry_run_1, dry_run_2, total_dataset_size=1000)
+    >>> assert result["estimated_time_seconds"] > 0
+    """
+    # Set defaults
+    if max_concurrency is None:
+        max_concurrency = DRY_RUN_MAX_CONCURRENT
+
+    if total_dataset_size is None:
+        total_dataset_size = dry_run_1.get(
+            "original_dataset_size", dry_run_1["sample_size"]
+        )
+
+    # Get sample sizes
+    samples_1 = dry_run_1["sample_size"]
+    samples_2 = (
+        dry_run_2["sample_size"] if dry_run_2 else 5
+    )  # Default to 5 if not provided
+
+    # If only one dry run, do simple scaling
+    if dry_run_2 is None:
+        # Process each block individually for synchronous execution
+        blocks_executed = dry_run_1.get("blocks_executed", [])
+        if not blocks_executed:
+            # Fallback to simple scaling if no block details available
+            total_time = dry_run_1["execution_time_seconds"]
+            simple_estimate = (total_time / samples_1) * total_dataset_size
+            # Apply conservative buffer
+            simple_estimate = simple_estimate * ESTIMATION_BUFFER_FACTOR
+            return {
+                "estimated_time_seconds": simple_estimate,
+                "total_estimated_requests": 0,
+                "note": "Synchronous execution - linear scaling from dry run",
+            }
+
+        # Calculate time for each block and sum them
+        total_estimated_time = 0
+        for block in blocks_executed:
+            block_time = block.get("execution_time_seconds", 0)
+            input_rows = block.get("input_rows", samples_1)
+
+            # Calculate time per row for this block
+            if input_rows > 0:
+                time_per_row = block_time / input_rows
+                block_total_time = time_per_row * total_dataset_size
+                total_estimated_time += block_total_time
+
+        # Apply conservative buffer
+        total_estimated_time = total_estimated_time * ESTIMATION_BUFFER_FACTOR
+        return {
+            "estimated_time_seconds": total_estimated_time,
+            "total_estimated_requests": 0,
+            "note": "Synchronous execution - no concurrency",
+        }
+
+    # Analyze each block with async execution
+    block_estimates = []
+    total_time = 0
+    total_requests = 0
+
+    # Process each block
+    for i, block_1 in enumerate(dry_run_1.get("blocks_executed", [])):
+        if i >= len(dry_run_2.get("blocks_executed", [])):
+            break
+
+        block_2 = dry_run_2["blocks_executed"][i]
+
+        # Only process LLM blocks
+        if not is_llm_using_block(block_1):
+            continue
+
+        # Calculate throughput and amplification
+        analysis = calculate_block_throughput(block_1, block_2, samples_1, samples_2)
+
+        # Estimate requests for full dataset
+        estimated_requests = total_dataset_size * analysis["amplification"]
+
+        # Calculate time with pipeline model
+        block_time = calculate_time_with_pipeline(
+            estimated_requests,
+            analysis["throughput"],
+            analysis["startup_overhead"],
+            max_concurrency,
+        )
+
+        total_time += block_time
+        total_requests += estimated_requests
+
+        block_estimates.append(
+            {
+                "block": block_1["block_name"],
+                "estimated_requests": estimated_requests,
+                "throughput": analysis["throughput"],
+                "estimated_time": block_time,
+                "amplification": analysis["amplification"],
+                "startup_overhead": analysis["startup_overhead"],
+            }
+        )
+
+    # Apply conservative buffer to account for API variability, network issues, etc.
+    total_time = total_time * ESTIMATION_BUFFER_FACTOR
+
+    return {
+        "estimated_time_seconds": total_time,
+        "total_estimated_requests": int(total_requests),
+        "block_estimates": block_estimates,
+    }

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/flow.yaml

@@ -17,7 +17,6 @@ metadata:
   - qa-pairs
   - detailed-summaries
   license: Apache-2.0
-  min_sdg_hub_version: 0.2.0
   dataset_requirements:
     required_columns:
     - document

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/doc_direct_qa/flow.yaml

@@ -17,7 +17,6 @@ metadata:
   - qa-pairs
   - detailed-summaries
   license: Apache-2.0
-  min_sdg_hub_version: 0.2.0
   dataset_requirements:
     required_columns:
     - document

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/flow.yaml

@@ -19,7 +19,6 @@ metadata:
   - qa-pairs
   - extractive-summaries
   license: Apache-2.0
-  min_sdg_hub_version: 0.2.0
   dataset_requirements:
     required_columns:
     - document

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/flow.yaml

@@ -17,7 +17,6 @@ metadata:
   - qa-pairs
   - key-facts
   license: Apache-2.0
-  min_sdg_hub_version: 0.2.0
   dataset_requirements:
     required_columns:
     - document

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/flow.yaml

@@ -18,8 +18,7 @@ metadata:
     - "educational"
   
   license: "Apache-2.0"
-  min_sdg_hub_version: "0.2.0"
-  
+
   dataset_requirements:
     required_columns:
       - "document"

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/flow.yaml

@@ -19,8 +19,7 @@ metadata:
     - "japanese"
   
   license: "Apache-2.0"
-  min_sdg_hub_version: "0.2.0"
-  
+
   dataset_requirements:
     required_columns:
       - "document"
@@ -55,17 +54,19 @@ blocks:
       output_cols: raw_summary_detailed
       max_tokens: 2048
       async_mode: true
+      # n: 2
 
   - block_type: LLMParserBlock
     block_config:
-      block_name: extract_detailed_summary
+      block_name: detailed_summary
       input_cols: raw_summary_detailed
       extract_content: true
+      # extract_reasoning_content: true
 
   - block_type: TextParserBlock
     block_config:
       block_name: parse_detailed_summary
-      input_cols: extract_detailed_summary_content
+      input_cols: detailed_summary_content
       output_cols: summary_detailed
       start_tags: [""]
       end_tags: [""]
@@ -87,14 +88,14 @@ blocks:
 
   - block_type: LLMParserBlock
     block_config:
-      block_name: extract_atomic_facts
+      block_name: atomic_facts
       input_cols: raw_atomic_facts
       extract_content: true
 
   - block_type: TextParserBlock
     block_config:
       block_name: parse_atomic_facts
-      input_cols: extract_atomic_facts_content
+      input_cols: atomic_facts_content
       output_cols: summary_atomic_facts
       start_tags: [""]
       end_tags: [""]
@@ -116,14 +117,14 @@ blocks:
 
   - block_type: LLMParserBlock
     block_config:
-      block_name: extract_extractive_summary
+      block_name: extractive_summary
       input_cols: raw_summary_extractive
       extract_content: true
 
   - block_type: TextParserBlock
     block_config:
       block_name: parse_extractive_summary
-      input_cols: extract_extractive_summary_content
+      input_cols: extractive_summary_content
       output_cols: summary_extractive
       start_tags: [""]
       end_tags: [""]
@@ -157,14 +158,14 @@ blocks:
 
   - block_type: LLMParserBlock
     block_config:
-      block_name: extract_knowledge_generation
+      block_name: get_knowledge_generation
       input_cols: raw_knowledge_generation
       extract_content: true
 
   - block_type: TextParserBlock
     block_config:
       block_name: parse_knowledge_generation
-      input_cols: extract_knowledge_generation_content
+      input_cols: get_knowledge_generation_content
       output_cols: [question, response]
       parsing_pattern: "\\[(?:Question|QUESTION)\\]\\s*(.*?)\\s*\\[(?:Answer|ANSWER)\\]\\s*(.*?)\\s*(?=\\[(?:Question|QUESTION)\\]|$)"
       parser_cleanup_tags: ["[END]"]

sdg_hub/flows/text_analysis/structured_insights/flow.yaml

@@ -24,7 +24,6 @@ metadata:
     - "entity-extraction"
     - "keyword-extraction"
   license: "Apache-2.0"
-  min_sdg_hub_version: "0.2.0"
   dataset_requirements:
     required_columns:
       - "text"

{sdg_hub-0.4.0.dist-info → sdg_hub-0.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.4.0
+Version: 0.4.2
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0
@@ -65,6 +65,7 @@ Requires-Dist: pytest-html; extra == "dev"
 Requires-Dist: tox<5,>=4.4.2; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pytest-env; extra == "dev"
+Requires-Dist: nbconvert>=7.0.0; extra == "dev"
 Dynamic: license-file
 
 # `sdg_hub`: Synthetic Data Generation Toolkit

{sdg_hub-0.4.0.dist-info → sdg_hub-0.4.2.dist-info}/RECORD

@@ -1,7 +1,7 @@
-sdg_hub/__init__.py,sha256=Tw-6R5a8_W1kJcTAsW3R9ltBDP1dy5-fe7Tvt3cSyCQ,550
-sdg_hub/_version.py,sha256=2_0GUP7yBCXRus-qiJKxQD62z172WSs1sQ6DVpPsbmM,704
+sdg_hub/__init__.py,sha256=TlkZT40-70urdcWLqv3kupaJj8s-SVgd2QyvlSFwb4A,510
+sdg_hub/_version.py,sha256=A45grTqzrHuDn1CT9K5GVUbY4_Q3OSTcXAl3zdHzcEI,704
 sdg_hub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sdg_hub/core/__init__.py,sha256=NwqB4fwhC29W50VW7QXZssLxx122YvgO9LHDLdgAnrI,496
+sdg_hub/core/__init__.py,sha256=e3BoejbqjYhasf9t__L4qE52lkD9EBjx4o--2kqKdro,460
 sdg_hub/core/blocks/__init__.py,sha256=5FsbkcO-dmBv6MqO96TPn9FKKPTQZQCv20j4wR7UvQw,1502
 sdg_hub/core/blocks/base.py,sha256=-SOdBpJwtRTMsrmCEuLjUBQMRCo_PLYlHEBRrz8sF9g,13031
 sdg_hub/core/blocks/registry.py,sha256=FuEN_pnq-nSH1LguY3_oCubT6Kz3SuJjk3TcUpLT-lw,10695
@@ -32,21 +32,22 @@ sdg_hub/core/blocks/transform/melt_columns.py,sha256=vaYa5Taq6GhNZYWFL4uPK3-SfN2
 sdg_hub/core/blocks/transform/rename_columns.py,sha256=qeB5L2utqDQnutUetH1VKZSqDiJSH_yUp5EFCV-XCVI,1998
 sdg_hub/core/blocks/transform/text_concat.py,sha256=_-B__Hob1WwgwkILPIZvTnsDzuwtoX1hKviyzHlnnes,3149
 sdg_hub/core/blocks/transform/uniform_col_val_setter.py,sha256=XnjiT29z3PzIPy8M-mmE2w-Miab6Ed5ahy32SaxTCTE,3263
-sdg_hub/core/flow/__init__.py,sha256=N2NZGngvd7qpT5FI_knKukUFM0IkD9K5jdTi-gDeUI4,475
-sdg_hub/core/flow/base.py,sha256=6UlQ7ymVNs03UQ4NNgD15Y6eFyKPcl5JpuWOZuY70Mo,56654
+sdg_hub/core/flow/__init__.py,sha256=0_m_htuZfPxk8xQ9IKfp0Pz-JRE4O7lYMUFrKyLNoLA,409
+sdg_hub/core/flow/base.py,sha256=4kR-dKXAlLFSwm3YWdT8EoedCIGJT56agcot3tQb6VY,59508
 sdg_hub/core/flow/checkpointer.py,sha256=stm5ZtjjEiLk9ZkAAnoQQn5Y8Yl_d7qCsQLZTrCXR48,11867
-sdg_hub/core/flow/metadata.py,sha256=h9jpvAzWsF5n4ztZMzwa9ZNgnzKTHmFWdn7YbyJLHCw,12977
+sdg_hub/core/flow/metadata.py,sha256=cFrpJjWOaK87aCuRFyC3Pdf83oYU93mrmZEMdUnhsN8,10540
 sdg_hub/core/flow/migration.py,sha256=6and-RBqV0t2gRipr1GiOOVnyBJdtyyjw1kO08Z--d4,7558
-sdg_hub/core/flow/registry.py,sha256=DzCqEEgwhvwnCBAGLogoMVdwXh4pCHrxOWqoxam7O8I,12162
+sdg_hub/core/flow/registry.py,sha256=N6KfX-L7QRkooznIFxDuhRZYuDA5g3N5zC-KRm2jVhk,12109
 sdg_hub/core/flow/validation.py,sha256=pUJvgaUjLpKNwvW6djcqVOF-HShOjegEmGOnUnoX4BA,9722
-sdg_hub/core/utils/__init__.py,sha256=C2FzLn3dHprwGJDEgI4fyFS3aoCJR-9PhHsunxropJ8,351
+sdg_hub/core/utils/__init__.py,sha256=KcT56JhobC5sBg0MKEMn5hc4OyKa9_Vnn45Mt_kS4jQ,610
 sdg_hub/core/utils/datautils.py,sha256=__HkUe1DxcJVHKrFX68z_hDXwxJygBlJDfjJLnj7rHc,4230
 sdg_hub/core/utils/error_handling.py,sha256=yku8cGj_nKCyXDsnb-mHCpgukkkAMucJ4iAUrIzqysc,5510
 sdg_hub/core/utils/flow_id_words.yaml,sha256=5QHpQdP7zwahRuooyAlJIwBY7WcDR7vtbJXxVJqujbg,2317
 sdg_hub/core/utils/flow_identifier.py,sha256=aAHfK_G9AwEtMglLRMdMpi_AI1dciub5UqBGm4yb2HE,2841
-sdg_hub/core/utils/flow_metrics.py,sha256=VOdreUzP0kPgnkPjuQk87tZsK5f1u6XGEPM8ugCt0CY,8824
+sdg_hub/core/utils/flow_metrics.py,sha256=3G-xbfr-rFA578wV4KUbQePTMVGZHr9-rXvyYL4Kt2Q,12604
 sdg_hub/core/utils/logger_config.py,sha256=6_cnsIHtSAdq1iTTZ7Q7nAJ1dmldlxSZ0AB49yLiQ20,2034
 sdg_hub/core/utils/path_resolution.py,sha256=yWof4kGNpQ5dKcrVHg0h9KfOKLZ6ROjdfsLAZsQT5rM,2000
+sdg_hub/core/utils/time_estimator.py,sha256=rM3_R-Ka5DEtvOtlJoA_5pXSyQ6tT6t4h6qh3_5BCZo,12639
 sdg_hub/core/utils/yaml_utils.py,sha256=tShCd-FFkp0xlKnLe7dXsMOR4AvT9d2qRUmu4ZnPSEY,1458
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/generate_answers.yaml,sha256=THRT3cY44KGI_69B2wqt2Q89EknnOSE7B4A_jdnxlIU,330
@@ -54,14 +55,14 @@ sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/gener
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/generate_question_list.yaml,sha256=qHOgUNrQz2vjUjJiEHNGWxDDXwjJlP1kofTxeGgLyPI,1461
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/detailed_summary.yaml,sha256=Ik6gAml0O-jPq8jpXBAkURzYkQuFOnDZb4LDwjmfAiE,381
-sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/flow.yaml,sha256=_h_EFdxen842BeJd20soaCeR4eccccxAerUV6myUefE,5567
+sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/flow.yaml,sha256=fUdzY9dtU69o99Uq8FIPycgVWdLD-1kbY97Bh-Vo2A0,5538
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/doc_direct_qa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/doc_direct_qa/flow.yaml,sha256=OJDlm8uGNqGPertACSG5pKKVGOKdfsQ6RMeh4UHZMJs,4442
+sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/doc_direct_qa/flow.yaml,sha256=smPWVUZRCt58EagWDmJVmTBQj8qMcjpzh-Q3GSuFrz0,4413
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/extractive_summary.yaml,sha256=SeapWoOx3fhN5SvWYuHss_9prLE8xSkOic7JkbDHSR0,4081
-sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/flow.yaml,sha256=Yy6-2Vytdr4FPxC5wTQkcv7Amy-DBMA3H8vOx9tBB9U,5735
+sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/flow.yaml,sha256=iNNIfofFE7awK7iivtIFWxjfjy8QviMugOPPnOTySKA,5706
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/flow.yaml,sha256=QYN-zNl0YtqKXCTpMJBD9vbYsTf-30cap9ziiDwxKk0,3248
+sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/flow.yaml,sha256=CIUZNYhvszT-jpz1Hvh6nS2y5W34P529ZOMp8thEQ9k,3219
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/key_facts_summary.yaml,sha256=YKMX_CuvcThG_bdNCAIXdVBkMvB72I89RGq2ltSSgc8,3298
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -71,24 +72,24 @@ sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/ev
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/evaluate_question.yaml,sha256=zwzklXup6khRkR88avgrJTcjaMcV1wnbeYaML5oPuNs,1767
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/evaluate_relevancy.yaml,sha256=cA8igo7jMrRXaWW6k0of6KOp7YnxLtPj0fP4DbrmZNQ,3647
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/extractive_summary.yaml,sha256=fcMV7LaCFZo4D29nwhGJXqFFuZMYVLo9XYjv8zcU6zs,364
-sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/flow.yaml,sha256=QOhucXsokNEXGdXtk38qxQnSDwiCngUciXRjBqDcnDU,9088
+sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/flow.yaml,sha256=HR8sf7RUZKr8UqKztBj-nlvyrve1UMUu8x8qgYM6O14,9055
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/generate_questions_responses.yaml,sha256=yX8aLY8dJSDML9ZJhnj9RzPbN8tH2xfcM4Gc6xZuwqQ,2596
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/atomic_facts_ja.yaml,sha256=OjPZaSCOSLxEWgW3pmNwF7mmLhGhFGTmKL_3rKdqeW4,2488
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/detailed_summary_ja.yaml,sha256=nEy_RcotHGiiENrmUANpKkbIFsrARAeSwECrBeHi2so,391
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/extractive_summary_ja.yaml,sha256=V90W0IeJQZTFThA8v0UOs3DtZbtU3BI9jkpChw1BULo,402
-sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/flow.yaml,sha256=ittFo_tyvG_1eqooO_9NK4jqepafgpHFGy2fuVfjFto,9207
+sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/flow.yaml,sha256=jumjKmKshSd8hoTYpyBJ0nMOADeQmxBmNPY7yfa_xQ8,9171
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/generate_questions_responses_ja.yaml,sha256=96SQqXG7fmb-50SdX85sgVtrFcQ-oNKe_0BoQdZmY5g,2638
 sdg_hub/flows/text_analysis/__init__.py,sha256=WStks4eM_KHNTVsHglcj8vFghmI0PH9P1hUrijBLbwc,125
 sdg_hub/flows/text_analysis/structured_insights/__init__.py,sha256=_DT4NR05JD9CZoSWROPr2lC6se0VjSqQPZJJlEV79mk,274
 sdg_hub/flows/text_analysis/structured_insights/analyze_sentiment.yaml,sha256=1YGPypFJYS8qfYFj2J6ERTgodKJvMF4YHNGt_vOF5qc,1000
 sdg_hub/flows/text_analysis/structured_insights/extract_entities.yaml,sha256=Q_SDy14Zu-qS2sbKfUBmGlYj3k7CUg6HzzXlFCXRKuU,1169
 sdg_hub/flows/text_analysis/structured_insights/extract_keywords.yaml,sha256=_nPPMdHnxag_lYbhYUjGJGo-CvRwWvwdGX7cQhdZ1S0,847
-sdg_hub/flows/text_analysis/structured_insights/flow.yaml,sha256=Qpo9WPtl0PWhBF1stIM8OjaTvhtw3dn4eDADt-xj5cA,4965
+sdg_hub/flows/text_analysis/structured_insights/flow.yaml,sha256=BBV18SdvuVTAESjwkJ7V1jbb-cSTBvNl3SCycd0oEQ4,4934
 sdg_hub/flows/text_analysis/structured_insights/summarize.yaml,sha256=WXwQak1pF8e1OwnOoI1EHu8QB6iUNW89rfkTdi1Oq54,687
-sdg_hub-0.4.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sdg_hub-0.4.0.dist-info/METADATA,sha256=SPjLdht-43yAyDwZzdk91SYoQn8jRbsCTr4qBkXVVlw,9735
-sdg_hub-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sdg_hub-0.4.0.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
-sdg_hub-0.4.0.dist-info/RECORD,,
+sdg_hub-0.4.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sdg_hub-0.4.2.dist-info/METADATA,sha256=5qbw9_DoVmfntmQlvz4VPdQXdUXoLO8Zhrxbc1uY7b0,9783
+sdg_hub-0.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sdg_hub-0.4.2.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
+sdg_hub-0.4.2.dist-info/RECORD,,