sdg-hub 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

sdg_hub/core/utils/time_estimator.py

@@ -0,0 +1,344 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Time estimation utility for predicting full dataset execution time from dry_run results."""
+
+# Standard
+from typing import Dict, Optional
+import math
+
+# Default max concurrent requests used during dry runs
+DRY_RUN_MAX_CONCURRENT = 100
+
+# Conservative estimation factor (20% buffer for API variability, network latency, etc.)
+ESTIMATION_BUFFER_FACTOR = 1.2
+
+
+def is_llm_using_block(block_info: Dict) -> bool:
+    """Detect if a block uses LLMs.
+
+    Identifies blocks that make LLM API calls based on their type or parameters.
+    This is used to calculate request amplification for LLM blocks.
+
+    Parameters
+    ----------
+    block_info : Dict
+        Block information from dry_run results containing block_type and parameters_used.
+
+    Returns
+    -------
+    bool
+        True if the block uses LLMs, False otherwise.
+
+    Examples
+    --------
+    >>> block = {"block_type": "LLMChatBlock", "parameters_used": {"model": "gpt-4"}}
+    >>> is_llm_using_block(block)
+    True
+    """
+    block_type = block_info.get("block_type", "")
+
+    # Direct LLM blocks or evaluation/verification blocks
+    if any(kw in block_type for kw in ["LLMChatBlock", "Evaluate", "Verify"]):
+        return True
+
+    # Check for model parameters
+    params = block_info.get("parameters_used", {})
+    if any(key in params for key in ["model", "api_base", "api_key"]):
+        return True
+
+    return False
+
+
+def calculate_block_throughput(
+    block_1: Dict, block_2: Dict, samples_1: int, samples_2: int
+) -> Dict:
+    """Calculate throughput and amplification from two dry runs.
+
+    Analyzes performance metrics from two dry runs with different sample sizes
+    to estimate throughput (requests/second), amplification factor, and startup overhead.
+
+    Parameters
+    ----------
+    block_1 : Dict
+        Block execution info from first dry run.
+    block_2 : Dict
+        Block execution info from second dry run.
+    samples_1 : int
+        Number of samples in first dry run.
+    samples_2 : int
+        Number of samples in second dry run.
+
+    Returns
+    -------
+    Dict
+        Dictionary containing:
+        - throughput: float, requests per second
+        - amplification: float, average requests per input sample
+        - startup_overhead: float, fixed startup time in seconds
+
+    Raises
+    ------
+    ValueError
+        If throughput cannot be calculated due to invalid measurements.
+
+    Examples
+    --------
+    >>> block1 = {"execution_time_seconds": 1.0, "input_rows": 1, "block_name": "test"}
+    >>> block2 = {"execution_time_seconds": 2.0, "input_rows": 5, "block_name": "test"}
+    >>> result = calculate_block_throughput(block1, block2, 1, 5)
+    >>> assert result["throughput"] > 0
+    """
+    time_1 = block_1.get("execution_time_seconds", 0)
+    time_2 = block_2.get("execution_time_seconds", 0)
+    requests_1 = block_1.get("input_rows", 0)
+    requests_2 = block_2.get("input_rows", 0)
+
+    # Calculate amplification (requests per sample)
+    amp_1 = requests_1 / samples_1 if samples_1 > 0 else 1
+    amp_2 = requests_2 / samples_2 if samples_2 > 0 else 1
+    avg_amplification = (amp_1 + amp_2) / 2
+
+    # Use linear scaling to extract throughput and overhead from two data points
+    # Model: time = startup_overhead + (requests / throughput)
+
+    if requests_2 > requests_1 and time_2 > time_1:
+        # Calculate marginal time per request (slope of the line)
+        marginal_time = (time_2 - time_1) / (requests_2 - requests_1)
+
+        # Throughput is the inverse of marginal time
+        measured_throughput = 1.0 / marginal_time if marginal_time > 0 else 0
+
+        # Y-intercept is the startup overhead
+        startup_overhead = max(0, time_1 - (requests_1 * marginal_time))
+    else:
+        # Fallback to simple calculation if we don't have good data for scaling
+        throughput_1 = requests_1 / time_1 if time_1 > 0 else 0
+        throughput_2 = requests_2 / time_2 if time_2 > 0 else 0
+        measured_throughput = max(throughput_1, throughput_2)
+
+        # Estimate overhead as a small fraction of time
+        startup_overhead = min(2.0, time_1 * 0.1)  # Assume 10% overhead, max 2 seconds
+
+    # If we have no valid measurements, raise an error
+    if measured_throughput == 0:
+        raise ValueError(
+            f"Cannot calculate throughput for block '{block_1.get('block_name', 'unknown')}': "
+            f"No valid measurements from dry runs (time_1={time_1}, time_2={time_2}, "
+            f"requests_1={requests_1}, requests_2={requests_2})"
+        )
+
+    return {
+        "throughput": measured_throughput,
+        "amplification": avg_amplification,
+        "startup_overhead": startup_overhead,
+    }
+
+
+def calculate_time_with_pipeline(
+    num_requests: float,
+    throughput: float,
+    startup_overhead: float,
+    max_concurrent: int = DRY_RUN_MAX_CONCURRENT,
+) -> float:
+    """Calculate time considering pipeline behavior and max concurrent limit.
+
+    Models the execution time for a given number of requests based on throughput,
+    startup overhead, and concurrency constraints. Applies non-linear scaling
+    for diminishing returns at high concurrency levels.
+
+    Parameters
+    ----------
+    num_requests : float
+        Total number of requests to process.
+    throughput : float
+        Base throughput in requests per second.
+    startup_overhead : float
+        Fixed startup time overhead in seconds.
+    max_concurrent : int, optional
+        Maximum number of concurrent requests, by default 100.
+
+    Returns
+    -------
+    float
+        Estimated total execution time in seconds.
+
+    Examples
+    --------
+    >>> time = calculate_time_with_pipeline(1000, 10.0, 0.5, 50)
+    >>> assert time > 0
+    """
+    if num_requests <= 0:
+        return 0
+
+    # Validate and clamp max_concurrent to avoid division by zero
+    if max_concurrent is None or max_concurrent <= 0:
+        max_concurrent = 1
+
+    # The throughput is what we measured - it represents the server's processing capability
+    if max_concurrent == 1:
+        # Sequential execution - no pipelining benefit
+        effective_throughput = throughput
+    else:
+        # Concurrent execution - small pipelining benefit
+        # At most 10% improvement from perfect pipelining (conservative estimate)
+        # Logarithmic growth to model diminishing returns
+        pipelining_factor = 1.0 + (0.1 * math.log(max_concurrent) / math.log(100))
+        pipelining_factor = min(pipelining_factor, 1.1)  # Cap at 10% improvement
+        effective_throughput = throughput * pipelining_factor
+
+    # Calculate total time
+    base_time = startup_overhead + (num_requests / effective_throughput)
+
+    return base_time
+
+
+def estimate_execution_time(
+    dry_run_1: Dict,
+    dry_run_2: Optional[Dict] = None,
+    total_dataset_size: Optional[int] = None,
+    max_concurrency: Optional[int] = None,
+) -> Dict:
+    """Estimate execution time based on dry run results.
+
+    Estimates the total execution time for a full dataset based on one or two
+    dry runs with smaller sample sizes. For async blocks (with two dry runs),
+    calculates throughput and concurrency benefits. For sync blocks (single dry run),
+    performs simple linear scaling.
+
+    The estimates include a conservative buffer (20%) to account for API variability,
+    network latency, and other real-world factors.
+
+    Parameters
+    ----------
+    dry_run_1 : Dict
+        Results from first dry run, must contain 'sample_size' and 'execution_time_seconds'.
+    dry_run_2 : Optional[Dict], optional
+        Results from second dry run for async estimation, by default None.
+    total_dataset_size : Optional[int], optional
+        Size of full dataset to estimate for. If None, uses original_dataset_size from dry_run_1.
+    max_concurrency : Optional[int], optional
+        Maximum concurrent requests allowed, by default 100.
+
+    Returns
+    -------
+    Dict
+        Estimation results containing:
+        - estimated_time_seconds: float, estimated time with current configuration (includes buffer)
+        - total_estimated_requests: int, total LLM requests (0 for sync blocks)
+        - block_estimates: list, per-block estimates (for async blocks)
+        - note: str, additional information about the estimation
+
+    Examples
+    --------
+    >>> dry_run = {"sample_size": 2, "execution_time_seconds": 10.0}
+    >>> result = estimate_execution_time(dry_run, total_dataset_size=100)
+    >>> assert result["estimated_time_seconds"] > 0
+    >>>
+    >>> # With two dry runs for async estimation
+    >>> dry_run_1 = {"sample_size": 1, "execution_time_seconds": 5.0, "blocks_executed": [...]}
+    >>> dry_run_2 = {"sample_size": 5, "execution_time_seconds": 20.0, "blocks_executed": [...]}
+    >>> result = estimate_execution_time(dry_run_1, dry_run_2, total_dataset_size=1000)
+    >>> assert result["estimated_time_seconds"] > 0
+    """
+    # Set defaults
+    if max_concurrency is None:
+        max_concurrency = DRY_RUN_MAX_CONCURRENT
+
+    if total_dataset_size is None:
+        total_dataset_size = dry_run_1.get(
+            "original_dataset_size", dry_run_1["sample_size"]
+        )
+
+    # Get sample sizes
+    samples_1 = dry_run_1["sample_size"]
+    samples_2 = (
+        dry_run_2["sample_size"] if dry_run_2 else 5
+    )  # Default to 5 if not provided
+
+    # If only one dry run, do simple scaling
+    if dry_run_2 is None:
+        # Process each block individually for synchronous execution
+        blocks_executed = dry_run_1.get("blocks_executed", [])
+        if not blocks_executed:
+            # Fallback to simple scaling if no block details available
+            total_time = dry_run_1["execution_time_seconds"]
+            simple_estimate = (total_time / samples_1) * total_dataset_size
+            # Apply conservative buffer
+            simple_estimate = simple_estimate * ESTIMATION_BUFFER_FACTOR
+            return {
+                "estimated_time_seconds": simple_estimate,
+                "total_estimated_requests": 0,
+                "note": "Synchronous execution - linear scaling from dry run",
+            }
+
+        # Calculate time for each block and sum them
+        total_estimated_time = 0
+        for block in blocks_executed:
+            block_time = block.get("execution_time_seconds", 0)
+            input_rows = block.get("input_rows", samples_1)
+
+            # Calculate time per row for this block
+            if input_rows > 0:
+                time_per_row = block_time / input_rows
+                block_total_time = time_per_row * total_dataset_size
+                total_estimated_time += block_total_time
+
+        # Apply conservative buffer
+        total_estimated_time = total_estimated_time * ESTIMATION_BUFFER_FACTOR
+        return {
+            "estimated_time_seconds": total_estimated_time,
+            "total_estimated_requests": 0,
+            "note": "Synchronous execution - no concurrency",
+        }
+
+    # Analyze each block with async execution
+    block_estimates = []
+    total_time = 0
+    total_requests = 0
+
+    # Process each block
+    for i, block_1 in enumerate(dry_run_1.get("blocks_executed", [])):
+        if i >= len(dry_run_2.get("blocks_executed", [])):
+            break
+
+        block_2 = dry_run_2["blocks_executed"][i]
+
+        # Only process LLM blocks
+        if not is_llm_using_block(block_1):
+            continue
+
+        # Calculate throughput and amplification
+        analysis = calculate_block_throughput(block_1, block_2, samples_1, samples_2)
+
+        # Estimate requests for full dataset
+        estimated_requests = total_dataset_size * analysis["amplification"]
+
+        # Calculate time with pipeline model
+        block_time = calculate_time_with_pipeline(
+            estimated_requests,
+            analysis["throughput"],
+            analysis["startup_overhead"],
+            max_concurrency,
+        )
+
+        total_time += block_time
+        total_requests += estimated_requests
+
+        block_estimates.append(
+            {
+                "block": block_1["block_name"],
+                "estimated_requests": estimated_requests,
+                "throughput": analysis["throughput"],
+                "estimated_time": block_time,
+                "amplification": analysis["amplification"],
+                "startup_overhead": analysis["startup_overhead"],
+            }
+        )
+
+    # Apply conservative buffer to account for API variability, network issues, etc.
+    total_time = total_time * ESTIMATION_BUFFER_FACTOR
+
+    return {
+        "estimated_time_seconds": total_time,
+        "total_estimated_requests": int(total_requests),
+        "block_estimates": block_estimates,
+    }

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/flow.yaml

@@ -77,9 +77,13 @@ blocks:
     - ''
 - block_type: RenameColumnsBlock
   block_config:
-    block_name: rename_to_document_column
+    block_name: rename_to_raw_document_column
     input_cols:
       document: raw_document
+- block_type: RenameColumnsBlock
+  block_config:
+    block_name: rename_to_document_column
+    input_cols:
       summary: document
 - block_type: PromptBuilderBlock
   block_config:

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/flow.yaml

@@ -79,9 +79,13 @@ blocks:
     - ''
 - block_type: RenameColumnsBlock
   block_config:
-    block_name: rename_to_document_column
+    block_name: rename_to_raw_document_column
     input_cols:
       document: raw_document
+- block_type: RenameColumnsBlock
+  block_config:
+    block_name: rename_to_document_column
+    input_cols:
       summary: document
 - block_type: PromptBuilderBlock
   block_config:

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/flow.yaml

@@ -72,9 +72,13 @@ blocks:
     parsing_pattern: '(?:^|\n)\s*\d+\.\s+(.*?)(?=\n\s*\d+\.\s+|\Z)'
 - block_type: RenameColumnsBlock
   block_config:
-    block_name: rename_to_document_column
+    block_name: rename_to_raw_document_column
     input_cols:
       document: raw_document
+- block_type: RenameColumnsBlock
+  block_config:
+    block_name: rename_to_document_column
+    input_cols:
       atomic_facts: document
 - block_type: PromptBuilderBlock
   block_config:

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/flow.yaml

@@ -134,10 +134,15 @@ blocks:
       input_cols: [summary_detailed, summary_extractive, summary_atomic_facts, base_document]
       output_cols: [summary, dataset_type]
 
+  - block_type: RenameColumnsBlock
+    block_config:
+      block_name: rename_to_raw_document_column
+      input_cols: {document: raw_document}
+
   - block_type: RenameColumnsBlock
     block_config:
       block_name: rename_to_document_column
-      input_cols: {document: raw_document, summary: document}
+      input_cols: {summary: document}
 
   - block_type: PromptBuilderBlock
     block_config:

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/flow.yaml

@@ -19,7 +19,7 @@ metadata:
     - "japanese"
   
   license: "Apache-2.0"
-  
+
   dataset_requirements:
     required_columns:
       - "document"
@@ -54,17 +54,19 @@ blocks:
       output_cols: raw_summary_detailed
       max_tokens: 2048
       async_mode: true
+      # n: 2
 
   - block_type: LLMParserBlock
     block_config:
-      block_name: extract_detailed_summary
+      block_name: detailed_summary
       input_cols: raw_summary_detailed
       extract_content: true
+      # extract_reasoning_content: true
 
   - block_type: TextParserBlock
     block_config:
       block_name: parse_detailed_summary
-      input_cols: extract_detailed_summary_content
+      input_cols: detailed_summary_content
       output_cols: summary_detailed
       start_tags: [""]
       end_tags: [""]
@@ -86,14 +88,14 @@ blocks:
 
   - block_type: LLMParserBlock
     block_config:
-      block_name: extract_atomic_facts
+      block_name: atomic_facts
       input_cols: raw_atomic_facts
       extract_content: true
 
   - block_type: TextParserBlock
     block_config:
       block_name: parse_atomic_facts
-      input_cols: extract_atomic_facts_content
+      input_cols: atomic_facts_content
       output_cols: summary_atomic_facts
       start_tags: [""]
       end_tags: [""]
@@ -115,14 +117,14 @@ blocks:
 
   - block_type: LLMParserBlock
     block_config:
-      block_name: extract_extractive_summary
+      block_name: extractive_summary
       input_cols: raw_summary_extractive
       extract_content: true
 
   - block_type: TextParserBlock
     block_config:
       block_name: parse_extractive_summary
-      input_cols: extract_extractive_summary_content
+      input_cols: extractive_summary_content
       output_cols: summary_extractive
       start_tags: [""]
       end_tags: [""]
@@ -133,10 +135,14 @@ blocks:
       input_cols: [summary_detailed, summary_extractive, summary_atomic_facts, base_document]
       output_cols: [summary, dataset_type]
 
+  - block_type: RenameColumnsBlock
+    block_config:
+      block_name: rename_to_raw_document_column
+      input_cols: {document: raw_document}
   - block_type: RenameColumnsBlock
     block_config:
       block_name: rename_to_document_column
-      input_cols: {document: raw_document, summary: document}
+      input_cols: {summary: document}
 
   - block_type: PromptBuilderBlock
     block_config:
@@ -156,14 +162,14 @@ blocks:
 
   - block_type: LLMParserBlock
     block_config:
-      block_name: extract_knowledge_generation
+      block_name: get_knowledge_generation
       input_cols: raw_knowledge_generation
       extract_content: true
 
   - block_type: TextParserBlock
     block_config:
       block_name: parse_knowledge_generation
-      input_cols: extract_knowledge_generation_content
+      input_cols: get_knowledge_generation_content
       output_cols: [question, response]
       parsing_pattern: "\\[(?:Question|QUESTION)\\]\\s*(.*?)\\s*\\[(?:Answer|ANSWER)\\]\\s*(.*?)\\s*(?=\\[(?:Question|QUESTION)\\]|$)"
       parser_cleanup_tags: ["[END]"]

{sdg_hub-0.4.1.dist-info → sdg_hub-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.4.1
+Version: 0.5.0
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0
@@ -23,7 +23,7 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: click<9.0.0,>=8.1.7
-Requires-Dist: datasets<4.0.0,>=2.18.0
+Requires-Dist: datasets>=4.0.0
 Requires-Dist: httpx<1.0.0,>=0.25.0
 Requires-Dist: jinja2
 Requires-Dist: litellm<1.75.0,>=1.73.0

{sdg_hub-0.4.1.dist-info → sdg_hub-0.5.0.dist-info}/RECORD

@@ -1,20 +1,10 @@
 sdg_hub/__init__.py,sha256=TlkZT40-70urdcWLqv3kupaJj8s-SVgd2QyvlSFwb4A,510
-sdg_hub/_version.py,sha256=k7cu0JKra64gmMNU_UfA5sw2eNc_GRvf3QmesiYAy8g,704
+sdg_hub/_version.py,sha256=fvHpBU3KZKRinkriKdtAt3crenOyysELF-M9y3ozg3U,704
 sdg_hub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/core/__init__.py,sha256=e3BoejbqjYhasf9t__L4qE52lkD9EBjx4o--2kqKdro,460
-sdg_hub/core/blocks/__init__.py,sha256=5FsbkcO-dmBv6MqO96TPn9FKKPTQZQCv20j4wR7UvQw,1502
+sdg_hub/core/blocks/__init__.py,sha256=8Rn1SglH8V3jGmTD_cG-h7qk9ktAab2eaBdyk7RN_hY,865
 sdg_hub/core/blocks/base.py,sha256=-SOdBpJwtRTMsrmCEuLjUBQMRCo_PLYlHEBRrz8sF9g,13031
 sdg_hub/core/blocks/registry.py,sha256=FuEN_pnq-nSH1LguY3_oCubT6Kz3SuJjk3TcUpLT-lw,10695
-sdg_hub/core/blocks/deprecated_blocks/__init__.py,sha256=RDu3MWFStDQko-TKkx8tGoB1UTatP_RSldZK43zHDvY,889
-sdg_hub/core/blocks/deprecated_blocks/combine_columns.py,sha256=HCvpaYsAwgx1Dm0vIshcWsKoVsRT0KrmKp9j4oqtByc,2757
-sdg_hub/core/blocks/deprecated_blocks/duplicate_columns.py,sha256=maCaaEs0EMMzt7L1xm7fAH3ylaFMHEkeC_dtOw3FrjU,2694
-sdg_hub/core/blocks/deprecated_blocks/filter_by_value.py,sha256=-fuuMKj2g2MrijMBTd0PWtYBbf9anQ2UkYXHigCxxJI,3328
-sdg_hub/core/blocks/deprecated_blocks/flatten_columns.py,sha256=IenCskrPEv09h2uT6aZKCQzaxgA_3kAzOeJSd-R_-EA,2839
-sdg_hub/core/blocks/deprecated_blocks/llmblock.py,sha256=34lzC43BODpMk5AwlWA1ctdYPmN7cA6WL5vMXaI0P0Y,20385
-sdg_hub/core/blocks/deprecated_blocks/rename_columns.py,sha256=thp-mHtkRmUw_nYKpldy_mLWR2AvC5YUhbqDETM6-T0,2620
-sdg_hub/core/blocks/deprecated_blocks/sample_populator.py,sha256=UdueMApxOmPWaxxMrw7b1v74fKJBfqqRATEBqgmVtNw,1737
-sdg_hub/core/blocks/deprecated_blocks/selector.py,sha256=nWecsVsW8DvBcqAF_LOqXmW-5MQ28uN3d1y6wkSy38c,2960
-sdg_hub/core/blocks/deprecated_blocks/set_to_majority_value.py,sha256=44TQu-rK5isia-otMVB1zHd8D-wWmu3C8CI1NLtfY5s,2729
 sdg_hub/core/blocks/filtering/__init__.py,sha256=isxSVSvDqkMjG8dQSl3Q2M4g5c1t9fTjBSA21icf-yA,275
 sdg_hub/core/blocks/filtering/column_value_filter.py,sha256=2Z9j_CiiTn5mHZ9gfXU-itLXDmeXSh0UI0x1x7j-LQ0,6001
 sdg_hub/core/blocks/llm/__init__.py,sha256=AyS0dd3pkPPXH5a9aj4mT5HsKjX2vjXfkmQc6rkFV4A,795
@@ -29,24 +19,24 @@ sdg_hub/core/blocks/transform/duplicate_columns.py,sha256=SaP7rIF4ZFEFFa50aU2xGN
 sdg_hub/core/blocks/transform/index_based_mapper.py,sha256=XC_a7Skbd3mu7f4ra8fGWPxMwqUMSjJkQ7Ag7vflwJA,8235
 sdg_hub/core/blocks/transform/json_structure_block.py,sha256=hm-0M0NAyUREgJRPyV1u-laorgX6MZ1o17E9rNBhN78,5010
 sdg_hub/core/blocks/transform/melt_columns.py,sha256=vaYa5Taq6GhNZYWFL4uPK3-SfN2BsKEm-wvjd2EYYoI,4382
-sdg_hub/core/blocks/transform/rename_columns.py,sha256=qeB5L2utqDQnutUetH1VKZSqDiJSH_yUp5EFCV-XCVI,1998
+sdg_hub/core/blocks/transform/rename_columns.py,sha256=W2hcDSJY6L73ZpElUhOML2sGLM9Y-v0gSo3xEF1LXDc,2749
 sdg_hub/core/blocks/transform/text_concat.py,sha256=_-B__Hob1WwgwkILPIZvTnsDzuwtoX1hKviyzHlnnes,3149
 sdg_hub/core/blocks/transform/uniform_col_val_setter.py,sha256=XnjiT29z3PzIPy8M-mmE2w-Miab6Ed5ahy32SaxTCTE,3263
 sdg_hub/core/flow/__init__.py,sha256=0_m_htuZfPxk8xQ9IKfp0Pz-JRE4O7lYMUFrKyLNoLA,409
-sdg_hub/core/flow/base.py,sha256=IRnNEZ3laDmR4sW_MTseL4syhLuUylyHY_0tS5QaS-A,54084
+sdg_hub/core/flow/base.py,sha256=64YJJujNRaSIbT1YKn9nAxij_hdJ9xRVH_uiUY1IUcI,55788
 sdg_hub/core/flow/checkpointer.py,sha256=stm5ZtjjEiLk9ZkAAnoQQn5Y8Yl_d7qCsQLZTrCXR48,11867
 sdg_hub/core/flow/metadata.py,sha256=cFrpJjWOaK87aCuRFyC3Pdf83oYU93mrmZEMdUnhsN8,10540
-sdg_hub/core/flow/migration.py,sha256=6and-RBqV0t2gRipr1GiOOVnyBJdtyyjw1kO08Z--d4,7558
 sdg_hub/core/flow/registry.py,sha256=N6KfX-L7QRkooznIFxDuhRZYuDA5g3N5zC-KRm2jVhk,12109
 sdg_hub/core/flow/validation.py,sha256=pUJvgaUjLpKNwvW6djcqVOF-HShOjegEmGOnUnoX4BA,9722
-sdg_hub/core/utils/__init__.py,sha256=C2FzLn3dHprwGJDEgI4fyFS3aoCJR-9PhHsunxropJ8,351
+sdg_hub/core/utils/__init__.py,sha256=KcT56JhobC5sBg0MKEMn5hc4OyKa9_Vnn45Mt_kS4jQ,610
 sdg_hub/core/utils/datautils.py,sha256=__HkUe1DxcJVHKrFX68z_hDXwxJygBlJDfjJLnj7rHc,4230
 sdg_hub/core/utils/error_handling.py,sha256=yku8cGj_nKCyXDsnb-mHCpgukkkAMucJ4iAUrIzqysc,5510
 sdg_hub/core/utils/flow_id_words.yaml,sha256=5QHpQdP7zwahRuooyAlJIwBY7WcDR7vtbJXxVJqujbg,2317
 sdg_hub/core/utils/flow_identifier.py,sha256=aAHfK_G9AwEtMglLRMdMpi_AI1dciub5UqBGm4yb2HE,2841
-sdg_hub/core/utils/flow_metrics.py,sha256=VOdreUzP0kPgnkPjuQk87tZsK5f1u6XGEPM8ugCt0CY,8824
+sdg_hub/core/utils/flow_metrics.py,sha256=3G-xbfr-rFA578wV4KUbQePTMVGZHr9-rXvyYL4Kt2Q,12604
 sdg_hub/core/utils/logger_config.py,sha256=6_cnsIHtSAdq1iTTZ7Q7nAJ1dmldlxSZ0AB49yLiQ20,2034
 sdg_hub/core/utils/path_resolution.py,sha256=yWof4kGNpQ5dKcrVHg0h9KfOKLZ6ROjdfsLAZsQT5rM,2000
+sdg_hub/core/utils/time_estimator.py,sha256=rM3_R-Ka5DEtvOtlJoA_5pXSyQ6tT6t4h6qh3_5BCZo,12639
 sdg_hub/core/utils/yaml_utils.py,sha256=tShCd-FFkp0xlKnLe7dXsMOR4AvT9d2qRUmu4ZnPSEY,1458
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/generate_answers.yaml,sha256=THRT3cY44KGI_69B2wqt2Q89EknnOSE7B4A_jdnxlIU,330
@@ -54,14 +44,14 @@ sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/gener
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/generate_question_list.yaml,sha256=qHOgUNrQz2vjUjJiEHNGWxDDXwjJlP1kofTxeGgLyPI,1461
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/detailed_summary.yaml,sha256=Ik6gAml0O-jPq8jpXBAkURzYkQuFOnDZb4LDwjmfAiE,381
-sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/flow.yaml,sha256=fUdzY9dtU69o99Uq8FIPycgVWdLD-1kbY97Bh-Vo2A0,5538
+sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/flow.yaml,sha256=cxNpPh60mcvzxfczMH8hw66Ql3S8O-cWCCDeauO736c,5649
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/doc_direct_qa/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/doc_direct_qa/flow.yaml,sha256=smPWVUZRCt58EagWDmJVmTBQj8qMcjpzh-Q3GSuFrz0,4413
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/extractive_summary.yaml,sha256=SeapWoOx3fhN5SvWYuHss_9prLE8xSkOic7JkbDHSR0,4081
-sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/flow.yaml,sha256=iNNIfofFE7awK7iivtIFWxjfjy8QviMugOPPnOTySKA,5706
+sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/flow.yaml,sha256=7dVc0_g7Ex5SfdX57pqtk9gmH_lC6Cdm3HC-lg8OiXQ,5817
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/flow.yaml,sha256=CIUZNYhvszT-jpz1Hvh6nS2y5W34P529ZOMp8thEQ9k,3219
+sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/flow.yaml,sha256=7X4N19TcyHUo7pNo3C6Zv3w6br7hjzEfgv06XUVDaQo,3330
 sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/key_facts_summary.yaml,sha256=YKMX_CuvcThG_bdNCAIXdVBkMvB72I89RGq2ltSSgc8,3298
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -71,14 +61,14 @@ sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/ev
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/evaluate_question.yaml,sha256=zwzklXup6khRkR88avgrJTcjaMcV1wnbeYaML5oPuNs,1767
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/evaluate_relevancy.yaml,sha256=cA8igo7jMrRXaWW6k0of6KOp7YnxLtPj0fP4DbrmZNQ,3647
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/extractive_summary.yaml,sha256=fcMV7LaCFZo4D29nwhGJXqFFuZMYVLo9XYjv8zcU6zs,364
-sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/flow.yaml,sha256=HR8sf7RUZKr8UqKztBj-nlvyrve1UMUu8x8qgYM6O14,9055
+sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/flow.yaml,sha256=km0ggcmFsZJGc2TfyYLkzPTrHGmcOB-jBAHInqySisk,9176
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/generate_questions_responses.yaml,sha256=yX8aLY8dJSDML9ZJhnj9RzPbN8tH2xfcM4Gc6xZuwqQ,2596
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/atomic_facts_ja.yaml,sha256=OjPZaSCOSLxEWgW3pmNwF7mmLhGhFGTmKL_3rKdqeW4,2488
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/detailed_summary_ja.yaml,sha256=nEy_RcotHGiiENrmUANpKkbIFsrARAeSwECrBeHi2so,391
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/extractive_summary_ja.yaml,sha256=V90W0IeJQZTFThA8v0UOs3DtZbtU3BI9jkpChw1BULo,402
-sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/flow.yaml,sha256=iY1N6CY97fEkqI5oqaamSfqmiXpHPhWH_aOppsMxVjY,9176
+sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/flow.yaml,sha256=Q6RusV-_HHMr5jlFNOP6UVuEf8d6btHENMOP3MnB3u0,9291
 sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/generate_questions_responses_ja.yaml,sha256=96SQqXG7fmb-50SdX85sgVtrFcQ-oNKe_0BoQdZmY5g,2638
 sdg_hub/flows/text_analysis/__init__.py,sha256=WStks4eM_KHNTVsHglcj8vFghmI0PH9P1hUrijBLbwc,125
 sdg_hub/flows/text_analysis/structured_insights/__init__.py,sha256=_DT4NR05JD9CZoSWROPr2lC6se0VjSqQPZJJlEV79mk,274
@@ -87,8 +77,8 @@ sdg_hub/flows/text_analysis/structured_insights/extract_entities.yaml,sha256=Q_S
 sdg_hub/flows/text_analysis/structured_insights/extract_keywords.yaml,sha256=_nPPMdHnxag_lYbhYUjGJGo-CvRwWvwdGX7cQhdZ1S0,847
 sdg_hub/flows/text_analysis/structured_insights/flow.yaml,sha256=BBV18SdvuVTAESjwkJ7V1jbb-cSTBvNl3SCycd0oEQ4,4934
 sdg_hub/flows/text_analysis/structured_insights/summarize.yaml,sha256=WXwQak1pF8e1OwnOoI1EHu8QB6iUNW89rfkTdi1Oq54,687
-sdg_hub-0.4.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sdg_hub-0.4.1.dist-info/METADATA,sha256=pLRs5oOsVI9515UEZxcUEZFZhCoZ0kli0KLpBPPPB7w,9783
-sdg_hub-0.4.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sdg_hub-0.4.1.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
-sdg_hub-0.4.1.dist-info/RECORD,,
+sdg_hub-0.5.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sdg_hub-0.5.0.dist-info/METADATA,sha256=z4tCCtWlTBzu5DF1K44RtWjIs7ZNL6__2Aae7I0EfxQ,9775
+sdg_hub-0.5.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sdg_hub-0.5.0.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
+sdg_hub-0.5.0.dist-info/RECORD,,

sdg_hub/core/blocks/deprecated_blocks/__init__.py

@@ -1,29 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-"""Deprecated blocks for backwards compatibility.
-
-This module contains deprecated block implementations that are maintained
-for backwards compatibility. These blocks should not be used in new code.
-"""
-
-# Local
-from .combine_columns import CombineColumnsBlock
-from .duplicate_columns import DuplicateColumns
-from .filter_by_value import FilterByValueBlock
-from .flatten_columns import FlattenColumnsBlock
-from .llmblock import LLMBlock
-from .rename_columns import RenameColumns
-from .sample_populator import SamplePopulatorBlock
-from .selector import SelectorBlock
-from .set_to_majority_value import SetToMajorityValue
-
-__all__ = [
-    "CombineColumnsBlock",
-    "DuplicateColumns",
-    "FilterByValueBlock",
-    "FlattenColumnsBlock",
-    "LLMBlock",
-    "RenameColumns",
-    "SamplePopulatorBlock",
-    "SelectorBlock",
-    "SetToMajorityValue",
-]