sdg-hub 0.4.2__py3-none-any.whl → 0.5.1__py3-none-any.whl

sdg_hub/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.4.2'
-__version_tuple__ = version_tuple = (0, 4, 2)
+__version__ = version = '0.5.1'
+__version_tuple__ = version_tuple = (0, 5, 1)
 
 __commit_id__ = commit_id = None

sdg_hub/core/blocks/__init__.py

@@ -5,17 +5,6 @@ This package provides various block implementations for data generation, process
 
 # Local
 from .base import BaseBlock
-from .deprecated_blocks import (
-    CombineColumnsBlock,
-    DuplicateColumns,
-    FilterByValueBlock,
-    FlattenColumnsBlock,
-    LLMBlock,
-    RenameColumns,
-    SamplePopulatorBlock,
-    SelectorBlock,
-    SetToMajorityValue,
-)
 from .filtering import ColumnValueFilterBlock
 from .llm import LLMChatBlock, LLMParserBlock, PromptBuilderBlock, TextParserBlock
 from .registry import BlockRegistry
@@ -28,8 +17,6 @@ from .transform import (
     UniformColumnValueSetter,
 )
 
-# All blocks moved to deprecated_blocks or transform modules
-
 __all__ = [
     "BaseBlock",
     "BlockRegistry",
@@ -40,15 +27,6 @@ __all__ = [
     "RenameColumnsBlock",
     "TextConcatBlock",
     "UniformColumnValueSetter",
-    "CombineColumnsBlock",  # Deprecated
-    "DuplicateColumns",  # Deprecated
-    "FilterByValueBlock",  # Deprecated
-    "FlattenColumnsBlock",  # Deprecated
-    "RenameColumns",  # Deprecated
-    "SamplePopulatorBlock",  # Deprecated
-    "SelectorBlock",  # Deprecated
-    "SetToMajorityValue",  # Deprecated
-    "LLMBlock",  # Deprecated
     "LLMChatBlock",
     "LLMParserBlock",
     "TextParserBlock",

sdg_hub/core/blocks/llm/llm_parser_block.py

@@ -7,13 +7,16 @@ This module provides the LLMParserBlock for extracting specific fields
 
 # Standard
 from typing import Any
+from weakref import finalize
+import json
 
 # Third Party
-from datasets import Dataset
+from datasets import Dataset, load_dataset
 from pydantic import Field, model_validator
 
 # Local
 from ...utils.logger_config import setup_logger
+from ...utils.temp_manager import cleanup_path, create_temp_dir, create_temp_file
 from ..base import BaseBlock
 from ..registry import BlockRegistry
 
@@ -26,6 +29,8 @@ logger = setup_logger(__name__)
     "Extracts specified fields from LLM response objects",
 )
 class LLMParserBlock(BaseBlock):
+    _flow_requires_jsonl_tmp: bool = True
+
     """Block for extracting fields from LLM response objects.
 
     This block extracts specified fields from chat completion response objects.
@@ -314,7 +319,54 @@ class LLMParserBlock(BaseBlock):
             logger.warning("No samples to process, returning empty dataset")
             return Dataset.from_list([])
 
-        new_data = []
-        for sample in samples:
-            new_data.extend(self._generate(sample))
-        return Dataset.from_list(new_data)
+        tmp_jsonl_path = kwargs.get("_flow_tmp_jsonl_path")
+        cleanup_locally = False
+
+        if tmp_jsonl_path is None:
+            tmp_jsonl_path = str(
+                create_temp_file(
+                    prefix=f"{self.block_name}_llm_parser", suffix=".jsonl"
+                )
+            )
+            cleanup_locally = True
+
+        rows_written = 0
+        batch = []
+        with open(tmp_jsonl_path, "w") as f:
+            for sample in samples:
+                out = self._generate(sample)
+                for row in out:
+                    batch.append(json.dumps(row) + "\n")
+                    rows_written += 1
+                    if len(batch) >= 5:
+                        f.writelines(batch)
+                        batch.clear()
+            if batch:
+                f.writelines(batch)
+
+        if rows_written == 0:
+            if cleanup_locally:
+                cleanup_path(tmp_jsonl_path)
+            return Dataset.from_list([])
+
+        hf_cache_dir = None
+        try:
+            hf_cache_dir = create_temp_dir(
+                prefix=f"{self.block_name}_llm_parser_hf_cache"
+            )
+            ret = load_dataset(
+                "json",
+                data_files=tmp_jsonl_path,
+                split="train",
+                keep_in_memory=False,
+                cache_dir=str(hf_cache_dir),
+            )
+            finalize(ret, cleanup_path, hf_cache_dir)
+            return ret
+        except Exception:
+            if hf_cache_dir is not None:
+                cleanup_path(hf_cache_dir)
+            raise
+        finally:
+            if cleanup_locally:
+                cleanup_path(tmp_jsonl_path)

sdg_hub/core/blocks/llm/text_parser_block.py

@@ -7,14 +7,17 @@ start/end tags, custom regex patterns, and cleanup operations.
 
 # Standard
 from typing import Any, Optional
+from weakref import finalize
+import json
 import re
 
 # Third Party
-from datasets import Dataset
+from datasets import Dataset, load_dataset
 from pydantic import Field, field_validator, model_validator
 
 # Local
 from ...utils.logger_config import setup_logger
+from ...utils.temp_manager import cleanup_path, create_temp_dir, create_temp_file
 from ..base import BaseBlock
 from ..registry import BlockRegistry
 
@@ -27,6 +30,8 @@ logger = setup_logger(__name__)
     "Parses and post-processes text content using tags or regex patterns",
 )
 class TextParserBlock(BaseBlock):
+    _flow_requires_jsonl_tmp: bool = True
+
     """Block for parsing and post-processing text content.
 
     This block handles text parsing using start/end tags, custom regex patterns,
@@ -317,7 +322,54 @@ class TextParserBlock(BaseBlock):
             logger.warning("No samples to parse, returning empty dataset")
             return Dataset.from_list([])
 
-        new_data = []
-        for sample in samples:
-            new_data.extend(self._generate(sample))
-        return Dataset.from_list(new_data)
+        tmp_jsonl_path = kwargs.get("_flow_tmp_jsonl_path")
+        cleanup_locally = False
+
+        if tmp_jsonl_path is None:
+            tmp_jsonl_path = str(
+                create_temp_file(
+                    prefix=f"{self.block_name}_text_parser", suffix=".jsonl"
+                )
+            )
+            cleanup_locally = True
+
+        rows_written = 0
+        batch = []
+        with open(tmp_jsonl_path, "w") as f:
+            for sample in samples:
+                out = self._generate(sample)
+                for row in out:
+                    batch.append(json.dumps(row) + "\n")
+                    rows_written += 1
+                    if len(batch) >= 5:
+                        f.writelines(batch)
+                        batch.clear()
+            if batch:
+                f.writelines(batch)
+
+        if rows_written == 0:
+            if cleanup_locally:
+                cleanup_path(tmp_jsonl_path)
+            return Dataset.from_list([])
+
+        hf_cache_dir = None
+        try:
+            hf_cache_dir = create_temp_dir(
+                prefix=f"{self.block_name}_text_parser_hf_cache"
+            )
+            ret = load_dataset(
+                "json",
+                data_files=tmp_jsonl_path,
+                split="train",
+                keep_in_memory=False,
+                cache_dir=str(hf_cache_dir),
+            )
+            finalize(ret, cleanup_path, hf_cache_dir)
+            return ret
+        except Exception:
+            if hf_cache_dir is not None:
+                cleanup_path(hf_cache_dir)
+            raise
+        finally:
+            if cleanup_locally:
+                cleanup_path(tmp_jsonl_path)

sdg_hub/core/blocks/transform/rename_columns.py

@@ -64,6 +64,25 @@ class RenameColumnsBlock(BaseBlock):
         -------
         Dataset
             Dataset with renamed columns.
+
+        Raises
+        ------
+        ValueError
+            If attempting to rename to a column name that already exists.
         """
+        # Check for column name collisions
+        # Strict validation: no target column name can be an existing column name
+        # This prevents chained/circular renames which can be confusing
+        existing_cols = set(samples.column_names)
+        target_cols = set(self.input_cols.values())
+
+        collision = target_cols & existing_cols
+        if collision:
+            raise ValueError(
+                f"Cannot rename to existing column names: {sorted(collision)}. "
+                "Target column names must not already exist in the dataset. "
+                "Chained renames are not supported."
+            )
+
         # Rename columns using HuggingFace datasets method
         return samples.rename_columns(self.input_cols)

sdg_hub/core/flow/base.py

@@ -5,6 +5,8 @@
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Optional, Union
+from weakref import finalize
+import gc
 import time
 import uuid
 
@@ -37,11 +39,15 @@ from ..utils.flow_metrics import (
 )
 from ..utils.logger_config import setup_logger
 from ..utils.path_resolution import resolve_path
+from ..utils.temp_manager import (
+    cleanup_path,
+    create_temp_dir,
+    create_temp_file,
+)
 from ..utils.time_estimator import estimate_execution_time
 from ..utils.yaml_utils import save_flow_yaml
 from .checkpointer import FlowCheckpointer
 from .metadata import DatasetRequirements, FlowMetadata
-from .migration import FlowMigration
 from .validation import FlowValidator
 
 logger = setup_logger(__name__)
@@ -73,8 +79,6 @@ class Flow(BaseModel):
     model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
 
     # Private attributes (not serialized)
-    _migrated_runtime_params: dict[str, dict[str, Any]] = {}
-    _llm_client: Any = None  # Only used for backward compatibility with old YAMLs
     _model_config_set: bool = False  # Track if model configuration has been set
     _block_metrics: list[dict[str, Any]] = PrivateAttr(
         default_factory=list
@@ -113,16 +117,13 @@ class Flow(BaseModel):
         return self
 
     @classmethod
-    def from_yaml(cls, yaml_path: str, client: Any = None) -> "Flow":
+    def from_yaml(cls, yaml_path: str) -> "Flow":
         """Load flow from YAML configuration file.
 
         Parameters
         ----------
         yaml_path : str
             Path to the YAML flow configuration file.
-        client : Any, optional
-            LLM client instance. Required for backward compatibility with old format YAMLs
-            that use deprecated LLMBlocks. Ignored for new format YAMLs.
 
         Returns
         -------
@@ -153,21 +154,6 @@ class Flow(BaseModel):
         except yaml.YAMLError as exc:
             raise FlowValidationError(f"Invalid YAML in {yaml_path}: {exc}") from exc
 
-        # Check if this is an old format flow and migrate if necessary
-        migrated_runtime_params = None
-        is_old_format = FlowMigration.is_old_format(flow_config)
-        if is_old_format:
-            logger.info(f"Detected old format flow, migrating: {yaml_path}")
-            if client is None:
-                logger.warning(
-                    "Old format YAML detected but no client provided. LLMBlocks may fail."
-                )
-            flow_config, migrated_runtime_params = FlowMigration.migrate_to_new_format(
-                flow_config, yaml_path
-            )
-            # Save migrated config back to YAML to persist id
-            save_flow_yaml(yaml_path, flow_config, "migrated to new format")
-
         # Validate YAML structure
         validator = FlowValidator()
         validation_errors = validator.validate_yaml_structure(flow_config)
@@ -194,19 +180,6 @@ class Flow(BaseModel):
 
         for i, block_config in enumerate(block_configs):
             try:
-                # Inject client for deprecated LLMBlocks if this is an old format flow
-                if (
-                    is_old_format
-                    and block_config.get("block_type") == "LLMBlock"
-                    and client is not None
-                ):
-                    if "block_config" not in block_config:
-                        block_config["block_config"] = {}
-                    block_config["block_config"]["client"] = client
-                    logger.debug(
-                        f"Injected client for deprecated LLMBlock: {block_config['block_config'].get('block_name')}"
-                    )
-
                 block = cls._create_block_from_config(block_config, yaml_dir)
                 blocks.append(block)
             except Exception as exc:
@@ -228,12 +201,6 @@ class Flow(BaseModel):
                 )
             else:
                 logger.debug(f"Flow already had id: {flow.metadata.id}")
-            # Store migrated runtime params and client for backward compatibility
-            if migrated_runtime_params:
-                flow._migrated_runtime_params = migrated_runtime_params
-            if is_old_format and client is not None:
-                flow._llm_client = client
-
             # Check if this is a flow without LLM blocks
             llm_blocks = flow._detect_llm_blocks()
             if not llm_blocks:
@@ -484,12 +451,6 @@ class Flow(BaseModel):
         self._block_metrics = []
         run_start = time.perf_counter()
 
-        # Merge migrated runtime params with provided ones (provided ones take precedence)
-        merged_runtime_params = self._migrated_runtime_params.copy()
-        if runtime_params:
-            merged_runtime_params.update(runtime_params)
-        runtime_params = merged_runtime_params
-
         # Execute flow with metrics capture, ensuring metrics are always displayed/saved
         final_dataset = None
         execution_successful = False
@@ -626,6 +587,7 @@ class Flow(BaseModel):
         # Use provided logger or fall back to global logger
         exec_logger = flow_logger if flow_logger is not None else logger
         current_dataset = dataset
+        current_dataset_temp_path: Optional[Path] = None
 
         # Execute blocks in sequence
         for i, block in enumerate(self.blocks):
@@ -637,6 +599,14 @@ class Flow(BaseModel):
             # Prepare block execution parameters
             block_kwargs = self._prepare_block_kwargs(block, runtime_params)
 
+            block_temp_jsonl_path: Optional[Path] = None
+            dataset_temp_dir: Optional[Path] = None
+            if getattr(block, "_flow_requires_jsonl_tmp", False):
+                block_temp_jsonl_path = create_temp_file(
+                    prefix=f"{block.block_name}_parser", suffix=".jsonl"
+                )
+                block_kwargs["_flow_tmp_jsonl_path"] = str(block_temp_jsonl_path)
+
             # Add max_concurrency to block kwargs if provided
             if max_concurrency is not None:
                 block_kwargs["_flow_max_concurrency"] = max_concurrency
@@ -647,22 +617,8 @@ class Flow(BaseModel):
             input_cols = set(current_dataset.column_names)
 
             try:
-                # Check if this is a deprecated block and skip validations
-                is_deprecated_block = (
-                    hasattr(block, "__class__")
-                    and hasattr(block.__class__, "__module__")
-                    and "deprecated_blocks" in block.__class__.__module__
-                )
-
-                if is_deprecated_block:
-                    exec_logger.debug(
-                        f"Skipping validations for deprecated block: {block.block_name}"
-                    )
-                    # Call generate() directly to skip validations, but keep the runtime params
-                    current_dataset = block.generate(current_dataset, **block_kwargs)
-                else:
-                    # Execute block with validation and logging
-                    current_dataset = block(current_dataset, **block_kwargs)
+                # Execute block with validation and logging
+                current_dataset = block(current_dataset, **block_kwargs)
 
                 # Validate output
                 if len(current_dataset) == 0:
@@ -670,6 +626,28 @@ class Flow(BaseModel):
                         f"Block '{block.block_name}' produced empty dataset"
                     )
 
+                # Here, we write and reload dataset object from and to disk.
+                # This is done because HF Datasets library creates a ton of intermediate
+                # objects, and holds on to them even after the objects have fulfilled
+                # their purpose. To get flush these objects, HF recommends to implement
+                # this `save_to_disk` and `load_from_disk` hack.
+                # https://github.com/huggingface/datasets/blob/main/src/datasets/arrow_dataset.py#L1029
+                previous_temp_path = current_dataset_temp_path
+                dataset_temp_dir = create_temp_dir(prefix=f"flow_{block.block_name}")
+                current_dataset.save_to_disk(str(dataset_temp_dir))
+                del current_dataset
+                gc.collect()
+                current_dataset = datasets.load_from_disk(
+                    str(dataset_temp_dir), keep_in_memory=False
+                )
+                finalize(current_dataset, cleanup_path, dataset_temp_dir)
+                current_dataset_temp_path = dataset_temp_dir
+                if previous_temp_path and previous_temp_path != dataset_temp_dir:
+                    cleanup_path(previous_temp_path)
+
+                if block_temp_jsonl_path is not None:
+                    cleanup_path(block_temp_jsonl_path)
+
                 # Capture metrics after successful execution
                 execution_time = time.perf_counter() - start_time
                 output_rows = len(current_dataset)
@@ -698,6 +676,10 @@ class Flow(BaseModel):
                 )
 
             except Exception as exc:
+                if block_temp_jsonl_path is not None:
+                    cleanup_path(block_temp_jsonl_path)
+                if dataset_temp_dir is not None:
+                    cleanup_path(dataset_temp_dir)
                 # Capture metrics for failed execution
                 execution_time = time.perf_counter() - start_time
                 self._block_metrics.append(
@@ -721,12 +703,21 @@ class Flow(BaseModel):
                     f"Block '{block.block_name}' execution failed: {exc}"
                 ) from exc
 
+        if current_dataset_temp_path is not None:
+            final_temp_path = current_dataset_temp_path
+            current_dataset = datasets.load_from_disk(
+                str(final_temp_path), keep_in_memory=True
+            )
+            cleanup_path(final_temp_path)
+
         return current_dataset
 
     def _prepare_block_kwargs(
-        self, block: BaseBlock, runtime_params: dict[str, dict[str, Any]]
+        self, block: BaseBlock, runtime_params: Optional[dict[str, dict[str, Any]]]
     ) -> dict[str, Any]:
         """Prepare execution parameters for a block."""
+        if runtime_params is None:
+            return {}
         return runtime_params.get(block.block_name, {})
 
     def set_model_config(
@@ -1114,22 +1105,8 @@ class Flow(BaseModel):
                 if max_concurrency is not None:
                     block_kwargs["_flow_max_concurrency"] = max_concurrency
 
-                # Check if this is a deprecated block and skip validations
-                is_deprecated_block = (
-                    hasattr(block, "__class__")
-                    and hasattr(block.__class__, "__module__")
-                    and "deprecated_blocks" in block.__class__.__module__
-                )
-
-                if is_deprecated_block:
-                    logger.debug(
-                        f"Dry run: Skipping validations for deprecated block: {block.block_name}"
-                    )
-                    # Call generate() directly to skip validations, but keep the runtime params
-                    current_dataset = block.generate(current_dataset, **block_kwargs)
-                else:
-                    # Execute block with validation and logging
-                    current_dataset = block(current_dataset, **block_kwargs)
+                # Execute block with validation and logging
+                current_dataset = block(current_dataset, **block_kwargs)
 
                 block_execution_time = (
                     time.perf_counter() - block_start_time

sdg_hub/core/utils/temp_manager.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Utilities for managing temporary files and directories used by the flow."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional, Union
+
+# Standard
+import os
+import shutil
+import tempfile
+
+TEMP_ROOT_DIR_NAME = ".tmp_sdg_buffer"
+
+
+def _get_temp_root() -> Path:
+    root = Path.cwd() / TEMP_ROOT_DIR_NAME
+    root.mkdir(parents=True, exist_ok=True)
+    return root
+
+
+def _format_prefix(prefix: str) -> str:
+    return f"{prefix}_" if prefix and not prefix.endswith("_") else prefix
+
+
+def create_temp_dir(prefix: str = "tmp", suffix: str = "") -> Path:
+    """Create a unique temporary directory."""
+    root = _get_temp_root()
+    name = tempfile.mkdtemp(prefix=_format_prefix(prefix), suffix=suffix, dir=root)
+    return Path(name)
+
+
+def create_temp_file(prefix: str = "tmp", suffix: str = "") -> Path:
+    """Create a unique temporary file."""
+    root = _get_temp_root()
+    fd, name = tempfile.mkstemp(prefix=_format_prefix(prefix), suffix=suffix, dir=root)
+    os.close(fd)
+    return Path(name)
+
+
+def cleanup_path(path: Optional[Union[str, os.PathLike]]) -> None:
+    """Remove a temporary file or directory if it exists."""
+    if not path:
+        return
+
+    target = Path(path)
+    if not target.exists():
+        return
+
+    if target.is_dir():
+        shutil.rmtree(target, ignore_errors=True)
+    else:
+        try:
+            target.unlink()
+        except FileNotFoundError:
+            pass

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/detailed_summary/flow.yaml

@@ -77,9 +77,13 @@ blocks:
     - ''
 - block_type: RenameColumnsBlock
   block_config:
-    block_name: rename_to_document_column
+    block_name: rename_to_raw_document_column
     input_cols:
       document: raw_document
+- block_type: RenameColumnsBlock
+  block_config:
+    block_name: rename_to_document_column
+    input_cols:
       summary: document
 - block_type: PromptBuilderBlock
   block_config:

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/extractive_summary/flow.yaml

@@ -79,9 +79,13 @@ blocks:
     - ''
 - block_type: RenameColumnsBlock
   block_config:
-    block_name: rename_to_document_column
+    block_name: rename_to_raw_document_column
     input_cols:
       document: raw_document
+- block_type: RenameColumnsBlock
+  block_config:
+    block_name: rename_to_document_column
+    input_cols:
       summary: document
 - block_type: PromptBuilderBlock
   block_config:

sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/key_facts/flow.yaml

@@ -72,9 +72,13 @@ blocks:
     parsing_pattern: '(?:^|\n)\s*\d+\.\s+(.*?)(?=\n\s*\d+\.\s+|\Z)'
 - block_type: RenameColumnsBlock
   block_config:
-    block_name: rename_to_document_column
+    block_name: rename_to_raw_document_column
     input_cols:
       document: raw_document
+- block_type: RenameColumnsBlock
+  block_config:
+    block_name: rename_to_document_column
+    input_cols:
       atomic_facts: document
 - block_type: PromptBuilderBlock
   block_config:

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/flow.yaml

@@ -134,10 +134,15 @@ blocks:
       input_cols: [summary_detailed, summary_extractive, summary_atomic_facts, base_document]
       output_cols: [summary, dataset_type]
 
+  - block_type: RenameColumnsBlock
+    block_config:
+      block_name: rename_to_raw_document_column
+      input_cols: {document: raw_document}
+
   - block_type: RenameColumnsBlock
     block_config:
       block_name: rename_to_document_column
-      input_cols: {document: raw_document, summary: document}
+      input_cols: {summary: document}
 
   - block_type: PromptBuilderBlock
     block_config:

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/multilingual/japanese/flow.yaml

@@ -135,10 +135,14 @@ blocks:
       input_cols: [summary_detailed, summary_extractive, summary_atomic_facts, base_document]
       output_cols: [summary, dataset_type]
 
+  - block_type: RenameColumnsBlock
+    block_config:
+      block_name: rename_to_raw_document_column
+      input_cols: {document: raw_document}
   - block_type: RenameColumnsBlock
     block_config:
       block_name: rename_to_document_column
-      input_cols: {document: raw_document, summary: document}
+      input_cols: {summary: document}
 
   - block_type: PromptBuilderBlock
     block_config:

{sdg_hub-0.4.2.dist-info → sdg_hub-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.4.2
+Version: 0.5.1
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0
@@ -23,7 +23,7 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: click<9.0.0,>=8.1.7
-Requires-Dist: datasets<4.0.0,>=2.18.0
+Requires-Dist: datasets>=4.0.0
 Requires-Dist: httpx<1.0.0,>=0.25.0
 Requires-Dist: jinja2
 Requires-Dist: litellm<1.75.0,>=1.73.0