sdg-hub 0.4.0__tar.gz → 0.4.1__tar.gz

sdg_hub-0.4.1/.github/workflows/integration-test.yml

@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Integration Test
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      # Only trigger on changes to relevant flows and examples (EXTEND THIS):
+      - 'src/sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/**'
+      - 'examples/knowledge_tuning/enhanced_summary_knowledge_tuning/**'
+      # Standard integration test triggers, DONT CHANGE THIS
+      - 'tests/integration/**/*.py'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - '.github/workflows/integration-test.yml'
+  pull_request:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      # Only trigger on changes to relevant flows and examples (EXTEND THIS):
+      - 'src/sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/**'
+      - 'examples/knowledge_tuning/enhanced_summary_knowledge_tuning/**'
+      # Standard integration test triggers, DONT CHANGE THIS
+      - 'tests/integration/**/*.py'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - '.github/workflows/integration-test.yml'
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  integration-test:
+    name: "Integration Tests - ${{ matrix.python }} on ${{ matrix.platform }}"
+    runs-on: "${{ matrix.platform }}"
+    # Require manual approval before running (via GitHub Environment)
+    environment: integration-tests
+    # Skip fork PRs (they can't access environment secrets anyway)
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'push' ||
+      (github.event_name == 'pull_request' &&
+       github.event.pull_request.head.repo.full_name == github.repository)
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+        platform:
+          - "ubuntu-latest"
+    steps:
+      - name: "Harden Runner"
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          # https://github.com/actions/checkout/issues/249
+          fetch-depth: 0
+
+      - name: Free disk space
+        uses: ./.github/actions/free-disk-space
+
+      - name: Install the expect package
+        run: |
+          sudo apt-get install -y expect
+
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/requirements*.txt
+
+      - name: Remove llama-cpp-python from cache
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Cache huggingface datasets
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/huggingface
+          # Invalidate cache when any example notebook changes (may affect dataset downloads)
+          key: huggingface-${{ hashFiles('examples/**/*.ipynb') }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh>=1.2
+
+      - name: Run integration tests with tox
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          tox -e py3-integrationcov
+
+      - name: Remove llama-cpp-python from cache
+        if: always()
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Upload integration test coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage-py3-integrationcov.xml
+          fail_ci_if_error: false
+          flags: integration
+
+      - name: Upload integration test artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-test-results-${{ matrix.python }}-${{ matrix.platform }}
+          path: |
+            coverage-py3-integrationcov/
+            coverage-py3-integrationcov.xml
+            durations/py3-integrationcov.html
+          retention-days: 30
+
+  integration-test-workflow-complete:
+    needs: ["integration-test"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Integration Test Workflow Complete
+        run: echo "Integration Test Workflow Complete"

{sdg_hub-0.4.0 → sdg_hub-0.4.1}/.github/workflows/test.yml

@@ -9,7 +9,8 @@ on:
       - "main"
       - "release-**"
     paths:
-      - '**.py'
+      - 'src/**/*.py'
+      - 'tests/**/*.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
       - 'tox.ini'
@@ -19,7 +20,8 @@ on:
       - "main"
       - "release-**"
     paths:
-      - '**.py'
+      - 'src/**/*.py'
+      - 'tests/**/*.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
       - 'tox.ini'
@@ -37,7 +39,7 @@ permissions:
 
 jobs:
   test:
-    name: "${{ matrix.python }} on ${{ matrix.platform }}"
+    name: "Unit Tests - ${{ matrix.python }} on ${{ matrix.platform }}"
     runs-on: "${{ matrix.platform }}"
     strategy:
       matrix:
@@ -104,6 +106,7 @@ jobs:
         run: |
           tox -e py3-unitcov
 
+
       - name: Remove llama-cpp-python from cache
         if: always()
         run: |

{sdg_hub-0.4.0 → sdg_hub-0.4.1}/.gitignore

@@ -84,6 +84,11 @@ target/
 # Jupyter Notebook
 .ipynb_checkpoints
 
+# Integration test artifacts
+tests/integration/**/converted_scripts/
+tests/integration/**/test_output/
+tests/integration/**/output_data/
+
 # IPython
 profile_default/
 ipython_config.py

{sdg_hub-0.4.0/src/sdg_hub.egg-info → sdg_hub-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.4.0
+Version: 0.4.1
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0
@@ -65,6 +65,7 @@ Requires-Dist: pytest-html; extra == "dev"
 Requires-Dist: tox<5,>=4.4.2; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pytest-env; extra == "dev"
+Requires-Dist: nbconvert>=7.0.0; extra == "dev"
 Dynamic: license-file
 
 # `sdg_hub`: Synthetic Data Generation Toolkit

{sdg_hub-0.4.0 → sdg_hub-0.4.1}/docs/blocks/llm-blocks.md

@@ -230,10 +230,244 @@ Constructs prompts from templates and data with validation and formatting suppor
 
 ## 🔍 TextParserBlock
 
-Extracts structured data from LLM responses using patterns, schemas, or custom parsers.
+Extracts structured data from LLM responses using tag-based parsing or custom regex patterns. Essential for parsing LLM outputs into structured fields.
 
-#TODO: Add text parser block example
+### Basic Tag-Based Parsing
 
+Extract content between start and end tags:
+
+```python
+from sdg_hub.core.blocks import TextParserBlock
+from datasets import Dataset
+
+# Single field extraction
+parser = TextParserBlock(
+    block_name="extract_answer",
+    input_cols=["llm_response"],
+    output_cols=["answer"],
+    start_tags=["<answer>"],
+    end_tags=["</answer>"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "Question analysis: ...\n<answer>Machine learning is a subset of AI.</answer>",
+        "Let me think...\n<answer>Neural networks process data in layers.</answer>"
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["answer"])
+# ['Machine learning is a subset of AI.', 'Neural networks process data in layers.']
+```
+
+### Multiple Field Extraction
+
+Extract multiple structured fields from a single response:
+
+```python
+# Extract multiple fields with tag pairs
+parser = TextParserBlock(
+    block_name="extract_qa",
+    input_cols=["llm_response"],
+    output_cols=["question", "answer", "confidence"],
+    start_tags=["<question>", "<answer>", "<confidence>"],
+    end_tags=["</question>", "</answer>", "</confidence>"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        """
+        <question>What is Python?</question>
+        <answer>Python is a high-level programming language.</answer>
+        <confidence>0.95</confidence>
+        """
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["question"])     # ['What is Python?']
+print(result["answer"])       # ['Python is a high-level programming language.']
+print(result["confidence"])   # ['0.95']
+```
+
+### Custom Regex Parsing
+
+Use regex patterns for flexible extraction:
+
+```python
+# Extract using regex pattern
+parser = TextParserBlock(
+    block_name="regex_parser",
+    input_cols=["llm_response"],
+    output_cols=["answer"],
+    parsing_pattern=r"Answer:\s*(.+?)(?:\n|$)"
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "Question: What is AI?\nAnswer: Artificial Intelligence is...\n",
+        "Let me answer:\nAnswer: Machine learning enables..."
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["answer"])
+# ['Artificial Intelligence is...', 'Machine learning enables...']
+```
+
+### Tag Cleanup
+
+Remove unwanted tags from extracted content:
+
+```python
+# Clean up markdown and code tags
+parser = TextParserBlock(
+    block_name="clean_parser",
+    input_cols=["llm_response"],
+    output_cols=["clean_answer"],
+    start_tags=["<answer>"],
+    end_tags=["</answer>"],
+    parser_cleanup_tags=["```", "###", "**"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "<answer>Here's the code: ```python\nprint('hello')```</answer>",
+        "<answer>**Important**: This is the ### answer</answer>"
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["clean_answer"])
+# ['Here\'s the code: python\nprint(\'hello\')', 'Important: This is the  answer']
+```
+
+### Handling Multiple Matches
+
+Extract all occurrences of a pattern:
+
+```python
+parser = TextParserBlock(
+    block_name="multi_extract",
+    input_cols=["llm_response"],
+    output_cols=["keywords"],
+    start_tags=["[KEY]"],
+    end_tags=["[/KEY]"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "Important terms: [KEY]machine learning[/KEY], [KEY]neural networks[/KEY], [KEY]deep learning[/KEY]"
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["keywords"])
+# [['machine learning', 'neural networks', 'deep learning']]
+```
+
+### Practical Example: Evaluation Response Parsing
+
+Common pattern for parsing LLM evaluation responses:
+
+```python
+# Parse structured evaluation output
+evaluation_parser = TextParserBlock(
+    block_name="parse_evaluation",
+    input_cols=["evaluation_response"],
+    output_cols=["explanation", "judgment"],
+    start_tags=["[Start of Explanation]", "[Start of Answer]"],
+    end_tags=["[End of Explanation]", "[End of Answer]"],
+    parser_cleanup_tags=["```", "###"]
+)
+
+dataset = Dataset.from_dict({
+    "evaluation_response": [
+        """
+        [Start of Explanation]
+        The response accurately reflects the information in the document.
+        No hallucinations or contradictions were found.
+        [End of Explanation]
+
+        [Start of Answer]
+        YES
+        [End of Answer]
+        """
+    ]
+})
+
+result = evaluation_parser.generate(dataset)
+print(result["explanation"])  # ['The response accurately reflects...']
+print(result["judgment"])     # ['YES']
+```
+
+### Integration with LLMChatBlock
+
+TextParserBlock is commonly used after LLMChatBlock to structure responses:
+
+```python
+from sdg_hub.core.blocks import LLMChatBlock, LLMParserBlock, TextParserBlock
+
+# Step 1: Generate LLM response
+chat_block = LLMChatBlock(
+    block_name="evaluator",
+    model="openai/gpt-4o",
+    input_cols=["messages"],
+    output_cols=["eval_response"]
+)
+
+# Step 2: Extract content from response object
+# Use field_prefix="" to get cleaner column names
+llm_parser = LLMParserBlock(
+    block_name="extract_eval",
+    input_cols=["eval_response"],
+    extract_content=True,
+    field_prefix="eval_"  # Results in "eval_content" instead of "extract_content"
+)
+
+# Step 3: Parse structured fields from text
+text_parser = TextParserBlock(
+    block_name="parse_fields",
+    input_cols=["eval_content"], 
+    output_cols=["score", "reasoning"],
+    start_tags=["[SCORE]", "[REASONING]"],
+    end_tags=["[/SCORE]", "[/REASONING]"]
+)
+
+# Execute in sequence (or use a Flow)
+dataset = Dataset.from_dict({
+    "messages": [[{"role": "user", "content": "Evaluate this text..."}]]
+})
+
+result = chat_block.generate(dataset)
+result = llm_parser.generate(result)
+result = text_parser.generate(result)
+
+print(result["score"])      # Extracted score
+print(result["reasoning"])  # Extracted reasoning
+```
+
+### Configuration Reference
+
+**Required Parameters:**
+- `block_name` - Unique identifier for the block
+- `input_cols` - Single column containing text to parse
+- `output_cols` - List of field names for extracted content
+
+**Parsing Methods (choose one):**
+- **Tag-based**: `start_tags` + `end_tags` (must have same length as `output_cols`)
+- **Regex**: `parsing_pattern` (single regex with capture groups)
+
+**Optional Parameters:**
+- `parser_cleanup_tags` - List of tags to remove from extracted text
+- `expand_lists` - Whether to expand list inputs into rows (default: `True`)
+
+**Tag Parsing Rules:**
+- Number of tag pairs must match number of output columns
+- Each tag pair extracts all matches for that field
+- Tags can be any string (XML-style, markdown-style, custom)
+- Missing tags result in empty lists for that field
 
 ## 🚀 Next Steps
 

{sdg_hub-0.4.0 → sdg_hub-0.4.1}/docs/flows/overview.md

@@ -292,34 +292,135 @@ print(f"Output columns: {dry_result['final_dataset']['columns']}")
 print(f"Sample output: {dry_result['sample_output']}")
 ```
 
-### Parameter Override
+### Runtime Parameters
 
-Customize flow behavior at runtime:
+Runtime parameters allow you to customize block behavior at execution time without modifying flow YAML files. You can override global parameters for all blocks or configure specific blocks individually.
+
+**Global Parameter Override:**
+
+Apply parameters to all compatible blocks in the flow:
 
 ```python
-# Override default runtime parameters
+# Override global parameters
 result = flow.generate(
     dataset,
     runtime_params={
+        "temperature": 0.7,
         "max_tokens": 200,
-        "temperature": 0.9,
+        "top_p": 0.95
     }
 )
 ```
 
-### Block-Specific Runtime Arguments
+**Block-Specific Configuration:**
+
+Target individual blocks by their `block_name` for fine-grained control:
 
-You can enable or disable advanced features—such as "thinking mode"—for individual blocks at runtime using the `runtime_params` argument. This allows fine-grained control over block behavior without modifying the flow YAML.
+```python
+# Configure different parameters for each block
+result = flow.generate(
+    dataset,
+    runtime_params={
+        # LLM blocks - control generation parameters
+        "question_generator": {
+            "temperature": 0.9,
+            "max_tokens": 100,
+            "top_p": 0.95,
+            "frequency_penalty": 0.5
+        },
+        "answer_generator": {
+            "temperature": 0.5,
+            "max_tokens": 300,
+            "presence_penalty": 0.3
+        },
+
+        # LLM parser blocks - configure extraction
+        "extract_eval_content": {
+            "extract_content": True,
+            "extract_reasoning_content": True,
+            "field_prefix": "llm_"
+        },
+
+        # Text parsing blocks - override parsing tags
+        "parse_evaluation": {
+            "start_tags": ["[Answer]", "[Explanation]", "[Score]"],
+            "end_tags": ["[/Answer]", "[/Explanation]", "[/Score]"],
+            "parser_cleanup_tags": ["```", "###", "---"]
+        },
+
+        # Filter blocks - adjust filter criteria
+        "quality_filter": {
+            "filter_value": 0.9,
+            "operation": "ge"
+        },
+        "faithfulness_filter": {
+            "filter_value": "YES",
+            "operation": "eq"
+        }
+    }
+)
+```
 
-For example, to disable "thinking mode" for several blocks:
+**Common Runtime Parameters by Block Type:**
+
+| Block Type | Parameter | Description | Example Values |
+|------------|-----------|-------------|----------------|
+| **LLMChatBlock** | `temperature` | Control randomness in generation | `0.0` - `2.0` |
+| | `max_tokens` | Maximum response length | `50`, `200`, `1000` |
+| | `top_p` | Nucleus sampling threshold | `0.0` - `1.0` |
+| | `frequency_penalty` | Penalize token repetition | `-2.0` - `2.0` |
+| | `presence_penalty` | Penalize new topics | `-2.0` - `2.0` |
+| **LLMParserBlock** | `extract_content` | Extract main content field | `True`, `False` |
+| | `extract_reasoning_content` | Extract reasoning/thinking | `True`, `False` |
+| | `extract_tool_calls` | Extract tool call data | `True`, `False` |
+| | `field_prefix` | Prefix for output fields | `"llm_"`, `"parsed_"` |
+| **TextParserBlock** | `start_tags` | Opening tags for extraction | `["<answer>", "[Q]"]` |
+| | `end_tags` | Closing tags for extraction | `["</answer>", "[/Q]"]` |
+| | `parsing_pattern` | Custom regex pattern | `r"Answer:\s*(.+)"` |
+| | `parser_cleanup_tags` | Tags to remove from output | `["```", "###"]` |
+| **ColumnValueFilterBlock** | `filter_value` | Value to filter by | `0.8`, `"YES"`, `[1, 2]` |
+| | `operation` | Comparison operation | `"eq"`, `"gt"`, `"contains"` |
+| | `convert_dtype` | Type conversion | `"float"`, `"int"` |
+
+**Practical Examples:**
 
 ```python
-# Set runtime_params for specific blocks
+# Experiment with different generation styles
+result = flow.generate(
+    dataset,
+    runtime_params={
+        "temperature": 0.9,  # More creative
+        "top_p": 0.95
+    }
+)
+
+# Adjust parsing for different prompt formats
+result = flow.generate(
+    dataset,
+    runtime_params={
+        "text_parser": {
+            "start_tags": ["<thinking>", "<answer>"],
+            "end_tags": ["</thinking>", "</answer>"]
+        }
+    }
+)
+
+# Increase quality thresholds for production
+result = flow.generate(
+    dataset,
+    runtime_params={
+        "quality_filter": {"filter_value": 0.95},
+        "relevancy_filter": {"filter_value": 0.90}
+    }
+)
+
+# Mix global and block-specific parameters
 result = flow.generate(
-    dataset, 
-    runtime_params = {
-    # LLMChatBlock blocks
-    "llm_chat_block_1": {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}},
+    dataset,
+    runtime_params={
+        "temperature": 0.7,  # Global default
+        "creative_generator": {"temperature": 1.0},  # Override for one block
+        "quality_filter": {"filter_value": 0.85}
     }
 )
 ```
@@ -369,6 +470,73 @@ result = flow.generate(dataset, max_concurrency=20)
 result = flow.generate(dataset)  # Default behavior
 ```
 
+### Checkpointing
+
+Flow checkpointing enables resuming interrupted executions by saving progress periodically. This is essential for long-running flows that process large datasets, preventing data loss from failures or interruptions.
+
+**Basic Checkpointing:**
+
+```python
+# Enable checkpointing with automatic resume
+result = flow.generate(
+    dataset,
+    checkpoint_dir="./my_flow_checkpoints",
+    save_freq=100  # Save every 100 completed samples
+)
+```
+
+**How It Works:**
+
+1. **Progress Tracking** - Flow saves completed samples to checkpoint files after every `save_freq` samples
+2. **Automatic Resume** - On restart, Flow detects existing checkpoints and processes only remaining samples
+3. **Final Merge** - Completed and newly processed samples are automatically combined in the final result
+
+**Use Cases:**
+
+- **Long-Running Flows** - Process thousands of samples safely over hours or days
+- **Unreliable Infrastructure** - Protect against network failures, rate limits, or system crashes
+- **Iterative Development** - Test and refine flows without reprocessing completed samples
+- **Cost Management** - Avoid wasting API credits by restarting from failures
+
+**Configuration Options:**
+
+```python
+# Save checkpoints every N samples (recommended for large datasets)
+result = flow.generate(
+    dataset,
+    checkpoint_dir="./checkpoints",
+    save_freq=50  # Checkpoint after each 50 samples
+)
+
+# Only save final result (minimal overhead)
+result = flow.generate(
+    dataset,
+    checkpoint_dir="./checkpoints"
+    # No save_freq - only saves at completion
+)
+
+# Combine with other execution features
+result = flow.generate(
+    dataset,
+    checkpoint_dir="./checkpoints",
+    save_freq=100,
+    max_concurrency=10
+)
+```
+
+**Checkpoint Structure:**
+
+Checkpoint directories contain:
+- `checkpoint_NNNN.jsonl` - Completed sample batches in JSONL format
+- `flow_metadata.json` - Flow ID, progress counters, and validation data
+
+**Important Notes:**
+
+- Checkpoints are flow-specific using `flow_id` to prevent mixing incompatible data
+- Remaining samples are identified by comparing input dataset with completed samples using common columns
+- If all samples are completed, Flow skips processing and returns merged results immediately
+- Clean up checkpoint directories manually when no longer needed
+
 ## 🚀 Next Steps
 
 Ready to master the flow system? Explore these detailed guides: