sdg-hub 0.4.0__tar.gz → 0.4.2__tar.gz

sdg_hub-0.4.2/.github/workflows/integration-test.yml

@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Integration Test
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      # Only trigger on changes to relevant flows and examples (EXTEND THIS):
+      - 'src/sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/**'
+      - 'examples/knowledge_tuning/enhanced_summary_knowledge_tuning/**'
+      # Standard integration test triggers, DONT CHANGE THIS
+      - 'tests/integration/**/*.py'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - '.github/workflows/integration-test.yml'
+  pull_request:
+    branches:
+      - "main"
+      - "release-**"
+    types: [opened, synchronize, reopened, labeled]
+    paths:
+      # Only trigger on changes to relevant flows and examples (EXTEND THIS):
+      - 'src/sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/**'
+      - 'examples/knowledge_tuning/enhanced_summary_knowledge_tuning/**'
+      # Standard integration test triggers, DONT CHANGE THIS
+      - 'tests/integration/**/*.py'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - '.github/workflows/integration-test.yml'
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  integration-test:
+    name: "Integration Tests - ${{ matrix.python }} on ${{ matrix.platform }}"
+    runs-on: "${{ matrix.platform }}"
+    # Require manual approval before running (via GitHub Environment)
+    environment: integration-tests
+    # Skip fork PRs (they can't access environment secrets anyway)
+    # Also check for 'run-integration-tests' label on labeled events
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'push' ||
+      (github.event_name == 'pull_request' &&
+       github.event.pull_request.head.repo.full_name == github.repository &&
+       (github.event.action != 'labeled' || contains(github.event.pull_request.labels.*.name, 'run-integration-tests')))
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+        platform:
+          - "ubuntu-latest"
+    steps:
+      - name: "Harden Runner"
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          # https://github.com/actions/checkout/issues/249
+          fetch-depth: 0
+
+      - name: Free disk space
+        uses: ./.github/actions/free-disk-space
+
+      - name: Install the expect package
+        run: |
+          sudo apt-get install -y expect
+
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/requirements*.txt
+
+      - name: Remove llama-cpp-python from cache
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Cache huggingface datasets
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/huggingface
+          # Invalidate cache when any example notebook changes (may affect dataset downloads)
+          key: huggingface-${{ hashFiles('examples/**/*.ipynb') }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh>=1.2
+
+      - name: Run integration tests with tox
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          tox -e py3-integrationcov
+
+      - name: Remove llama-cpp-python from cache
+        if: always()
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Upload integration test coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage-py3-integrationcov.xml
+          fail_ci_if_error: false
+          flags: integration
+
+      - name: Upload integration test artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-test-results-${{ matrix.python }}-${{ matrix.platform }}
+          path: |
+            coverage-py3-integrationcov/
+            coverage-py3-integrationcov.xml
+            durations/py3-integrationcov.html
+          retention-days: 30
+
+  integration-test-workflow-complete:
+    needs: ["integration-test"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Integration Test Workflow Complete
+        run: echo "Integration Test Workflow Complete"

{sdg_hub-0.4.0 → sdg_hub-0.4.2}/.github/workflows/test.yml

@@ -9,7 +9,8 @@ on:
       - "main"
       - "release-**"
     paths:
-      - '**.py'
+      - 'src/**/*.py'
+      - 'tests/**/*.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
       - 'tox.ini'
@@ -19,7 +20,8 @@ on:
       - "main"
       - "release-**"
     paths:
-      - '**.py'
+      - 'src/**/*.py'
+      - 'tests/**/*.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
       - 'tox.ini'
@@ -37,7 +39,7 @@ permissions:
 
 jobs:
   test:
-    name: "${{ matrix.python }} on ${{ matrix.platform }}"
+    name: "Unit Tests - ${{ matrix.python }} on ${{ matrix.platform }}"
     runs-on: "${{ matrix.platform }}"
     strategy:
       matrix:
@@ -104,6 +106,7 @@ jobs:
         run: |
           tox -e py3-unitcov
 
+
       - name: Remove llama-cpp-python from cache
         if: always()
         run: |

{sdg_hub-0.4.0 → sdg_hub-0.4.2}/.gitignore

@@ -84,6 +84,11 @@ target/
 # Jupyter Notebook
 .ipynb_checkpoints
 
+# Integration test artifacts
+tests/integration/**/converted_scripts/
+tests/integration/**/test_output/
+tests/integration/**/output_data/
+
 # IPython
 profile_default/
 ipython_config.py

{sdg_hub-0.4.0/src/sdg_hub.egg-info → sdg_hub-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.4.0
+Version: 0.4.2
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0
@@ -65,6 +65,7 @@ Requires-Dist: pytest-html; extra == "dev"
 Requires-Dist: tox<5,>=4.4.2; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pytest-env; extra == "dev"
+Requires-Dist: nbconvert>=7.0.0; extra == "dev"
 Dynamic: license-file
 
 # `sdg_hub`: Synthetic Data Generation Toolkit

{sdg_hub-0.4.0 → sdg_hub-0.4.2}/docs/blocks/llm-blocks.md

@@ -230,10 +230,244 @@ Constructs prompts from templates and data with validation and formatting suppor
 
 ## 🔍 TextParserBlock
 
-Extracts structured data from LLM responses using patterns, schemas, or custom parsers.
+Extracts structured data from LLM responses using tag-based parsing or custom regex patterns. Essential for parsing LLM outputs into structured fields.
 
-#TODO: Add text parser block example
+### Basic Tag-Based Parsing
 
+Extract content between start and end tags:
+
+```python
+from sdg_hub.core.blocks import TextParserBlock
+from datasets import Dataset
+
+# Single field extraction
+parser = TextParserBlock(
+    block_name="extract_answer",
+    input_cols=["llm_response"],
+    output_cols=["answer"],
+    start_tags=["<answer>"],
+    end_tags=["</answer>"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "Question analysis: ...\n<answer>Machine learning is a subset of AI.</answer>",
+        "Let me think...\n<answer>Neural networks process data in layers.</answer>"
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["answer"])
+# ['Machine learning is a subset of AI.', 'Neural networks process data in layers.']
+```
+
+### Multiple Field Extraction
+
+Extract multiple structured fields from a single response:
+
+```python
+# Extract multiple fields with tag pairs
+parser = TextParserBlock(
+    block_name="extract_qa",
+    input_cols=["llm_response"],
+    output_cols=["question", "answer", "confidence"],
+    start_tags=["<question>", "<answer>", "<confidence>"],
+    end_tags=["</question>", "</answer>", "</confidence>"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        """
+        <question>What is Python?</question>
+        <answer>Python is a high-level programming language.</answer>
+        <confidence>0.95</confidence>
+        """
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["question"])     # ['What is Python?']
+print(result["answer"])       # ['Python is a high-level programming language.']
+print(result["confidence"])   # ['0.95']
+```
+
+### Custom Regex Parsing
+
+Use regex patterns for flexible extraction:
+
+```python
+# Extract using regex pattern
+parser = TextParserBlock(
+    block_name="regex_parser",
+    input_cols=["llm_response"],
+    output_cols=["answer"],
+    parsing_pattern=r"Answer:\s*(.+?)(?:\n|$)"
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "Question: What is AI?\nAnswer: Artificial Intelligence is...\n",
+        "Let me answer:\nAnswer: Machine learning enables..."
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["answer"])
+# ['Artificial Intelligence is...', 'Machine learning enables...']
+```
+
+### Tag Cleanup
+
+Remove unwanted tags from extracted content:
+
+```python
+# Clean up markdown and code tags
+parser = TextParserBlock(
+    block_name="clean_parser",
+    input_cols=["llm_response"],
+    output_cols=["clean_answer"],
+    start_tags=["<answer>"],
+    end_tags=["</answer>"],
+    parser_cleanup_tags=["```", "###", "**"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "<answer>Here's the code: ```python\nprint('hello')```</answer>",
+        "<answer>**Important**: This is the ### answer</answer>"
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["clean_answer"])
+# ['Here\'s the code: python\nprint(\'hello\')', 'Important: This is the  answer']
+```
+
+### Handling Multiple Matches
+
+Extract all occurrences of a pattern:
+
+```python
+parser = TextParserBlock(
+    block_name="multi_extract",
+    input_cols=["llm_response"],
+    output_cols=["keywords"],
+    start_tags=["[KEY]"],
+    end_tags=["[/KEY]"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "Important terms: [KEY]machine learning[/KEY], [KEY]neural networks[/KEY], [KEY]deep learning[/KEY]"
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["keywords"])
+# [['machine learning', 'neural networks', 'deep learning']]
+```
+
+### Practical Example: Evaluation Response Parsing
+
+Common pattern for parsing LLM evaluation responses:
+
+```python
+# Parse structured evaluation output
+evaluation_parser = TextParserBlock(
+    block_name="parse_evaluation",
+    input_cols=["evaluation_response"],
+    output_cols=["explanation", "judgment"],
+    start_tags=["[Start of Explanation]", "[Start of Answer]"],
+    end_tags=["[End of Explanation]", "[End of Answer]"],
+    parser_cleanup_tags=["```", "###"]
+)
+
+dataset = Dataset.from_dict({
+    "evaluation_response": [
+        """
+        [Start of Explanation]
+        The response accurately reflects the information in the document.
+        No hallucinations or contradictions were found.
+        [End of Explanation]
+
+        [Start of Answer]
+        YES
+        [End of Answer]
+        """
+    ]
+})
+
+result = evaluation_parser.generate(dataset)
+print(result["explanation"])  # ['The response accurately reflects...']
+print(result["judgment"])     # ['YES']
+```
+
+### Integration with LLMChatBlock
+
+TextParserBlock is commonly used after LLMChatBlock to structure responses:
+
+```python
+from sdg_hub.core.blocks import LLMChatBlock, LLMParserBlock, TextParserBlock
+
+# Step 1: Generate LLM response
+chat_block = LLMChatBlock(
+    block_name="evaluator",
+    model="openai/gpt-4o",
+    input_cols=["messages"],
+    output_cols=["eval_response"]
+)
+
+# Step 2: Extract content from response object
+# Use field_prefix="" to get cleaner column names
+llm_parser = LLMParserBlock(
+    block_name="extract_eval",
+    input_cols=["eval_response"],
+    extract_content=True,
+    field_prefix="eval_"  # Results in "eval_content" instead of "extract_content"
+)
+
+# Step 3: Parse structured fields from text
+text_parser = TextParserBlock(
+    block_name="parse_fields",
+    input_cols=["eval_content"], 
+    output_cols=["score", "reasoning"],
+    start_tags=["[SCORE]", "[REASONING]"],
+    end_tags=["[/SCORE]", "[/REASONING]"]
+)
+
+# Execute in sequence (or use a Flow)
+dataset = Dataset.from_dict({
+    "messages": [[{"role": "user", "content": "Evaluate this text..."}]]
+})
+
+result = chat_block.generate(dataset)
+result = llm_parser.generate(result)
+result = text_parser.generate(result)
+
+print(result["score"])      # Extracted score
+print(result["reasoning"])  # Extracted reasoning
+```
+
+### Configuration Reference
+
+**Required Parameters:**
+- `block_name` - Unique identifier for the block
+- `input_cols` - Single column containing text to parse
+- `output_cols` - List of field names for extracted content
+
+**Parsing Methods (choose one):**
+- **Tag-based**: `start_tags` + `end_tags` (must have same length as `output_cols`)
+- **Regex**: `parsing_pattern` (single regex with capture groups)
+
+**Optional Parameters:**
+- `parser_cleanup_tags` - List of tags to remove from extracted text
+- `expand_lists` - Whether to expand list inputs into rows (default: `True`)
+
+**Tag Parsing Rules:**
+- Number of tag pairs must match number of output columns
+- Each tag pair extracts all matches for that field
+- Tags can be any string (XML-style, markdown-style, custom)
+- Missing tags result in empty lists for that field
 
 ## 🚀 Next Steps
 

{sdg_hub-0.4.0 → sdg_hub-0.4.2}/docs/concepts.md

@@ -148,9 +148,22 @@ Every block validates data at runtime:
 ## 🚀 Best Practices
 
 ### 1. Start Small
-- Use `dry_run()` to test with small samples
+- Use `dry_run()` to test with small samples before processing full datasets
+- Add `enable_time_estimation=True` to predict execution time for the complete dataset
 - Validate your pipeline before scaling up
 
+```python
+# Test AND estimate in one call
+result = flow.dry_run(dataset, sample_size=5, enable_time_estimation=True, max_concurrency=100)
+
+# Access dry run results
+print(f"Tested with {result['sample_size']} samples")
+print(f"Output columns: {result['final_dataset']['columns']}")
+
+# Time estimation is automatically displayed in a Rich table format
+# No need to access it programmatically - the table shows all estimation details
+```
+
 ### 2. Layer Validation
 - Use basic block composition (PromptBuilder → LLMChat → Parser → Filter) to assess quality
 - Implement filtering to maintain data standards

{sdg_hub-0.4.0 → sdg_hub-0.4.2}/docs/flows/discovery.md

@@ -67,7 +67,44 @@ for flow_name in all_flows:
 
 ### Getting Flow Information
 
-#TODO: Add flow info example
+Access detailed flow metadata and configuration:
+
+```python
+from sdg_hub.core.flow import FlowRegistry, Flow
+
+# Get metadata for a specific flow
+flow_name = "Advanced Document Grounded Question-Answer Generation Flow for Knowledge Tuning"
+metadata = FlowRegistry.get_flow_metadata(flow_name)
+
+if metadata:
+    print(f"Flow: {metadata.name}")
+    print(f"Version: {metadata.version}")
+    print(f"Author: {metadata.author}")
+    print(f"Description: {metadata.description}")
+    print(f"Tags: {', '.join(metadata.tags)}")
+    print(f"Recommended model: {metadata.recommended_models.get('default', 'Not specified')}")
+
+# Load flow and get detailed information
+flow_path = FlowRegistry.get_flow_path(flow_name)
+flow = Flow.from_yaml(flow_path)
+
+# Get comprehensive flow info
+info = flow.get_info()
+print(f"Total blocks: {info['total_blocks']}")
+print(f"Block sequence: {', '.join(info['block_names'])}")
+
+# Get dataset requirements
+requirements = flow.get_dataset_requirements()
+if requirements:
+    print(f"Required columns: {requirements.required_columns}")
+    print(f"Description: {requirements.description}")
+    print(f"Min samples: {requirements.min_samples}")
+
+# Get model recommendations
+recommendations = flow.get_model_recommendations()
+print(f"Default model: {recommendations.get('default')}")
+print(f"Compatible models: {recommendations.get('compatible', [])}")
+```
 
 ### Getting Flow Paths