sdg-hub 0.3.1__tar.gz → 0.4.1__tar.gz

sdg_hub-0.4.1/.github/workflows/integration-test.yml

@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Integration Test
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      # Only trigger on changes to relevant flows and examples (EXTEND THIS):
+      - 'src/sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/**'
+      - 'examples/knowledge_tuning/enhanced_summary_knowledge_tuning/**'
+      # Standard integration test triggers, DONT CHANGE THIS
+      - 'tests/integration/**/*.py'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - '.github/workflows/integration-test.yml'
+  pull_request:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      # Only trigger on changes to relevant flows and examples (EXTEND THIS):
+      - 'src/sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/**'
+      - 'examples/knowledge_tuning/enhanced_summary_knowledge_tuning/**'
+      # Standard integration test triggers, DONT CHANGE THIS
+      - 'tests/integration/**/*.py'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - '.github/workflows/integration-test.yml'
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  integration-test:
+    name: "Integration Tests - ${{ matrix.python }} on ${{ matrix.platform }}"
+    runs-on: "${{ matrix.platform }}"
+    # Require manual approval before running (via GitHub Environment)
+    environment: integration-tests
+    # Skip fork PRs (they can't access environment secrets anyway)
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'push' ||
+      (github.event_name == 'pull_request' &&
+       github.event.pull_request.head.repo.full_name == github.repository)
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+        platform:
+          - "ubuntu-latest"
+    steps:
+      - name: "Harden Runner"
+        uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          # https://github.com/actions/checkout/issues/249
+          fetch-depth: 0
+
+      - name: Free disk space
+        uses: ./.github/actions/free-disk-space
+
+      - name: Install the expect package
+        run: |
+          sudo apt-get install -y expect
+
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/requirements*.txt
+
+      - name: Remove llama-cpp-python from cache
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Cache huggingface datasets
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/huggingface
+          # Invalidate cache when any example notebook changes (may affect dataset downloads)
+          key: huggingface-${{ hashFiles('examples/**/*.ipynb') }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh>=1.2
+
+      - name: Run integration tests with tox
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          tox -e py3-integrationcov
+
+      - name: Remove llama-cpp-python from cache
+        if: always()
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Upload integration test coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage-py3-integrationcov.xml
+          fail_ci_if_error: false
+          flags: integration
+
+      - name: Upload integration test artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-test-results-${{ matrix.python }}-${{ matrix.platform }}
+          path: |
+            coverage-py3-integrationcov/
+            coverage-py3-integrationcov.xml
+            durations/py3-integrationcov.html
+          retention-days: 30
+
+  integration-test-workflow-complete:
+    needs: ["integration-test"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Integration Test Workflow Complete
+        run: echo "Integration Test Workflow Complete"

sdg_hub-0.4.1/.github/workflows/packer.yml

@@ -0,0 +1,15 @@
+name: Build AMI with Packer
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-ami:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # This is required for OIDC
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/.github/workflows/test.yml

@@ -9,7 +9,8 @@ on:
       - "main"
       - "release-**"
     paths:
-      - '**.py'
+      - 'src/**/*.py'
+      - 'tests/**/*.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
       - 'tox.ini'
@@ -19,7 +20,8 @@ on:
       - "main"
       - "release-**"
     paths:
-      - '**.py'
+      - 'src/**/*.py'
+      - 'tests/**/*.py'
       - 'pyproject.toml'
       - 'requirements*.txt'
       - 'tox.ini'
@@ -37,7 +39,7 @@ permissions:
 
 jobs:
   test:
-    name: "${{ matrix.python }} on ${{ matrix.platform }}"
+    name: "Unit Tests - ${{ matrix.python }} on ${{ matrix.platform }}"
     runs-on: "${{ matrix.platform }}"
     strategy:
       matrix:
@@ -104,6 +106,7 @@ jobs:
         run: |
           tox -e py3-unitcov
 
+
       - name: Remove llama-cpp-python from cache
         if: always()
         run: |

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/.gitignore

@@ -84,6 +84,11 @@ target/
 # Jupyter Notebook
 .ipynb_checkpoints
 
+# Integration test artifacts
+tests/integration/**/converted_scripts/
+tests/integration/**/test_output/
+tests/integration/**/output_data/
+
 # IPython
 profile_default/
 ipython_config.py

{sdg_hub-0.3.1/src/sdg_hub.egg-info → sdg_hub-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.3.1
+Version: 0.4.1
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0
@@ -65,6 +65,7 @@ Requires-Dist: pytest-html; extra == "dev"
 Requires-Dist: tox<5,>=4.4.2; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pytest-env; extra == "dev"
+Requires-Dist: nbconvert>=7.0.0; extra == "dev"
 Dynamic: license-file
 
 # `sdg_hub`: Synthetic Data Generation Toolkit

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/docs/README.md

@@ -49,7 +49,6 @@ Learn about the modular block architecture that powers SDG Hub:
 - **[LLM Blocks](blocks/llm-blocks.md)** - Chat, prompt building, and text parsing
 - **[Transform Blocks](blocks/transform-blocks.md)** - Data transformation and manipulation
 - **[Filtering Blocks](blocks/filtering-blocks.md)** - Quality filtering and data validation
-- **[Evaluation Blocks](blocks/evaluation-blocks.md)** - Faithfulness and relevancy assessment
 - **[Custom Blocks](blocks/custom-blocks.md)** - Building your own processing blocks
 
 ### Flow System

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/docs/_sidebar.md

@@ -9,7 +9,6 @@
   * [LLM Blocks](blocks/llm-blocks.md)
   * [Transform Blocks](blocks/transform-blocks.md)
   * [Filtering Blocks](blocks/filtering-blocks.md)
-  * [Evaluation Blocks](blocks/evaluation-blocks.md)
   * [Custom Blocks](blocks/custom-blocks.md)
 
 * **Flow System**

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/docs/blocks/filtering-blocks.md

@@ -10,7 +10,6 @@ Filters dataset rows based on column values using flexible comparison operators
 
 ## 🚀 Next Steps
 
-- **[Evaluation Blocks](evaluation-blocks.md)** - Quality assessment and scoring
 - **[LLM Blocks](llm-blocks.md)** - AI-powered text generation
 - **[Transform Blocks](transform-blocks.md)** - Data manipulation and reshaping
 - **[Flow Integration](../flows/overview.md)** - Combine filtering into complete pipelines

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/docs/blocks/llm-blocks.md

@@ -230,14 +230,247 @@ Constructs prompts from templates and data with validation and formatting suppor
 
 ## 🔍 TextParserBlock
 
-Extracts structured data from LLM responses using patterns, schemas, or custom parsers.
+Extracts structured data from LLM responses using tag-based parsing or custom regex patterns. Essential for parsing LLM outputs into structured fields.
 
-#TODO: Add text parser block example
+### Basic Tag-Based Parsing
 
+Extract content between start and end tags:
+
+```python
+from sdg_hub.core.blocks import TextParserBlock
+from datasets import Dataset
+
+# Single field extraction
+parser = TextParserBlock(
+    block_name="extract_answer",
+    input_cols=["llm_response"],
+    output_cols=["answer"],
+    start_tags=["<answer>"],
+    end_tags=["</answer>"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "Question analysis: ...\n<answer>Machine learning is a subset of AI.</answer>",
+        "Let me think...\n<answer>Neural networks process data in layers.</answer>"
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["answer"])
+# ['Machine learning is a subset of AI.', 'Neural networks process data in layers.']
+```
+
+### Multiple Field Extraction
+
+Extract multiple structured fields from a single response:
+
+```python
+# Extract multiple fields with tag pairs
+parser = TextParserBlock(
+    block_name="extract_qa",
+    input_cols=["llm_response"],
+    output_cols=["question", "answer", "confidence"],
+    start_tags=["<question>", "<answer>", "<confidence>"],
+    end_tags=["</question>", "</answer>", "</confidence>"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        """
+        <question>What is Python?</question>
+        <answer>Python is a high-level programming language.</answer>
+        <confidence>0.95</confidence>
+        """
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["question"])     # ['What is Python?']
+print(result["answer"])       # ['Python is a high-level programming language.']
+print(result["confidence"])   # ['0.95']
+```
+
+### Custom Regex Parsing
+
+Use regex patterns for flexible extraction:
+
+```python
+# Extract using regex pattern
+parser = TextParserBlock(
+    block_name="regex_parser",
+    input_cols=["llm_response"],
+    output_cols=["answer"],
+    parsing_pattern=r"Answer:\s*(.+?)(?:\n|$)"
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "Question: What is AI?\nAnswer: Artificial Intelligence is...\n",
+        "Let me answer:\nAnswer: Machine learning enables..."
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["answer"])
+# ['Artificial Intelligence is...', 'Machine learning enables...']
+```
+
+### Tag Cleanup
+
+Remove unwanted tags from extracted content:
+
+```python
+# Clean up markdown and code tags
+parser = TextParserBlock(
+    block_name="clean_parser",
+    input_cols=["llm_response"],
+    output_cols=["clean_answer"],
+    start_tags=["<answer>"],
+    end_tags=["</answer>"],
+    parser_cleanup_tags=["```", "###", "**"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "<answer>Here's the code: ```python\nprint('hello')```</answer>",
+        "<answer>**Important**: This is the ### answer</answer>"
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["clean_answer"])
+# ['Here\'s the code: python\nprint(\'hello\')', 'Important: This is the  answer']
+```
+
+### Handling Multiple Matches
+
+Extract all occurrences of a pattern:
+
+```python
+parser = TextParserBlock(
+    block_name="multi_extract",
+    input_cols=["llm_response"],
+    output_cols=["keywords"],
+    start_tags=["[KEY]"],
+    end_tags=["[/KEY]"]
+)
+
+dataset = Dataset.from_dict({
+    "llm_response": [
+        "Important terms: [KEY]machine learning[/KEY], [KEY]neural networks[/KEY], [KEY]deep learning[/KEY]"
+    ]
+})
+
+result = parser.generate(dataset)
+print(result["keywords"])
+# [['machine learning', 'neural networks', 'deep learning']]
+```
+
+### Practical Example: Evaluation Response Parsing
+
+Common pattern for parsing LLM evaluation responses:
+
+```python
+# Parse structured evaluation output
+evaluation_parser = TextParserBlock(
+    block_name="parse_evaluation",
+    input_cols=["evaluation_response"],
+    output_cols=["explanation", "judgment"],
+    start_tags=["[Start of Explanation]", "[Start of Answer]"],
+    end_tags=["[End of Explanation]", "[End of Answer]"],
+    parser_cleanup_tags=["```", "###"]
+)
+
+dataset = Dataset.from_dict({
+    "evaluation_response": [
+        """
+        [Start of Explanation]
+        The response accurately reflects the information in the document.
+        No hallucinations or contradictions were found.
+        [End of Explanation]
+
+        [Start of Answer]
+        YES
+        [End of Answer]
+        """
+    ]
+})
+
+result = evaluation_parser.generate(dataset)
+print(result["explanation"])  # ['The response accurately reflects...']
+print(result["judgment"])     # ['YES']
+```
+
+### Integration with LLMChatBlock
+
+TextParserBlock is commonly used after LLMChatBlock to structure responses:
+
+```python
+from sdg_hub.core.blocks import LLMChatBlock, LLMParserBlock, TextParserBlock
+
+# Step 1: Generate LLM response
+chat_block = LLMChatBlock(
+    block_name="evaluator",
+    model="openai/gpt-4o",
+    input_cols=["messages"],
+    output_cols=["eval_response"]
+)
+
+# Step 2: Extract content from response object
+# Use field_prefix="" to get cleaner column names
+llm_parser = LLMParserBlock(
+    block_name="extract_eval",
+    input_cols=["eval_response"],
+    extract_content=True,
+    field_prefix="eval_"  # Results in "eval_content" instead of "extract_content"
+)
+
+# Step 3: Parse structured fields from text
+text_parser = TextParserBlock(
+    block_name="parse_fields",
+    input_cols=["eval_content"], 
+    output_cols=["score", "reasoning"],
+    start_tags=["[SCORE]", "[REASONING]"],
+    end_tags=["[/SCORE]", "[/REASONING]"]
+)
+
+# Execute in sequence (or use a Flow)
+dataset = Dataset.from_dict({
+    "messages": [[{"role": "user", "content": "Evaluate this text..."}]]
+})
+
+result = chat_block.generate(dataset)
+result = llm_parser.generate(result)
+result = text_parser.generate(result)
+
+print(result["score"])      # Extracted score
+print(result["reasoning"])  # Extracted reasoning
+```
+
+### Configuration Reference
+
+**Required Parameters:**
+- `block_name` - Unique identifier for the block
+- `input_cols` - Single column containing text to parse
+- `output_cols` - List of field names for extracted content
+
+**Parsing Methods (choose one):**
+- **Tag-based**: `start_tags` + `end_tags` (must have same length as `output_cols`)
+- **Regex**: `parsing_pattern` (single regex with capture groups)
+
+**Optional Parameters:**
+- `parser_cleanup_tags` - List of tags to remove from extracted text
+- `expand_lists` - Whether to expand list inputs into rows (default: `True`)
+
+**Tag Parsing Rules:**
+- Number of tag pairs must match number of output columns
+- Each tag pair extracts all matches for that field
+- Tags can be any string (XML-style, markdown-style, custom)
+- Missing tags result in empty lists for that field
 
 ## 🚀 Next Steps
 
 - **[Transform Blocks](transform-blocks.md)** - Data manipulation and reshaping
 - **[Filtering Blocks](filtering-blocks.md)** - Quality control and validation
-- **[Evaluation Blocks](evaluation-blocks.md)** - Quality assessment and scoring
 - **[Flow Integration](../flows/overview.md)** - Combine LLM blocks into complete pipelines

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/docs/blocks/overview.md

@@ -65,11 +65,6 @@ Data manipulation and transformation:
 Quality control and data validation:
 - **ColumnValueFilterBlock** - Filter rows based on column values
 
-### 📊 Evaluation Blocks (`evaluation/`)
-Quality assessment and scoring:
-- **EvaluateFaithfulnessBlock** - Assess factual accuracy
-- **EvaluateRelevancyBlock** - Measure relevance scores
-- **VerifyQuestionBlock** - Validate question quality
 
 ## 🔧 Block Lifecycle
 
@@ -149,5 +144,4 @@ Ready to dive deeper? Explore specific block categories:
 - **[LLM Blocks](llm-blocks.md)** - AI-powered language model operations
 - **[Transform Blocks](transform-blocks.md)** - Data manipulation and reshaping
 - **[Filtering Blocks](filtering-blocks.md)** - Quality control and validation
-- **[Evaluation Blocks](evaluation-blocks.md)** - Quality assessment and scoring
 - **[Custom Blocks](custom-blocks.md)** - Build your own processing blocks

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/docs/blocks/transform-blocks.md

@@ -26,6 +26,5 @@ Sets uniform values across specified columns, useful for adding metadata or defa
 ## 🚀 Next Steps
 
 - **[Filtering Blocks](filtering-blocks.md)** - Quality control and data validation
-- **[Evaluation Blocks](evaluation-blocks.md)** - Quality assessment and scoring
 - **[LLM Blocks](llm-blocks.md)** - AI-powered text generation
 - **[Flow Integration](../flows/overview.md)** - Combine transform blocks into complete pipelines

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/docs/concepts.md

@@ -152,7 +152,7 @@ Every block validates data at runtime:
 - Validate your pipeline before scaling up
 
 ### 2. Layer Validation
-- Use evaluation blocks to assess quality
+- Use basic block composition (PromptBuilder → LLMChat → Parser → Filter) to assess quality
 - Implement filtering to maintain data standards
 
 ### 3. Monitor Performance

{sdg_hub-0.3.1 → sdg_hub-0.4.1}/docs/development.md

@@ -206,13 +206,6 @@ class TestMyNewBlock:
   - Comprehensive operator support
   - Good performance on large datasets
 
-#### Evaluation Blocks (`src/sdg_hub/core/blocks/evaluation/`)
-- **Purpose**: Quality assessment and scoring
-- **Examples**: Faithfulness evaluation, relevancy scoring
-- **Requirements**:
-  - Consistent scoring methodology
-  - Support for different evaluation criteria
-  - Clear documentation of scoring rubrics
 
 ## 🌊 Contributing Flows