sdg-hub 0.4.1__tar.gz → 0.5.0__tar.gz

{sdg_hub-0.4.1 → sdg_hub-0.5.0}/.github/workflows/integration-test.yml

@@ -7,28 +7,11 @@ on:
     branches:
       - "main"
       - "release-**"
-    paths:
-      # Only trigger on changes to relevant flows and examples (EXTEND THIS):
-      - 'src/sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/**'
-      - 'examples/knowledge_tuning/enhanced_summary_knowledge_tuning/**'
-      # Standard integration test triggers, DONT CHANGE THIS
-      - 'tests/integration/**/*.py'
-      - 'pyproject.toml'
-      - 'tox.ini'
-      - '.github/workflows/integration-test.yml'
   pull_request:
     branches:
       - "main"
       - "release-**"
-    paths:
-      # Only trigger on changes to relevant flows and examples (EXTEND THIS):
-      - 'src/sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/**'
-      - 'examples/knowledge_tuning/enhanced_summary_knowledge_tuning/**'
-      # Standard integration test triggers, DONT CHANGE THIS
-      - 'tests/integration/**/*.py'
-      - 'pyproject.toml'
-      - 'tox.ini'
-      - '.github/workflows/integration-test.yml'
+    types: [opened, synchronize, reopened, labeled]
 
 env:
   LC_ALL: en_US.UTF-8
@@ -41,17 +24,58 @@ permissions:
   contents: read
 
 jobs:
+  check-trigger:
+    name: "Check If Integration Should Run"
+    runs-on: ubuntu-latest
+    outputs:
+      should_run: ${{ steps.check.outputs.should_run }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dorny/paths-filter@v3
+        id: filter
+        if: github.event_name == 'pull_request'
+        with:
+          filters: |
+            relevant:
+              # Only trigger on changes to relevant flows and examples (EXTEND THIS):
+              - 'src/sdg_hub/flows/qa_generation/document_grounded_qa/enhanced_multi_summary_qa/**'
+              - 'examples/knowledge_tuning/enhanced_summary_knowledge_tuning/**'
+              # Standard integration test triggers, DONT CHANGE THIS
+              - 'tests/integration/**/*.py'
+              - 'pyproject.toml'
+              - 'tox.ini'
+              - '.github/workflows/integration-test.yml'
+
+      - name: Determine if tests should run
+        id: check
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] || [[ "${{ github.event_name }}" == "push" ]]; then
+            echo "should_run=true" >> "$GITHUB_OUTPUT"
+          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            # Check if from fork
+            if [[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]]; then
+              echo "should_run=false" >> "$GITHUB_OUTPUT"
+            # Check if labeled event with correct label
+            elif [[ "${{ github.event.action }}" == "labeled" ]] && [[ "${{ contains(github.event.pull_request.labels.*.name, 'run-integration-tests') }}" == "true" ]]; then
+              echo "should_run=true" >> "$GITHUB_OUTPUT"
+            # Check if relevant paths changed for non-labeled events
+            elif [[ "${{ github.event.action }}" != "labeled" ]] && [[ "${{ steps.filter.outputs.relevant }}" == "true" ]]; then
+              echo "should_run=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "should_run=false" >> "$GITHUB_OUTPUT"
+            fi
+          else
+            echo "should_run=false" >> "$GITHUB_OUTPUT"
+          fi
+
   integration-test:
     name: "Integration Tests - ${{ matrix.python }} on ${{ matrix.platform }}"
     runs-on: "${{ matrix.platform }}"
+    needs: check-trigger
+    if: needs.check-trigger.outputs.should_run == 'true'
     # Require manual approval before running (via GitHub Environment)
     environment: integration-tests
-    # Skip fork PRs (they can't access environment secrets anyway)
-    if: |
-      github.event_name == 'workflow_dispatch' ||
-      github.event_name == 'push' ||
-      (github.event_name == 'pull_request' &&
-       github.event.pull_request.head.repo.full_name == github.repository)
     strategy:
       matrix:
         python:
@@ -86,12 +110,9 @@ jobs:
             **/pyproject.toml
             **/requirements*.txt
 
-      - name: Remove llama-cpp-python from cache
-        run: |
-          pip cache remove llama_cpp_python
 
       - name: Cache huggingface datasets
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: ~/.cache/huggingface
           # Invalidate cache when any example notebook changes (may affect dataset downloads)
@@ -108,10 +129,6 @@ jobs:
         run: |
           tox -e py3-integrationcov
 
-      - name: Remove llama-cpp-python from cache
-        if: always()
-        run: |
-          pip cache remove llama_cpp_python
 
       - name: Upload integration test coverage to Codecov
         uses: codecov/codecov-action@v4

sdg_hub-0.5.0/.github/workflows/packer.yml

@@ -0,0 +1,33 @@
+name: Build AMI with Packer
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-ami:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # This is required for OIDC
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c
+        with:
+          role-to-assume: arn:aws:iam::851725220677:role/github-actions-packer-role
+          aws-region: us-east-2
+          role-session-name: github-actions-packer  # For tracking in CloudTrail
+
+      - name: Setup Packer
+        uses: hashicorp/setup-packer@1aa358be5cf73883762b302a3a03abd66e75b232
+
+      - name: Build and create AMI
+        run: |
+          set -euo pipefail
+          cd scripts/packer
+          packer init .
+          packer validate .
+          packer build .

{sdg_hub-0.4.1 → sdg_hub-0.5.0}/.github/workflows/test.yml

@@ -86,16 +86,7 @@ jobs:
             **/pyproject.toml
             **/requirements*.txt
 
-      - name: Remove llama-cpp-python from cache
-        run: |
-          pip cache remove llama_cpp_python
 
-      - name: Cache huggingface
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
-        with:
-          path: ~/.cache/huggingface
-          # config contains DEFAULT_MODEL
-          key: huggingface-${{ hashFiles('src/instructlab/configuration.py') }}
 
       - name: Install dependencies
         run: |
@@ -107,10 +98,6 @@ jobs:
           tox -e py3-unitcov
 
 
-      - name: Remove llama-cpp-python from cache
-        if: always()
-        run: |
-          pip cache remove llama_cpp_python
       
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

{sdg_hub-0.4.1 → sdg_hub-0.5.0}/CLAUDE.md

@@ -86,7 +86,6 @@ The framework is built around a modular block system with **composability at its
   - `transform/`: Data transformation blocks (column operations, text manipulation)
   - `filtering/`: Data filtering blocks with quality thresholds
   - `evaluation/`: Quality evaluation blocks (faithfulness, relevancy assessment)
-  - `deprecated_blocks/`: Legacy blocks maintained for backward compatibility
 
 **Key Benefits**: Type-safe composition, automatic validation, rich logging, and high-performance async processing.
 
@@ -97,7 +96,6 @@ Flows orchestrate multiple blocks into data processing pipelines:
 - **FlowRegistry** (`src/sdg_hub/core/flow/registry.py`): Registry for flow discovery
 - **FlowMetadata** (`src/sdg_hub/core/flow/metadata.py`): Metadata and parameter definitions
 - **FlowValidator** (`src/sdg_hub/core/flow/validation.py`): YAML structure validation
-- **FlowMigration** (`src/sdg_hub/core/flow/migration.py`): Backward compatibility for old flow formats
 
 ### Flow Configuration
 Flows are defined in YAML files with this structure:
@@ -148,11 +146,6 @@ All blocks operate on HuggingFace `datasets.Dataset` objects:
 - Rich logging provides processing summaries
 - Empty dataset handling with appropriate errors
 
-### Backward Compatibility
-The framework maintains compatibility with legacy formats:
-- Deprecated blocks are preserved in `deprecated_blocks/`
-- Flow migration automatically converts old YAML formats
-- Legacy LLMBlocks receive special handling during execution
 
 ## Testing Guidelines
 

{sdg_hub-0.4.1/src/sdg_hub.egg-info → sdg_hub-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.4.1
+Version: 0.5.0
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0
@@ -23,7 +23,7 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: click<9.0.0,>=8.1.7
-Requires-Dist: datasets<4.0.0,>=2.18.0
+Requires-Dist: datasets>=4.0.0
 Requires-Dist: httpx<1.0.0,>=0.25.0
 Requires-Dist: jinja2
 Requires-Dist: litellm<1.75.0,>=1.73.0

{sdg_hub-0.4.1 → sdg_hub-0.5.0}/docs/concepts.md

@@ -148,9 +148,22 @@ Every block validates data at runtime:
 ## 🚀 Best Practices
 
 ### 1. Start Small
-- Use `dry_run()` to test with small samples
+- Use `dry_run()` to test with small samples before processing full datasets
+- Add `enable_time_estimation=True` to predict execution time for the complete dataset
 - Validate your pipeline before scaling up
 
+```python
+# Test AND estimate in one call
+result = flow.dry_run(dataset, sample_size=5, enable_time_estimation=True, max_concurrency=100)
+
+# Access dry run results
+print(f"Tested with {result['sample_size']} samples")
+print(f"Output columns: {result['final_dataset']['columns']}")
+
+# Time estimation is automatically displayed in a Rich table format
+# No need to access it programmatically - the table shows all estimation details
+```
+
 ### 2. Layer Validation
 - Use basic block composition (PromptBuilder → LLMChat → Parser → Filter) to assess quality
 - Implement filtering to maintain data standards

{sdg_hub-0.4.1 → sdg_hub-0.5.0}/docs/flows/discovery.md

@@ -67,7 +67,44 @@ for flow_name in all_flows:
 
 ### Getting Flow Information
 
-#TODO: Add flow info example
+Access detailed flow metadata and configuration:
+
+```python
+from sdg_hub.core.flow import FlowRegistry, Flow
+
+# Get metadata for a specific flow
+flow_name = "Advanced Document Grounded Question-Answer Generation Flow for Knowledge Tuning"
+metadata = FlowRegistry.get_flow_metadata(flow_name)
+
+if metadata:
+    print(f"Flow: {metadata.name}")
+    print(f"Version: {metadata.version}")
+    print(f"Author: {metadata.author}")
+    print(f"Description: {metadata.description}")
+    print(f"Tags: {', '.join(metadata.tags)}")
+    print(f"Recommended model: {metadata.recommended_models.get('default', 'Not specified')}")
+
+# Load flow and get detailed information
+flow_path = FlowRegistry.get_flow_path(flow_name)
+flow = Flow.from_yaml(flow_path)
+
+# Get comprehensive flow info
+info = flow.get_info()
+print(f"Total blocks: {info['total_blocks']}")
+print(f"Block sequence: {', '.join(info['block_names'])}")
+
+# Get dataset requirements
+requirements = flow.get_dataset_requirements()
+if requirements:
+    print(f"Required columns: {requirements.required_columns}")
+    print(f"Description: {requirements.description}")
+    print(f"Min samples: {requirements.min_samples}")
+
+# Get model recommendations
+recommendations = flow.get_model_recommendations()
+print(f"Default model: {recommendations.get('default')}")
+print(f"Compatible models: {recommendations.get('compatible', [])}")
+```
 
 ### Getting Flow Paths
 

{sdg_hub-0.4.1 → sdg_hub-0.5.0}/docs/flows/overview.md

@@ -292,6 +292,41 @@ print(f"Output columns: {dry_result['final_dataset']['columns']}")
 print(f"Sample output: {dry_result['sample_output']}")
 ```
 
+### Time Estimation
+
+Predict execution time for your full dataset before running:
+
+```python
+# Get dry run results AND time estimation in one call
+result = flow.dry_run(
+    dataset, 
+    sample_size=5, 
+    enable_time_estimation=True,
+    max_concurrency=100
+)
+
+# Time estimation is automatically displayed in a Rich table format
+# The table shows estimated time, total API requests, and per-block breakdowns
+print(f"Dry run completed with {result['sample_size']} samples")
+print(f"Output columns: {result['final_dataset']['columns']}")
+```
+
+**How It Works:**
+
+The estimation uses 2 dry runs to accurately predict execution time:
+- Extracts startup overhead (one-time costs)
+- Calculates per-sample throughput (variable costs)
+- Uses linear regression to separate fixed from variable costs
+
+**Accuracy:**
+- Includes a 20% conservative buffer to account for API variability
+- Typical accuracy: within 15-40% of actual runtime depending on workload
+- Better to finish early than run over time!
+
+**When to Use:**
+- Before processing with your full dataset
+- To identify bottleneck blocks and optimize your pipeline
+
 ### Runtime Parameters
 
 Runtime parameters allow you to customize block behavior at execution time without modifying flow YAML files. You can override global parameters for all blocks or configure specific blocks individually.

{sdg_hub-0.4.1 → sdg_hub-0.5.0}/docs/quick-start.md

@@ -62,11 +62,14 @@ dataset = Dataset.from_dict({
     'icl_response_3': ['Java provides platform independence and strong object-oriented features.']
 })
 
-# Test with a small sample first (recommended!)
-print("🧪 Running dry run...")
-dry_result = flow.dry_run(dataset, sample_size=1)
+# Test with a small sample AND get time estimate (recommended!)
+print("🧪 Running dry run with time estimation...")
+dry_result = flow.dry_run(dataset, sample_size=5, enable_time_estimation=True, max_concurrency=100)
 print(f"✅ Dry run completed in {dry_result['execution_time_seconds']:.2f}s")
 print(f"📊 Output columns: {list(dry_result['final_dataset']['columns'])}")
+
+# Time estimation is automatically displayed in a Rich table format
+# The table shows estimated time, total API calls, and per-block breakdowns
 ```
 
 ## 📊 Step 3: Generate Synthetic Data

{sdg_hub-0.4.1 → sdg_hub-0.5.0}/examples/knowledge_tuning/enhanced_summary_knowledge_tuning/.env.example

@@ -16,6 +16,7 @@ VLLM_MODEL=hosted_vllm/meta-llama/Llama-3.3-70B-Instruct
 VLLM_API_BASE=http://localhost:8000/v1
 VLLM_API_KEY=EMPTY
 ENABLE_REASONING=false
+MAX_CONCURRENCY=50
 # =============================================================================
 # OPENAI CONFIGURATION
 # =============================================================================
@@ -38,7 +39,9 @@ MAAS_API_KEY=your-maas-api-key-here
 # =============================================================================
 # DATA CONFIGURATION
 # =============================================================================
-SEED_DATA_PATH=seed_data_val.jsonl
+SEED_DATA_PATH=seed_data.jsonl
+# Set this for subsampling the seed data. Useful for debugging or running validation
+SEED_DATA_SUBSAMPLE=24
 OUTPUT_DATA_FOLDER=output_data
 RUN_ON_VALIDATION_SET=true
 NUMBER_OF_SUMMARIES=50

sdg_hub-0.5.0/examples/knowledge_tuning/enhanced_summary_knowledge_tuning/document_pre_processing.ipynb

@@ -0,0 +1,214 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "83f458de",
+   "metadata": {},
+   "source": [
+    "# Document Pre-processing for Knowledge Tuning\n",
+    "\n",
+    "## Overview\n",
+    "\n",
+    "This notebook demonstrates a complete document preprocessing pipeline designed specifically for **knowledge tuning** with sdg-hub. \n",
+    "\n",
+    "## What This Notebook Does\n",
+    "\n",
+    "This preprocessing pipeline transforms raw documents (PDFs, Word docs, etc.) into seed data for data generation:\n",
+    "\n",
+    "1. **Document Parsing**: Converts raw documents to structured markdown format\n",
+    "2. **Chunking**: Splits documents into manageable chunks while preserving structure and context\n",
+    "3. **Seed Data Creation**: Formats chunks with in-context learning (ICL) templates for effective knowledge tuning\n",
+    "\n",
+    "## Prerequisites\n",
+    "\n",
+    "- We will use the existing InstructLab document parser (`docparser_v2.py`) and Document parsing configuration (`docling_v2_config.yaml`)\n",
+    "- Raw pdf documents in the `document_collection/` directory\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "daa22c74",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Step 1: Document Processing Pipeline\n",
+    "# Define the directory containing raw documents to be processed\n",
+    "data_dir = 'document_collection/'\n",
+    "\n",
+    "# Run the document parser to convert documents to markdown\n",
+    "# - input-dir: Directory containing source documents\n",
+    "# - output-dir: Directory where processed markdown files will be saved\n",
+    "# - c: Configuration file specifying parsing parameters\n",
+    "!python ../instructlab/docparser_v2.py --input-dir {data_dir} --output-dir {data_dir} -c ../instructlab/docling_v2_config.yaml"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "295749b5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Step 2: Install Required Dependencies\n",
+    "# Install packages needed for document processing and text chunking\n",
+    "\n",
+    "%pip install docling markdown-it-py\n",
+    "%pip install --upgrade transformers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dd8a4a2a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Step 3: Load Processed Document\n",
+    "import glob\n",
+    "\n",
+    "# In our example above docling step produces markdown of all the pdf files in the document_collection\n",
+    "with open(glob.glob(f'{data_dir}/*.md')[0], 'r') as f:\n",
+    "    text = f.read()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7614dc73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Step 4: Text Chunking and Dataset Creation\n",
+    "\n",
+    "from markdown_it import MarkdownIt  \n",
+    "from typing import List\n",
+    "import datasets  \n",
+    "\n",
+    "\n",
+    "def chunk_markdown(\n",
+    "    text: str,\n",
+    "    max_tokens: int = 200,\n",
+    "    overlap: int = 50\n",
+    ") -> List[str]:\n",
+    "    \"\"\"\n",
+    "    Splits Markdown text into chunks at block-level elements\n",
+    "    (headings, paragraphs, lists, tables, code, blockquotes).\n",
+    "    Adds overlap (in words) between all consecutive chunks.\n",
+    "    \n",
+    "    Args:\n",
+    "        text: The markdown text to be chunked\n",
+    "        max_tokens: Maximum number of words per chunk\n",
+    "        overlap: Number of overlapping words between consecutive chunks\n",
+    "    \n",
+    "    Returns:\n",
+    "        List of text chunks with specified overlap\n",
+    "    \"\"\"\n",
+    "\n",
+    "    # Initialize markdown parser to understand document structure\n",
+    "    md = MarkdownIt()\n",
+    "    tokens = md.parse(text)\n",
+    "\n",
+    "    # Group tokens into block-level segments to preserve markdown structure\n",
+    "    # This ensures we don't split in the middle of headings, lists, etc.\n",
+    "    blocks = []\n",
+    "    buf = []\n",
+    "    for tok in tokens:\n",
+    "        if tok.block and tok.type.endswith(\"_open\"):\n",
+    "            buf = []\n",
+    "        elif tok.block and tok.type.endswith(\"_close\"):\n",
+    "            if buf:\n",
+    "                blocks.append(\"\\n\".join(buf).strip())\n",
+    "                buf = []\n",
+    "        elif tok.content:\n",
+    "            buf.append(tok.content)\n",
+    "    if buf:\n",
+    "        blocks.append(\"\\n\".join(buf).strip())\n",
+    "\n",
+    "    # Split blocks into chunks with overlap to maintain context continuity\n",
+    "    chunks = []\n",
+    "    current_words = []\n",
+    "    for block in blocks:\n",
+    "        words = block.split()\n",
+    "        for w in words:\n",
+    "            current_words.append(w)\n",
+    "            if len(current_words) >= max_tokens:\n",
+    "                # Emit a complete chunk\n",
+    "                chunks.append(\" \".join(current_words))\n",
+    "                # Prepare next buffer with overlap from the end of this chunk\n",
+    "                # This ensures context continuity between chunks\n",
+    "                current_words = current_words[-overlap:] if overlap > 0 else []\n",
+    "\n",
+    "    # Add any remaining words as the final chunk\n",
+    "    if current_words:\n",
+    "        chunks.append(\" \".join(current_words))\n",
+    "\n",
+    "    return chunks\n",
+    "\n",
+    "\n",
+    "chunks = chunk_markdown(text, max_tokens=5000, overlap=1000)\n",
+    "\n",
+    "\n",
+    "# Prepare seed data for the SDG-Hub knowledge pipeline.\n",
+    "# \n",
+    "# The seed data requires the following fields:\n",
+    "#   - document_outline: A concise title or summary that accurately represents the entire document.\n",
+    "#     For documents covering multiple themes, consider providing multiple outlines (one per section).\n",
+    "#   - icl_document: A representative sample extract from the document. This may include tables, code snippets, definitions, etc.\n",
+    "#   - icl_query_1, icl_query_2, icl_query_3: Three questions based on the icl_document sample.\n",
+    "#   - domain: The domain or subject area of the document.\n",
+    "#\n",
+    "# The code below creates a HuggingFace Dataset from the document chunks,\n",
+    "# then maps the required ICL fields to each entry, and finally saves the result as a JSONL file.\n",
+    "\n",
+    "seed_data = datasets.Dataset.from_dict({'document': chunks})\n",
+    "\n",
+    "icl = {\n",
+    "    \"document_outline\": \"The document contains excerpts from FINTRAC regulations designed to combat money laundering and terrorist financing in Canada\",\n",
+    "    \"icl_document\": \"## Overview\\n\\nThis guidance came into effect on June 1, 2021.\\n\\n\\nThis guidance explains the methods that can be used by reporting entities\\n(REs) to verify the identity of a person or an entity.\\n\\n\\n## 1. Meaning of verifying the identity of a person or an entity\\n\\nIt means to use the methods described in this guidance to ensure that the\\ninformation in an identification document or from other informational\\nsources matches the information that the person or entity provided.\\n\\n\\nVerifying identity is a foundational element of Canada's anti-money\\nlaundering and anti-terrorist financing regime and a key component of an\\nRE's relationship with clients. It helps you to know your clients and to\\nunderstand and assess any risk that may be associated to their\\ntransactions or activities.\\n\\n\\n## 2. How to verify the identity of a person\\n\\nYou can use any of the 5 methods described below to identify a person:\\n\\n- 2.1 Government-issued photo identification method\\n\\n- 2.2 Credit file method\\n\\n- 2.3 Dual-process method\\n\\n- 2.4 Affiliate or member method\\n\\n- 2.5 Reliance method\\n\",\n",
+    "    \"icl_query_1\": \"In Canada, what are the methods for verifying someone's identity?\",\n",
+    "    \"icl_query_2\": \"In Canada, why is it important to confirm a client's identity?\",\n",
+    "    \"icl_query_3\": \"In Canada, can I use Reliance method to verify identity of a person?\",\n",
+    "    \"domain\": \"Finance\"\n",
+    "}\n",
+    "\n",
+    "# Map the ICL fields to each document chunk (if you want to use the same ICL for all, as shown here)\n",
+    "seed_data = seed_data.map(lambda x: icl)\n",
+    "\n",
+    "# Save the seed data to a JSONL file for downstream use\n",
+    "seed_data.to_json('seed_data.jsonl', orient='records', lines=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "44f3ff7f",
+   "metadata": {},
+   "source": [
+    "### Next Steps:\n",
+    "- The seed_data.jsonl file is now ready for the knowledge tuning pipeline.\n",
+    "- You can now refer to the [knowledge generation](knowledge_generation.ipynb) notebook"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "sdg_hub",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}