sdg-hub 0.1.1__tar.gz → 0.1.3__tar.gz

{sdg_hub-0.1.1 → sdg_hub-0.1.3}/.github/workflows/pypi.yaml

@@ -110,7 +110,7 @@ jobs:
                   path: dist
 
             - name: "Sigstore sign package"
-              uses: sigstore/gh-action-sigstore-python@f514d46b907ebcd5bedc05145c03b69c1edd8b46 # v3.0.0
+              uses: sigstore/gh-action-sigstore-python@f7ad0af51a5648d09a20d00370f0a91c3bdf8f84 # v3.0.1
               with:
                   inputs: |
                       ./dist/*.tar.gz

{sdg_hub-0.1.1/src/sdg_hub.egg-info → sdg_hub-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.1.1
+Version: 0.1.3
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0
@@ -36,7 +36,7 @@ Requires-Dist: flask>=3.0.2; extra == "web-interface"
 Requires-Dist: pyyaml>=6.0.1; extra == "web-interface"
 Requires-Dist: flask-wtf>=1.2.2; extra == "web-interface"
 Provides-Extra: vllm
-Requires-Dist: vllm<0.8.4,>=0.8.0; extra == "vllm"
+Requires-Dist: vllm>=0.9.1; extra == "vllm"
 Requires-Dist: torch>=2.0.0; extra == "vllm"
 Requires-Dist: transformers>=4.37.0; extra == "vllm"
 Requires-Dist: accelerate>=0.21.0; extra == "vllm"

{sdg_hub-0.1.1 → sdg_hub-0.1.3}/docs/blocks.md

@@ -22,6 +22,184 @@ Blocks are the fundamental processing units in SDG Hub. Each block performs a sp
 
 ## LLM Blocks
 
+### OpenAIChatBlock
+- **Registered Name**: `OpenAIChatBlock`
+- **Purpose**: Modern chat completion block using OpenAI Chat Completions API
+- **Key Features**:
+  - Direct OpenAI message format support (system/user/assistant roles)
+  - All OpenAI Chat Completions API parameters supported
+  - Automatic retry logic for rate limits and API errors
+  - Comprehensive structured logging for monitoring
+  - Works with any OpenAI-compatible endpoint
+
+**Parameters:**
+- `block_name: str` - Name of the block
+- `input_cols: Union[str, List[str]]` - Input column containing messages (must be exactly one)
+- `output_cols: Union[str, List[str]]` - Output column for responses (must be exactly one)
+- `client: openai.OpenAI` - OpenAI client instance
+- `model_id: str` - Model ID to use (e.g., "gpt-4", "gpt-3.5-turbo")
+- **OpenAI API Parameters** (all optional):
+  - `frequency_penalty: Optional[float]` - Penalize frequent tokens (-2.0 to 2.0)
+  - `logit_bias: Optional[Dict[str, int]]` - Modify likelihood of specified tokens
+  - `logprobs: Optional[bool]` - Whether to return log probabilities
+  - `max_completion_tokens: Optional[int]` - Maximum tokens in completion
+  - `max_tokens: Optional[int]` - Maximum tokens in completion (legacy)
+  - `n: Optional[int]` - Number of completions to generate
+  - `presence_penalty: Optional[float]` - Penalize repeated tokens (-2.0 to 2.0)
+  - `response_format: Optional[Dict[str, Any]]` - Response format (e.g., JSON mode)
+  - `seed: Optional[int]` - Seed for deterministic outputs
+  - `stop: Optional[Union[str, List[str]]]` - Stop sequences
+  - `stream: Optional[bool]` - Whether to stream responses
+  - `temperature: Optional[float]` - Sampling temperature (0.0 to 2.0)
+  - `tool_choice: Optional[Union[str, Dict[str, Any]]]` - Tool selection strategy
+  - `tools: Optional[List[Dict[str, Any]]]` - Available tools for function calling
+  - `top_logprobs: Optional[int]` - Number of top log probabilities to return
+  - `top_p: Optional[float]` - Nucleus sampling parameter (0.0 to 1.0)
+  - `user: Optional[str]` - End-user identifier
+  - `extra_body: Optional[dict]` - Additional parameters for custom endpoints
+
+**Example Usage:**
+```yaml
+- block_type: OpenAIChatBlock
+  block_config:
+    block_name: chat_generator
+    input_cols: messages
+    output_cols: response
+    model_id: gpt-4
+    temperature: 0.7
+    max_tokens: 500
+```
+
+**Example with Messages Dataset:**
+```python
+import openai
+from datasets import Dataset
+from sdg_hub.blocks import OpenAIChatBlock
+
+# Create client
+client = openai.OpenAI(api_key="your-api-key")
+
+# Prepare dataset with messages in OpenAI format
+messages_data = [
+    [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Explain quantum computing in simple terms."}
+    ],
+    [
+        {"role": "user", "content": "What is the capital of France?"}
+    ]
+]
+dataset = Dataset.from_dict({"messages": messages_data})
+
+# Create and use block
+block = OpenAIChatBlock(
+    block_name="qa_generator",
+    input_cols="messages",
+    output_cols="response", 
+    client=client,
+    model_id="gpt-4",
+    temperature=0.7,
+    max_tokens=150
+)
+
+result = block.generate(dataset)
+print(result["response"])
+```
+
+### OpenAIAsyncChatBlock
+- **Registered Name**: `OpenAIAsyncChatBlock`
+- **Purpose**: Async version of OpenAIChatBlock for concurrent processing and better performance
+- **Key Features**:
+  - Concurrent async requests for improved throughput
+  - All features of OpenAIChatBlock
+  - Better performance for large batches
+  - Automatic concurrency management
+
+**Parameters:**
+- Same as `OpenAIChatBlock` except:
+  - `async_client: openai.AsyncOpenAI` - Async OpenAI client instance
+
+**Example Usage:**
+```yaml
+- block_type: OpenAIAsyncChatBlock
+  block_config:
+    block_name: async_chat_generator
+    input_cols: messages
+    output_cols: response
+    model_id: gpt-4
+    temperature: 0.7
+    max_tokens: 500
+```
+
+**Example with Async Client:**
+```python
+import asyncio
+import openai
+from datasets import Dataset
+from sdg_hub.blocks import OpenAIAsyncChatBlock
+
+# Create async client
+async_client = openai.AsyncOpenAI(api_key="your-api-key")
+
+# Same dataset format as sync version
+messages_data = [
+    [{"role": "user", "content": f"Generate a creative story about topic {i}"}]
+    for i in range(100)  # Large batch for demonstration
+]
+dataset = Dataset.from_dict({"messages": messages_data})
+
+# Create and use async block
+block = OpenAIAsyncChatBlock(
+    block_name="async_story_generator",
+    input_cols="messages",
+    output_cols="story",
+    async_client=async_client,
+    model_id="gpt-4",
+    temperature=0.8,
+    max_tokens=200
+)
+
+# Process large batch concurrently
+result = block.generate(dataset)
+print(f"Generated {len(result)} stories concurrently")
+```
+
+**OpenAI-Compatible Endpoints:**
+Both blocks work with any OpenAI-compatible endpoint:
+
+```python
+# Example with local endpoint
+client = openai.OpenAI(
+    api_key="not-needed-for-local",
+    base_url="http://localhost:8000/v1"
+)
+
+# Example with other providers (Azure, Anthropic, etc.)
+client = openai.OpenAI(
+    api_key="your-provider-key",
+    base_url="https://your-provider-endpoint.com/v1"
+)
+```
+
+**Monitoring and Logging:**
+Both blocks provide comprehensive structured logging:
+- Initialization logs with model and parameters
+- Generation start/completion logs with batch metrics
+- Effective parameter tracking (including runtime overrides)
+- Error tracking and retry information
+
+Log output example:
+```
+INFO: Initialized OpenAIChatBlock 'chat_generator' with model 'gpt-4'
+  {"block_name": "chat_generator", "model_id": "gpt-4", "generation_params": {"temperature": 0.7}}
+
+INFO: Starting generation for 10 samples
+  {"block_name": "chat_generator", "model_id": "gpt-4", "batch_size": 10, "effective_params": {"temperature": 0.9}}
+
+INFO: Generation completed successfully for 10 samples
+  {"block_name": "chat_generator", "model_id": "gpt-4", "batch_size": 10}
+```
+
 ### LLMBlock
 - **Registered Name**: `LLMBlock`
 - **Purpose**: Core block for text generation using language models

{sdg_hub-0.1.1 → sdg_hub-0.1.3}/examples/knowledge_tuning/knowledge_utils.py

@@ -1,25 +1,25 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Standard
-import json
-import random
-import uuid
-import os
-import yaml
 from pathlib import Path
 from typing import List
+import json
+import os
+import random
 import re
+import uuid
 
 # Third Party
-from datasets import Dataset
+from datasets import Dataset, concatenate_datasets
+from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
 from tabulate import tabulate
 from transformers import AutoTokenizer
-from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
+import yaml
 
-# Local
-import sdg_hub
+# First Party
 from sdg_hub.logger_config import setup_logger
 from sdg_hub.utils.datautils import safe_concatenate_datasets
+import sdg_hub
 
 logger = setup_logger(__name__)
 _DEFAULT_CHUNK_OVERLAP = 100
@@ -98,9 +98,70 @@ def _conv_pretrain(rec):
     return rec
 
 
+def mask_qa_per_doc(ds: Dataset, keep_no_qa_per_doc: int = 3) -> Dataset:
+    """
+    Mark QA entries per document for pre-training vs fine-tuning.
+
+    Parameters
+    ----------
+    ds : Dataset
+        Input dataset containing documents and QA pairs
+    keep_no_qa_per_doc : int, default=3
+        Number of QA entries per document to mark as unmask (pre-training)
+
+    Returns
+    -------
+    Dataset
+        Dataset with added 'unmask' boolean column indicating pre-training entries
+    """
+
+    unmask_entries = []
+    mask_entries = []
+    doc_count = {}
+
+    for i, doc in enumerate(ds["document"]):
+        if doc not in doc_count:
+            doc_count[doc] = 1
+        else:
+            doc_count[doc] += 1
+
+        entry = ds[i].copy()
+        if doc_count[doc] <= keep_no_qa_per_doc:
+            entry["unmask"] = True
+            unmask_entries.append(entry)
+        else:
+            entry["unmask"] = False
+            mask_entries.append(entry)
+
+    ds_new = concatenate_datasets(
+        [Dataset.from_list(unmask_entries), Dataset.from_list(mask_entries)]
+    )
+    return ds_new
+
+
 def generate_knowledge_qa_dataset(
-    generated_dataset: Dataset, keep_context_separate=False, keep_document_outline=False
+    generated_dataset: Dataset,
+    keep_context_separate: bool = False,
+    keep_document_outline: bool = False,
+    keep_columns: List[str] = None,
+    filter_non_pre_training: bool = True,
+    keep_no_qa_per_doc: int = 3,
 ):
+    generated_dataset = generated_dataset.map(
+        lambda x: {
+            "response": x["response"]
+            .replace("[END]", "")
+            .replace("[ANSWER]", "")
+            .strip()
+        },
+        num_proc=10,
+    )
+    generated_dataset = mask_qa_per_doc(
+        generated_dataset, keep_no_qa_per_doc=keep_no_qa_per_doc
+    )
+    if filter_non_pre_training:
+        generated_dataset = generated_dataset.filter(lambda x: x["unmask"])
+
     def __create_qa_row(rec):
         context = rec["document"]
         instruction = rec["question"]
@@ -146,7 +207,12 @@ def generate_knowledge_qa_dataset(
             return {"messages": messages, "metadata": metadata, "id": str(uuid.uuid4())}
 
     knowledge_ds = generated_dataset.map(
-        __create_qa_row, remove_columns=generated_dataset.column_names
+        __create_qa_row,
+        remove_columns=[
+            e
+            for e in generated_dataset.column_names
+            if e not in keep_columns + ["unmask"]
+        ],
     )
     return knowledge_ds
 

{sdg_hub-0.1.1 → sdg_hub-0.1.3}/pyproject.toml

@@ -55,7 +55,7 @@ web_interface = [
     "flask-wtf>=1.2.2",
 ]
 vllm = [
-    "vllm>=0.8.0,<0.8.4",
+    "vllm>=0.9.1",
     "torch>=2.0.0",
     "transformers>=4.37.0",
     "accelerate>=0.21.0",

{sdg_hub-0.1.1 → sdg_hub-0.1.3}/src/sdg_hub/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.1'
-__version_tuple__ = version_tuple = (0, 1, 1)
+__version__ = version = '0.1.3'
+__version_tuple__ = version_tuple = (0, 1, 3)

{sdg_hub-0.1.1 → sdg_hub-0.1.3}/src/sdg_hub/blocks/__init__.py

@@ -6,6 +6,10 @@ This package provides various block implementations for data generation, process
 # Local
 from .block import Block
 from .llmblock import LLMBlock, ConditionalLLMBlock
+from .openaichatblock import (
+    OpenAIChatBlock,
+    OpenAIAsyncChatBlock
+)
 from .utilblocks import (
     SamplePopulatorBlock,
     SelectorBlock,
@@ -33,4 +37,6 @@ __all__ = [
     "RenameColumns",
     "SetToMajorityValue",
     "BlockRegistry",
+    "OpenAIChatBlock",
+    "OpenAIAsyncChatBlock"
 ]