sdg-hub 0.1.0a3__py3-none-any.whl → 0.1.0a4__py3-none-any.whl

sdg_hub/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.0a3'
-__version_tuple__ = version_tuple = (0, 1, 0)
+__version__ = version = '0.1.0a4'
+__version_tuple__ = version_tuple = (0, 1, 0, 'a4')

sdg_hub/blocks/llmblock.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # Standard
-from collections import Counter
 from typing import Any, Dict, List
+from typing import Optional
 import json
 import re
 
@@ -84,6 +84,27 @@ class LLMBlock(Block):
         # and supports the n parameter to generate n outputs per input
         self.server_supports_batched = server_supports_batched(client, self.model)
 
+
+    def _extract_matches(
+        self, text: str, start_tag: Optional[str], end_tag: Optional[str]
+    ) -> List[str]:
+        if not text:
+            return []
+        if not start_tag and not end_tag:
+            return [text.strip()]
+
+        pattern = ""
+        if start_tag:
+            pattern += re.escape(start_tag)
+        pattern += r"(.*?)"
+        if end_tag:
+            pattern += re.escape(end_tag)
+        elif start_tag:
+            # Enforce matching till end of string when only start_tag is provided.
+            pattern += "$"
+
+        return [match.strip() for match in re.findall(pattern, text, re.DOTALL)]
+
     def _parse(self, generated_string) -> dict:
         matches = {}
 
@@ -108,16 +129,9 @@ class LLMBlock(Block):
                 self.block_config.get("end_tags", []),
                 self.output_cols,
             ):
-                if not start_tag and not end_tag:
-                    matches[output_col] = [
-                        generated_string.strip() if generated_string else None
-                    ]
-                else:
-                    pattern = re.escape(start_tag) + r"(.*?)" + re.escape(end_tag)
-                    all_matches = re.findall(pattern, generated_string, re.DOTALL)
-                    matches[output_col] = (
-                        [match.strip() for match in all_matches] if all_matches else []
-                    )
+                matches[output_col] = self._extract_matches(
+                    generated_string, start_tag, end_tag
+                )
 
         return matches
 
@@ -247,11 +261,12 @@ class ConditionalLLMBlock(LLMBlock):
             for config_key, config in config_paths.items():
                 # Template(self.prompt_struct.format(**filtered_config))
                 filtered_config = {
-                    k: (v if v is not None else "") for k, v in self.block_config.items()
+                    k: (v if v is not None else "")
+                    for k, v in self.block_config.items()
                 }
-                self.prompt_template[config_key] = Template(self.prompt_struct.format(
-                    **self._load_config(config)
-                ))
+                self.prompt_template[config_key] = Template(
+                    self.prompt_struct.format(**self._load_config(config))
+                )
 
     def _format_prompt(self, sample: Dict) -> str:
         if isinstance(self.prompt_template, dict):
@@ -380,7 +395,7 @@ class LLMMessagesBlock(Block):
         client,
         input_col,
         output_col,
-        model_prompt=None, 
+        model_prompt=None,
         model_id=None,
         **batch_kwargs,
     ) -> None:
@@ -395,7 +410,7 @@ class LLMMessagesBlock(Block):
             self.model = model_id
         else:
             self.model = self.client.models.list().data[0].id
-        
+
         self.defaults = {
             "model": self.model,
             "temperature": 0,
@@ -417,7 +432,9 @@ class LLMMessagesBlock(Block):
         results = []
         n = gen_kwargs.get("n", 1)
         for message in messages:
-            responses = self.client.chat.completions.create(messages=message, **generate_args)
+            responses = self.client.chat.completions.create(
+                messages=message, **generate_args
+            )
             if n > 1:
                 results.append([choice.message.content for choice in responses.choices])
             else:

sdg_hub/prompts.py

@@ -15,3 +15,8 @@ def instructlab_chat_template():
 @PromptRegistry.register("mistralai")
 def mistral_chat_template():
     return """{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n\n<s>\n{%- for message in loop_messages %}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n        {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif %}\n    {%- if message['role'] == 'user' %}\n        {%- if loop.first and system_message is defined %}\n            {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n        {%- else %}\n            {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n        {%- endif %}\n    {%- elif message['role'] == 'assistant' %}\n        {{- ' ' + message['content'] + '</s>'}}\n    {%- else %}\n        {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n    {%- endif %}\n{%- endfor %}\n"""
+
+
+@PromptRegistry.register("meta-llama/Llama-3.3")
+def meta_llama_chat_template():
+    return """{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n"""

sdg_hub-0.1.0a4.dist-info/METADATA

@@ -0,0 +1,309 @@
+Metadata-Version: 2.4
+Name: sdg_hub
+Version: 0.1.0a4
+Summary: Synthetic Data Generation
+Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
+License: Apache-2.0
+Project-URL: homepage, https://ai-innovation.team/
+Project-URL: source, https://github.com/Red-Hat-AI-Innovation-Team/sdg_hub
+Project-URL: issues, https://github.com/Red-Hat-AI-Innovation-Team/sdg_hub/issues
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click<9.0.0,>=8.1.7
+Requires-Dist: datasets<4.0.0,>=2.18.0
+Requires-Dist: httpx<1.0.0,>=0.25.0
+Requires-Dist: jinja2
+Requires-Dist: langchain-text-splitters
+Requires-Dist: openai<2.0.0,>=1.13.3
+Requires-Dist: rich
+Requires-Dist: tenacity!=8.4.0,>=8.3.0
+Requires-Dist: tqdm<5.0.0,>=4.66.2
+Dynamic: license-file
+
+# sdg_hub: Synthetic Data Generation Toolkit for LLMs
+
+![Build](https://github.com/Red-Hat-AI-Innovation-Team/sdg_hub/actions/workflows/pypi.yaml/badge.svg?branch=main)
+![Release](https://img.shields.io/github/v/release/Red-Hat-AI-Innovation-Team/sdg_hub)
+![License](https://img.shields.io/github/license/Red-Hat-AI-Innovation-Team/sdg_hub)
+[![Tests](https://github.com/Red-Hat-AI-Innovation-Team/sdg_hub/actions/workflows/test.yml/badge.svg)](https://github.com/Red-Hat-AI-Innovation-Team/sdg_hub/actions/workflows/test.yml)
+[![codecov](https://codecov.io/gh/Red-Hat-AI-Innovation-Team/sdg_hub/graph/badge.svg?token=SP75BCXWO2)](https://codecov.io/gh/Red-Hat-AI-Innovation-Team/sdg_hub)
+
+sdg_hub is a modular, scalable, and efficient solution for creating synthetic data generation workflows in a "no-code" manner. At its core, this framework is designed to simplify data creation for LLMs, allowing users to chain computational units and build powerful pipelines for generating data and processing tasks.
+
+
+## Installation
+
+Latest release from PyPI
+
+```sh
+pip install sdg-hub
+```
+
+Latest main branch
+```sh
+pip install git+https://github.com/Red-Hat-AI-Innovation-Team/sdg_hub.git
+```
+
+## Core Design Principles
+
+The framework is built around the following principles:
+
+1. **Modular Design**: Highly composable blocks form the building units of the framework, allowing users to build workflows effortlessly.
+2. **No-Code Workflow Creation**: Specify workflows using simple YAML configuration files.
+3. **Scalability and Performance**: Optimized for handling large-scale workflows with millions of records.
+
+---
+
+## Framework Architecture
+
+![overview](assets/imgs/overview.png)
+
+### Blocks: The Fundamental Unit
+
+At the heart of the framework is the **Block**. Each block is a self-contained computational unit that performs specific tasks, such as:
+
+- Making LLM calls
+- Performing data transformations
+- Applying filters
+
+Blocks are designed to be:
+- **Modular**: Reusable across multiple pipelines.
+- **Composable**: Easily chained together to create workflows.
+
+These blocks are implemented in the [src/sdg_hub/blocks](src/sdg_hub/blocks) directory.
+
+### Prompts
+
+Prompts are at the core of how LLMs are instructed within SDG Hub. Each `LLMBlock` is associated with a prompt configuration file written in YAML, allowing users to define the exact behavior of the language model — including system instructions, generation principles, and output formatting.
+
+#### Prompt YAML Structure
+
+A typical prompt YAML file looks like this:
+
+```yaml
+system: You are a helpful assistant that can summarize text.
+introduction: Give me a short summary of the text.
+principles:
+  - Do not add any new information.
+  - Do not miss any key points from the provided text.
+examples:
+  - input: Red Hat announced the acquisition of Neural Magic...
+    output: Red Hat acquired Neural Magic to enhance its AI optimization capabilities.
+generation: Here is the document to summarize: {{document}}
+```
+
+#### Key Fields
+* `system`: A high-level instruction that sets the persona or behavior of the model.
+* `introduction`: Optional introduction to set context for the user.
+* `principles`: A list of guiding constraints or rules the model should follow during generation.
+* `examples`: Few-shot examples (optional) to guide output format or tone.
+* `generation`: The actual template used to generate the model input. This supports variable injection using {{variable_name}}.
+
+### YAML-Based Workflow: The Flow
+
+The YAML configuration file, known as the **Flow**, is central to defining data generation workflows in the SDG Framework. A Flow describes how blocks and pipelines are orchestrated to process and generate data efficiently. By leveraging YAML, users can create highly customizable and modular workflows without writing any code.
+
+#### Key Features of a Flow
+
+1. **Modular Design**:
+   - Flows are composed of blocks, which can be chained together into pipelines.
+   - Each block performs a specific task, such as generating, filtering, or transforming data.
+
+2. **Reusability**:
+   - Blocks and configurations defined in a Flow can be reused across different workflows.
+   - YAML makes it easy to tweak or extend workflows without significant changes.
+
+3. **Ease of Configuration**:
+   - Users can specify block types, configurations, and data processing details in a simple and intuitive manner.
+
+
+
+## Hello World Example
+
+Let’s say you have a document and want to generate a concise summary using an LLM. Here’s how simple that is in sdg\_hub:
+
+```yaml
+- block_type: LLMBlock
+  block_config:
+    block_name: gen_summary
+    config_path: prompts/summarization.yaml
+    model_id: meta-llama/Llama-3.3-70B-Instruct
+    output_cols:
+      - summary
+  gen_kwargs:
+    max_tokens: 512
+```
+
+Want to go further? Add another block to extract keywords from the summary:
+
+```yaml
+- block_type: LLMBlock
+  block_config:
+    block_name: gen_keywords
+    config_path: prompts/keywords.yaml
+    model_id: meta-llama/Llama-3.3-70B-Instruct
+    output_cols:
+      - keywords
+  gen_kwargs:
+    max_tokens: 64
+```
+
+Just like that, you’ve built a multi-step LLM workflow using nothing but YAML.
+
+## Available Blocks
+
+The SDG Framework provides a rich set of blocks for different data processing needs. Here's a comprehensive overview of the available blocks and when to use them:
+
+### Base Block Class
+
+The framework is built around the abstract `Block` class, which serves as the foundation for all other blocks:
+
+- **Purpose**: Provides core functionality and interface for all blocks
+- **Key Features**:
+  - Template validation for input data
+  - Configuration loading from YAML files
+  - Standardized block initialization
+  - Common interface for all blocks
+- **Core Methods**:
+  - `_validate`: Validates input data against templates
+  - `_load_config`: Loads configuration from YAML files
+  - `generate`: Abstract method for block execution
+
+All blocks inherit from this base class, ensuring consistent behavior and interface across the framework.
+
+### LLM Blocks
+
+1. **LLMBlock**
+   - **Purpose**: Generate text using language models
+   - **Use Cases**: 
+     - Generating questions, responses, or any text content
+     - Single-prompt generation with structured outputs
+   - **Features**: 
+     - Supports batched processing
+     - Configurable output parsing
+     - Template-based prompt generation
+
+2. **ConditionalLLMBlock**
+   - **Purpose**: Generate text based on conditional logic
+   - **Use Cases**:
+     - Different prompt templates based on input conditions
+     - Multi-path text generation workflows
+   - **Features**:
+     - Multiple config paths for different conditions
+     - Dynamic prompt selection
+
+3. **LLMLogProbBlock**
+   - **Purpose**: Generate text with log probabilities
+   - **Use Cases**:
+     - Analyzing model confidence
+     - Quality scoring of generations
+   - **Features**:
+     - Returns top-k log probabilities
+     - JSON-formatted output
+
+4. **LLMMessagesBlock**
+   - **Purpose**: Chat-based text generation
+   - **Use Cases**:
+     - Multi-turn conversations
+     - Chat-based interactions
+   - **Features**:
+     - Supports message history
+     - Chat completion API
+
+### Filtering and Processing Blocks
+
+1. **FilterByValueBlock**
+   - **Purpose**: Filter datasets based on column values
+   - **Use Cases**:
+     - Removing unwanted samples
+     - Data cleaning
+     - Quality filtering
+   - **Features**:
+     - Multiple filter operations
+     - Type conversion support
+     - Parallel processing
+
+2. **IterBlock**
+   - **Purpose**: Iterative processing of data
+   - **Use Cases**:
+     - Multiple generation attempts
+     - Iterative refinement
+   - **Features**:
+     - Configurable number of iterations
+     - Nested block execution
+
+
+
+### Utility Blocks
+
+1. **SamplePopulatorBlock**
+   - **Purpose**: Populate samples with configuration data
+   - **Use Cases**:
+     - Adding metadata
+     - Configuration injection
+
+2. **SelectorBlock**
+   - **Purpose**: Select data based on mapping
+   - **Use Cases**:
+     - Conditional data selection
+     - Data routing
+
+3. **CombineColumnsBlock**
+   - **Purpose**: Merge multiple columns
+   - **Use Cases**:
+     - Text concatenation
+     - Feature combination
+
+4. **FlattenColumnsBlock**
+   - **Purpose**: Convert wide to long format
+   - **Use Cases**:
+     - Data reshaping
+     - Variable-value pairs
+
+5. **DuplicateColumns**
+   - **Purpose**: Create column copies
+   - **Use Cases**:
+     - Data preservation
+     - Multiple processing paths
+
+6. **RenameColumns**
+   - **Purpose**: Rename dataset columns
+   - **Use Cases**:
+     - Standardizing column names
+     - Data reorganization
+
+7. **SetToMajorityValue**
+   - **Purpose**: Replace values with majority
+   - **Use Cases**:
+     - Data normalization
+     - Outlier handling
+
+---
+### Dataflow and Storage
+
+- **Data Representation**: Dataflow between blocks and pipelines is handled using **Hugging Face Datasets**, which are based on Arrow tables. This provides:
+  - Native parallelization capabilities (e.g., maps, filters).
+  - Support for efficient data transformations.
+
+- **Data Checkpoints**: Intermediate caches of generated data. Checkpoints allow users to:
+  - Resume workflows from the last successful state if interrupted.
+  - Improve reliability for long-running workflows.
+
+
+## Examples
+
+For sample use cases and implementation examples, please refer to the [examples](examples) directory. This directory contains various examples demonstrating different workflows and use cases of the SDG Framework.

{sdg_hub-0.1.0a3.dist-info → sdg_hub-0.1.0a4.dist-info}/RECORD

@@ -1,9 +1,9 @@
 sdg_hub/__init__.py,sha256=5Wa6onDndPvG4iwnjq2jK747t3-7XKdQn2WfHfq1sFc,67
-sdg_hub/_version.py,sha256=wrhrM1UZdxROWn7XOHbbPZa5jOBzV8tlSBMw233huBg,513
+sdg_hub/_version.py,sha256=c_h4q533sxFKNKMu0mPWtg9zCSI76KH0QdADDgQNFCY,519
 sdg_hub/flow.py,sha256=3b97fMei1rWuQWeNfv-xyHKUbcMaf-d_b9Xms9J3BCQ,5425
 sdg_hub/logger_config.py,sha256=7uHEJVRfym1c4n95DOKHelLXqAus8uHsZYmzLsEjqpo,422
 sdg_hub/pipeline.py,sha256=u24ccryfy_nOSvsrWiynNmq1rOmOOkw1L5-TqJvuRSo,2339
-sdg_hub/prompts.py,sha256=dOiC9CsNbMt5Km9PnwyuW0v9zUs3cVXE5jZYwtXZTwc,1957
+sdg_hub/prompts.py,sha256=jdpTUaVfsTeKTc7xUHa8ONFbv8l8Cd33Ozc9rFYFIAU,6891
 sdg_hub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sdg_hub/registry.py,sha256=Sc_HNxo4n0pgWMiEDd_sLjxaSXAMZFiHJIhQKqjywwk,3772
 sdg_hub/sdg.py,sha256=SXXnDGA3MpYlNpsw4XyImL97l0pXiF5P9jrDkZNlDJc,6492
@@ -11,7 +11,7 @@ sdg_hub/blocks/__init__.py,sha256=OwPWofuBBWG7n0nYAXNtFXdq4rPf7FyvKkPfjUBlqec,13
 sdg_hub/blocks/block.py,sha256=ObJp8JaAhQ3lQK6SOYoqHPc7b2hBZMhOXEmIap_qa1k,1788
 sdg_hub/blocks/filterblock.py,sha256=leH0k3stcRzdCWoy8kI2hFruGJ0VUemeA4QBW1eQcdQ,2650
 sdg_hub/blocks/iterblock.py,sha256=7UZnK_JyQfbMhVNVzZ79TtEtADLuosI0z62LhoP63s4,958
-sdg_hub/blocks/llmblock.py,sha256=Jy5vWvcMpXphtv4JEc9Nyjs7lgcoF-Yp0gYx4d_Iopc,16156
+sdg_hub/blocks/llmblock.py,sha256=Diyd-BLsZchDH6-w4pXSUiyFHK-EpY64cNYSVrrteos,16427
 sdg_hub/blocks/rmblocks.py,sha256=nw0p1LytHO7Dmc8RGfJ5uajDQWM93-oNoYrzhaY2QEY,6222
 sdg_hub/blocks/utilblocks.py,sha256=nAehqcDKiDE5W3REGApytYAXztRm9AW65cAy95Ufb8U,4926
 sdg_hub/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -83,8 +83,8 @@ sdg_hub/utils/chunking.py,sha256=VSPQ8dSFI5LF4sefcI0tzWG0Vc1rM_FSMTO6xg_iFzA,255
 sdg_hub/utils/datautils.py,sha256=0t_SZ_UXBKl8uL6rVp3SUh8YKRbzKlh2oO5gr2cKyEw,389
 sdg_hub/utils/docprocessor.py,sha256=Z4J2DfLhRxMCeIeMKttwi-FdivmPqI-hjEwq6-Ub35c,12485
 sdg_hub/utils/parse_and_convert.py,sha256=I27FdS-H2mSoZ07SsKZmNYM2F_Cg7GHTBXD7YNgASNw,13443
-sdg_hub-0.1.0a3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sdg_hub-0.1.0a3.dist-info/METADATA,sha256=vUusH0jLACOcoxvTL-e5dAPfhoTV--zgs_MJ-6IYQfQ,5847
-sdg_hub-0.1.0a3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-sdg_hub-0.1.0a3.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
-sdg_hub-0.1.0a3.dist-info/RECORD,,
+sdg_hub-0.1.0a4.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sdg_hub-0.1.0a4.dist-info/METADATA,sha256=2Zs4Sbcg5sG5iBsITJ7wt5nxpN1RiAEtRWgG6P81AUY,11012
+sdg_hub-0.1.0a4.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+sdg_hub-0.1.0a4.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
+sdg_hub-0.1.0a4.dist-info/RECORD,,

{sdg_hub-0.1.0a3.dist-info → sdg_hub-0.1.0a4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.3.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

sdg_hub-0.1.0a3.dist-info/METADATA

@@ -1,154 +0,0 @@
-Metadata-Version: 2.4
-Name: sdg_hub
-Version: 0.1.0a3
-Summary: Synthetic Data Generation
-Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
-License: Apache-2.0
-Project-URL: homepage, https://ai-innovation.team/
-Project-URL: source, https://github.com/Red-Hat-AI-Innovation-Team/sdg_hub
-Project-URL: issues, https://github.com/Red-Hat-AI-Innovation-Team/sdg_hub/issues
-Classifier: Development Status :: 3 - Alpha
-Classifier: Environment :: Console
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: MacOS :: MacOS X
-Classifier: Operating System :: POSIX :: Linux
-Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: Implementation :: CPython
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: click<9.0.0,>=8.1.7
-Requires-Dist: datasets<4.0.0,>=2.18.0
-Requires-Dist: httpx<1.0.0,>=0.25.0
-Requires-Dist: jinja2
-Requires-Dist: langchain-text-splitters
-Requires-Dist: openai<2.0.0,>=1.13.3
-Requires-Dist: rich
-Requires-Dist: tenacity!=8.4.0,>=8.3.0
-Requires-Dist: tqdm<5.0.0,>=4.66.2
-Dynamic: license-file
-
-# Synthetic Data Generation for LLMs
-
-The SDG Framework is a modular, scalable, and efficient solution for creating synthetic data generation workflows in a "no-code" manner. At its core, this framework is designed to simplify data creation for LLMs, allowing users to chain computational units and build powerful pipelines for generating data and processing tasks.
-
-
-
-## Core Design Principles
-
-The framework is built around the following principles:
-
-1. **Modular Design**: Highly composable blocks form the building units of the framework, allowing users to build workflows effortlessly.
-2. **No-Code Workflow Creation**: Specify workflows using simple YAML configuration files.
-3. **Scalability and Performance**: Optimized for handling large-scale workflows with millions of records.
-
----
-
-## Framework Architecture
-
-![overview](assets/imgs/overview.png)
-
-### Blocks: The Fundamental Unit
-
-At the heart of the framework is the **Block**. Each block is a self-contained computational unit that performs specific tasks, such as:
-
-- Making LLM calls
-- Performing data transformations
-- Applying filters
-
-Blocks are designed to be:
-- **Modular**: Reusable across multiple pipelines.
-- **Composable**: Easily chained together to create workflows.
-
-These blocks are implemented in the [src/sdg_hub/blocks](src/sdg_hub/blocks) directory.
-
-### Pipelines: Higher-Level Abstraction
-
-Blocks can be chained together to form a **Pipeline**. Pipelines enable:
-- Linear or recursive chaining of blocks.
-- Execution of complex workflows by chaining multiple pipelines together.
-
-### SDG Workflow: Full Workflow Automation
-
-Pipelines are further orchestrated into **SDG Workflows**, enabling seamless end-to-end processing. When invoking `sdg_hub.generate`, it triggers a pipeline/ or multiple pipelines that processes data through all the configured blocks.
-
----
-
-### YAML-Based Workflow: The Flow
-
-The YAML configuration file, known as the **Flow**, is central to defining data generation workflows in the SDG Framework. A Flow describes how blocks and pipelines are orchestrated to process and generate data efficiently. By leveraging YAML, users can create highly customizable and modular workflows without writing any code.
-
-#### Key Features of a Flow
-
-1. **Modular Design**:
-   - Flows are composed of blocks, which can be chained together into pipelines.
-   - Each block performs a specific task, such as generating, filtering, or transforming data.
-
-2. **Reusability**:
-   - Blocks and configurations defined in a Flow can be reused across different workflows.
-   - YAML makes it easy to tweak or extend workflows without significant changes.
-
-3. **Ease of Configuration**:
-   - Users can specify block types, configurations, and data processing details in a simple and intuitive manner.
-
----
-
-### Sample Flow
-
-Here is an example of a Flow configuration:
-
-```yaml
-- block_type: LLMBlock
-  block_config:
-    block_name: gen_questions
-    config_path: configs/skills/freeform_questions.yaml
-    model_id: mistralai/Mixtral-8x7B-Instruct-v0.1
-    output_cols:
-      - question
-    batch_kwargs:
-      num_samples: 30
-  drop_duplicates:
-    - question
-- block_type: FilterByValueBlock
-  block_config:
-    block_name: filter_questions
-    filter_column: score
-    filter_value: 1.0
-    operation: operator.eq
-    convert_dtype: float
-    batch_kwargs:
-      num_procs: 8
-  drop_columns:
-    - evaluation
-    - score
-    - num_samples
-- block_type: LLMBlock
-  block_config:
-    block_name: gen_responses
-    config_path: configs/skills/freeform_responses.yaml
-    model_id: mistralai/Mixtral-8x7B-Instruct-v0.1
-    output_cols:
-      - response
-```
-
-### Dataflow and Storage
-
-- **Data Representation**: Dataflow between blocks and pipelines is handled using **Hugging Face Datasets**, which are based on Arrow tables. This provides:
-  - Native parallelization capabilities (e.g., maps, filters).
-  - Support for efficient data transformations.
-
-- **Data Checkpoints**: Intermediate caches of generated data. Checkpoints allow users to:
-  - Resume workflows from the last successful state if interrupted.
-  - Improve reliability for long-running workflows.
-
----
-
-## Examples
-
-For sample use cases and implementation examples, please refer to the [examples](examples) directory. This directory contains various examples demonstrating different workflows and use cases of the SDG Framework.