sdg-hub 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

sdg_hub/flow_runner.py

@@ -1,18 +1,31 @@
 """Script for running data generation flows with configurable parameters."""
 
 # Standard
+from importlib import resources
+from typing import Optional
 import os
+import sys
+import traceback
 
 # Third Party
 from datasets import load_dataset
 from openai import OpenAI
 import click
+import yaml
 
 # First Party
 from sdg_hub.flow import Flow
 from sdg_hub.logger_config import setup_logger
 from sdg_hub.sdg import SDG
-
+from sdg_hub.utils.error_handling import (
+    APIConnectionError,
+    DataGenerationError,
+    DataSaveError,
+    DatasetLoadError,
+    FlowConfigurationError,
+    FlowRunnerError,
+)
+from sdg_hub.utils.path_resolution import resolve_path
 
 logger = setup_logger(__name__)
 
@@ -28,7 +41,7 @@ def run_flow(
     save_freq: int = 2,
     debug: bool = False,
     dataset_start_index: int = 0,
-    dataset_end_index: int = None,
+    dataset_end_index: Optional[int] = None,
 ) -> None:
     """Process the dataset using the specified configuration.
 
@@ -52,6 +65,10 @@ def run_flow(
         Frequency (in batches) at which to save checkpoints, by default 2.
     debug : bool, optional
         If True, enables debug mode with a smaller dataset subset, by default False.
+    dataset_start_index : int, optional
+        Start index for dataset slicing, by default 0.
+    dataset_end_index : Optional[int], optional
+        End index for dataset slicing, by default None.
 
     Returns
     -------
@@ -59,42 +76,214 @@ def run_flow(
 
     Raises
     ------
-    FileNotFoundError
-        If the flow configuration file is not found.
+    DatasetLoadError
+        If the dataset cannot be loaded or processed.
+    FlowConfigurationError
+        If the flow configuration is invalid or cannot be loaded.
+    APIConnectionError
+        If connection to the API endpoint fails.
+    DataGenerationError
+        If data generation fails during processing.
+    DataSaveError
+        If saving the generated data fails.
     """
     logger.info(f"Generation configuration: {locals()}\n\n")
-    ds = load_dataset("json", data_files=ds_path, split="train")
-    if dataset_start_index is not None and dataset_end_index is not None:
-        ds = ds.select(range(dataset_start_index, dataset_end_index))
-        logger.info(f"Dataset sliced from {dataset_start_index} to {dataset_end_index}")
-    if debug:
-        ds = ds.shuffle(seed=42).select(range(30))
-        logger.info("Debug mode enabled. Using a subset of the dataset.")
-
-    openai_api_key = os.environ.get("OPENAI_API_KEY", "EMPTY")
-    openai_api_base = endpoint
-
-    client = OpenAI(
-        api_key=openai_api_key,
-        base_url=openai_api_base,
-    )
-
-    if not os.path.exists(flow_path):
-        raise FileNotFoundError(f"Flow file not found: {flow_path}")
-
-    flow = Flow(client).get_flow_from_file(flow_path)
-    sdg = SDG(
-        flows=[flow],
-        num_workers=num_workers,
-        batch_size=batch_size,
-        save_freq=save_freq,
-    )
-
-    generated_data = sdg.generate(ds, checkpoint_dir=checkpoint_dir)
-    if dataset_end_index is not None and dataset_start_index is not None:
-        save_path = save_path.replace(".jsonl", f"_{dataset_start_index}_{dataset_end_index}.jsonl")
-    generated_data.to_json(save_path, orient="records", lines=True)
-    logger.info(f"Data saved to {save_path}")
+
+    try:
+        # Load and validate dataset
+        try:
+            ds = load_dataset("json", data_files=ds_path, split="train")
+            logger.info(
+                f"Successfully loaded dataset from {ds_path} with {len(ds)} rows"
+            )
+        except Exception as e:
+            raise DatasetLoadError(
+                f"Failed to load dataset from '{ds_path}'. "
+                f"Please check if the file exists and is a valid JSON file.",
+                details=str(e),
+            ) from e
+
+        # Apply dataset slicing if specified
+        try:
+            if dataset_start_index is not None and dataset_end_index is not None:
+                if dataset_start_index >= len(ds) or dataset_end_index > len(ds):
+                    raise DatasetLoadError(
+                        f"Dataset slice indices ({dataset_start_index}, {dataset_end_index}) "
+                        f"are out of bounds for dataset with {len(ds)} rows"
+                    )
+                if dataset_start_index >= dataset_end_index:
+                    raise DatasetLoadError(
+                        f"Start index ({dataset_start_index}) must be less than end index ({dataset_end_index})"
+                    )
+                ds = ds.select(range(dataset_start_index, dataset_end_index))
+                logger.info(
+                    f"Dataset sliced from {dataset_start_index} to {dataset_end_index}"
+                )
+
+            if debug:
+                if len(ds) < 30:
+                    logger.warning(
+                        f"Debug mode requested 30 samples but dataset only has {len(ds)} rows"
+                    )
+                ds = ds.shuffle(seed=42).select(range(min(30, len(ds))))
+                logger.info(
+                    f"Debug mode enabled. Using {len(ds)} samples from the dataset."
+                )
+        except DatasetLoadError:
+            raise
+        except Exception as e:
+            raise DatasetLoadError(
+                "Failed to process dataset slicing or debug mode.", details=str(e)
+            ) from e
+
+        # Validate API configuration
+        openai_api_key = os.environ.get("OPENAI_API_KEY")
+        if not openai_api_key or openai_api_key == "EMPTY":
+            logger.warning("OPENAI_API_KEY not set or is 'EMPTY'. API calls may fail.")
+
+        openai_api_base = endpoint
+        if not openai_api_base:
+            raise APIConnectionError("API endpoint cannot be empty")
+
+        # Initialize OpenAI client
+        try:
+            client = OpenAI(
+                api_key=openai_api_key or "EMPTY",
+                base_url=openai_api_base,
+            )
+            # test connection with a model list
+            models = client.models.list()
+            logger.info(f"Initialized OpenAI client with endpoint: {openai_api_base}")
+            logger.info(f"Available models: {[model.id for model in models.data]}")
+        except Exception as e:
+            raise APIConnectionError(
+                f"Failed to initialize OpenAI client with endpoint '{openai_api_base}'. "
+                f"Please check if the endpoint is valid and accessible.",
+                details=str(e),
+            ) from e
+
+        # Load and validate flow configuration
+        try:
+            base_path = str(resources.files(__package__))
+            flow_path = resolve_path(flow_path, [".", base_path])
+            if not os.path.exists(flow_path):
+                raise FlowConfigurationError(
+                    f"Flow configuration file not found: {flow_path}"
+                )
+
+            # Validate flow file is readable YAML
+            try:
+                with open(flow_path, "r", encoding="utf-8") as f:
+                    flow_config = yaml.safe_load(f)
+                if not flow_config:
+                    raise FlowConfigurationError(
+                        f"Flow configuration file is empty: {flow_path}"
+                    )
+                logger.info(f"Successfully loaded flow configuration from {flow_path}")
+            except yaml.YAMLError as e:
+                raise FlowConfigurationError(
+                    f"Flow configuration file '{flow_path}' contains invalid YAML.",
+                    details=str(e),
+                ) from e
+            except Exception as e:
+                raise FlowConfigurationError(
+                    f"Failed to read flow configuration file '{flow_path}'.",
+                    details=str(e),
+                ) from e
+
+            flow = Flow(client).get_flow_from_file(flow_path)
+            logger.info("Successfully initialized flow from configuration")
+        except FlowConfigurationError:
+            raise
+        except Exception as e:
+            raise FlowConfigurationError(
+                f"Failed to create flow from configuration file '{flow_path}'. "
+                f"Please check the flow configuration format and block definitions.",
+                details=str(e),
+            ) from e
+
+        # Initialize SDG and generate data
+        try:
+            sdg = SDG(
+                flows=[flow],
+                num_workers=num_workers,
+                batch_size=batch_size,
+                save_freq=save_freq,
+            )
+            logger.info(
+                f"Initialized SDG with {num_workers} workers, batch size {batch_size}"
+            )
+
+            # Ensure checkpoint directory exists if specified
+            if checkpoint_dir and not os.path.exists(checkpoint_dir):
+                os.makedirs(checkpoint_dir, exist_ok=True)
+                logger.info(f"Created checkpoint directory: {checkpoint_dir}")
+
+            generated_data = sdg.generate(ds, checkpoint_dir=checkpoint_dir)
+
+            if generated_data is None or len(generated_data) == 0:
+                raise DataGenerationError(
+                    "Data generation completed but no data was generated. "
+                    "This may indicate issues with the flow configuration or input data."
+                )
+
+            logger.info(f"Successfully generated {len(generated_data)} rows of data")
+
+        except Exception as e:
+            if isinstance(e, DataGenerationError):
+                raise
+            raise DataGenerationError(
+                "Data generation failed during processing. This could be due to:"
+                "\n- API connection issues with the endpoint"
+                "\n- Invalid flow configuration or block parameters"
+                "\n- Insufficient system resources (try reducing batch_size or num_workers)"
+                "\n- Input data format incompatibility",
+                details=f"Endpoint: {openai_api_base}, Error: {e}",
+            ) from e
+
+        # Save generated data
+        try:
+            # Adjust save path for dataset slicing
+            final_save_path = save_path
+            if dataset_end_index is not None and dataset_start_index is not None:
+                final_save_path = save_path.replace(
+                    ".jsonl", f"_{dataset_start_index}_{dataset_end_index}.jsonl"
+                )
+
+            # Ensure save directory exists
+            save_dir = os.path.dirname(final_save_path)
+            if save_dir and not os.path.exists(save_dir):
+                os.makedirs(save_dir, exist_ok=True)
+                logger.info(f"Created save directory: {save_dir}")
+
+            generated_data.to_json(final_save_path, orient="records", lines=True)
+            logger.info(f"Data successfully saved to {final_save_path}")
+
+        except Exception as e:
+            raise DataSaveError(
+                f"Failed to save generated data to '{final_save_path}'. "
+                f"Please check write permissions and disk space.",
+                details=str(e),
+            ) from e
+
+    except (
+        DatasetLoadError,
+        FlowConfigurationError,
+        APIConnectionError,
+        DataGenerationError,
+        DataSaveError,
+    ):
+        # Re-raise our custom exceptions with their detailed messages
+        raise
+    except Exception as e:
+        # Catch any unexpected errors
+        logger.error(f"Unexpected error during flow execution: {e}")
+        logger.error(f"Traceback: {traceback.format_exc()}")
+        raise FlowRunnerError(
+            "An unexpected error occurred during flow execution. "
+            "Please check the logs for more details.",
+            details=str(e),
+        ) from e
 
 
 @click.command()
@@ -154,8 +343,12 @@ def run_flow(
     is_flag=True,
     help="Enable debug mode with a smaller dataset subset.",
 )
-@click.option("--dataset_start_index", type=int, default=0, help="Start index of the dataset.")
-@click.option("--dataset_end_index", type=int, default=None, help="End index of the dataset.")
+@click.option(
+    "--dataset_start_index", type=int, default=0, help="Start index of the dataset."
+)
+@click.option(
+    "--dataset_end_index", type=int, default=None, help="End index of the dataset."
+)
 def main(
     ds_path: str,
     bs: int,
@@ -167,7 +360,7 @@ def main(
     save_freq: int,
     debug: bool,
     dataset_start_index: int,
-    dataset_end_index: int,
+    dataset_end_index: Optional[int],
 ) -> None:
     """CLI entry point for running data generation flows.
 
@@ -191,24 +384,52 @@ def main(
         Frequency (in batches) at which to save checkpoints.
     debug : bool
         If True, enables debug mode with a smaller dataset subset.
+    dataset_start_index : int
+        Start index for dataset slicing.
+    dataset_end_index : Optional[int]
+        End index for dataset slicing.
 
     Returns
     -------
     None
     """
-    run_flow(
-        ds_path=ds_path,
-        batch_size=bs,
-        num_workers=num_workers,
-        save_path=save_path,
-        endpoint=endpoint,
-        flow_path=flow,
-        checkpoint_dir=checkpoint_dir,
-        save_freq=save_freq,
-        debug=debug,
-        dataset_start_index=dataset_start_index,
-        dataset_end_index=dataset_end_index,
-    )
+    try:
+        run_flow(
+            ds_path=ds_path,
+            batch_size=bs,
+            num_workers=num_workers,
+            save_path=save_path,
+            endpoint=endpoint,
+            flow_path=flow,
+            checkpoint_dir=checkpoint_dir,
+            save_freq=save_freq,
+            debug=debug,
+            dataset_start_index=dataset_start_index,
+            dataset_end_index=dataset_end_index,
+        )
+    except (
+        DatasetLoadError,
+        FlowConfigurationError,
+        APIConnectionError,
+        DataGenerationError,
+        DataSaveError,
+        FlowRunnerError,
+    ) as e:
+        logger.error(f"Flow execution failed: {e}")
+        click.echo(f"Error: {e}", err=True)
+        sys.exit(1)
+    except KeyboardInterrupt:
+        logger.info("Flow execution interrupted by user")
+        click.echo("Flow execution interrupted by user", err=True)
+        sys.exit(130)  # Standard exit code for SIGINT
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}")
+        logger.error(f"Traceback: {traceback.format_exc()}")
+        click.echo(
+            f"Unexpected error occurred. Please check the logs for details. Error: {e}",
+            err=True,
+        )
+        sys.exit(1)
 
 
 if __name__ == "__main__":

sdg_hub/flows/generation/knowledge/synth_knowledge1.5.yaml

@@ -14,7 +14,6 @@
   gen_kwargs:
     max_tokens: 4096
     temperature: 0.7
-    seed: 7452
     n: 50
 
 - block_type: LLMBlock
@@ -27,7 +26,6 @@
   gen_kwargs:
     max_tokens: 4096
     temperature: 0.7
-    seed: 7452
 
 - block_type: LLMBlock
   block_config:
@@ -39,7 +37,6 @@
   gen_kwargs:
     max_tokens: 4096
     temperature: 0.7
-    seed: 7452
 
 - block_type: FlattenColumnsBlock
   block_config:
@@ -72,7 +69,6 @@
   gen_kwargs:
     temperature: 0.7
     max_tokens: 100
-    seed: 7452
 
 - block_type: LLMBlock
   block_config:
@@ -84,7 +80,6 @@
   gen_kwargs:
     temperature: 0.7
     max_tokens: 2048
-    seed: 7452
 
 - block_type: LLMBlock
   block_config:

sdg_hub/prompts.py

@@ -28,6 +28,11 @@ def microsoft_phi_chat_template():
 
 @PromptRegistry.register("nvidia/Llama-3_3-Nemotron-Super-49B-v1")
 def nemotron_chat_template():
+    """
+    Format chat messages for the Nemotron model, including a system prompt and structured message headers.
+    
+    The template starts with a system message containing "detailed thinking on", then iterates over messages, wrapping each with start and end header tokens and an end-of-text token. For assistant messages containing a `</think>` tag, only the content after this tag is included. Optionally appends an assistant prompt if generation is requested.
+    """
     return """{{- bos_token }}
 {{- "<|start_header_id|>system<|end_header_id|>\n\n" }}detailed thinking on{{- "<|eot_id|>" }}
 {%- for message in messages %}
@@ -41,3 +46,29 @@ def nemotron_chat_template():
 {%- if add_generation_prompt %}
   {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
 {%- endif %}"""
+
+
+@PromptRegistry.register("Qwen/Qwen2.5")
+def qwen_2_5_chat_template():
+    """
+    Formats chat messages into the prompt structure required by the Qwen 2.5 model family, supporting system messages, tool descriptions, function call instructions, and role-based message formatting.
+    
+    If tools are provided, includes tool signatures and instructions for function calls in the system prompt. User, assistant, and tool messages are wrapped with special tokens, and assistant tool calls are serialized as JSON within XML tags. Optionally appends a generation prompt for the assistant.
+    """
+    return """{%- if tools %}\n    {{- \'<|im_start|>system\\n\' }}\n    {%- if messages[0][\'role\'] == \'system\' %}\n        {{- messages[0][\'content\'] }}\n    {%- else %}\n        {{- \'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.\' }}\n    {%- endif %}\n    {{- "\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>" }}\n    {%- for tool in tools %}\n        {{- "\\n" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- "\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\"name\\": <function-name>, \\"arguments\\": <args-json-object>}\\n</tool_call><|im_end|>\\n" }}\n{%- else %}\n    {%- if messages[0][\'role\'] == \'system\' %}\n        {{- \'<|im_start|>system\\n\' + messages[0][\'content\'] + \'<|im_end|>\\n\' }}\n    {%- else %}\n        {{- \'<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n\' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}\n        {{- \'<|im_start|>\' + message.role + \'\\n\' + message.content + \'<|im_end|>\' + \'\\n\' }}\n    {%- elif message.role == "assistant" %}\n        {{- \'<|im_start|>\' + message.role }}\n        {%- if message.content %}\n            {{- \'\\n\' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- \'\\n<tool_call>\\n{"name": "\' }}\n            {{- tool_call.name }}\n            {{- \'", "arguments": \' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \'}\\n</tool_call>\' }}\n        {%- endfor %}\n        {{- \'<|im_end|>\\n\' }}\n    {%- elif message.role == "tool" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}\n            {{- \'<|im_start|>user\' }}\n        {%- endif %}\n        {{- \'\\n<tool_response>\\n\' }}\n        {{- message.content }}\n        {{- \'\\n</tool_response>\' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}\n            {{- \'<|im_end|>\\n\' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- \'<|im_start|>assistant\\n\' }}\n{%- endif %}\n"""
+
+
+@PromptRegistry.register("Qwen/Qwen3")
+def qwen_3_chat_template():
+    """
+    Formats chat messages for the Qwen 3 model family, supporting multi-step tool usage, reasoning content, and special XML tags for tool calls and responses.
+    
+    This template handles system messages, user and assistant roles, and tool interactions. When tools are provided, it outputs their signatures and instructions for function calls. It tracks the last user query to determine where to insert assistant reasoning content within `<think>` tags. Assistant tool calls are serialized as JSON within `<tool_call>` tags, and tool responses are grouped inside `<tool_response>` tags. Optionally, a generation prompt and empty reasoning block can be added.
+    
+    Parameters:
+        tools (optional): List of tool signature objects to be included in the prompt.
+        messages: List of message objects, each with a role and content, and optionally tool_calls or reasoning_content.
+        add_generation_prompt (optional): If true, appends an assistant prompt for generation.
+        enable_thinking (optional): If false, inserts an empty reasoning block in the assistant prompt.
+    """
+    return """{%- if tools %}\n    {{- \'<|im_start|>system\\n\' }}\n    {%- if messages[0].role == \'system\' %}\n        {{- messages[0].content + \'\\n\\n\' }}\n    {%- endif %}\n    {{- "# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>" }}\n    {%- for tool in tools %}\n        {{- "\\n" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- "\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\"name\\": <function-name>, \\"arguments\\": <args-json-object>}\\n</tool_call><|im_end|>\\n" }}\n{%- else %}\n    {%- if messages[0].role == \'system\' %}\n        {{- \'<|im_start|>system\\n\' + messages[0].content + \'<|im_end|>\\n\' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith(\'<tool_response>\') and message.content.endswith(\'</tool_response>\')) %}\n        {%- set ns.multi_step_tool = false %}\n        {%- set ns.last_query_index = index %}\n    {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n    {%- if message.content is string %}\n        {%- set content = message.content %}\n    {%- else %}\n        {%- set content = \'\' %}\n    {%- endif %}\n    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}\n        {{- \'<|im_start|>\' + message.role + \'\\n\' + content + \'<|im_end|>\' + \'\\n\' }}\n    {%- elif message.role == "assistant" %}\n        {%- set reasoning_content = \'\' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if \'</think>\' in content %}\n                {%- set reasoning_content = content.split(\'</think>\')[0].rstrip(\'\\n\').split(\'<think>\')[-1].lstrip(\'\\n\') %}\n                {%- set content = content.split(\'</think>\')[-1].lstrip(\'\\n\') %}\n            {%- endif %}\n        {%- endif %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {%- if loop.last or (not loop.last and reasoning_content) %}\n                {{- \'<|im_start|>\' + message.role + \'\\n<think>\\n\' + reasoning_content.strip(\'\\n\') + \'\\n</think>\\n\\n\' + content.lstrip(\'\\n\') }}\n            {%- else %}\n                {{- \'<|im_start|>\' + message.role + \'\\n\' + content }}\n            {%- endif %}\n        {%- else %}\n            {{- \'<|im_start|>\' + message.role + \'\\n\' + content }}\n        {%- endif %}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- \'\\n\' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- \'<tool_call>\\n{"name": "\' }}\n                {{- tool_call.name }}\n                {{- \'", "arguments": \' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- \'}\\n</tool_call>\' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- \'<|im_end|>\\n\' }}\n    {%- elif message.role == "tool" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}\n            {{- \'<|im_start|>user\' }}\n        {%- endif %}\n        {{- \'\\n<tool_response>\\n\' }}\n        {{- content }}\n        {{- \'\\n</tool_response>\' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}\n            {{- \'<|im_end|>\\n\' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- \'<|im_start|>assistant\\n\' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- \'<think>\\n\\n</think>\\n\\n\' }}\n    {%- endif %}\n{%- endif %}"""

sdg_hub/utils/__init__.py

@@ -3,3 +3,8 @@
 # This is part of the public API, and used by instructlab
 class GenerateException(Exception):
     """An exception raised during generate step."""
+
+
+from .path_resolution import resolve_path
+
+__all__ = ["GenerateException", "resolve_path"]

sdg_hub/utils/error_handling.py

@@ -0,0 +1,94 @@
+"""Custom exception classes for SDG Hub error handling."""
+
+
+class SDGHubError(Exception):
+    """Base exception class for all SDG Hub errors."""
+
+    def __init__(self, message: str, details: str = None):
+        """Initialize SDGHubError.
+
+        Parameters
+        ----------
+        message : str
+            The main error message.
+        details : str, optional
+            Additional details about the error.
+        """
+        self.message = message
+        self.details = details
+        full_message = message
+        if details:
+            full_message = f"{message}\nDetails: {details}"
+        super().__init__(full_message)
+
+
+class FlowRunnerError(SDGHubError):
+    """Base exception class for flow runner errors."""
+
+    pass
+
+
+class DatasetLoadError(FlowRunnerError):
+    """Raised when dataset loading fails."""
+
+    pass
+
+
+class FlowConfigurationError(FlowRunnerError):
+    """Raised when flow configuration is invalid."""
+
+    pass
+
+
+class APIConnectionError(FlowRunnerError):
+    """Raised when API connection fails."""
+
+    pass
+
+
+class DataGenerationError(FlowRunnerError):
+    """Raised when data generation fails."""
+
+    pass
+
+
+class DataSaveError(FlowRunnerError):
+    """Raised when saving generated data fails."""
+
+    pass
+
+
+class BlockError(SDGHubError):
+    """Base exception class for block-related errors."""
+
+    pass
+
+
+class BlockConfigurationError(BlockError):
+    """Raised when block configuration is invalid."""
+
+    pass
+
+
+class BlockExecutionError(BlockError):
+    """Raised when block execution fails."""
+
+    pass
+
+
+class FlowError(SDGHubError):
+    """Base exception class for flow-related errors."""
+
+    pass
+
+
+class FlowValidationError(FlowError):
+    """Raised when flow validation fails."""
+
+    pass
+
+
+class FlowExecutionError(FlowError):
+    """Raised when flow execution fails."""
+
+    pass

sdg_hub/utils/path_resolution.py

@@ -0,0 +1,62 @@
+"""
+Path resolution utilities for SDG Hub.
+
+This module provides utilities for resolving file paths relative to one or more
+base directories, with support for both single directory and multiple directory
+search paths.
+"""
+
+# Standard
+from typing import List, Union
+import os
+
+
+def resolve_path(filename: str, search_dirs: Union[str, List[str]]) -> str:
+    """Resolve a file path relative to one or more search directories.
+
+    Files are checked in the following order:
+        1. Absolute path is always used as-is
+        2. Checked relative to each directory in search_dirs (in order)
+        3. If not found, returns the original filename (assumes relative to current directory)
+
+    Parameters
+    ----------
+    filename : str
+        The path to the file to resolve.
+    search_dirs : Union[str, List[str]]
+        Directory or list of directories in which to search for the file.
+
+    Returns
+    -------
+    str
+        Resolved file path.
+
+    Examples
+    --------
+    >>> resolve_path("config.yaml", "/path/to/base")
+    '/path/to/base/config.yaml'  # if file exists
+
+    >>> resolve_path("config.yaml", ["/path1", "/path2"])
+    '/path1/config.yaml'  # if file exists in path1
+    '/path2/config.yaml'  # if file exists in path2 but not path1
+
+    >>> resolve_path("/absolute/path/file.yaml", ["/path1", "/path2"])
+    '/absolute/path/file.yaml'  # absolute path always used as-is
+    """
+    # Handle absolute paths - always use as-is
+    if os.path.isabs(filename):
+        return filename
+
+    # Convert single directory to list for uniform handling
+    if isinstance(search_dirs, str):
+        search_dirs = [search_dirs]
+
+    # Check each directory in order
+    for directory in search_dirs:
+        full_file_path = os.path.join(directory, filename)
+        if os.path.isfile(full_file_path):
+            return full_file_path
+
+    # If not found in any search directory, return the original filename
+    # This assumes the path is relative to the current directory
+    return filename

{sdg_hub-0.1.1.dist-info → sdg_hub-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.1.1
+Version: 0.1.2
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0