sdg-hub 0.1.4__py3-none-any.whl → 0.2.1__py3-none-any.whl

sdg_hub/core/utils/datautils.py

@@ -0,0 +1,63 @@
+# Third Party
+from datasets import Dataset, concatenate_datasets
+
+# Local
+from .error_handling import FlowValidationError
+
+
+def safe_concatenate_datasets(datasets: list):
+    """Concatenate datasets safely, ignoring any datasets that are None or empty."""
+    filtered_datasets = [ds for ds in datasets if ds is not None and ds.num_rows > 0]
+
+    if not filtered_datasets:
+        return None
+
+    return concatenate_datasets(filtered_datasets)
+
+
+def safe_concatenate_with_validation(
+    datasets: list, context: str = "datasets"
+) -> Dataset:
+    """Safely concatenate datasets with schema validation and clear error messages.
+
+    Parameters
+    ----------
+    datasets : list[Dataset]
+        List of datasets to concatenate
+    context : str
+        Description of what's being concatenated for error messages
+
+    Returns
+    -------
+    Dataset
+        Concatenated dataset
+
+    Raises
+    ------
+    FlowValidationError
+        If schema mismatch prevents concatenation or no valid datasets
+    """
+    # Filter out None and empty datasets first
+    valid_datasets = [ds for ds in datasets if ds is not None and len(ds) > 0]
+
+    if not valid_datasets:
+        raise FlowValidationError(f"No valid datasets to concatenate in {context}")
+
+    if len(valid_datasets) == 1:
+        return valid_datasets[0]
+
+    try:
+        return concatenate_datasets(valid_datasets)
+    except Exception as e:
+        # Schema mismatch or other concatenation error
+        schema_info = []
+        for i, ds in enumerate(valid_datasets):
+            schema_info.append(f"Dataset {i}: columns={ds.column_names}")
+
+        schema_details = "\n".join(schema_info)
+        raise FlowValidationError(
+            f"Schema mismatch when concatenating {context}. "
+            f"All datasets must have compatible schemas (same columns/types). "
+            f"Original error: {e}\n"
+            f"Dataset schemas:\n{schema_details}"
+        ) from e

sdg_hub/core/utils/error_handling.py

@@ -0,0 +1,208 @@
+"""Custom exception classes for SDG Hub error handling."""
+
+# Standard
+from typing import Optional
+
+
+class SDGHubError(Exception):
+    """Base exception class for all SDG Hub errors."""
+
+    def __init__(self, message: str, details: Optional[str] = None):
+        """Initialize SDGHubError.
+
+        Parameters
+        ----------
+        message : str
+            The main error message.
+        details : str, optional
+            Additional details about the error.
+        """
+        self.message = message
+        self.details = details
+        full_message = message
+        if details:
+            full_message = f"{message}\nDetails: {details}"
+        super().__init__(full_message)
+
+
+class FlowRunnerError(SDGHubError):
+    """Base exception class for flow runner errors."""
+
+    pass
+
+
+class DatasetLoadError(FlowRunnerError):
+    """Raised when dataset loading fails."""
+
+    pass
+
+
+class FlowConfigurationError(FlowRunnerError):
+    """Raised when flow configuration is invalid."""
+
+    pass
+
+
+class APIConnectionError(FlowRunnerError):
+    """Raised when API connection fails."""
+
+    pass
+
+
+class DataGenerationError(FlowRunnerError):
+    """Raised when data generation fails."""
+
+    pass
+
+
+class DataSaveError(FlowRunnerError):
+    """Raised when saving generated data fails."""
+
+    pass
+
+
+class BlockError(SDGHubError):
+    """Base exception class for block-related errors."""
+
+    pass
+
+
+class BlockConfigurationError(BlockError):
+    """Raised when block configuration is invalid."""
+
+    pass
+
+
+class BlockExecutionError(BlockError):
+    """Raised when block execution fails."""
+
+    pass
+
+
+class BlockValidationError(BlockError):
+    """Base exception class for block validation errors."""
+
+    pass
+
+
+class MissingColumnError(BlockValidationError):
+    """Raised when required input columns are missing from dataset."""
+
+    def __init__(
+        self, block_name: str, missing_columns: list[str], available_columns: list[str]
+    ):
+        """Initialize MissingColumnError.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block that failed validation.
+        missing_columns : List[str]
+            List of missing column names.
+        available_columns : List[str]
+            List of available column names in the dataset.
+        """
+        self.block_name = block_name
+        self.missing_columns = missing_columns
+        self.available_columns = available_columns
+
+        message = (
+            f"Block '{block_name}' missing required input columns: {missing_columns}"
+        )
+        details = f"Available columns: {available_columns}"
+
+        super().__init__(message, details)
+
+
+class EmptyDatasetError(BlockValidationError):
+    """Raised when an empty dataset is provided to a block."""
+
+    def __init__(self, block_name: str):
+        """Initialize EmptyDatasetError.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block that received the empty dataset.
+        """
+        self.block_name = block_name
+
+        message = f"Block '{block_name}' received an empty dataset"
+        details = "Dataset must contain at least one sample for processing"
+
+        super().__init__(message, details)
+
+
+class OutputColumnCollisionError(BlockValidationError):
+    """Raised when output columns would overwrite existing dataset columns."""
+
+    def __init__(
+        self, block_name: str, collision_columns: list[str], existing_columns: list[str]
+    ):
+        """Initialize OutputColumnCollisionError.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block that has column collisions.
+        collision_columns : List[str]
+            List of output columns that collide with existing columns.
+        existing_columns : List[str]
+            List of existing column names in the dataset.
+        """
+        self.block_name = block_name
+        self.collision_columns = collision_columns
+        self.existing_columns = existing_columns
+
+        message = f"Block '{block_name}' output columns would overwrite existing data: {collision_columns}"
+        details = f"Existing columns: {existing_columns}"
+
+        super().__init__(message, details)
+
+
+class TemplateValidationError(BlockValidationError):
+    """Raised when template validation fails due to missing variables."""
+
+    def __init__(
+        self,
+        block_name: str,
+        missing_variables: list[str],
+        available_variables: list[str],
+    ):
+        """Initialize TemplateValidationError.
+
+        Parameters
+        ----------
+        block_name : str
+            Name of the block that failed template validation.
+        missing_variables : List[str]
+            List of missing template variable names.
+        available_variables : List[str]
+            List of available template variable names.
+        """
+        self.block_name = block_name
+        self.missing_variables = missing_variables
+        self.available_variables = available_variables
+
+        message = f"Block '{block_name}' template validation failed - missing required variables: {missing_variables}"
+        details = f"Available variables: {available_variables}"
+
+        super().__init__(message, details)
+
+
+class FlowError(SDGHubError):
+    """Base exception class for flow-related errors."""
+
+    pass
+
+
+class FlowValidationError(FlowError):
+    """Raised when flow validation fails."""
+
+    pass
+
+
+class FlowExecutionError(FlowError):
+    """Raised when flow execution fails."""
+
+    pass

sdg_hub/core/utils/flow_id_words.yaml

@@ -0,0 +1,231 @@
+# Flow ID word lists for wandb-style deterministic generation
+# Format: adjective-noun-number (e.g., "bright-river-123")
+
+adjectives:
+  - able
+  - ancient
+  - autumn
+  - bold
+  - brave
+  - bright
+  - calm
+  - clean
+  - clever
+  - cool
+  - cosmic
+  - daily
+  - dark
+  - deep
+  - divine
+  - dry
+  - eager
+  - early
+  - earnest
+  - easy
+  - epic
+  - even
+  - exact
+  - fair
+  - fast
+  - fine
+  - firm
+  - first
+  - fresh
+  - full
+  - gentle
+  - glad
+  - golden
+  - good
+  - great
+  - green
+  - happy
+  - hard
+  - heavy
+  - high
+  - holy
+  - huge
+  - jolly
+  - keen
+  - kind
+  - large
+  - late
+  - light
+  - live
+  - long
+  - loud
+  - lucky
+  - major
+  - mild
+  - new
+  - nice
+  - noble
+  - old
+  - open
+  - plain
+  - proud
+  - pure
+  - quick
+  - quiet
+  - rapid
+  - rare
+  - real
+  - rich
+  - right
+  - rough
+  - round
+  - safe
+  - sharp
+  - short
+  - simple
+  - slow
+  - small
+  - smart
+  - smooth
+  - soft
+  - solid
+  - strong
+  - sure
+  - swift
+  - tall
+  - thick
+  - thin
+  - tiny
+  - vast
+  - warm
+  - weak
+  - whole
+  - wide
+  - wild
+  - wise
+  - young
+  - exalted
+  - legendary
+  - resilient
+  - vibrant
+  - stellar
+  - graceful
+  - radiant
+  - serene
+  - brilliant
+  - majestic
+  - elegant
+
+nouns:
+  - abyss
+  - angel
+  - arrow
+  - atom
+  - ball
+  - band
+  - bark
+  - beam
+  - bear
+  - bell
+  - bird
+  - bloom
+  - blue
+  - boat
+  - bone
+  - book
+  - brook
+  - brush
+  - calm
+  - cave
+  - cell
+  - chant
+  - chord
+  - clay
+  - cliff
+  - cloud
+  - coal
+  - coast
+  - coin
+  - colt
+  - coral
+  - core
+  - creek
+  - crop
+  - crown
+  - cube
+  - dawn
+  - day
+  - dew
+  - disk
+  - dove
+  - dream
+  - drop
+  - dust
+  - eagle
+  - earth
+  - echo
+  - edge
+  - ember
+  - field
+  - fire
+  - fish
+  - flame
+  - flight
+  - flow
+  - foam
+  - fog
+  - forest
+  - frost
+  - glow
+  - gold
+  - grass
+  - grove
+  - haze
+  - heart
+  - hill
+  - ice
+  - iris
+  - jade
+  - lake
+  - land
+  - leaf
+  - light
+  - lion
+  - moon
+  - moss
+  - night
+  - oak
+  - ocean
+  - path
+  - peak
+  - pearl
+  - pine
+  - pond
+  - rain
+  - reef
+  - river
+  - rock
+  - rose
+  - sage
+  - sand
+  - sea
+  - shadow
+  - shore
+  - sky
+  - snow
+  - song
+  - star
+  - stone
+  - storm
+  - stream
+  - sun
+  - sunset
+  - surf
+  - tide
+  - tree
+  - vale
+  - wave
+  - wind
+  - wing
+  - wolf
+  - wood
+  - darkness
+  - meadow
+  - thunder
+  - crystal
+  - valley
+  - mountain

sdg_hub/core/utils/flow_identifier.py

@@ -0,0 +1,94 @@
+# Standard
+from pathlib import Path
+from typing import Dict, List
+import hashlib
+import random
+
+# Third Party
+import yaml
+
+# Cache for loaded word lists to avoid repeated file I/O
+_WORD_CACHE: Dict[str, List[str]] = {}
+
+
+def _load_word_lists() -> Dict[str, List[str]]:
+    """Load word lists from YAML configuration file.
+
+    Returns:
+        Dictionary containing 'adjectives' and 'nouns' lists
+
+    Raises:
+        FileNotFoundError: If the word list file is not found
+        yaml.YAMLError: If the YAML file is malformed
+    """
+    global _WORD_CACHE
+
+    if _WORD_CACHE:
+        return _WORD_CACHE
+
+    # Get path to word list file relative to this module
+    current_dir = Path(__file__).parent
+    words_file = current_dir / "flow_id_words.yaml"
+
+    try:
+        with open(words_file, "r", encoding="utf-8") as f:
+            word_data = yaml.safe_load(f)
+
+        _WORD_CACHE = {
+            "adjectives": word_data["adjectives"],
+            "nouns": word_data["nouns"],
+        }
+
+        return _WORD_CACHE
+
+    except FileNotFoundError:
+        # Fallback to minimal word lists if configuration file is not found
+        _WORD_CACHE = {
+            "adjectives": ["bright", "calm", "fast", "smart", "quick"],
+            "nouns": ["river", "star", "cloud", "moon", "rock"],
+        }
+        return _WORD_CACHE
+    except yaml.YAMLError as e:
+        raise yaml.YAMLError(f"Error parsing word list YAML: {e}")
+    except KeyError as e:
+        raise KeyError(f"Missing required key in word list YAML: {e}")
+
+
+def get_flow_identifier(name: str) -> str:
+    """Generate a deterministic wandb-style flow identifier.
+
+    Creates a human-readable identifier in the format "adjective-noun-number"
+    that is deterministic based on the input name. Same name will always
+    produce the same identifier.
+
+    Args:
+        name: Flow name to generate identifier from
+
+    Returns:
+        A string in the format "adjective-noun-number" (e.g., "bright-river-123")
+
+    Examples:
+        >>> get_flow_identifier("My Document QA Flow")
+        "bright-river-123"
+        >>> get_flow_identifier("My Document QA Flow")  # Same input
+        "bright-river-123"  # Same output
+
+    Raises:
+        FileNotFoundError: If the word list configuration file is not found
+        yaml.YAMLError: If the word list YAML file is malformed
+    """
+    # Load word lists from YAML configuration
+    word_lists = _load_word_lists()
+    adjectives = word_lists["adjectives"]
+    nouns = word_lists["nouns"]
+
+    # Create deterministic seed from name
+    seed_value = int(hashlib.sha256(name.encode()).hexdigest()[:8], 16)
+    rng = random.Random(seed_value)
+
+    # Select words and number deterministically
+    adjective = rng.choice(adjectives)
+    noun = rng.choice(nouns)
+    number = rng.randint(1, 999)
+
+    return f"{adjective}-{noun}-{number}"

sdg_hub/{utils → core/utils}/path_resolution.py

@@ -7,11 +7,11 @@ search paths.
 """
 
 # Standard
-from typing import List, Union
+from typing import Union
 import os
 
 
-def resolve_path(filename: str, search_dirs: Union[str, List[str]]) -> str:
+def resolve_path(filename: str, search_dirs: Union[str, list[str]]) -> str:
     """Resolve a file path relative to one or more search directories.
 
     Files are checked in the following order:

sdg_hub/core/utils/yaml_utils.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: Apache-2.0
+"""YAML utilities for flow configuration."""
+
+# Standard
+from pathlib import Path
+from typing import Any, Dict
+
+# Third Party
+import yaml
+
+# Local
+from .logger_config import setup_logger
+
+logger = setup_logger(__name__)
+
+
+def save_flow_yaml(
+    yaml_path: str,
+    flow_config: Dict[str, Any],
+    reason: str = "",
+    sort_keys: bool = False,
+    width: int = 240,
+    indent: int = 2,
+) -> None:
+    """
+    Save flow configuration to a YAML file.
+
+    This utility function saves flow configurations to YAML files,
+    ensuring consistent formatting and logging across the codebase.
+
+    Parameters
+    ----------
+    yaml_path : str
+        Path to the YAML file to write.
+    flow_config : Dict[str, Any]
+        Flow configuration to save.
+    reason : str, optional
+        Reason for saving, used in log message.
+    width : int, optional
+        Maximum line width for YAML output.
+    indent : int, optional
+        Indentation level for YAML output.
+    """
+    yaml_path = str(Path(yaml_path))  # Normalize path
+
+    with open(yaml_path, "w", encoding="utf-8") as f:
+        yaml.dump(
+            flow_config,
+            f,
+            default_flow_style=False,
+            sort_keys=sort_keys,
+            width=width,
+            indent=indent,
+        )
+
+    log_msg = f"Saved flow configuration to YAML: {yaml_path}"
+    if reason:
+        log_msg = f"{log_msg} ({reason})"
+    logger.debug(log_msg)

sdg_hub/flows/qa_generation/document_grounded_qa/multi_summary_qa/instructlab/atomic_facts.yaml

@@ -0,0 +1,40 @@
+- role: system
+  content: You are an AI assistant knowledgeable about {{domain}} domain. Be accurate but concise in response.
+
+- role: user
+  content: |
+    Please break down the following snippet from an article about {{domain}} into atomic facts.
+
+    1. Makesure each fact is grounded in the given text.
+    2. Include any necessary information needed to explain the fact or concept
+    3. The atomic facts should be as simple as possible, if it's compound sentence, break down one more time
+    4. For clarity, avoid using pronouns like 'it', 'he', 'she', 'this', 'that' etc., and instead use the full names or titles.
+    5. Focus only on key concepts and facts. Skip any question or problems mentioned in the passage.
+
+    To help you understand the task, here is an example:
+    [Passage]
+    The tournament was contested by ten national teams, maintaining the same format used in 2019. After six weeks of round-robin matches, India, South Africa, Australia, and New Zealand finished as the top four and qualified for the knockout stage. In the knockout stage, India and Australia beat New Zealand and South Africa, respectively, to advance to the final, played on 19 November at the Narendra Modi Stadium in Ahmedabad. Australia won the final by six wickets, winning their sixth Cricket World Cup title.
+    [Facts]
+    1. The tournament was contested by ten national teams.
+    2. The tournament maintained the same format used in 2019.
+    3. The round-robin matches lasted for six weeks.
+    4. India finished as one of the top four teams.
+    5. South Africa finished as one of the top four teams.
+    6. Australia finished as one of the top four teams.
+    7. New Zealand finished as one of the top four teams.
+    8. India, South Africa, Australia, and New Zealand qualified for the knockout stage.
+    9. In the knockout stage, India beat New Zealand.
+    10. In the knockout stage, Australia beat South Africa.
+    11. India advanced to the final.
+    12. Australia advanced to the final.
+    13. The final was played on 19 November.
+    14. The final was held at the Narendra Modi Stadium in Ahmedabad.
+    15. Australia won the final by six wickets.
+    16. Australia won their sixth Cricket World Cup title.
+    [End]
+
+    Now it's your turn breakdown following snippet from article about {{domain}} into atomic facts following similar style as above examples
+    [Passage]
+    {{document_outline}}
+    {{document}}
+    [Facts]