sdg-hub 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

sdg_hub/core/flow/registry.py

@@ -0,0 +1,330 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Flow registry for managing contributed flows."""
+
+# Standard
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+import os
+
+# Third Party
+import yaml
+
+# Local
+from ..utils.logger_config import setup_logger
+from .metadata import FlowMetadata
+
+logger = setup_logger(__name__)
+
+
+@dataclass
+class FlowRegistryEntry:
+    """Entry in the flow registry.
+
+    Parameters
+    ----------
+    path : str
+        Path to the flow YAML file.
+    metadata : FlowMetadata
+        Flow metadata extracted from the file.
+    """
+
+    path: str
+    metadata: FlowMetadata
+
+
+class FlowRegistry:
+    """Registry for managing contributed flows."""
+
+    _entries: dict[str, FlowRegistryEntry] = {}
+    _search_paths: list[str] = []
+    _initialized: bool = False
+
+    @classmethod
+    def _ensure_initialized(cls) -> None:
+        """Ensure the registry is initialized with built-in flows."""
+        if cls._initialized:
+            return
+
+        try:
+            # Find the sdg_hub package directory
+            # First Party
+            import sdg_hub
+
+            package_path = Path(sdg_hub.__file__).parent
+            flows_dir = package_path / "flows"
+
+            # Register built-in flows directory if it exists
+            if flows_dir.exists():
+                flows_dir_str = str(flows_dir)
+                if flows_dir_str not in cls._search_paths:
+                    cls._search_paths.append(flows_dir_str)
+                    logger.debug(
+                        f"Auto-registered built-in flows directory: {flows_dir}"
+                    )
+            else:
+                logger.debug(f"Built-in flows directory not found: {flows_dir}")
+
+        except Exception as exc:
+            logger.warning(f"Failed to auto-register built-in flows: {exc}")
+
+        cls._initialized = True
+
+    @classmethod
+    def register_search_path(cls, path: str) -> None:
+        """Add a directory to search for flows.
+
+        Parameters
+        ----------
+        path : str
+            Path to directory containing flow YAML files.
+        """
+        if path not in cls._search_paths:
+            cls._search_paths.append(path)
+            logger.debug(f"Added flow search path: {path}")
+
+    @classmethod
+    def _discover_flows(cls, force_refresh: bool = False) -> None:
+        """Discover and register flows from search paths (private method).
+
+        Parameters
+        ----------
+        force_refresh : bool, optional
+            Whether to force refresh the registry.
+        """
+        # Ensure built-in flows are registered
+        cls._ensure_initialized()
+
+        if cls._entries and not force_refresh:
+            return
+
+        cls._entries.clear()
+
+        for search_path in cls._search_paths:
+            if not os.path.exists(search_path):
+                logger.warning(f"Flow search path does not exist: {search_path}")
+                continue
+
+            cls._discover_flows_in_directory(search_path)
+
+        logger.info(f"Discovered {len(cls._entries)} flows")
+
+    @classmethod
+    def _discover_flows_in_directory(cls, directory: str) -> None:
+        """Discover flows in a specific directory."""
+        path = Path(directory)
+
+        for yaml_file in path.rglob("*.yaml"):
+            try:
+                with open(yaml_file, encoding="utf-8") as f:
+                    flow_config = yaml.safe_load(f)
+
+                # Check if this is a flow file
+                if "metadata" in flow_config and "blocks" in flow_config:
+                    metadata_dict = flow_config["metadata"]
+                    metadata = FlowMetadata(**metadata_dict)
+
+                    entry = FlowRegistryEntry(path=str(yaml_file), metadata=metadata)
+
+                    cls._entries[metadata.name] = entry
+                    logger.debug(f"Registered flow: {metadata.name} from {yaml_file}")
+
+            except Exception as exc:
+                logger.debug(f"Skipped {yaml_file}: {exc}")
+
+    @classmethod
+    def get_flow_path(cls, flow_name: str) -> Optional[str]:
+        """Get the path to a registered flow.
+
+        Parameters
+        ----------
+        flow_name : str
+            Name of the flow to find.
+
+        Returns
+        -------
+        Optional[str]
+            Path to the flow file, or None if not found.
+        """
+        cls._ensure_initialized()
+        cls._discover_flows()
+
+        if flow_name in cls._entries:
+            return cls._entries[flow_name].path
+        return None
+
+    @classmethod
+    def get_flow_metadata(cls, flow_name: str) -> Optional[FlowMetadata]:
+        """Get metadata for a registered flow.
+
+        Parameters
+        ----------
+        flow_name : str
+            Name of the flow.
+
+        Returns
+        -------
+        Optional[FlowMetadata]
+            Flow metadata, or None if not found.
+        """
+        cls._ensure_initialized()
+        cls._discover_flows()
+
+        if flow_name in cls._entries:
+            return cls._entries[flow_name].metadata
+        return None
+
+    @classmethod
+    def list_flows(cls) -> list[str]:
+        """List all registered flow names.
+
+        Returns
+        -------
+        List[str]
+            List of flow names.
+        """
+        cls._ensure_initialized()
+        cls._discover_flows()
+        return list(cls._entries.keys())
+
+    @classmethod
+    def search_flows(
+        cls, tag: Optional[str] = None, author: Optional[str] = None
+    ) -> list[str]:
+        """Search flows by criteria.
+
+        Parameters
+        ----------
+        tag : Optional[str]
+            Tag to filter by.
+        author : Optional[str]
+            Author to filter by.
+
+        Returns
+        -------
+        List[str]
+            List of matching flow names.
+        """
+        cls._ensure_initialized()
+        cls._discover_flows()
+
+        matching_flows = []
+
+        for name, entry in cls._entries.items():
+            metadata = entry.metadata
+
+            # Filter by tag
+            if tag and tag not in metadata.tags:
+                continue
+
+            # Filter by author
+            if author and author.lower() not in metadata.author.lower():
+                continue
+
+            matching_flows.append(name)
+
+        return matching_flows
+
+    @classmethod
+    def get_flows_by_category(cls) -> dict[str, list[str]]:
+        """Get flows organized by their primary tag.
+
+        Returns
+        -------
+        Dict[str, List[str]]
+            Dictionary mapping tags to flow names.
+        """
+        cls._ensure_initialized()
+        cls._discover_flows()
+
+        categories = {}
+
+        for name, entry in cls._entries.items():
+            metadata = entry.metadata
+
+            # Use first tag as primary category, or "uncategorized"
+            category = metadata.tags[0] if metadata.tags else "uncategorized"
+
+            if category not in categories:
+                categories[category] = []
+
+            categories[category].append(name)
+
+        return categories
+
+    @classmethod
+    def discover_flows(cls, show_all_columns: bool = False) -> None:
+        """Discover and display all flows in a formatted table.
+
+        This is the main public API for flow discovery. It finds all flows
+        in registered search paths and displays them in a beautiful Rich table.
+
+        Parameters
+        ----------
+        show_all_columns : bool, optional
+            Whether to show extended table with all columns, by default False
+        """
+        cls._ensure_initialized()
+        cls._discover_flows()
+
+        if not cls._entries:
+            print(
+                "No flows discovered. Try adding search paths with register_search_path()"
+            )
+            print("Note: Only flows with 'metadata' section are discoverable.")
+            return
+
+        # Prepare data with fallbacks
+        flow_data = []
+        for name, entry in cls._entries.items():
+            metadata = entry.metadata
+            flow_data.append(
+                {
+                    "name": name,
+                    "author": metadata.author or "Unknown",
+                    "tags": ", ".join(metadata.tags) if metadata.tags else "-",
+                    "description": metadata.description or "No description",
+                    "version": metadata.version,
+                    "cost": metadata.estimated_cost,
+                }
+            )
+
+        # Sort by name for consistency
+        flow_data.sort(key=lambda x: x["name"])
+
+        # Display Rich table
+        # Third Party
+        from rich.console import Console
+        from rich.table import Table
+
+        console = Console()
+        table = Table(show_header=True, header_style="bold magenta")
+
+        # Add columns
+        table.add_column("Name", style="cyan", no_wrap=True)
+        table.add_column("Author", style="green")
+
+        if show_all_columns:
+            table.add_column("Version", style="blue")
+            table.add_column("Cost", style="yellow")
+
+        table.add_column("Tags", style="dim")
+        table.add_column("Description")
+
+        # Add rows
+        for flow in flow_data:
+            if show_all_columns:
+                table.add_row(
+                    flow["name"],
+                    flow["author"],
+                    flow["version"],
+                    flow["cost"],
+                    flow["tags"],
+                    flow["description"],
+                )
+            else:
+                table.add_row(
+                    flow["name"], flow["author"], flow["tags"], flow["description"]
+                )
+
+        console.print(table)

sdg_hub/core/flow/validation.py

@@ -0,0 +1,265 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Flow validation utilities."""
+
+# Standard
+from typing import TYPE_CHECKING, Any
+
+# Third Party
+from datasets import Dataset
+
+if TYPE_CHECKING:
+    # Local
+    from .base import Flow
+
+
+class FlowValidator:
+    """Validator for flow configurations and execution readiness."""
+
+    def validate_yaml_structure(self, flow_config: dict[str, Any]) -> list[str]:
+        """Validate the structure of a flow YAML configuration.
+
+        Parameters
+        ----------
+        flow_config : Dict[str, Any]
+            The loaded YAML configuration.
+
+        Returns
+        -------
+        List[str]
+            List of validation error messages. Empty if valid.
+        """
+        errors = []
+
+        # Check required top-level keys
+        if "blocks" not in flow_config:
+            errors.append("Flow configuration must contain 'blocks' section")
+            return errors  # Can't continue without blocks
+
+        blocks = flow_config["blocks"]
+        if not isinstance(blocks, list):
+            errors.append("'blocks' must be a list")
+            return errors
+
+        if not blocks:
+            errors.append("Flow must contain at least one block")
+            return errors
+
+        # Validate each block configuration
+        for i, block_config in enumerate(blocks):
+            block_errors = self._validate_block_config(block_config, i)
+            errors.extend(block_errors)
+
+        # Validate metadata if present
+        if "metadata" in flow_config:
+            metadata_errors = self._validate_metadata_config(flow_config["metadata"])
+            errors.extend(metadata_errors)
+
+        # Validate parameters if present
+        if "parameters" in flow_config:
+            param_errors = self._validate_parameters_config(flow_config["parameters"])
+            errors.extend(param_errors)
+
+        return errors
+
+    def _validate_block_config(
+        self, block_config: dict[str, Any], index: int
+    ) -> list[str]:
+        """Validate a single block configuration."""
+        errors = []
+        prefix = f"Block {index}"
+
+        if not isinstance(block_config, dict):
+            errors.append(f"{prefix}: Block configuration must be a dictionary")
+            return errors
+
+        # Check required fields
+        if "block_type" not in block_config:
+            errors.append(f"{prefix}: Missing required field 'block_type'")
+
+        if "block_config" not in block_config:
+            errors.append(f"{prefix}: Missing required field 'block_config'")
+        else:
+            # Validate block_config structure
+            inner_config = block_config["block_config"]
+            if not isinstance(inner_config, dict):
+                errors.append(f"{prefix}: 'block_config' must be a dictionary")
+            elif "block_name" not in inner_config:
+                errors.append(f"{prefix}: 'block_config' must contain 'block_name'")
+
+        # Validate optional fields
+        if "runtime_overrides" in block_config:
+            overrides = block_config["runtime_overrides"]
+            if not isinstance(overrides, list):
+                errors.append(f"{prefix}: 'runtime_overrides' must be a list")
+            elif not all(isinstance(item, str) for item in overrides):
+                errors.append(
+                    f"{prefix}: All 'runtime_overrides' items must be strings"
+                )
+
+        return errors
+
+    def _validate_metadata_config(self, metadata: dict[str, Any]) -> list[str]:
+        """Validate metadata configuration."""
+        errors = []
+
+        if not isinstance(metadata, dict):
+            errors.append("'metadata' must be a dictionary")
+            return errors
+
+        # Check required name field
+        if "name" not in metadata:
+            errors.append("Metadata must contain 'name' field")
+        elif not isinstance(metadata["name"], str) or not metadata["name"].strip():
+            errors.append("Metadata 'name' must be a non-empty string")
+
+        # Validate optional fields
+        string_fields = [
+            "description",
+            "version",
+            "author",
+            "recommended_model",
+            "license",
+        ]
+        for field in string_fields:
+            if field in metadata and not isinstance(metadata[field], str):
+                errors.append(f"Metadata '{field}' must be a string")
+
+        if "tags" in metadata:
+            tags = metadata["tags"]
+            if not isinstance(tags, list):
+                errors.append("Metadata 'tags' must be a list")
+            elif not all(isinstance(tag, str) for tag in tags):
+                errors.append("All metadata 'tags' must be strings")
+
+        return errors
+
+    def _validate_parameters_config(self, parameters: dict[str, Any]) -> list[str]:
+        """Validate parameters configuration."""
+        errors = []
+
+        if not isinstance(parameters, dict):
+            errors.append("'parameters' must be a dictionary")
+            return errors
+
+        for param_name, param_config in parameters.items():
+            if not isinstance(param_name, str):
+                errors.append("Parameter names must be strings")
+                continue
+
+            if isinstance(param_config, dict):
+                # Full parameter specification
+                if "default" not in param_config:
+                    errors.append(f"Parameter '{param_name}' must have 'default' value")
+
+                # Validate optional fields
+                if "description" in param_config and not isinstance(
+                    param_config["description"], str
+                ):
+                    errors.append(
+                        f"Parameter '{param_name}' description must be a string"
+                    )
+
+                if "required" in param_config and not isinstance(
+                    param_config["required"], bool
+                ):
+                    errors.append(
+                        f"Parameter '{param_name}' required field must be boolean"
+                    )
+
+        return errors
+
+    def validate_flow_execution(self, flow: "Flow", dataset: Dataset) -> list[str]:
+        """Validate that a flow can be executed with the given dataset.
+
+        Parameters
+        ----------
+        flow : Flow
+            The flow to validate.
+        dataset : Dataset
+            Dataset to validate against.
+
+        Returns
+        -------
+        List[str]
+            List of validation error messages. Empty if validation passes.
+        """
+        errors = []
+
+        if not flow.blocks:
+            errors.append("Flow contains no blocks")
+            return errors
+
+        if len(dataset) == 0:
+            errors.append("Dataset is empty")
+            return errors
+
+        # Track available columns as we progress through blocks
+        current_columns = set(dataset.column_names)
+
+        for _i, block in enumerate(flow.blocks):
+            block_name = block.block_name
+
+            # Check input columns
+            if hasattr(block, "input_cols") and block.input_cols:
+                missing_cols = self._check_missing_columns(
+                    block.input_cols, current_columns
+                )
+                if missing_cols:
+                    errors.append(
+                        f"Block '{block_name}' missing input columns: {missing_cols}"
+                    )
+
+            # Update available columns for next block
+            if hasattr(block, "output_cols") and block.output_cols:
+                new_columns = self._extract_column_names(block.output_cols)
+                current_columns.update(new_columns)
+
+        return errors
+
+    def _check_missing_columns(
+        self, required_cols: Any, available_cols: set[str]
+    ) -> list[str]:
+        """Check which required columns are missing."""
+        if isinstance(required_cols, (list, dict)):
+            return [col for col in required_cols if col not in available_cols]
+        return []
+
+    def _extract_column_names(self, output_cols: Any) -> list[str]:
+        """Extract column names from output specification."""
+        if isinstance(output_cols, list):
+            return output_cols
+        elif isinstance(output_cols, dict):
+            return list(output_cols.keys())
+        return []
+
+    def validate_block_chain(self, blocks: list[Any]) -> list[str]:
+        """Validate that blocks can be chained together.
+
+        Parameters
+        ----------
+        blocks : List[Any]
+            List of block instances to validate.
+
+        Returns
+        -------
+        List[str]
+            List of validation error messages.
+        """
+        errors = []
+
+        if not blocks:
+            errors.append("Block chain is empty")
+            return errors
+
+        # Check that all blocks have unique names
+        block_names = []
+        for i, block in enumerate(blocks):
+            if hasattr(block, "block_name"):
+                name = block.block_name
+                if name in block_names:
+                    errors.append(f"Duplicate block name '{name}' at index {i}")
+                block_names.append(name)
+            else:
+                errors.append(f"Block at index {i} missing 'block_name' attribute")
+
+        return errors

sdg_hub/{utils → core/utils}/__init__.py

@@ -1,10 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
+# Local
+from .path_resolution import resolve_path
+
+
 # This is part of the public API, and used by instructlab
-class GenerateException(Exception):
+class GenerateError(Exception):
     """An exception raised during generate step."""
 
 
-from .path_resolution import resolve_path
-
-__all__ = ["GenerateException", "resolve_path"]
+__all__ = ["GenerateError", "resolve_path"]

sdg_hub/{utils → core/utils}/datautils.py

@@ -3,9 +3,7 @@ from datasets import concatenate_datasets
 
 
 def safe_concatenate_datasets(datasets: list):
-    """
-    Concatenate datasets safely, ignoring any datasets that are None or empty.
-    """
+    """Concatenate datasets safely, ignoring any datasets that are None or empty."""
     filtered_datasets = [ds for ds in datasets if ds is not None and ds.num_rows > 0]
 
     if not filtered_datasets: