sdg-hub 0.1.4__py3-none-any.whl → 0.2.1__py3-none-any.whl

sdg_hub/core/flow/metadata.py

@@ -0,0 +1,389 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Flow metadata and parameter definitions."""
+
+# Standard
+from datetime import datetime
+from enum import Enum
+from typing import Any, Optional
+
+# Third Party
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+# Local
+from ..utils.flow_identifier import get_flow_identifier
+
+
+class ModelCompatibility(str, Enum):
+    """Model compatibility levels."""
+
+    REQUIRED = "required"
+    RECOMMENDED = "recommended"
+    COMPATIBLE = "compatible"
+    EXPERIMENTAL = "experimental"
+
+
+class ModelOption(BaseModel):
+    """Represents a model option with compatibility level.
+
+    Attributes
+    ----------
+    name : str
+        Model identifier (e.g., "gpt-4", "claude-3-sonnet")
+    compatibility : ModelCompatibility
+        Compatibility level with the flow
+    """
+
+    name: str = Field(..., description="Model identifier")
+    compatibility: ModelCompatibility = Field(
+        default=ModelCompatibility.COMPATIBLE,
+        description="Compatibility level with the flow",
+    )
+
+    @field_validator("name")
+    @classmethod
+    def validate_name(cls, v: str) -> str:
+        """Validate model name is not empty."""
+        if not v.strip():
+            raise ValueError("Model name cannot be empty")
+        return v.strip()
+
+
+class RecommendedModels(BaseModel):
+    """Simplified recommended models structure.
+
+    Attributes
+    ----------
+    default : str
+        The default model to use
+    compatible : List[str]
+        List of compatible models
+    experimental : List[str]
+        List of experimental models
+    """
+
+    default: str = Field(..., description="Default model to use")
+    compatible: list[str] = Field(default_factory=list, description="Compatible models")
+    experimental: list[str] = Field(
+        default_factory=list, description="Experimental models"
+    )
+
+    @field_validator("default")
+    @classmethod
+    def validate_default(cls, v: str) -> str:
+        """Validate default model name is not empty."""
+        if not v.strip():
+            raise ValueError("Default model name cannot be empty")
+        return v.strip()
+
+    @field_validator("compatible", "experimental")
+    @classmethod
+    def validate_model_lists(cls, v: list[str]) -> list[str]:
+        """Validate model lists contain non-empty names."""
+        return [model.strip() for model in v if model.strip()]
+
+    def get_all_models(self) -> list[str]:
+        """Get all models (default + compatible + experimental)."""
+        return [self.default] + self.compatible + self.experimental
+
+    def get_best_model(
+        self, available_models: Optional[list[str]] = None
+    ) -> Optional[str]:
+        """Get the best model based on availability.
+
+        Parameters
+        ----------
+        available_models : Optional[List[str]]
+            List of available model names. If None, returns default.
+
+        Returns
+        -------
+        Optional[str]
+            Best model name or None if no models available.
+        """
+        if available_models is None:
+            return self.default
+
+        # Check in priority order: default, compatible, experimental
+        if self.default in available_models:
+            return self.default
+
+        for model in self.compatible:
+            if model in available_models:
+                return model
+
+        for model in self.experimental:
+            if model in available_models:
+                return model
+
+        return None
+
+
+class FlowParameter(BaseModel):
+    """Represents a runtime parameter for a flow.
+
+    Attributes
+    ----------
+    default : Any
+        Default value for the parameter.
+    description : str
+        Human-readable description of the parameter.
+    type_hint : str
+        Type hint as string (e.g., "float", "str").
+    required : bool
+        Whether this parameter is required at runtime.
+    constraints : Dict[str, Any]
+        Additional constraints for the parameter.
+    """
+
+    default: Any = Field(..., description="Default value for the parameter")
+    description: str = Field(default="", description="Human-readable description")
+    type_hint: str = Field(default="Any", description="Type hint as string")
+    required: bool = Field(default=False, description="Whether parameter is required")
+    constraints: dict[str, Any] = Field(
+        default_factory=dict, description="Additional constraints for the parameter"
+    )
+
+    @model_validator(mode="after")
+    def validate_required_default(self) -> "FlowParameter":
+        """Validate that required parameters have appropriate defaults."""
+        if self.required and self.default is None:
+            raise ValueError("Required parameters cannot have None as default")
+        return self
+
+
+class DatasetRequirements(BaseModel):
+    """Dataset requirements for flow execution.
+
+    Attributes
+    ----------
+    required_columns : List[str]
+        Column names that must be present in the input dataset.
+    optional_columns : List[str]
+        Column names that are optional but can enhance flow performance.
+    min_samples : int
+        Minimum number of samples required in the dataset.
+    max_samples : Optional[int]
+        Maximum number of samples to process (for resource management).
+    column_types : Dict[str, str]
+        Expected types for specific columns.
+    description : str
+        Human-readable description of dataset requirements.
+    """
+
+    required_columns: list[str] = Field(
+        default_factory=list, description="Column names that must be present"
+    )
+    optional_columns: list[str] = Field(
+        default_factory=list,
+        description="Optional columns that can enhance performance",
+    )
+    min_samples: int = Field(
+        default=1, ge=1, description="Minimum number of samples required"
+    )
+    max_samples: Optional[int] = Field(
+        default=None, gt=0, description="Maximum number of samples to process"
+    )
+    column_types: dict[str, str] = Field(
+        default_factory=dict, description="Expected types for specific columns"
+    )
+    description: str = Field(default="", description="Human-readable description")
+
+    @field_validator("required_columns", "optional_columns")
+    @classmethod
+    def validate_column_names(cls, v: list[str]) -> list[str]:
+        """Validate column names are not empty."""
+        return [col.strip() for col in v if col.strip()]
+
+    @model_validator(mode="after")
+    def validate_sample_limits(self) -> "DatasetRequirements":
+        """Validate sample limits are consistent."""
+        if self.max_samples is not None and self.max_samples < self.min_samples:
+            raise ValueError("max_samples must be greater than or equal to min_samples")
+        return self
+
+    def validate_dataset(
+        self, dataset_columns: list[str], dataset_size: int
+    ) -> list[str]:
+        """Validate a dataset against these requirements.
+
+        Parameters
+        ----------
+        dataset_columns : List[str]
+            Column names in the dataset.
+        dataset_size : int
+            Number of samples in the dataset.
+
+        Returns
+        -------
+        List[str]
+            List of validation error messages. Empty if valid.
+        """
+        errors = []
+
+        # Check required columns
+        missing_columns = [
+            col for col in self.required_columns if col not in dataset_columns
+        ]
+        if missing_columns:
+            errors.append(f"Missing required columns: {missing_columns}")
+
+        # Check minimum samples
+        if dataset_size < self.min_samples:
+            errors.append(
+                f"Dataset has {dataset_size} samples, minimum required: {self.min_samples}"
+            )
+
+        return errors
+
+
+class FlowMetadata(BaseModel):
+    """Metadata for flow configuration and open source contributions.
+
+    Attributes
+    ----------
+    id : str
+        Unique identifier for the flow.
+    name : str
+        Human-readable name of the flow.
+    description : str
+        Detailed description of what the flow does.
+    version : str
+        Semantic version (e.g., "1.0.0").
+    author : str
+        Author or contributor name.
+    recommended_models : Optional[RecommendedModels]
+        Simplified recommended models structure with default, compatible, and experimental lists.
+    tags : List[str]
+        Tags for categorization and search.
+    created_at : str
+        Creation timestamp.
+    updated_at : str
+        Last update timestamp.
+    license : str
+        License identifier.
+    min_sdg_hub_version : str
+        Minimum required SDG Hub version.
+    dataset_requirements : Optional[DatasetRequirements]
+        Requirements for input datasets.
+    estimated_cost : str
+        Estimated cost tier for running the flow.
+    estimated_duration : str
+        Estimated duration for flow execution.
+    """
+
+    name: str = Field(..., min_length=1, description="Human-readable name")
+    id: str = Field(
+        default="", description="Unique identifier for the flow, generated from name"
+    )
+    description: str = Field(default="", description="Detailed description")
+    version: str = Field(
+        default="1.0.0",
+        pattern=r"^\d+\.\d+\.\d+(-[a-zA-Z0-9.-]+)?$",
+        description="Semantic version",
+    )
+    author: str = Field(default="", description="Author or contributor name")
+    recommended_models: Optional[RecommendedModels] = Field(
+        default=None, description="Simplified recommended models structure"
+    )
+    tags: list[str] = Field(
+        default_factory=list, description="Tags for categorization and search"
+    )
+    created_at: str = Field(
+        default_factory=lambda: datetime.now().isoformat(),
+        description="Creation timestamp",
+    )
+    updated_at: str = Field(
+        default_factory=lambda: datetime.now().isoformat(),
+        description="Last update timestamp",
+    )
+    license: str = Field(default="Apache-2.0", description="License identifier")
+    min_sdg_hub_version: str = Field(
+        default="", description="Minimum required SDG Hub version"
+    )
+    dataset_requirements: Optional[DatasetRequirements] = Field(
+        default=None, description="Requirements for input datasets"
+    )
+    estimated_cost: str = Field(
+        default="medium",
+        pattern="^(low|medium|high)$",
+        description="Estimated cost tier for running the flow",
+    )
+    estimated_duration: str = Field(
+        default="", description="Estimated duration for flow execution"
+    )
+
+    @field_validator("id")
+    @classmethod
+    def validate_id(cls, v: str) -> str:
+        """Validate flow id."""
+        # Note: Auto-generation is handled in the model_validator since field_validator
+        # doesn't have access to other field values in Pydantic v2
+
+        # Validate id format if provided
+        if v:
+            # Must be lowercase
+            if not v.islower():
+                raise ValueError("id must be lowercase")
+
+            # Must contain only alphanumeric characters and hyphens
+            if not v.replace("-", "").isalnum():
+                raise ValueError(
+                    "id must contain only alphanumeric characters and hyphens"
+                )
+
+            # Must not start or end with a hyphen
+            if v.startswith("-") or v.endswith("-"):
+                raise ValueError("id must not start or end with a hyphen")
+
+        return v
+
+    @field_validator("tags")
+    @classmethod
+    def validate_tags(cls, v: list[str]) -> list[str]:
+        """Validate and clean tags."""
+        return [tag.strip().lower() for tag in v if tag.strip()]
+
+    @field_validator("recommended_models")
+    @classmethod
+    def validate_recommended_models(
+        cls, v: Optional[RecommendedModels]
+    ) -> Optional[RecommendedModels]:
+        """Validate recommended models structure."""
+        # Validation is handled within RecommendedModels class
+        return v
+
+    def update_timestamp(self) -> None:
+        """Update the updated_at timestamp."""
+        self.updated_at = datetime.now().isoformat()
+
+    @model_validator(mode="after")
+    def ensure_id(self) -> "FlowMetadata":
+        """Ensure id is set.
+
+        Note: YAML persistence is handled by Flow.from_yaml() and FlowRegistry
+        to maintain proper separation of concerns.
+        """
+        if not self.id and self.name:
+            self.id = get_flow_identifier(self.name)
+
+        return self
+
+    def get_best_model(
+        self, available_models: Optional[list[str]] = None
+    ) -> Optional[str]:
+        """Get the best recommended model based on availability.
+
+        Parameters
+        ----------
+        available_models : Optional[List[str]]
+            List of available model names. If None, returns default model.
+
+        Returns
+        -------
+        Optional[str]
+            Best model name or None if no models available.
+        """
+        if not self.recommended_models:
+            return None
+
+        return self.recommended_models.get_best_model(available_models)

sdg_hub/core/flow/migration.py

@@ -0,0 +1,198 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Migration utilities for backward compatibility with old flow formats."""
+
+# Standard
+from pathlib import Path
+from typing import Any, Union
+
+# Local
+from ..utils.logger_config import setup_logger
+
+logger = setup_logger(__name__)
+
+
+class FlowMigration:
+    """Utility class for migrating old flow formats to new format."""
+
+    @staticmethod
+    def is_old_format(flow_config: Union[list[dict[str, Any]], dict[str, Any]]) -> bool:
+        """Detect if a flow configuration is in the old format.
+
+        Parameters
+        ----------
+        flow_config : Union[List[Dict[str, Any]], Dict[str, Any]]
+            The loaded YAML configuration.
+
+        Returns
+        -------
+        bool
+            True if the configuration is in old format, False otherwise.
+        """
+        # Old format: Direct array of blocks
+        # New format: Dictionary with 'metadata' and 'blocks' keys
+        if isinstance(flow_config, list):
+            return True
+
+        if isinstance(flow_config, dict):
+            # Check if it has the new format structure
+            has_metadata = "metadata" in flow_config
+            has_blocks = "blocks" in flow_config
+
+            # If it has both metadata and blocks, it's new format
+            if has_metadata and has_blocks:
+                return False
+
+            # If it doesn't have the expected new format structure but is a dict,
+            # check if it looks like old format (all keys are block configs)
+            if not has_metadata and not has_blocks:
+                # Check first few items to see if they look like old block configs
+                for value in flow_config.values():
+                    if isinstance(value, dict) and "block_type" in value:
+                        return True
+                # If it's a dict but doesn't look like blocks, assume new format
+                return False
+
+        # If we can't determine, assume new format
+        return False
+
+    @staticmethod
+    def migrate_to_new_format(
+        flow_config: list[dict[str, Any]], yaml_path: str
+    ) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]:
+        """Migrate old format flow configuration to new format.
+
+        Parameters
+        ----------
+        flow_config : List[Dict[str, Any]]
+            Old format flow configuration (array of blocks).
+        yaml_path : str
+            Path to the original YAML file for generating metadata.
+
+        Returns
+        -------
+        tuple[Dict[str, Any], Dict[str, Dict[str, Any]]]
+            Tuple of (new format flow configuration, extracted runtime_params).
+        """
+        logger.info(f"Migrating old flow format from: {yaml_path}")
+
+        # Generate default metadata
+        flow_name = Path(yaml_path).stem
+        metadata = FlowMigration._generate_default_metadata(flow_name)
+
+        # Process blocks and extract runtime parameters
+        migrated_blocks = []
+        runtime_params = {}
+
+        for i, block_config in enumerate(flow_config):
+            try:
+                migrated_block, block_runtime_params = (
+                    FlowMigration._migrate_block_config(block_config)
+                )
+                migrated_blocks.append(migrated_block)
+
+                # Add block's runtime params if any
+                if block_runtime_params:
+                    block_name = migrated_block.get("block_config", {}).get(
+                        "block_name"
+                    )
+                    if block_name:
+                        runtime_params[block_name] = block_runtime_params
+
+            except Exception as exc:
+                logger.warning(f"Failed to migrate block at index {i}: {exc}")
+                # Keep original block config as fallback
+                migrated_blocks.append(block_config)
+
+        # Create new format structure
+        new_config = {"metadata": metadata, "blocks": migrated_blocks}
+
+        logger.info(f"Successfully migrated flow with {len(migrated_blocks)} blocks")
+        logger.info(f"Extracted runtime_params for {len(runtime_params)} blocks")
+
+        return new_config, runtime_params
+
+    @staticmethod
+    def _generate_default_metadata(flow_name: str) -> dict[str, Any]:
+        """Generate default metadata for migrated flows."""
+        # Import here to avoid circular import
+        from ..utils.flow_identifier import get_flow_identifier
+
+        metadata = {
+            "name": flow_name,
+            "description": f"Migrated flow: {flow_name}",
+            "version": "1.0.0",
+            "author": "SDG_Hub",
+            "tags": ["migrated"],
+            "recommended_models": {
+                "default": "meta-llama/Llama-3.3-70B-Instruct",
+                "compatible": [],
+                "experimental": [],
+            },
+        }
+
+        # Generate id for migrated flows
+        flow_id = get_flow_identifier(flow_name)
+        if flow_id:
+            metadata["id"] = flow_id
+            logger.debug(f"Generated id for migrated flow: {flow_id}")
+
+        return metadata
+
+    @staticmethod
+    def _migrate_block_config(
+        block_config: dict[str, Any],
+    ) -> tuple[dict[str, Any], dict[str, Any]]:
+        """Migrate individual block configuration from old to new format.
+
+        Parameters
+        ----------
+        block_config : Dict[str, Any]
+            Old format block configuration.
+
+        Returns
+        -------
+        tuple[Dict[str, Any], Dict[str, Any]]
+            Tuple of (migrated block configuration, extracted runtime_params).
+        """
+        if not isinstance(block_config, dict):
+            return block_config, {}
+
+        # Start with the original config
+        migrated_config = block_config.copy()
+        runtime_params = {}
+
+        # Extract gen_kwargs as runtime_params
+        if "gen_kwargs" in migrated_config:
+            runtime_params = migrated_config.pop("gen_kwargs")
+            logger.debug(f"Extracted gen_kwargs as runtime_params: {runtime_params}")
+
+        # Remove unsupported fields
+        for unsupported_field in ["drop_columns", "drop_duplicates", "batch_kwargs"]:
+            if unsupported_field in migrated_config:
+                migrated_config.pop(unsupported_field)
+                logger.debug(
+                    f"Ignoring {unsupported_field} as it's not supported in new flow format"
+                )
+
+        # Handle parser_kwargs for LLMBlock (keep in block_config)
+        if migrated_config.get("block_type") == "LLMBlock":
+            block_config_section = migrated_config.get("block_config", {})
+            if "parser_kwargs" in block_config_section:
+                parser_kwargs = block_config_section["parser_kwargs"]
+                logger.debug(f"Preserving parser_kwargs for LLMBlock: {parser_kwargs}")
+
+        # Handle operator string conversion for FilterByValueBlock
+        if migrated_config.get("block_type") == "FilterByValueBlock":
+            block_config_section = migrated_config.get("block_config", {})
+            if "operation" in block_config_section:
+                operation = block_config_section["operation"]
+                if isinstance(operation, str) and operation.startswith("operator."):
+                    # Convert "operator.eq" to "eq"
+                    block_config_section["operation"] = operation.replace(
+                        "operator.", ""
+                    )
+                    logger.debug(
+                        f"Converted operation from {operation} to {block_config_section['operation']}"
+                    )
+
+        return migrated_config, runtime_params