expops 0.1.3__py3-none-any.whl

mlops/adapters/custom/custom_adapter.py

@@ -0,0 +1,447 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from mlops.core import StepStateManager, get_context_factory, get_step_registry, set_current_context
+from mlops.core.custom_model_base import MLOpsCustomModelBase
+from mlops.core.dask_networkx_executor import DaskNetworkXExecutor
+from mlops.core.networkx_parser import parse_networkx_pipeline_from_config
+
+from ..base import ModelAdapter
+from ..config_schema import AdapterConfig
+
+
+def _as_dict(value: Any) -> Dict[str, Any]:
+    if isinstance(value, dict):
+        return value
+    try:
+        if hasattr(value, "model_dump"):
+            return value.model_dump()  # type: ignore[attr-defined]
+    except Exception:
+        pass
+    try:
+        return dict(value)  # type: ignore[arg-type]
+    except Exception:
+        return {}
+
+
+def _to_int(value: Any) -> Optional[int]:
+    try:
+        if value is None:
+            return None
+        return int(value)
+    except Exception:
+        return None
+
+
+def _to_float(value: Any) -> Optional[float]:
+    try:
+        if value is None:
+            return None
+        return float(value)
+    except Exception:
+        return None
+
+
+class CustomModelAdapter(ModelAdapter):
+    """Adapter for user-provided Python model scripts.
+
+    Responsibilities:
+    - Load the custom script and register `@step` functions
+    - Build a StepStateManager (KV + optional object store)
+    - Execute the configured NetworkX pipeline via DaskNetworkXExecutor
+    """
+
+    def __init__(
+        self,
+        config: AdapterConfig,
+        python_interpreter: Optional[str] = None,
+        environment_name: Optional[str] = None,
+        conda_env_name: Optional[str] = None,
+        project_path: Optional[Path] = None,
+        run_context: Optional[object] = None,
+    ) -> None:
+        super().__init__(config, python_interpreter=python_interpreter, environment_name=environment_name, conda_env_name=conda_env_name)
+        self.project_path = project_path
+        self.run_context = run_context
+        self.step_registry = get_step_registry()
+        self.step_state_manager: StepStateManager | None = None
+        self.networkx_executor: DaskNetworkXExecutor | None = None
+        self.tracker: Any = None
+        self.logger = logging.getLogger(__name__)
+
+    def set_tracker(self, tracker: Any) -> None:
+        self.tracker = tracker
+
+    def _repo_root(self) -> Path:
+        """Best-effort workspace root resolution (where projects/ lives)."""
+        try:
+            from mlops.core.workspace import get_workspace_root
+            return get_workspace_root()
+        except Exception:
+            return Path.cwd()
+
+    def _resolve_custom_script_path(self, custom_script_path: str) -> Path:
+        p = Path(custom_script_path)
+        if p.is_absolute() and p.exists():
+            return p
+        if p.exists():
+            return p
+        # Try relative to project path (for configs that use "models/foo.py")
+        try:
+            if self.project_path and (self.project_path / p).exists():
+                return self.project_path / p
+        except Exception:
+            pass
+        # Try relative to repo root (for configs that use "projects/<id>/models/foo.py")
+        try:
+            repo_root = self._repo_root()
+            cand = repo_root / p
+            if cand.exists():
+                return cand
+        except Exception:
+            pass
+        return p
+        
+    def initialize(self) -> None:
+        """Initialize the adapter: import model module, configure caching, and set up the executor."""
+        self.logger = logging.getLogger(__name__)
+
+        custom_script_path = getattr(self.config.parameters, "custom_script_path", None)
+        if not custom_script_path:
+            raise ValueError("custom_script_path must be specified in configuration")
+        custom_target = getattr(self.config.parameters, "custom_target", None)
+
+        script_path = self._resolve_custom_script_path(str(custom_script_path))
+        if not script_path.exists():
+            raise FileNotFoundError(f"Custom script not found: {custom_script_path}")
+
+        spec = importlib.util.spec_from_file_location("custom_model", str(script_path.resolve()))
+        if spec is None or spec.loader is None:
+            raise ImportError(f"Failed to create import spec for: {script_path}")
+        custom_module = importlib.util.module_from_spec(spec)
+        sys.modules["custom_model"] = custom_module
+        spec.loader.exec_module(custom_module)  # type: ignore[attr-defined]
+
+        if custom_target:
+            if not hasattr(custom_module, custom_target):
+                raise AttributeError(f"Target '{custom_target}' not found in {custom_script_path}")
+            self.model_class = getattr(custom_module, custom_target)
+            try:
+                if not issubclass(self.model_class, MLOpsCustomModelBase):
+                    self.logger.info(f"Target '{custom_target}' does not inherit MLOpsCustomModelBase; proceeding anyway.")
+            except TypeError:
+                pass
+
+        self.logger.info(f"Imported model module from: {custom_script_path}")
+
+        # Steps are registered via decorators during import.
+        self.step_registry = get_step_registry()
+        try:
+            step_names = list(getattr(self.step_registry, "_steps", {}).keys())
+            if not step_names:
+                self.logger.warning("No steps found in registry. Ensure your model uses @step decorators.")
+            else:
+                self.logger.info(f"Found {len(step_names)} registered step(s)")
+        except Exception:
+            pass
+
+        cache_config = getattr(self.config.parameters, "cache", {}) or {}
+        cache_config = _as_dict(cache_config)
+        cache_ttl_hours = cache_config.get("ttl_hours", 24) if isinstance(cache_config, dict) else 24
+
+        step_cache_dir = (self.project_path / "cache" / "steps") if self.project_path else Path("step_cache")
+
+        project_id_for_ns = self.project_path.name if self.project_path else "default"
+        backend_cfg = _as_dict(cache_config.get("backend", {}))
+
+        # Centralized KV/object-store creation (config -> env override -> safe fallback).
+        try:
+            from mlops.storage.factory import create_kv_store, create_object_store
+        except Exception:
+            create_kv_store = None  # type: ignore[assignment]
+            create_object_store = None  # type: ignore[assignment]
+
+        try:
+            ws_root = self._repo_root()
+        except Exception:
+            ws_root = None
+
+        if create_kv_store:
+            kv_store = create_kv_store(
+                project_id_for_ns,
+                backend_cfg if isinstance(backend_cfg, dict) else {},
+                env=os.environ,
+                workspace_root=ws_root,
+                project_root=self.project_path,
+            )
+        else:
+            from mlops.storage.adapters.memory_store import InMemoryStore
+            kv_store = InMemoryStore(project_id_for_ns)
+
+        obj_store = None
+        if create_object_store:
+            try:
+                obj_store = create_object_store(cache_config if isinstance(cache_config, dict) else {}, env=os.environ)
+            except Exception:
+                obj_store = None
+        obj_prefix = None
+
+        self.step_state_manager = StepStateManager(
+            cache_dir=step_cache_dir,
+            kv_store=kv_store,
+            logger=self.logger,
+            cache_ttl_hours=cache_ttl_hours,
+            object_store=obj_store,
+            object_prefix=obj_prefix,
+        )
+
+        executor_config = _as_dict(getattr(self.config.parameters, "executor", {}) or {})
+        env_workers = os.environ.get("MLOPS_N_WORKERS")
+        try:
+            n_workers = int(env_workers) if env_workers else int(executor_config.get("n_workers", 2))
+        except Exception:
+            n_workers = 2
+
+        dask_tuning_cfg = _as_dict(executor_config.get("dask") or {})
+        if isinstance(dask_tuning_cfg, dict) and not dask_tuning_cfg and executor_config.get("dask_config"):
+            dask_tuning_cfg = _as_dict(executor_config.get("dask_config"))
+
+        min_workers_override = _to_int(executor_config.get("min_workers")) if isinstance(executor_config, dict) else None
+        wait_for_workers_override = _to_float(executor_config.get("wait_for_workers_sec")) if isinstance(executor_config, dict) else None
+        if isinstance(dask_tuning_cfg, dict):
+            if min_workers_override is None:
+                min_workers_override = _to_int(dask_tuning_cfg.get("min_workers"))
+            if wait_for_workers_override is None:
+                wait_for_workers_override = _to_float(dask_tuning_cfg.get("wait_for_workers_sec"))
+
+        dask_overrides: Dict[str, Any] = {}
+        compression_setting = None
+        if isinstance(dask_tuning_cfg, dict):
+            comm_cfg = _as_dict(dask_tuning_cfg.get("comm") or {})
+            if isinstance(comm_cfg, dict):
+                compression_setting = comm_cfg.get("compression") or comm_cfg.get("codec")
+            if compression_setting:
+                dask_overrides["distributed.comm.compression"] = str(compression_setting)
+            memory_cfg = _as_dict(dask_tuning_cfg.get("memory") or {})
+            if isinstance(memory_cfg, dict):
+                mem_map = {
+                    "worker_target_fraction": "distributed.worker.memory.target",
+                    "worker_spill_fraction": "distributed.worker.memory.spill",
+                    "worker_pause_fraction": "distributed.worker.memory.pause",
+                }
+                for src_key, dst_key in mem_map.items():
+                    val = memory_cfg.get(src_key)
+                    if val is not None:
+                        try:
+                            dask_overrides[dst_key] = float(val)
+                        except Exception:
+                            dask_overrides[dst_key] = val
+            overrides_block = dask_tuning_cfg.get("overrides")
+            if isinstance(overrides_block, dict):
+                for key, value in overrides_block.items():
+                    if isinstance(key, str):
+                        dask_overrides[key] = value
+
+        if compression_setting:
+            try:
+                os.environ.setdefault("DASK_DISTRIBUTED__COMM__COMPRESSION", str(compression_setting))
+            except Exception:
+                pass
+
+        scheduler_address = executor_config.get("scheduler_address") or os.environ.get("DASK_SCHEDULER_ADDRESS")
+        scheduler_mode = "distributed" if scheduler_address else "threads"
+
+        extra_files_to_upload: list[str] = []
+        try:
+            extra_files_to_upload = [str(script_path)]
+        except Exception:
+            extra_files_to_upload = []
+        # Upload reporting entrypoint for worker-side chart imports (best-effort).
+        try:
+            rep_cfg_text = os.environ.get("MLOPS_REPORTING_CONFIG") or ""
+            if rep_cfg_text:
+                rep_cfg = json.loads(rep_cfg_text) or {}
+                ep = rep_cfg.get("static_entrypoint") or rep_cfg.get("entrypoint")
+                if isinstance(ep, str) and ep.strip():
+                    p = Path(ep)
+                    if not p.is_absolute():
+                        p = self._repo_root() / p
+                    if p.exists():
+                        extra_files_to_upload.append(str(p))
+        except Exception:
+            pass
+
+        self.networkx_executor = DaskNetworkXExecutor(
+            step_registry=self.step_registry,
+            state_manager=self.step_state_manager,
+            logger=self.logger,
+            n_workers=n_workers,
+            scheduler_mode=scheduler_mode,
+            scheduler_address=scheduler_address,
+            client=None,
+            extra_files_to_upload=extra_files_to_upload,
+            min_workers=min_workers_override,
+            wait_for_workers_sec=wait_for_workers_override,
+            dask_config_overrides=dask_overrides,
+        )
+        mode_label = "distributed" if scheduler_mode == "distributed" else "threads"
+        self.logger.info(f"Initialized Dask NetworkX executor with {n_workers} workers ({mode_label} scheduler)")
+        
+        self.logger.info("Enhanced caching with function hashing: ENABLED")
+
+        # Make state manager available for manual step-level caching in decorators
+        try:
+            from mlops.core.step_system import set_state_manager as _set_sm
+            _set_sm(self.step_state_manager)
+        except Exception:
+            pass
+        
+        
+
+    def _execute_step_graph(self, run_id: str, data_paths: Dict[str, Path], tracker: Any = None, **kwargs) -> Dict[str, Any]:
+        """Execute the configured pipeline once."""
+        if not self.networkx_executor or not self.step_state_manager:
+            raise RuntimeError("NetworkX execution not properly initialized")
+            
+        pipeline_config = self.config.parameters.pipeline
+        if not pipeline_config:
+            raise ValueError("NetworkX executor requires pipeline configuration with 'processes' and 'steps' sections")
+            
+        pipeline_dict = pipeline_config.model_dump() if hasattr(pipeline_config, 'model_dump') else pipeline_config
+        
+        if "processes" not in pipeline_dict:
+            raise ValueError(
+                "NetworkX executor requires 'processes' section in pipeline configuration. "
+                "Legacy 'main_flow' syntax is no longer supported. "
+                "Please use the NetworkX format with processes and steps."
+            )
+        
+        networkx_config = parse_networkx_pipeline_from_config(pipeline_dict)
+        
+        try:
+            project_id = getattr(getattr(self, "run_context", None), "project_id", None) or (
+                self.project_path.name if self.project_path else "default"
+            )
+        except Exception:
+            project_id = self.project_path.name if self.project_path else "default"
+        
+        context_factory = get_context_factory()
+        global_cfg = _as_dict(self.config.parameters)
+        overrides = kwargs.get("global_config_overrides")
+        if isinstance(overrides, dict) and overrides:
+            global_cfg = {**global_cfg, **overrides}
+
+        context = context_factory.create_context(
+            project_id=project_id,
+            run_id=run_id,
+            tracker=tracker,
+            global_config=global_cfg,
+            data_paths=data_paths,
+            checkpoint_dir=self.project_path / "artifacts" / "checkpoints" if self.project_path else Path("artifacts/checkpoints")
+        )
+        
+        set_current_context(context)
+        
+        try:
+            resume_from_process = kwargs.get("resume_from_process")
+            config_hash = self.step_state_manager._compute_config_hash(pipeline_dict)
+            
+            if resume_from_process and self.step_state_manager.can_resume_from_step(run_id, resume_from_process, config_hash):
+                self.logger.info(f"Resuming execution from process: {resume_from_process}")
+                context.step_results = self.step_state_manager.get_step_results(run_id)
+            else:
+                self.step_state_manager.start_pipeline_execution(run_id, pipeline_dict)
+                
+            single_process = kwargs.get("single_process", False)
+            execution_results = self.networkx_executor.execute_graph(
+                networkx_config,
+                context,
+                run_id=run_id,
+                resume_from_process=resume_from_process,
+                stop_after_process=single_process,
+            )
+            
+            results = {}
+            
+            for process_name, process_result in execution_results.items():
+                if process_result.error is None:  # success check for simplified result
+                    if process_result.result:
+                        result_dict = process_result.result
+                        results[f"{process_name}_result"] = result_dict
+                        
+                        if process_name == "model_training" and result_dict:
+                            training_summary = {}
+                            if 'model' in result_dict:
+                                model = result_dict['model']
+                                if hasattr(model, 'get_training_metrics'):
+                                    training_summary['model_metrics'] = model.get_training_metrics()
+                                training_summary['model_type'] = type(model).__name__
+                            results[f"{process_name}_summary"] = training_summary
+                        
+                        elif process_name == "evaluate_model" and result_dict:
+                            evaluation_summary = {}
+                            if 'evaluation_metrics' in result_dict:
+                                evaluation_summary['final_metrics'] = result_dict['evaluation_metrics']
+                            if 'predictions' in result_dict:
+                                evaluation_summary['prediction_count'] = len(result_dict['predictions'])
+                            results[f"{process_name}_summary"] = evaluation_summary
+                
+                else:
+                    results[f"{process_name}_error"] = process_result.error
+                    self.logger.error(f"Process {process_name} failed: {process_result.error}")
+            
+            return results
+            
+        finally:
+            set_current_context(None)
+
+    def run(self, data_paths: Dict[str, Path] | None = None, **kwargs) -> Dict[str, Any]:
+        """Run the pipeline once according to the configured processes/steps."""
+        run_id = kwargs.pop("run_id", f"run_{int(time.time())}")
+        tracker = kwargs.pop("tracker", self.tracker)
+        normalized_paths: Dict[str, Path] = data_paths or {}
+        return self._execute_step_graph(run_id=run_id, data_paths=normalized_paths, tracker=tracker, **kwargs)
+
+    def save_model(self, model_path: str, **kwargs) -> None:
+        """Save model artifacts (handled automatically by step system)."""
+        self.logger.info(f"Model artifacts will be saved automatically by the step system to: {model_path}")
+
+    def load_model(self, model_path: str, **kwargs) -> Any:
+        """Load model artifacts (handled automatically by step system)."""
+        self.logger.info(f"Model artifacts will be loaded automatically by the step system from: {model_path}")
+        return None
+
+        
+
+    @classmethod
+    def validate_config(cls, config: AdapterConfig) -> bool:
+        """Validate the adapter configuration."""
+        try:
+            if not config.parameters.custom_script_path:
+                return False
+            if not config.parameters.pipeline:
+                return False
+            pipeline_dict = config.parameters.pipeline.model_dump() if hasattr(config.parameters.pipeline, 'model_dump') else config.parameters.pipeline
+            if "processes" not in pipeline_dict:
+                return False
+            return True
+        except Exception:
+            return False
+
+    def save(self, path: Path) -> None:
+        """Save adapter state."""
+        pass
+
+    def load(self, path: Path) -> None:
+        """Load adapter state."""
+        pass
+
+Adapter = CustomModelAdapter 

mlops/adapters/plugin_manager.py

@@ -0,0 +1,113 @@
+import importlib
+import pkgutil
+from pathlib import Path
+from typing import Dict, Type, Optional
+from .base import ModelAdapter, ModelConfig
+from .config_schema import AdapterConfig
+
+class AdapterPluginManager:
+    """Manages the discovery and loading of model adapters."""
+    
+    def __init__(self):
+        self._adapters: Dict[str, Type[ModelAdapter]] = {}
+    
+    def discover_adapters(self, package_path: str) -> None:
+        """Discover and load all adapter plugins in the given package.
+
+        Tries both the provided path and a 'src.'-prefixed or de-prefixed variant
+        to support source-tree layouts (src/mlops) and installed packages (mlops).
+
+        Args:
+            package_path: Dotted path to the package containing adapters
+                          (e.g., 'mlops.adapters' or 'src.mlops.adapters').
+        """
+        candidate_paths = [package_path]
+        # Add fallback variants for common source layout differences
+        if package_path.startswith("src."):
+            candidate_paths.append(package_path[len("src."):])
+        else:
+            candidate_paths.append(f"src.{package_path}")
+
+        for pkg in candidate_paths:
+            try:
+                package = importlib.import_module(pkg)
+            except Exception:
+                continue
+            for _, name, is_pkg in pkgutil.iter_modules(package.__path__):
+                if not is_pkg:
+                    continue
+                try:
+                    module = importlib.import_module(f"{pkg}.{name}")
+                    if hasattr(module, "Adapter"):
+                        adapter_class = getattr(module, "Adapter")
+                        if issubclass(adapter_class, ModelAdapter):
+                            self._adapters[name] = adapter_class
+                except Exception as e:
+                    print(f"Failed to load adapter {name}: {e}")
+    
+    def get_adapter(self, name: str) -> Optional[Type[ModelAdapter]]:
+        """Get an adapter class by name.
+        
+        Args:
+            name: Name of the adapter
+            
+        Returns:
+            The adapter class if found, None otherwise
+        """
+        return self._adapters.get(name)
+    
+    def list_adapters(self) -> Dict[str, Type[ModelAdapter]]:
+        """List all available adapters.
+        
+        Returns:
+            Dictionary mapping adapter names to their classes
+        """
+        return self._adapters.copy()
+    
+    def create_adapter(
+        self,
+        name: str,
+        config: AdapterConfig,
+        python_interpreter: Optional[str] = None,
+        environment_name: Optional[str] = None,
+        conda_env_name: Optional[str] = None,
+        project_path: Optional[Path] = None,
+        run_context: Optional[object] = None,
+    ) -> Optional[ModelAdapter]:
+        """
+        Create an adapter by name with the given configuration.
+        
+        Args:
+            name: Name of the adapter.
+            config: Configuration for the adapter.
+            python_interpreter: Path to the python interpreter in the environment.
+            environment_name: Name of the environment (supports all types: conda, venv, etc.).
+            conda_env_name: Legacy parameter name for backward compatibility.
+            project_path: Path to the project directory for project-specific artifact storage.
+            run_context: Optional run-scoped context object (passed through when supported by the adapter).
+        
+        Returns:
+            An instance of the adapter or None if not found.
+        """
+        if name not in self._adapters:
+            print(f"Adapter '{name}' not found.")
+            return None
+        
+        adapter_class = self._adapters[name]
+        
+        # Support both new and legacy parameter names
+        env_name = environment_name or conda_env_name
+        
+        # Check if the adapter class supports optional parameters
+        import inspect
+        signature = inspect.signature(adapter_class.__init__)
+        kwargs = {
+            "python_interpreter": python_interpreter,
+            "environment_name": env_name,
+            "conda_env_name": env_name,
+        }
+        if "project_path" in signature.parameters:
+            kwargs["project_path"] = project_path
+        if "run_context" in signature.parameters:
+            kwargs["run_context"] = run_context
+        return adapter_class(config, **kwargs)

mlops/adapters/sklearn/__init__.py

@@ -0,0 +1,3 @@
+from .adapter import Adapter
+
+__all__ = ["Adapter"] 

mlops/adapters/sklearn/adapter.py

@@ -0,0 +1,94 @@
+from pathlib import Path
+from typing import Any, Dict, Optional
+import joblib
+import numpy as np
+from sklearn.base import BaseEstimator
+from sklearn.model_selection import train_test_split
+import importlib
+
+from ..base import ModelAdapter
+from ..config_schema import AdapterConfig
+
+class Adapter(ModelAdapter):
+    """Adapter for scikit-learn models."""
+    
+    def __init__(self, config: AdapterConfig, python_interpreter: Optional[str] = None, environment_name: Optional[str] = None, conda_env_name: Optional[str] = None):
+        super().__init__(config, python_interpreter=python_interpreter, environment_name=environment_name, conda_env_name=conda_env_name)
+        self.model: BaseEstimator = None
+    
+    def initialize(self) -> None:
+        """Initialize the scikit-learn model."""
+        module_name, class_name_val = self.config.parameters.class_name.rsplit('.', 1)
+        try:
+            ModelClass = getattr(importlib.import_module(module_name), class_name_val)
+        except (ImportError, AttributeError) as e:
+            raise ValueError(f"Could not import class_name '{self.config.parameters.class_name}': {e}")
+
+        if not issubclass(ModelClass, BaseEstimator):
+            raise ValueError(f"class_name '{self.config.parameters.class_name}' must be a scikit-learn BaseEstimator.")
+
+        model_hyperparams = self.config.parameters.hyperparameters if self.config.parameters.hyperparameters is not None else {}
+        self.model = ModelClass(**model_hyperparams)
+    
+    def run(self, data_paths: Dict[str, Path] | None = None, **kwargs) -> Dict[str, Any]:
+        """Run a minimal fit/evaluate loop depending on provided data paths."""
+        if self.model is None:
+            self.initialize()
+
+        data_paths = data_paths or {}
+        results: Dict[str, Any] = {}
+
+        def _load_xy(csv_path: Path):
+            import pandas as pd
+            df = pd.read_csv(csv_path)
+            if 'label' not in df.columns:
+                raise ValueError("Target column 'label' not found in the CSV.")
+            X = df.drop('label', axis=1).values
+            y = df['label'].values
+            return X, y
+
+        if 'training' in data_paths and data_paths['training'] is not None:
+            X, y = _load_xy(data_paths['training'])
+            self.model.fit(X, y)
+            results['model_type'] = self.model.__class__.__name__
+            results['parameters'] = self.model.get_params()
+
+        if 'validation' in data_paths and data_paths['validation'] is not None:
+            Xv, yv = _load_xy(data_paths['validation'])
+            results['validation_score'] = self.model.score(Xv, yv)
+
+        return results
+    
+    def save(self, path: Path) -> None:
+        """Save the scikit-learn model."""
+        if self.model is None:
+            raise ValueError("No model to save")
+        path.mkdir(parents=True, exist_ok=True)
+        joblib.dump(self.model, path / "model.joblib")
+    
+    def load(self, path: Path) -> None:
+        """Load the scikit-learn model."""
+        self.model = joblib.load(path / "model.joblib")
+    
+    def predict(self, data: Any) -> Any:
+        """Make predictions using the scikit-learn model."""
+        if self.model is None:
+            raise ValueError("Model not initialized")
+        return self.model.predict(data)
+    
+    @classmethod
+    def validate_config(cls, config: AdapterConfig) -> bool:
+        """Validate the configuration for this adapter."""
+        if not hasattr(config, 'parameters') or config.parameters is None:
+            print("Validation failed: config has no parameters attribute or it is None")
+            return False
+
+        if not hasattr(config.parameters, 'class_name') or not isinstance(config.parameters.class_name, str) or not config.parameters.class_name:
+            print(f"Validation failed: class_name issue. Has attr: {hasattr(config.parameters, 'class_name')}, Is str: {isinstance(config.parameters.class_name, str) if hasattr(config.parameters, 'class_name') else 'N/A'}, Value: {config.parameters.class_name if hasattr(config.parameters, 'class_name') else 'N/A'}")
+            return False
+
+        if not hasattr(config.parameters, 'hyperparameters') or not isinstance(config.parameters.hyperparameters, dict):
+            print(f"Validation failed: hyperparameters issue. Has attr: {hasattr(config.parameters, 'hyperparameters')}, Is dict: {isinstance(config.parameters.hyperparameters, dict) if hasattr(config.parameters, 'hyperparameters') else 'N/A'}")
+            return False
+            
+        return True 

mlops/cluster/__init__.py

@@ -0,0 +1,3 @@
+# Cluster providers package
+
+