ob-metaflow-extensions 1.1.175rc1__py2.py3-none-any.whl → 1.1.175rc3__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/core/app_config.py

@@ -2,7 +2,12 @@ import json
 import os
 
 from typing import Dict, Any
-from .cli_to_config import build_config_from_options
+from .config import (
+    CoreConfig,
+    MergingNotAllowedFieldsException,
+    ConfigValidationFailedException,
+    RequiredFieldMissingException,
+)
 
 CODE_PACKAGE_PREFIX = "mf.obp-apps"
 
@@ -53,18 +58,18 @@ class AuthType:
 class AppConfig:
     """Class representing an Outerbounds App configuration."""
 
-    def __init__(self, config_dict: Dict[str, Any]):
+    def __init__(self, core_config: CoreConfig):
         """Initialize configuration from a dictionary."""
-        self.config = config_dict or {}
-        self.schema = self._load_schema()
+        self._core_config = core_config
         self._final_state: Dict[str, Any] = {}
+        self.config = {}
 
     def set_state(self, key, value):
         self._final_state[key] = value
         return self
 
     def get_state(self, key, default=None):
-        return self._final_state.get(key, self.config.get(key, default))
+        return self._final_state.get(key, self.get(key, default))
 
     def dump_state(self):
         x = {k: v for k, v in self.config.items()}
@@ -72,187 +77,35 @@ class AppConfig:
             x[k] = v
         return x
 
-    @staticmethod
-    def _load_schema():
-        """Load the configuration schema from the YAML file."""
-        schema_path = os.path.join(os.path.dirname(__file__), "config_schema.yaml")
-        auto_gen_schema_path = os.path.join(
-            os.path.dirname(__file__), "config_schema_autogen.json"
-        )
-
-        with open(schema_path, "r") as f:
-            schema = _try_loading_yaml(f)
-        if schema is None:
-            with open(auto_gen_schema_path, "r") as f:
-                schema = json.load(f)
-        return schema
+    def commit(self):
+        try:
+            self._core_config.commit()
+            self.config = self._core_config.to_dict()
+        except RequiredFieldMissingException as e:
+            raise AppConfigError(
+                "The configuration is missing the following required fields: %s. \n\tException: %s"
+                % (e.field_name, e.message)
+            )
+        except ConfigValidationFailedException as e:
+            raise AppConfigError(
+                "The configuration is invalid. \n\n\tException: %s" % (e.message)
+            )
 
     def get(self, key: str, default: Any = None) -> Any:
         """Get a configuration value by key."""
-        return self.config.get(key, default)
-
-    def validate(self) -> None:
-        """Validate the configuration against the schema."""
-        self._validate_required_fields()
-        self._validate_field_types()
-        self._validate_field_constraints()
-
-    def set_deploy_defaults(self, packaging_directory: str) -> None:
-        """Set default values for fields that are not provided."""
-        if not self.config.get("auth"):
-            self.config["auth"] = {}
-        if not self.config["auth"].get("public"):
-            self.config["auth"]["public"] = True
-        if not self.config["auth"].get("type"):
-            self.config["auth"]["type"] = AuthType.BROWSER
-
-        if not self.config.get("health_check"):
-            self.config["health_check"] = {}
-        if not self.config["health_check"].get("enabled"):
-            self.config["health_check"]["enabled"] = False
-
-        if not self.config.get("resources"):
-            self.config["resources"] = {}
-        if not self.config["resources"].get("cpu"):
-            self.config["resources"]["cpu"] = 1
-        if not self.config["resources"].get("memory"):
-            self.config["resources"]["memory"] = "4096Mi"
-        if not self.config["resources"].get("disk"):
-            self.config["resources"]["disk"] = "20Gi"
-
-        if not self.config.get("replicas", None):
-            self.config["replicas"] = {
-                "min": 1,
-                "max": 1,
-            }
-        else:
-            # TODO: The replicas related code blocks will change as we add autoscaling
-            # configurations
-            max_is_set = self.config["replicas"].get("max", None) is not None
-            min_is_set = self.config["replicas"].get("min", None) is not None
-            if max_is_set and not min_is_set:
-                # If users want to set 0 replicas for min,
-                # then they need explicitly specify min to 0
-                self.config["replicas"]["min"] = self.config["replicas"]["max"]
-            if min_is_set and not max_is_set:
-                # In the situations where we dont have min/max replicas, we can
-                # set max to min.
-                self.config["replicas"]["max"] = self.config["replicas"].get("min")
-
-    def _validate_required_fields(self) -> None:
-        """Validate that all required fields are present."""
-        required_fields = self.schema.get("required", [])
-        for field in required_fields:
-            if field not in self.config:
-                raise AppConfigError(
-                    f"Required field '{field}' is missing from the configuration."
-                )
-
-    def _validate_field_types(self) -> None:
-        """Validate that fields have correct types."""
-        properties = self.schema.get("properties", {})
-
-        for field, value in self.config.items():
-            if field not in properties:
-                raise AppConfigError(f"Unknown field '{field}' in configuration.")
-
-            field_schema = properties[field]
-            field_type = field_schema.get("type")
-
-            if field_type == "string" and not isinstance(value, str):
-                raise AppConfigError(f"Field '{field}' must be a string.")
+        config_value = self.config.get(key, default)
+        if config_value is None:
+            return default
+        return config_value
 
-            elif field_type == "integer" and not isinstance(value, int):
-                raise AppConfigError(f"Field '{field}' must be an integer.")
-
-            elif field_type == "boolean" and not isinstance(value, bool):
-                raise AppConfigError(f"Field '{field}' must be a boolean.")
-
-            elif field_type == "array" and not isinstance(value, list):
-                raise AppConfigError(f"Field '{field}' must be an array.")
-
-            elif field_type == "object" and not isinstance(value, dict):
-                raise AppConfigError(f"Field '{field}' must be an object.")
-
-    def _validate_field_constraints(self) -> None:
-        """Validate field-specific constraints."""
-        properties = self.schema.get("properties", {})
-
-        # Validate name
-        if "name" in self.config:
-            name = self.config["name"]
-            max_length = properties["name"].get("maxLength", 20)
-            if len(name) > max_length:
-                raise AppConfigError(
-                    f"App name '{name}' exceeds maximum length of {max_length} characters."
-                )
-
-        # Validate port
-        if "port" in self.config:
-            port = self.config["port"]
-            min_port = properties["port"].get("minimum", 1)
-            max_port = properties["port"].get("maximum", 65535)
-            if port < min_port or port > max_port:
-                raise AppConfigError(
-                    f"Port number {port} is outside valid range ({min_port}-{max_port})."
-                )
-
-        # Validate dependencies (only one type allowed)
-        if "dependencies" in self.config:
-            deps = self.config["dependencies"]
-            if not isinstance(deps, dict):
-                raise AppConfigError("Dependencies must be an object.")
-
-            valid_dep_types = [
-                "from_requirements_file",
-                "from_pyproject_toml",
-            ]
-
-            found_types = [dep_type for dep_type in valid_dep_types if dep_type in deps]
-
-            if len(found_types) > 1:
-                raise AppConfigError(
-                    f"You can only specify one mode of specifying dependencies. You have specified : {found_types} . Please only set one."
-                )
-
-        # Validate that each tag has exactly one key
-        if "tags" in self.config:
-            tags = self.config["tags"]
-            for tag in tags:
-                if not isinstance(tag, dict):
-                    raise AppConfigError(
-                        "Each tag must be a dictionary. %s is of type %s"
-                        % (str(tag), type(tag))
-                    )
-                if len(tag.keys()) != 1:
-                    raise AppConfigError(
-                        "Each tag must have exactly one key-value pair. Tag %s has %d key-value pairs."
-                        % (str(tag), len(tag.keys()))
-                    )
-        if "replicas" in self.config:
-            replicas = self.config["replicas"]
-            if not isinstance(replicas, dict):
-                raise AppConfigError("Replicas must be an object.")
-            max_is_set = self.config["replicas"].get("max", None) is not None
-            min_is_set = self.config["replicas"].get("min", None) is not None
-            if max_is_set and min_is_set:
-                if replicas.get("min") > replicas.get("max"):
-                    raise AppConfigError(
-                        "Min replicas must be less than equals max replicas. %s > %s"
-                        % (replicas.get("min"), replicas.get("max"))
-                    )
-
-    def to_dict(self) -> Dict[str, Any]:
-        """Return the configuration as a dictionary."""
-        return self.config
+    def to_json(self):
+        return json.dumps(self.config, indent=2)
 
-    def to_yaml(self) -> str:
-        """Return the configuration as a YAML string."""
+    def to_yaml(self):
         return self.to_json()
 
-    def to_json(self) -> str:
-        """Return the configuration as a JSON string."""
-        return json.dumps(self.config, indent=2)
+    def to_dict(self):
+        return self.config
 
     @classmethod
     def from_file(cls, file_path: str) -> "AppConfig":
@@ -273,53 +126,20 @@ class AppConfig:
             except Exception as e:
                 raise AppConfigError(f"Failed to parse configuration file: {e}")
 
-        return cls(config_dict)
-
-    def update_from_cli_options(self, options):
-        """
-        Update configuration from CLI options using the same logic as build_config_from_options.
-        This ensures consistent handling of CLI options whether they come from a config file
-        or direct CLI input.
-        """
-        cli_config = build_config_from_options(options)
-
-        # Process each field using allow_union property
-        for key, value in cli_config.items():
-            if key in self.schema.get("properties", {}):
-                self._update_field(key, value)
+        return cls(CoreConfig.from_dict(config_dict))
 
-        return self
+    @classmethod
+    def from_cli(cls, options: Dict[str, Any]):
+        return cls(CoreConfig.from_cli(options))
 
-    def _update_field(self, field_name, new_value):
-        """Update a field based on its allow_union property."""
-        properties = self.schema.get("properties", {})
-
-        # Skip if field doesn't exist in schema
-        if field_name not in properties:
-            return
-
-        field_schema = properties[field_name]
-        allow_union = field_schema.get("allow_union", False)
-
-        # If field doesn't exist in config, just set it
-        if field_name not in self.config:
-            self.config[field_name] = new_value
-            return
-
-        # If allow_union is True, merge values based on type
-        if allow_union:
-            current_value = self.config[field_name]
-
-            if isinstance(current_value, list) and isinstance(new_value, list):
-                # For lists, append new items
-                self.config[field_name].extend(new_value)
-            elif isinstance(current_value, dict) and isinstance(new_value, dict):
-                # For dicts, update with new values
-                self.config[field_name].update(new_value)
-            else:
-                # For other types, replace with new value
-                self.config[field_name] = new_value
-        else:
+    def update_from_cli_options(self, options):
+        cli_options_config = CoreConfig.from_cli(options)
+        try:
+            self._core_config = CoreConfig.merge_configs(
+                self._core_config, cli_options_config
+            )
+        except MergingNotAllowedFieldsException as e:
             raise AppConfigError(
-                f"Field '{field_name}' does not allow union. Current value: {self.config[field_name]}, new value: {new_value}"
+                "CLI Overrides are not allowed for the following fields: %s. \n\tException: %s"
+                % (e.field_name, e.message)
             )

metaflow_extensions/outerbounds/plugins/apps/core/capsule.py

@@ -246,7 +246,16 @@ class CapsuleInput:
         return _return
 
     @classmethod
-    def from_app_config(self, app_config: AppConfig):
+    def from_app_config(cls, app_config: AppConfig):
+        ## Replica settings
+        replicas = app_config.get_state("replicas", {})
+        fixed, _min, _max = (
+            replicas.get("fixed"),
+            replicas.get("min"),
+            replicas.get("max"),
+        )
+        if fixed:
+            _min, _max = fixed, fixed
         gpu_resource = app_config.get_state("resources").get("gpu")
         resources = {}
         shared_memory = app_config.get_state("resources").get("shared_memory")
@@ -284,16 +293,16 @@ class CapsuleInput:
                 **resources,
             },
             "autoscalingConfig": {
-                "minReplicas": app_config.get_state("replicas", {}).get("min"),
-                "maxReplicas": app_config.get_state("replicas", {}).get("max"),
+                "minReplicas": _min,
+                "maxReplicas": _max,
             },
             **_scheduling_config,
             "containerStartupConfig": {
-                "entrypoint": self.construct_exec_command(
+                "entrypoint": cls.construct_exec_command(
                     app_config.get_state("commands")
                 )
             },
-            "environmentVariables": self._marshal_environment_variables(app_config),
+            "environmentVariables": cls._marshal_environment_variables(app_config),
             # "assets": [{"name": "startup-script.sh"}],
             "authConfig": {
                 "authType": app_config.get_state("auth").get("type"),
@@ -699,6 +708,13 @@ class CapsuleDeployer:
         logs = self.capsule_api.logs(self.identifier, worker_id, previous=True)
         return logs, worker_id
 
+    def _get_min_replicas(self):
+        replicas = self._app_config.get_state("replicas", {})
+        fixed, _min, _ = replicas.get("fixed"), replicas.get("min"), replicas.get("max")
+        if fixed:
+            return fixed
+        return _min
+
     def wait_for_terminal_state(
         self,
     ):
@@ -708,7 +724,7 @@ class CapsuleDeployer:
             self.identifier, self.current_deployment_instance_version
         )
         # min_replicas will always be present
-        min_replicas = self._app_config.get_state("replicas", {}).get("min")
+        min_replicas = self._get_min_replicas()
         workers_state_machine = CapsuleWorkersStateMachine(
             self.identifier,
             self.current_deployment_instance_version,

metaflow_extensions/outerbounds/plugins/apps/core/code_package/code_packager.py

@@ -15,6 +15,21 @@ from metaflow.metaflow_config import (
     DATASTORE_SYSROOT_LOCAL,
 )
 
+DEFAULT_FILE_SUFFIXES = [
+    ".py",
+    ".txt",
+    ".yaml",
+    ".yml",
+    ".json",
+    ".html",
+    ".css",
+    ".js",
+    ".jsx",
+    ".ts",
+    ".tsx",
+    ".md",
+    ".rst",
+]
 # Default prefix for code packages in content addressed store
 CODE_PACKAGE_PREFIX = "apps-code-packages"
 
@@ -155,21 +170,7 @@ class CodePackager:
         """
         # Prepare default values
         _paths_to_include = paths_to_include or []
-        _file_suffixes = file_suffixes or [
-            ".py",
-            ".txt",
-            ".yaml",
-            ".yml",
-            ".json",
-            ".html",
-            ".css",
-            ".js",
-            ".jsx",
-            ".ts",
-            ".tsx",
-            ".md",
-            ".rst",
-        ]
+        _file_suffixes = file_suffixes or DEFAULT_FILE_SUFFIXES
         _metadata = metadata or {}
 
         # If no package_create_fn provided, use default_package_create

metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py

@@ -0,0 +1,12 @@
+from .unified_config import CoreConfig
+from .cli_generator import auto_cli_options
+from .config_utils import (
+    PureStringKVPairType,
+    JsonFriendlyKeyValuePairType,
+    CommaSeparatedListType,
+    MergingNotAllowedFieldsException,
+    ConfigValidationFailedException,
+    RequiredFieldMissingException,
+)
+from . import schema_export
+from .typed_configs import TypedCoreConfig

metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py

@@ -0,0 +1,161 @@
+"""
+CLI Generator for Unified Configuration System
+
+This module automatically generates Click CLI options from the CoreConfig,
+eliminating the need for manual CLI option definitions and ensuring consistency
+between configuration structure and CLI interface.
+
+It also provides machinery for merging configurations from different sources
+(CLI options, config files) with proper precedence and behavior handling.
+"""
+
+from typing import Any, List, Optional
+import json
+
+from ..click_importer import click
+from .unified_config import (
+    CoreConfig,
+    CLIOption,
+    ConfigMeta,
+)
+from .config_utils import (
+    PureStringKVPairType,
+    JsonFriendlyKeyValuePairType,
+    CommaSeparatedListType,
+)
+
+
+class CLIGenerator:
+    """Generates Click CLI options from CoreConfig dataclass."""
+
+    def __init__(self, config_class: type = CoreConfig):
+        self.config_class = config_class
+        self._type_mapping = {
+            str: str,
+            int: int,
+            float: float,
+            bool: bool,
+            list: CommaSeparatedListType,
+            dict: JsonFriendlyKeyValuePairType,
+        }
+
+    def generate_options(self):
+        """Generate all CLI options from the configuration class."""
+        options = []
+
+        # Generate options for all fields automatically
+        options.extend(self._generate_all_options(self.config_class))
+
+        return options
+
+    def _generate_all_options(self, config_class: type):
+        """Generate all options from a config class. Returns a list of click.Options"""
+
+        def _options_from_cfg_cls(_config_class):
+            options = []
+            for field_name, field_info in _config_class._fields.items():
+                if ConfigMeta.is_instance(field_info.field_type):
+                    _subfield_options = _options_from_cfg_cls(field_info.field_type)
+                    options.extend(_subfield_options)
+                    continue
+
+                cli_meta = field_info.cli_meta
+                if not cli_meta or cli_meta.hidden:
+                    continue
+
+                option = self._create_option(field_name, field_info, cli_meta)
+                if option:
+                    options.append(option)
+            return options
+
+        return _options_from_cfg_cls(config_class)
+
+    def _create_option(self, field_name: str, field_info, cli_meta: CLIOption):
+        """Create a Click option from field info and CLI metadata."""
+        # Use the cli_option_str from the CLIOption
+        option_str = cli_meta.cli_option_str
+        param_name = cli_meta.name
+
+        # Determine Click type
+        click_type = self._get_click_type(field_info, cli_meta)
+
+        # Build option parameters
+        help_text = cli_meta.help or field_info.help or f"Set {field_name}"
+        option_params = {
+            "help": help_text,
+            "default": cli_meta.default if cli_meta.default is not None else None,
+            "type": click_type,
+        }
+
+        # Handle multiple values
+        if cli_meta.multiple:
+            option_params["multiple"] = True
+
+        # Handle choices
+        if cli_meta.choices:
+            option_params["type"] = click.Choice(cli_meta.choices)
+
+        # Handle flags
+        if cli_meta.is_flag:
+            option_params["is_flag"] = True
+            option_params.pop("type", None)
+
+        # Handle special flag patterns (e.g., --public-access/--private-access)
+        return click.option(option_str, param_name, **option_params)
+
+    def _get_click_type(self, field_info, cli_meta: CLIOption) -> Any:
+        """Determine the appropriate Click type for a field."""
+        if cli_meta.click_type:
+            return cli_meta.click_type
+
+        # Get the field type
+        field_type = field_info.field_type
+
+        # Handle basic types
+        if field_type == list:
+            return CommaSeparatedListType
+        elif field_type == dict:
+            return JsonFriendlyKeyValuePairType
+        elif field_type == str:
+            return str
+        elif field_type == int:
+            return int
+        elif field_type == bool:
+            return bool
+        elif field_type == float:
+            return float
+
+        # Handle custom config types
+        if hasattr(field_type, "__name__") and field_type.__name__.endswith("Config"):
+            return str  # Default to string for complex types
+
+        # Use type mapping
+        return self._type_mapping.get(field_type, str)
+
+    def create_decorator(self, command_type: str = "deploy") -> callable:
+        """Create a decorator that applies all CLI options to a command."""
+
+        def decorator(func):
+            # Apply options in reverse order since decorators are applied bottom-up
+            for option in reversed(self.generate_options()):
+                func = option(func)
+            return func
+
+        return decorator
+
+
+def auto_cli_options(config_class: type = CoreConfig, command_type: str = "deploy"):
+    """
+    Decorator that automatically adds CLI options from CoreConfig.
+
+    Args:
+        command_type: Type of command (e.g., "deploy", "list", "delete")
+
+    Usage:
+        @auto_cli_options("deploy")
+        def deploy_command(**kwargs):
+            config = CoreConfig.from_cli(kwargs)
+            # ... use config
+    """
+    generator = CLIGenerator(config_class)
+    return generator.create_decorator(command_type)