ob-metaflow-extensions 1.1.175rc2__py2.py3-none-any.whl → 1.1.175rc3__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py

@@ -0,0 +1,317 @@
+"""
+Typed Init Generator for ConfigMeta Classes
+
+This module provides a mechanism to dynamically generate explicit typed classes
+from ConfigMeta classes that IDEs can understand and provide autocomplete for.
+"""
+
+from typing import Any, Dict, List, Optional, Union, Type
+
+from .config_utils import ConfigMeta
+
+import os
+
+current_dir = os.path.dirname(__file__)
+
+
+def generate_typed_class_code(config_class: Type) -> str:
+    """
+    Generate the actual Python code for a typed class that IDEs can understand.
+
+    Args:
+        config_class: A class that inherits from ConfigMeta
+
+    Returns:
+        Python code string for the typed class
+    """
+    if not hasattr(config_class, "_fields"):
+        raise ValueError(f"Class {config_class.__name__} is not a ConfigMeta class")
+
+    class_name = f"Typed{config_class.__name__}"
+
+    # Generate TypedDict for nested configs
+    nested_typeddict_code = []
+
+    # First pass: collect all nested configs
+    nested_configs = {}
+    for field_name, field_info in config_class._fields.items():
+        if ConfigMeta.is_instance(field_info.field_type):
+            nested_configs[field_info.field_type.__name__] = field_info.field_type
+
+    # Generate TypedDict classes for nested configs
+    for nested_name, nested_class in nested_configs.items():
+        dict_name = f"{nested_name}Dict"
+        fields = []
+
+        for field_name, field_info in nested_class._fields.items():
+            field_type = _get_type_string(field_info.field_type)
+            if not field_info.required:
+                field_type = f"Optional[{field_type}]"
+            fields.append(f"    {field_name}: {field_type}")
+
+        typeddict_code = f"""class {dict_name}(TypedDict, total=False):
+{chr(10).join(fields)}"""
+        nested_typeddict_code.append(typeddict_code)
+
+    # Generate __init__ method signature
+    required_params = []
+    optional_params = []
+    all_assignments = []
+
+    for field_name, field_info in config_class._fields.items():
+        field_type = field_info.field_type
+
+        # Handle nested ConfigMeta classes
+        if ConfigMeta.is_instance(field_type):
+            type_hint = f"Optional[{field_type.__name__}Dict]"
+            param_line = f"        {field_name}: {type_hint} = None"
+            optional_params.append(param_line)
+        else:
+            # All params will be set as options here even if the are required in the
+            # configMeta
+            type_hint = _get_type_string(field_type)
+            param_line = f"        {field_name}: Optional[{type_hint}] = None"
+            optional_params.append(param_line)
+
+        all_assignments.append(f'            "{field_name}": {field_name}')
+
+    # Combine required params first, then optional params
+    all_params = required_params + optional_params
+
+    # Generate the class code
+    newline = "\n"
+    comma_newline = ",\n"
+
+    # Add **kwargs to the parameter list
+    if all_params:
+        params_with_kwargs = all_params + ["        **kwargs"]
+    else:
+        params_with_kwargs = ["        **kwargs"]
+
+    class_code = f"""class {class_name}:
+    def __init__(
+        self,
+{comma_newline.join(params_with_kwargs)}
+    ) -> None:
+        self._kwargs = {{
+{comma_newline.join(all_assignments)}
+        }}
+        # Add any additional kwargs
+        self._kwargs.update(kwargs)
+        # Remove None values
+        self._kwargs = {{k: v for k, v in self._kwargs.items() if v is not None}}
+        self._config_class = {config_class.__name__}
+        self._config = self.create_config()
+
+    def create_config(self) -> {config_class.__name__}:
+        return {config_class.__name__}.from_dict(self._kwargs)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return self._config.to_dict()"""
+
+    # Combine all code
+    full_code = []
+    if nested_typeddict_code:
+        full_code.extend(nested_typeddict_code)
+        full_code.append("")  # Empty line
+    full_code.append(class_code)
+
+    return (newline + newline).join(full_code)
+
+
+def _get_type_string(field_type: Type) -> str:
+    """Convert a type to its string representation for code generation."""
+    if field_type == str:
+        return "str"
+    elif field_type == int:
+        return "int"
+    elif field_type == float:
+        return "float"
+    elif field_type == bool:
+        return "bool"
+    elif hasattr(field_type, "__origin__"):
+        # Handle generic types like List[str], Dict[str, str], etc.
+        origin = field_type.__origin__
+        args = getattr(field_type, "__args__", ())
+
+        if origin == list:
+            if args:
+                return f"List[{_get_type_string(args[0])}]"
+            return "List[Any]"
+        elif origin == dict:
+            if len(args) == 2:
+                return f"Dict[{_get_type_string(args[0])}, {_get_type_string(args[1])}]"
+            return "Dict[str, Any]"
+        elif origin == Union:
+            # Handle Optional types
+            if len(args) == 2 and type(None) in args:
+                non_none_type = args[0] if args[1] is type(None) else args[1]
+                return f"Optional[{_get_type_string(non_none_type)}]"
+            return f"Union[{', '.join(_get_type_string(arg) for arg in args)}]"
+
+    # Default case - use the type name
+    return getattr(field_type, "__name__", str(field_type))
+
+
+def generate_typed_classes_module(
+    config_classes: List[Type], module_name: str = "typed_configs"
+) -> str:
+    """
+    Generate a complete Python module with typed classes for multiple ConfigMeta classes.
+
+    Args:
+        config_classes: List of ConfigMeta classes
+        module_name: Name for the generated module
+
+    Returns:
+        Complete Python module code
+    """
+    imports = [
+        "from typing import Optional, List, Dict, Any, TypedDict",
+        "from .unified_config import "
+        + ", ".join(cls.__name__ for cls in config_classes),
+    ]
+
+    class_codes = []
+    for config_class in config_classes:
+        class_codes.append(generate_typed_class_code(config_class))
+
+    # Use string concatenation instead of f-string with backslashes
+    newline = "\n"
+    module_code = (
+        '"""'
+        + newline
+        + "Auto-generated typed classes for ConfigMeta classes."
+        + newline
+        + newline
+        + "This module provides IDE-friendly typed interfaces for all configuration classes."
+        + newline
+        + '"""'
+        + newline
+        + newline
+        + newline.join(imports)
+        + newline
+        + newline
+        + (newline + newline).join(class_codes)
+        + newline
+    )
+
+    return module_code
+
+
+def create_typed_init_class_dynamic(config_class: Type) -> Type:
+    """
+    Dynamically create a typed init class with proper IDE support.
+
+    This creates the class at runtime but with proper type annotations
+    that IDEs can understand.
+    """
+    if not hasattr(config_class, "_fields"):
+        raise ValueError(f"Class {config_class.__name__} is not a ConfigMeta class")
+
+    class_name = f"Typed{config_class.__name__}"
+
+    # Create the init method with proper signature
+    def create_init_method():
+        # Build the signature dynamically
+        sig_params = []
+        annotations = {"return": None}
+
+        for field_name, field_info in config_class._fields.items():
+            field_type = field_info.field_type
+
+            # Handle nested ConfigMeta classes
+            if ConfigMeta.is_instance(field_type):
+                field_type = Dict[str, Any]  # Use Dict for nested configs
+
+            # Handle Optional fields
+            if not field_info.required:
+                field_type = Optional[field_type]
+
+            annotations[field_name] = field_type
+
+        def __init__(self, **kwargs):
+            # Validate kwargs
+            required_fields = {
+                name for name, info in config_class._fields.items() if info.required
+            }
+            provided_fields = set(kwargs.keys())
+            valid_fields = set(config_class._fields.keys())
+
+            # Check required fields
+            missing_fields = required_fields - provided_fields
+            if missing_fields:
+                raise ValueError(
+                    f"Missing required fields: {', '.join(missing_fields)}"
+                )
+
+            # Check for unknown fields - but allow them for flexibility
+            unknown_fields = provided_fields - valid_fields
+            if unknown_fields:
+                print(
+                    f"Warning: Unknown fields will be passed through: {', '.join(unknown_fields)}"
+                )
+
+            self._kwargs = kwargs
+            self._config_class = config_class
+
+        # Set annotations
+        __init__.__annotations__ = annotations
+        return __init__
+
+    def create_config(self) -> config_class:
+        """Create and return the ConfigMeta class instance."""
+        return config_class.from_dict(self._kwargs)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return the raw kwargs as a dictionary."""
+        return self._kwargs.copy()
+
+    def __repr__(self) -> str:
+        return f"{class_name}({self._kwargs})"
+
+    # Create the class
+    init_method = create_init_method()
+
+    TypedClass = type(
+        class_name,
+        (object,),
+        {
+            "__init__": init_method,
+            "create_config": create_config,
+            "to_dict": to_dict,
+            "__repr__": __repr__,
+            "__module__": __name__,
+            "__qualname__": class_name,
+        },
+    )
+
+    return TypedClass
+
+
+# Auto-generate and write typed classes to a file
+def generate_typed_classes_file(output_file: str = None):
+    """
+    Generate typed classes and write them to a file for IDE support.
+
+    Args:
+        output_file: Path to write the generated classes. If None, prints to stdout.
+    """
+    from .unified_config import CoreConfig
+
+    config_classes = [CoreConfig]
+
+    module_code = generate_typed_classes_module(config_classes)
+
+    if output_file:
+        with open(output_file, "w") as f:
+            f.write(module_code)
+        print(f"Generated typed classes written to {output_file}")
+    else:
+        print(module_code)
+
+
+# Example usage and testing
+if __name__ == "__main__":
+    # Generate typed classes file
+    generate_typed_classes_file(os.path.join(current_dir, "typed_configs.py"))

metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py

@@ -12,7 +12,7 @@ No external dependencies required - uses only Python standard library.
 import os
 import json
 from typing import Any, Dict, List, Optional, Union, Type
-
+import re
 
 from .config_utils import (
     ConfigField,
@@ -41,9 +41,142 @@ class AuthType:
         return [cls.BROWSER, cls.API]
 
 
+class UnitParser:
+    UNIT_FREE_REGEX = r"^\d+$"
+
+    metrics = {
+        "memory": {
+            "default_unit": "Mi",
+            "requires_unit": True,  # if a Unit free value is provided then we will add the default unit to it.
+            # Regex to match values with units (e.g., "512Mi", "4Gi", "1024Ki")
+            "unit_regex": r"^\d+(\.\d+)?(Ki|Mi|Gi|Ti|Pi|Ei)$",
+        },
+        "cpu": {
+            "default_unit": None,
+            "requires_unit": False,  # if a Unit free value is provided then we will not add the default unit to it.
+            # Accepts values like 400m, 4, 0.4, 1000n, etc.
+            # Regex to match values with units (e.g., "400m", "1000n", "2", "0.5")
+            "unit_regex": r"^(\d+(\.\d+)?(m|n)?|\d+(\.\d+)?)$",
+        },
+        "disk": {
+            "default_unit": "Mi",
+            "requires_unit": True,  # if a Unit free value is provided then we will add the default unit to it.
+            # Regex to match values with units (e.g., "100Mi", "1Gi", "500Ki")
+            "unit_regex": r"^\d+(\.\d+)?(Ki|Mi|Gi|Ti|Pi|Ei)$",
+        },
+        "gpu": {
+            "default_unit": None,
+            "requires_unit": False,
+            # Regex to match values with units (usually just integer count, e.g., "1", "2")
+            "unit_regex": r"^\d+$",
+        },
+    }
+
+    def __init__(self, metric_name: str):
+        self.metric_name = metric_name
+
+    def validate(self, value: str):
+        if self.metrics[self.metric_name]["requires_unit"]:
+            if not re.match(self.metrics[self.metric_name]["unit_regex"], value):
+                return False
+        return True
+
+    def process(self, value: str):
+        value = str(value)
+        if self.metrics[self.metric_name]["requires_unit"]:
+            if re.match(self.UNIT_FREE_REGEX, value):
+                # This means the value is unit free and we need to add the default unit to it.
+                value = "%s%s" % (
+                    value.strip(),
+                    self.metrics[self.metric_name]["default_unit"],
+                )
+                return value
+
+            if re.match(self.metrics[self.metric_name]["unit_regex"], value):
+                return value
+
+        return value
+
+    def parse(self, value: str):
+        if value is None:
+            return None
+        return self.process(value)
+
+    @staticmethod
+    def validation_wrapper_fn(
+        metric_name: str,
+    ):
+        def validation_fn(value: str):
+            if value is None:
+                return True
+            field_info = ResourceConfig._get_field(ResourceConfig, metric_name)  # type: ignore
+            parser = UnitParser(metric_name)
+            validation = parser.validate(value)
+            if not validation:
+                raise ConfigValidationFailedException(
+                    field_name=metric_name,
+                    field_info=field_info,
+                    current_value=value,
+                    message=f"Invalid value for `{metric_name}`. Must be of the format {parser.metrics[metric_name]['unit_regex']}.",
+                )
+            return validation
+
+        return validation_fn
+
+
+class BasicValidations:
+    def __init__(self, config_meta_class, field_name):
+        self.config_meta_class = config_meta_class
+        self.field_name = field_name
+
+    def _get_field(self):
+        return self.config_meta_class._get_field(self.config_meta_class, self.field_name)  # type: ignore
+
+    def enum_validation(self, enums: List[str], current_value):
+        if current_value not in enums:
+            raise ConfigValidationFailedException(
+                field_name=self.field_name,
+                field_info=self._get_field(),
+                current_value=current_value,
+                message=f"Configuration field {self.field_name} has invalid value {current_value}. Value must be one of: {' '.join(enums)}",
+            )
+        return True
+
+    def range_validation(self, min_value, max_value, current_value):
+        if current_value < min_value or current_value > max_value:
+            raise ConfigValidationFailedException(
+                field_name=self.field_name,
+                field_info=self._get_field(),
+                current_value=current_value,
+                message=f"Configuration field {self.field_name} has invalid value {current_value}. Value must be between {min_value} and {max_value}",
+            )
+        return True
+
+    def length_validation(self, max_length, current_value):
+        if len(current_value) > max_length:
+            raise ConfigValidationFailedException(
+                field_name=self.field_name,
+                field_info=self._get_field(),
+                current_value=current_value,
+                message=f"Configuration field {self.field_name} has invalid value {current_value}. Value must be less than {max_length}",
+            )
+        return True
+
+    def regex_validation(self, regex, current_value):
+        if not re.match(regex, current_value):
+            raise ConfigValidationFailedException(
+                field_name=self.field_name,
+                field_info=self._get_field(),
+                current_value=current_value,
+                message=f"Configuration field {self.field_name} has invalid value {current_value}. Value must match regex {regex}",
+            )
+        return True
+
+
 class ResourceConfig(metaclass=ConfigMeta):
     """Resource configuration for the app."""
 
+    # TODO: Add Unit Validation/Parsing Support for the Fields.
     cpu = ConfigField(
         default="1",
         cli_meta=CLIOption(
@@ -53,6 +186,8 @@ class ResourceConfig(metaclass=ConfigMeta):
         ),
         field_type=str,
         example="500m",
+        validation_fn=UnitParser.validation_wrapper_fn("cpu"),
+        parsing_fn=UnitParser("cpu").parse,
     )
     memory = ConfigField(
         default="4Gi",
@@ -63,6 +198,8 @@ class ResourceConfig(metaclass=ConfigMeta):
         ),
         field_type=str,
         example="512Mi",
+        validation_fn=UnitParser.validation_wrapper_fn("memory"),
+        parsing_fn=UnitParser("memory").parse,
     )
     gpu = ConfigField(
         cli_meta=CLIOption(
@@ -72,6 +209,8 @@ class ResourceConfig(metaclass=ConfigMeta):
         ),
         field_type=str,
         example="1",
+        validation_fn=UnitParser.validation_wrapper_fn("gpu"),
+        parsing_fn=UnitParser("gpu").parse,
     )
     disk = ConfigField(
         default="20Gi",
@@ -82,6 +221,8 @@ class ResourceConfig(metaclass=ConfigMeta):
         ),
         field_type=str,
         example="1Gi",
+        validation_fn=UnitParser.validation_wrapper_fn("disk"),
+        parsing_fn=UnitParser("disk").parse,
     )
 
 
@@ -156,14 +297,11 @@ class AuthConfig(metaclass=ConfigMeta):
 
     @staticmethod
     def validate(auth_config: "AuthConfig"):
-        if auth_config.type is not None and auth_config.type not in AuthType.choices():
-            raise ConfigValidationFailedException(
-                field_name="type",
-                field_info=auth_config._get_field("type"),
-                current_value=auth_config.type,
-                message=f"Invalid auth type: {auth_config.type}. Must be one of {AuthType.choices()}",
-            )
-        return True
+        if auth_config.type is None:
+            return True
+        return BasicValidations(AuthConfig, "type").enum_validation(
+            AuthType.choices(), auth_config.type
+        )
 
 
 class ReplicaConfig(metaclass=ConfigMeta):
@@ -226,17 +364,17 @@ class ReplicaConfig(metaclass=ConfigMeta):
         def _greater_than_equals_zero(x):
             return x is not None and x >= 0
 
-        if both_min_max_set and replica_config.min > replica_config.max:
+        if both_min_max_set and replica_config.min > replica_config.max:  # type: ignore
             raise ConfigValidationFailedException(
                 field_name="min",
-                field_info=replica_config._get_field("min"),
+                field_info=replica_config._get_field("min"),  # type: ignore
                 current_value=replica_config.min,
                 message="Min replicas cannot be greater than max replicas",
             )
         if fixed_set and any_min_max_set:
             raise ConfigValidationFailedException(
                 field_name="fixed",
-                field_info=replica_config._get_field("fixed"),
+                field_info=replica_config._get_field("fixed"),  # type: ignore
                 current_value=replica_config.fixed,
                 message="Fixed replicas cannot be set when min or max replicas are set",
             )
@@ -244,15 +382,15 @@ class ReplicaConfig(metaclass=ConfigMeta):
         if max_is_set and not min_is_set:
             raise ConfigValidationFailedException(
                 field_name="min",
-                field_info=replica_config._get_field("min"),
+                field_info=replica_config._get_field("min"),  # type: ignore
                 current_value=replica_config.min,
                 message="If max replicas is set then min replicas must be set too.",
             )
 
-        if fixed_set and replica_config.fixed < 0:
+        if fixed_set and replica_config.fixed < 0:  # type: ignore
             raise ConfigValidationFailedException(
                 field_name="fixed",
-                field_info=replica_config._get_field("fixed"),
+                field_info=replica_config._get_field("fixed"),  # type: ignore
                 current_value=replica_config.fixed,
                 message="Fixed replicas cannot be less than 0",
             )
@@ -260,7 +398,7 @@ class ReplicaConfig(metaclass=ConfigMeta):
         if min_is_set and not _greater_than_equals_zero(replica_config.min):
             raise ConfigValidationFailedException(
                 field_name="min",
-                field_info=replica_config._get_field("min"),
+                field_info=replica_config._get_field("min"),  # type: ignore
                 current_value=replica_config.min,
                 message="Min replicas cannot be less than 0",
             )
@@ -268,7 +406,7 @@ class ReplicaConfig(metaclass=ConfigMeta):
         if max_is_set and not _greater_than_equals_zero(replica_config.max):
             raise ConfigValidationFailedException(
                 field_name="max",
-                field_info=replica_config._get_field("max"),
+                field_info=replica_config._get_field("max"),  # type: ignore
                 current_value=replica_config.max,
                 message="Max replicas cannot be less than 0",
             )
@@ -346,7 +484,7 @@ class DependencyConfig(metaclass=ConfigMeta):
         ):
             raise ConfigValidationFailedException(
                 field_name="from_requirements_file",
-                field_info=dependency_config._get_field("from_requirements_file"),
+                field_info=dependency_config._get_field("from_requirements_file"),  # type: ignore
                 current_value=dependency_config.from_requirements_file,
                 message="Cannot set from_requirements_file and from_pyproject_toml at the same time",
             )
@@ -358,7 +496,7 @@ class DependencyConfig(metaclass=ConfigMeta):
         ):
             raise ConfigValidationFailedException(
                 field_name="pypi" if dependency_config.pypi else "conda",
-                field_info=dependency_config._get_field(
+                field_info=dependency_config._get_field(  # type: ignore
                     "pypi" if dependency_config.pypi else "conda"
                 ),
                 current_value=dependency_config.pypi or dependency_config.conda,
@@ -395,27 +533,17 @@ class BasicAppValidations:
     def name(name):
         if name is None:
             return True
-        if len(name) > 128:
-            raise ConfigValidationFailedException(
-                field_name="name",
-                field_info=CoreConfig._get_field(CoreConfig, "name"),
-                current_value=name,
-                message="Name cannot be longer than 128 characters",
-            )
-        return True
+        regex = r"^[a-z0-9-]+$"  # Only allow lowercase letters, numbers, and hyphens
+        validator = BasicValidations(CoreConfig, "name")
+        return validator.length_validation(20, name) and validator.regex_validation(
+            regex, name
+        )
 
     @staticmethod
     def port(port):
         if port is None:
             return True
-        if port < 1 or port > 65535:
-            raise ConfigValidationFailedException(
-                field_name="port",
-                field_info=CoreConfig._get_field(CoreConfig, "port"),
-                current_value=port,
-                message="Port must be between 1 and 65535",
-            )
-        return True
+        return BasicValidations(CoreConfig, "port").range_validation(1, 65535, port)
 
     @staticmethod
     def tags(tags):
@@ -424,7 +552,7 @@ class BasicAppValidations:
         if not all(isinstance(tag, dict) and len(tag) == 1 for tag in tags):
             raise ConfigValidationFailedException(
                 field_name="tags",
-                field_info=CoreConfig._get_field(CoreConfig, "tags"),
+                field_info=CoreConfig._get_field(CoreConfig, "tags"),  # type: ignore
                 current_value=tags,
                 message="Tags must be a list of dictionaries with one key. Currently they are set to %s "
                 % (str(tags)),
@@ -439,9 +567,10 @@ class BasicAppValidations:
         if not isinstance(secrets, list):
             raise ConfigValidationFailedException(
                 field_name="secrets",
-                field_info=CoreConfig._get_field(CoreConfig, "secrets"),
+                field_info=CoreConfig._get_field(CoreConfig, "secrets"),  # type: ignore
                 current_value=secrets,
-                message="Secrets must be a list of strings",
+                message="Secrets must be a list of strings. Currently they are set to %s "
+                % (str(secrets)),
             )
         from ..validations import secrets_validator
 
@@ -450,7 +579,7 @@ class BasicAppValidations:
         except Exception as e:
             raise ConfigValidationFailedException(
                 field_name="secrets",
-                field_info=CoreConfig._get_field(CoreConfig, "secrets"),
+                field_info=CoreConfig._get_field(CoreConfig, "secrets"),  # type: ignore
                 current_value=secrets,
                 message=f"Secrets validation failed, {e}",
             )
@@ -460,14 +589,9 @@ class BasicAppValidations:
     def persistence(persistence):
         if persistence is None:
             return True
-        if persistence not in ["none", "postgres"]:
-            raise ConfigValidationFailedException(
-                field_name="persistence",
-                field_info=CoreConfig._get_field(CoreConfig, "persistence"),
-                current_value=persistence,
-                message=f"Persistence must be one of: {['none', 'postgres']}",
-            )
-        return True
+        return BasicValidations(CoreConfig, "persistence").enum_validation(
+            ["none", "postgres"], persistence
+        )
 
 
 class CoreConfig(metaclass=ConfigMeta):
@@ -787,7 +911,7 @@ How to read this schema:
         merged_config = CoreConfig()
 
         # Process each field according to its behavior
-        for field_name, field_info in CoreConfig._fields.items():
+        for field_name, field_info in CoreConfig._fields.items():  # type: ignore
             base_value = getattr(base_config, field_name, None)
             override_value = getattr(override_config, field_name, None)
 
@@ -852,7 +976,7 @@ How to read this schema:
                 for v in value:
                     _env_dict.update(v)
                 return _env_dict
-            if type(value) == tuple:
+            if type(value) == tuple or type(value) == list:
                 obj = list(x for x in source_data[key])
                 if len(obj) == 0:
                     return None  # Dont return Empty Lists so that we can set Nones