ob-metaflow-extensions 1.1.45rc3__py2.py3-none-any.whl → 1.5.1__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py

@@ -0,0 +1,299 @@
+"""
+Schema Export Module for Unified Configuration System
+
+This module provides standalone functions to export configuration schemas in various formats:
+- OpenAPI schemas in YAML or JSON format
+- JSON schemas in YAML or JSON format
+
+Usage:
+    from schema_export import export_schema, to_openapi_schema, to_json_schema, to_dict
+
+    # Export schema to file
+    export_schema(CoreConfig, "schema.yaml")
+    export_schema(CoreConfig, "schema.json", schema_type="json", format="json")
+
+    # Generate schema in memory
+    openapi_schema = to_openapi_schema(CoreConfig)
+    json_schema = to_json_schema(CoreConfig)
+
+    # Export config instance to dict
+    config_instance = CoreConfig(name="myapp", port=8000)
+    config_dict = to_dict(config_instance)
+
+No external dependencies required for basic functionality.
+PyYAML is optional for YAML export support.
+"""
+
+import json
+import textwrap
+from collections import OrderedDict
+from typing import Any, Dict
+
+try:
+    import yaml
+
+    HAS_YAML = True
+except ImportError:
+    HAS_YAML = False
+
+
+def to_openapi_schema(config_class) -> Dict[str, Any]:
+    """Generate OpenAPI schema for a configuration class.
+
+    Args:
+        config_class: The configuration class to generate schema for
+
+    Returns:
+        OpenAPI schema dictionary
+    """
+    return _generate_openapi_schema(config_class)
+
+
+def to_json_schema(config_class) -> Dict[str, Any]:
+    """Generate JSON schema for a configuration class.
+
+    Args:
+        config_class: The configuration class to generate schema for
+
+    Returns:
+        JSON schema dictionary
+    """
+    return _generate_json_schema(config_class)
+
+
+def export_schema(
+    config_class, filepath: str, schema_type: str = "openapi", format: str = "yaml"
+) -> None:
+    """Export configuration schema to file.
+
+    Args:
+        config_class: The configuration class to export schema for
+        filepath: Path to save the schema file
+        schema_type: Type of schema to generate ('openapi' or 'json')
+        format: Output format ('yaml' or 'json')
+
+    Examples:
+        # Export OpenAPI schema as YAML (default)
+        export_schema(CoreConfig, "schema.yaml")
+
+        # Export JSON schema as YAML
+        export_schema(CoreConfig, "schema.yaml", schema_type="json")
+
+        # Export OpenAPI schema as JSON
+        export_schema(CoreConfig, "schema.json", schema_type="openapi", format="json")
+
+        # Export JSON schema as JSON
+        export_schema(CoreConfig, "schema.json", schema_type="json", format="json")
+    """
+    # Validate inputs
+    if schema_type.lower() not in ["openapi", "json"]:
+        raise ValueError(
+            f"Unsupported schema type: {schema_type}. Use 'openapi' or 'json'."
+        )
+
+    if format.lower() not in ["yaml", "json"]:
+        raise ValueError(f"Unsupported format: {format}. Use 'yaml' or 'json'.")
+
+    # Generate the appropriate schema
+    if schema_type.lower() == "openapi":
+        base_schema = _generate_openapi_schema(config_class)
+        # Wrap in OpenAPI document structure with proper ordering
+        schema_data = OrderedDict(
+            [
+                ("openapi", "3.0.0"),
+                (
+                    "info",
+                    OrderedDict(
+                        [
+                            ("title", f"{config_class.__name__} Configuration Schema"),
+                            ("version", "1.0.0"),
+                        ]
+                    ),
+                ),
+                (
+                    "components",
+                    OrderedDict(
+                        [
+                            (
+                                "schemas",
+                                OrderedDict([(config_class.__name__, base_schema)]),
+                            )
+                        ]
+                    ),
+                ),
+            ]
+        )
+    else:  # json schema
+        schema_data = _generate_json_schema(config_class)
+
+    # Export in the requested format
+    if format.lower() == "yaml":
+        if not HAS_YAML:
+            raise ImportError(
+                "PyYAML is required for YAML export. Install with: pip install pyyaml"
+            )
+
+        # Custom YAML representer for multiline strings
+        def multiline_representer(dumper, data):
+            if "\n" in data or len(data) > 80:
+                style = "|"  # use literal block
+            else:
+                style = None  # normal flow
+            return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style)
+
+        # Custom YAML representer for OrderedDict to preserve order
+        def ordered_dict_representer(dumper, data):
+            return dumper.represent_dict(data.items())
+
+        yaml.add_representer(str, multiline_representer)
+        yaml.add_representer(OrderedDict, ordered_dict_representer)
+
+        with open(filepath, "w") as f:
+            yaml.dump(
+                schema_data,
+                f,
+                default_flow_style=False,
+                sort_keys=False,
+                allow_unicode=True,
+                width=120,
+                indent=2,
+            )
+    else:  # json format
+        with open(filepath, "w") as f:
+            json.dump(schema_data, f, indent=2)
+
+
+# Private helper functions
+def _generate_openapi_schema(cls) -> Dict[str, Any]:
+    """Generate OpenAPI schema for a configuration class.
+
+    Note: Schema is intended for CLI/config file usage, so PROGRAMMATIC-only
+    fields are excluded.
+    """
+    # Clean up class docstring for better YAML formatting
+    description = f"{cls.__name__} configuration"
+    get_description = getattr(cls, "SCHEMA_DOC", None)
+    if get_description:
+        description = get_description
+    elif cls.__doc__:
+        # Remove common indentation and clean up whitespace
+        cleaned_doc = textwrap.dedent(cls.__doc__).strip()
+        # Replace multiple spaces with single spaces but preserve line breaks
+        lines = [line.strip() for line in cleaned_doc.split("\n") if line.strip()]
+        description = "\n".join(lines)
+
+    # Create ordered schema with specific order: title, description, type, required, then properties
+    schema = OrderedDict(
+        [
+            ("title", cls.__name__),
+            ("description", description),
+            ("type", "object"),
+            ("required", []),
+            ("properties", OrderedDict()),
+        ]
+    )
+
+    for field_name, field_info in cls._fields.items():
+        if field_name.startswith("_"):
+            continue
+
+        # Skip PROGRAMMATIC-only fields - schema is for CLI/config file usage
+        if (
+            hasattr(field_info, "is_available_in_cli")
+            and not field_info.is_available_in_cli()
+        ):
+            continue
+
+        field_schema = _get_field_schema(field_info)
+        schema["properties"][field_name] = field_schema
+
+        # Add to required if field is required
+        if field_info.required:
+            schema["required"].append(field_name)
+
+    return schema
+
+
+def _generate_json_schema(cls) -> Dict[str, Any]:
+    """Generate JSON schema for a configuration class."""
+    openapi_schema = _generate_openapi_schema(cls)
+
+    # Create ordered JSON schema with $schema first
+    schema = OrderedDict(
+        [
+            ("$schema", "https://json-schema.org/draft/2020-12/schema"),
+            ("title", openapi_schema["title"]),
+            ("description", openapi_schema["description"]),
+            ("type", openapi_schema["type"]),
+            ("required", openapi_schema["required"]),
+            ("properties", openapi_schema["properties"]),
+        ]
+    )
+
+    return schema
+
+
+def _get_field_schema(field_info) -> Dict[str, Any]:
+    """Generate schema for a single field."""
+    field_schema = OrderedDict()
+
+    # Get description from ConfigField.help first, then CLI metadata
+    description = field_info.help or (
+        field_info.cli_meta.help if field_info.cli_meta else None
+    )
+    if description:
+        field_schema["description"] = description
+
+    # Handle field type
+    if field_info.field_type == str:
+        field_schema["type"] = "string"
+    elif field_info.field_type == int:
+        field_schema["type"] = "integer"
+    elif field_info.field_type == float:
+        field_schema["type"] = "number"
+    elif field_info.field_type == bool:
+        field_schema["type"] = "boolean"
+    elif field_info.field_type == list:
+        field_schema["type"] = "array"
+        field_schema["items"] = {"type": "string"}  # Default to string items
+    elif field_info.field_type == dict:
+        field_schema["type"] = "object"
+        field_schema["additionalProperties"] = True
+    elif hasattr(field_info.field_type, "_fields"):
+        # Nested configuration object
+        field_schema = _generate_openapi_schema(field_info.field_type)
+    else:
+        # Fallback to string for unknown types
+        field_schema["type"] = "string"
+
+    # Add default value
+    if field_info.default is not None and not callable(field_info.default):
+        field_schema["default"] = field_info.default
+
+    # Handle choices from CLI metadata
+    if field_info.cli_meta and field_info.cli_meta.choices:
+        if "type" in field_schema:
+            field_schema["enum"] = field_info.cli_meta.choices
+
+    # Add examples from ConfigField
+    if field_info.example is not None:
+        field_schema["example"] = field_info.example
+
+    # Add experimental flag
+    if field_info.is_experimental:
+        field_schema["experimental"] = True
+
+    # Add Field Behavior
+    if field_info.behavior:
+        field_schema["mutation_behavior"] = field_info.behavior
+
+    if field_info.cli_meta is not None:
+        field_schema["cli_option"] = field_info.cli_meta.cli_option_str
+
+    # Handle validation from CLI metadata
+    if field_info.cli_meta and field_info.validation_fn:
+        # Add validation hints in description
+        if "description" in field_schema:
+            field_schema["description"] += " (validation applied)"
+
+    return field_schema

metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py

@@ -0,0 +1,233 @@
+"""
+Auto-generated typed classes for ConfigMeta classes.
+
+This module provides IDE-friendly typed interfaces for all configuration classes.
+The reason we auto-generate this file is because we want to provide a bridge between what is the ConfigMeta classes and the typed programmatic interface.
+The CoreConfig class is setup in a way that if any additionally params are missed out from being auto-generated then it will not affect the core functionality of the programmatic API.
+The new parameters will just not show up in IDE autocompletions.
+It is fine if this file is not regularly updated by running the script in the .pre-commit-config.app-changes.yaml
+but it is recommended that this file not be deleted or manually edited.
+
+"""
+
+from typing import Optional, List, Dict, Any
+from .unified_config import CoreConfig
+
+import sys
+from typing import TYPE_CHECKING
+
+# on 3.8+ use the stdlib TypedDict;
+# in TYPE_CHECKING blocks mypy/pyright still pick it up on older Pythons
+if sys.version_info >= (3, 8):
+    from typing import TypedDict
+else:
+    if TYPE_CHECKING:
+        # for the benefit of type-checkers
+        from typing import TypedDict  # noqa: F401
+    # runtime no-op TypedDict shim
+    class _TypedDictMeta(type):
+        def __new__(cls, name, bases, namespace, total=True):
+            # ignore total at runtime
+            return super().__new__(cls, name, bases, namespace)
+
+    class TypedDict(dict, metaclass=_TypedDictMeta):
+        # Runtime stand-in for typing.TypedDict on <3.8.
+        pass
+
+
+class ResourceConfigDict(TypedDict, total=False):
+    cpu: Optional[str]
+    memory: Optional[str]
+    gpu: Optional[str]
+    disk: Optional[str]
+    shared_memory: Optional[str]
+
+
+class AuthConfigDict(TypedDict, total=False):
+    type: Optional[str]
+    public: Optional[bool]
+
+
+class ReplicaConfigDict(TypedDict, total=False):
+    fixed: Optional[int]
+    min: Optional[int]
+    max: Optional[int]
+    scaling_policy: Optional["ScalingPolicyConfigDict"]
+
+
+class ScalingPolicyConfigDict(TypedDict, total=False):
+    rpm: Optional[int]
+
+
+class DependencyConfigDict(TypedDict, total=False):
+    from_requirements_file: Optional[str]
+    from_pyproject_toml: Optional[str]
+    python: Optional[str]
+    pypi: Optional[dict]
+    conda: Optional[dict]
+
+
+class PackageConfigDict(TypedDict, total=False):
+    src_paths: Optional[list]
+    suffixes: Optional[list]
+
+
+class TypedCoreConfig:
+    """
+    Parameters
+    ----------
+    name : str, optional
+        The name of the app to deploy.
+
+    port : int, optional
+        Port where the app is hosted. When deployed this will be port on which we will deploy the app.
+
+    description : str, optional
+        The description of the app to deploy.
+
+    app_type : str, optional
+        The User defined type of app to deploy. Its only used for bookkeeping purposes.
+
+    image : str, optional
+        The Docker image to deploy with the App.
+
+    tags : list, optional
+        The tags of the app to deploy.
+
+    secrets : list, optional
+        Outerbounds integrations to attach to the app. You can use the value you set in the `@secrets` decorator in your code.
+
+    compute_pools : list, optional
+        A list of compute pools to deploy the app to.
+
+    environment : dict, optional
+        Environment variables to deploy with the App.
+
+    commands : list, optional
+        A list of commands to run the app with.
+
+    resources : ResourceConfigDict, optional
+        Resource configuration for the app.
+            - cpu (str)
+                CPU requests
+            - memory (str)
+                Memory requests
+            - gpu (str)
+                GPU requests
+            - disk (str)
+                Storage disk size.
+            - shared_memory (str)
+                Shared memory
+
+    auth : AuthConfigDict, optional
+        Auth related configurations.
+            - type (str)
+                The type of authentication to use for the app.
+            - public (bool)
+                Whether the app is public or not.
+
+    replicas : ReplicaConfigDict, optional
+        The number of replicas to deploy the app with.
+            - fixed (int)
+                The fixed number of replicas to deploy the app with. If min and max are set, this will raise an error.
+            - min (int)
+                The minimum number of replicas to deploy the app with.
+            - max (int)
+                The maximum number of replicas to deploy the app with.
+            - scaling_policy (ScalingPolicyConfigDict)
+                Scaling policy defines the the metric based on which the replicas will horizontally scale. If min and max replicas are set and are not the same, then a scaling policy will be applied. Default scaling policies can be 60 rpm (ie 1 rps).
+                - rpm (int)
+                    Scale up replicas when the requests per minute crosses this threshold. If nothing is provided and the replicas.max and replicas.min is set then the default rpm would be 60.
+
+    code_package : tuple, optional
+        Pre-packaged code from package_code(). A PackagedCode namedtuple containing url and key.
+
+    force_upgrade : bool, optional
+        Force upgrade the app even if it is currently being upgraded.
+
+    persistence : str, optional
+        The persistence mode to deploy the app with.
+        [Experimental] May change in the future.
+
+    project : str, optional
+        The project name to deploy the app to.
+        [Experimental] May change in the future.
+
+    branch : str, optional
+        The branch name to deploy the app to.
+        [Experimental] May change in the future.
+
+    models : list, optional
+        [Experimental] May change in the future.
+
+    data : list, optional
+        [Experimental] May change in the future.
+
+    generate_static_url : bool, optional
+        Generate a static URL for the app based on its name.
+    """
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        port: Optional[int] = None,
+        description: Optional[str] = None,
+        app_type: Optional[str] = None,
+        image: Optional[str] = None,
+        tags: Optional[list] = None,
+        secrets: Optional[list] = None,
+        compute_pools: Optional[list] = None,
+        environment: Optional[dict] = None,
+        commands: Optional[list] = None,
+        resources: Optional[ResourceConfigDict] = None,
+        auth: Optional[AuthConfigDict] = None,
+        replicas: Optional[ReplicaConfigDict] = None,
+        code_package: Optional[tuple] = None,
+        force_upgrade: Optional[bool] = None,
+        persistence: Optional[str] = None,
+        project: Optional[str] = None,
+        branch: Optional[str] = None,
+        models: Optional[list] = None,
+        data: Optional[list] = None,
+        generate_static_url: Optional[bool] = None,
+        **kwargs
+    ) -> None:
+        self._kwargs = {
+            "name": name,
+            "port": port,
+            "description": description,
+            "app_type": app_type,
+            "image": image,
+            "tags": tags,
+            "secrets": secrets,
+            "compute_pools": compute_pools,
+            "environment": environment,
+            "commands": commands,
+            "resources": resources,
+            "auth": auth,
+            "replicas": replicas,
+            "code_package": code_package,
+            "force_upgrade": force_upgrade,
+            "persistence": persistence,
+            "project": project,
+            "branch": branch,
+            "models": models,
+            "data": data,
+            "generate_static_url": generate_static_url,
+        }
+        # Add any additional kwargs
+        self._kwargs.update(kwargs)
+        # Remove None values
+        self._kwargs = {k: v for k, v in self._kwargs.items() if v is not None}
+        self._config_class = CoreConfig
+        self._config = self.create_config()
+        self._init()
+
+    def create_config(self) -> CoreConfig:
+        return CoreConfig.from_dict(self._kwargs)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return self._config.to_dict()
+
+    def _init(self):
+        raise NotImplementedError