ob-metaflow-extensions 1.1.130__py2.py3-none-any.whl → 1.5.1__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/core/code_package/examples.py

@@ -0,0 +1,125 @@
+"""
+Examples demonstrating how to use the code packager abstraction.
+
+This file provides usage examples for the code packager classes.
+These examples are for documentation purposes and are not meant to be run directly.
+"""
+
+import os
+import sys
+from io import BytesIO
+from typing import List, Dict, Any, Callable, Optional
+
+from .code_packager import CodePackager
+
+
+def basic_usage_example(datastore_type: str = "s3") -> None:
+    """
+    Basic usage example with ContentAddressedStore.
+
+    This example shows how to:
+    1. Define paths to include in a package
+    2. Create a package using CodePackager's default packaging
+    3. Store the package using ContentAddressedStore directly
+    4. Generate a download command
+
+    Parameters
+    ----------
+    datastore_type : str, default "s3"
+        The type of datastore to use: "s3", "azure", "gs", or "local"
+    """
+    # Define the paths to include in the package
+    paths_to_include = ["./"]
+
+    # Define which file suffixes to include
+    file_suffixes = [".py", ".md"]
+
+    # Create metadata for the package
+    metadata = {"example": True, "timestamp": "2023-01-01T00:00:00Z"}
+
+    # Initialize the packager with datastore configuration
+    packager = CodePackager(
+        datastore_type=datastore_type,
+        code_package_prefix="my-custom-prefix",  # Optional
+    )
+
+    # Store the package with packaging parameters
+    package_url, package_key = packager.store(
+        paths_to_include=paths_to_include,
+        file_suffixes=file_suffixes,
+        metadata=metadata,
+    )
+
+    # Generate a download command
+    download_cmd = CodePackager.get_download_cmd(
+        package_url=package_url,
+        datastore_type=datastore_type,
+        target_file="my_package.tar",
+    )
+
+    # Generate complete package commands for downloading and setup
+    package_commands = packager.get_package_commands(
+        code_package_url=package_url,
+        target_file="my_package.tar",
+        working_dir="my_app",
+    )
+
+    # Print some information
+    print(f"Package URL: {package_url}")
+    print(f"Package Key: {package_key}")
+    print(f"Download Command: {download_cmd}")
+    print(f"Complete package commands: {package_commands}")
+
+
+def usage_with_custom_package_function(datastore_type: str = "s3") -> None:
+    """
+    Example of using the CodePackager with a custom package creation function.
+
+    Parameters
+    ----------
+    datastore_type : str, default "s3"
+        The type of datastore to use: "s3", "azure", "gs", or "local"
+    """
+
+    # Define a custom package function
+    def create_custom_package():
+        # This is a simple example - in real use, you might create a more complex package
+        from io import BytesIO
+        import tarfile
+        import time
+
+        buf = BytesIO()
+        with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+            # Add a simple file to the tarball
+            content = b"print('Hello, custom package!')"
+            info = tarfile.TarInfo(name="hello.py")
+            info.size = len(content)
+            info.mtime = int(time.time())
+            file_object = BytesIO(content)
+            tar.addfile(info, file_object)
+
+        return bytearray(buf.getvalue())
+
+    # Initialize the packager with datastore configuration
+    packager = CodePackager(datastore_type=datastore_type)
+
+    # Store the package with the custom package function
+    package_url, package_key = packager.store(package_create_fn=create_custom_package)
+
+    # Generate a download command
+    download_cmd = CodePackager.get_download_cmd(
+        package_url=package_url,
+        datastore_type=datastore_type,
+        target_file="custom_package.tar",
+    )
+
+    # Generate complete package commands
+    package_commands = packager.get_package_commands(
+        code_package_url=package_url,
+    )
+
+    # Print some information
+    print(f"Package URL: {package_url}")
+    print(f"Package Key: {package_key}")
+    print(f"Download Command: {download_cmd}")
+    print(f"Complete commands: {package_commands}")

metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py

@@ -0,0 +1,15 @@
+from .unified_config import CoreConfig, PackagedCode, BakedImage, AuthType
+from .cli_generator import auto_cli_options
+from .config_utils import (
+    PureStringKVPairType,
+    JsonFriendlyKeyValuePairType,
+    CommaSeparatedListType,
+    MergingNotAllowedFieldsException,
+    ConfigValidationFailedException,
+    RequiredFieldMissingException,
+    ConfigFieldContext,
+    ConfigField,
+    FieldBehavior,
+)
+from . import schema_export
+from .typed_configs import TypedCoreConfig, TypedDict

metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py

@@ -0,0 +1,165 @@
+"""
+CLI Generator for Unified Configuration System
+
+This module automatically generates Click CLI options from the CoreConfig,
+eliminating the need for manual CLI option definitions and ensuring consistency
+between configuration structure and CLI interface.
+
+It also provides machinery for merging configurations from different sources
+(CLI options, config files) with proper precedence and behavior handling.
+"""
+
+from typing import Any, List, Optional
+import json
+
+from ..click_importer import click
+from .unified_config import (
+    CoreConfig,
+    CLIOption,
+    ConfigMeta,
+)
+from .config_utils import (
+    PureStringKVPairType,
+    JsonFriendlyKeyValuePairType,
+    CommaSeparatedListType,
+)
+
+
+class CLIGenerator:
+    """Generates Click CLI options from CoreConfig dataclass."""
+
+    def __init__(self, config_class: type = CoreConfig):
+        self.config_class = config_class
+        self._type_mapping = {
+            str: str,
+            int: int,
+            float: float,
+            bool: bool,
+            list: CommaSeparatedListType,
+            dict: JsonFriendlyKeyValuePairType,
+        }
+
+    def generate_options(self):
+        """Generate all CLI options from the configuration class."""
+        options = []
+
+        # Generate options for all fields automatically
+        options.extend(self._generate_all_options(self.config_class))
+
+        return options
+
+    def _generate_all_options(self, config_class: type):
+        """Generate all options from a config class. Returns a list of click.Options"""
+
+        def _options_from_cfg_cls(_config_class):
+            options = []
+            for field_name, field_info in _config_class._fields.items():
+                # Skip fields not available in CLI context
+                if not field_info.is_available_in_cli():
+                    continue
+
+                if ConfigMeta.is_instance(field_info.field_type):
+                    _subfield_options = _options_from_cfg_cls(field_info.field_type)
+                    options.extend(_subfield_options)
+                    continue
+
+                cli_meta = field_info.cli_meta
+                if not cli_meta or cli_meta.hidden:
+                    continue
+
+                option = self._create_option(field_name, field_info, cli_meta)
+                if option:
+                    options.append(option)
+            return options
+
+        return _options_from_cfg_cls(config_class)
+
+    def _create_option(self, field_name: str, field_info, cli_meta: CLIOption):
+        """Create a Click option from field info and CLI metadata."""
+        # Use the cli_option_str from the CLIOption
+        option_str = cli_meta.cli_option_str
+        param_name = cli_meta.name
+
+        # Determine Click type
+        click_type = self._get_click_type(field_info, cli_meta)
+
+        # Build option parameters
+        help_text = cli_meta.help or field_info.help or f"Set {field_name}"
+        option_params = {
+            "help": help_text,
+            "default": cli_meta.default if cli_meta.default is not None else None,
+            "type": click_type,
+        }
+
+        # Handle multiple values
+        if cli_meta.multiple:
+            option_params["multiple"] = True
+
+        # Handle choices
+        if cli_meta.choices:
+            option_params["type"] = click.Choice(cli_meta.choices)
+
+        # Handle flags
+        if cli_meta.is_flag:
+            option_params["is_flag"] = True
+            option_params.pop("type", None)
+
+        # Handle special flag patterns (e.g., --public-access/--private-access)
+        return click.option(option_str, param_name, **option_params)
+
+    def _get_click_type(self, field_info, cli_meta: CLIOption) -> Any:
+        """Determine the appropriate Click type for a field."""
+        if cli_meta.click_type:
+            return cli_meta.click_type
+
+        # Get the field type
+        field_type = field_info.field_type
+
+        # Handle basic types
+        if field_type == list:
+            return CommaSeparatedListType
+        elif field_type == dict:
+            return JsonFriendlyKeyValuePairType
+        elif field_type == str:
+            return str
+        elif field_type == int:
+            return int
+        elif field_type == bool:
+            return bool
+        elif field_type == float:
+            return float
+
+        # Handle custom config types
+        if hasattr(field_type, "__name__") and field_type.__name__.endswith("Config"):
+            return str  # Default to string for complex types
+
+        # Use type mapping
+        return self._type_mapping.get(field_type, str)
+
+    def create_decorator(self, command_type: str = "deploy") -> callable:
+        """Create a decorator that applies all CLI options to a command."""
+
+        def decorator(func):
+            # Apply options in reverse order since decorators are applied bottom-up
+            for option in reversed(self.generate_options()):
+                func = option(func)
+            return func
+
+        return decorator
+
+
+def auto_cli_options(config_class: type = CoreConfig, command_type: str = "deploy"):
+    """
+    Decorator that automatically adds CLI options from CoreConfig.
+
+    Args:
+        command_type: Type of command (e.g., "deploy", "list", "delete")
+
+    Usage:
+        @auto_cli_options("deploy")
+        def deploy_command(**kwargs):
+            config = CoreConfig.from_cli(kwargs)
+            # ... use config
+    """
+    generator = CLIGenerator(config_class)
+    return generator.create_decorator(command_type)