ob-metaflow-extensions 1.1.175rc1__py2.py3-none-any.whl → 1.1.175rc3__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/__init__.py

@@ -339,7 +339,7 @@ STEP_DECORATORS_DESC = [
     ("nim", ".nim.nim_decorator.NimDecorator"),
     ("ollama", ".ollama.OllamaDecorator"),
     ("vllm", ".vllm.VLLMDecorator"),
-    ("app_deploy", ".apps.deploy_decorator.WorkstationAppDeployDecorator"),
+    ("app_deploy", ".apps.app_deploy_decorator.AppDeployDecorator"),
 ]
 
 TOGGLE_STEP_DECORATOR = [

metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py

@@ -0,0 +1,112 @@
+from metaflow.exception import MetaflowException
+from metaflow.decorators import StepDecorator
+from metaflow import current
+from .core import AppDeployer, apps
+from .core.perimeters import PerimeterExtractor
+import os
+import hashlib
+
+
+class AppDeployDecorator(StepDecorator):
+
+    """
+    MF Add To Current
+    -----------------
+    apps -> metaflow_extensions.outerbounds.plugins.apps.core.apps
+
+        @@ Returns
+        ----------
+        apps
+            The object carrying the Deployer class to deploy apps.
+    """
+
+    name = "app_deploy"
+    defaults = {}
+
+    package_url = None
+    package_sha = None
+
+    def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
+        self.logger = logger
+        self.environment = environment
+        self.step = step
+        self.flow_datastore = flow_datastore
+
+    def _resolve_package_url_and_sha(self):
+        return os.environ.get("METAFLOW_CODE_URL", self.package_url), os.environ.get(
+            "METAFLOW_CODE_SHA", self.package_sha
+        )
+
+    def task_pre_step(
+        self,
+        step_name,
+        task_datastore,
+        metadata,
+        run_id,
+        task_id,
+        flow,
+        graph,
+        retry_count,
+        max_user_code_retries,
+        ubf_context,
+        inputs,
+    ):
+        perimeter, api_server = PerimeterExtractor.during_metaflow_execution()
+        package_url, package_sha = self._resolve_package_url_and_sha()
+        if package_url is None or package_sha is None:
+            raise MetaflowException(
+                "METAFLOW_CODE_URL or METAFLOW_CODE_SHA is not set. "
+                "Please set METAFLOW_CODE_URL and METAFLOW_CODE_SHA in your environment."
+            )
+        default_name = "-".join(current.pathspec.split("/")).lower()
+        image = os.environ.get("FASTBAKERY_IMAGE", None)
+
+        hash_key = hashlib.sha256(package_url.encode()).hexdigest()[:6]
+
+        default_name = (
+            (current.flow_name + "-" + current.step_name)[:12] + "-" + hash_key
+        ).lower()
+
+        AppDeployer._set_state(
+            perimeter,
+            api_server,
+            code_package_url=package_url,
+            code_package_key=package_sha,
+            name=default_name,
+            image=image,
+        )
+        current._update_env(
+            {
+                "apps": apps(),
+            }
+        )
+
+    def task_post_step(
+        self, step_name, flow, graph, retry_count, max_user_code_retries
+    ):
+        pass
+
+    def runtime_init(self, flow, graph, package, run_id):
+        # Set some more internal state.
+        self.flow = flow
+        self.graph = graph
+        self.package = package
+        self.run_id = run_id
+
+    def runtime_task_created(
+        self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context
+    ):
+        # To execute the Kubernetes job, the job container needs to have
+        # access to the code package. We store the package in the datastore
+        # which the pod is able to download as part of it's entrypoint.
+        if not is_cloned:
+            self._save_package_once(self.flow_datastore, self.package)
+
+    @classmethod
+    def _save_package_once(cls, flow_datastore, package):
+        if cls.package_url is None:
+            cls.package_url, cls.package_sha = flow_datastore.save_data(
+                [package.blob], len_hint=1
+            )[0]
+            os.environ["METAFLOW_CODE_URL"] = cls.package_url
+            os.environ["METAFLOW_CODE_SHA"] = cls.package_sha

metaflow_extensions/outerbounds/plugins/apps/core/__init__.py

@@ -1 +1,10 @@
 from . import app_cli
+from . import config
+from .deployer import AppDeployer, apps
+from .config.typed_configs import (
+    ReplicaConfigDict,
+    ResourceConfigDict,
+    AuthConfigDict,
+    DependencyConfigDict,
+    PackageConfigDict,
+)

metaflow_extensions/outerbounds/plugins/apps/core/app_cli.py

@@ -28,13 +28,11 @@ from .app_config import (
     CAPSULE_DEBUG,
     AuthType,
 )
+from .config import auto_cli_options, CoreConfig
 from .perimeters import PerimeterExtractor
-from .cli_to_config import build_config_from_options
 from .utils import (
     MultiStepSpinner,
 )
-from . import experimental
-from .validations import deploy_validations
 from .code_package import CodePackager
 from .capsule import (
     CapsuleDeployer,
@@ -101,101 +99,6 @@ class KeyValuePair(click.ParamType):
         return "KV-PAIR"
 
 
-class MountMetaflowArtifact(click.ParamType):
-    name = "MOUNT-METAFLOW-ARTIFACT"  # type: ignore
-
-    def convert(self, value, param, ctx):
-        """
-        Convert a string like "flow=MyFlow,artifact=my_model,path=/tmp/abc" or
-        "pathspec=MyFlow/123/foo/345/my_model,path=/tmp/abc" to a dict.
-        """
-        artifact_dict = {}
-        parts = value.split(",")
-
-        for part in parts:
-            if "=" not in part:
-                self.fail(
-                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
-                )
-
-            key, val = part.split("=", 1)
-            artifact_dict[key.strip()] = val.strip()
-
-        # Validate required fields
-        if "pathspec" in artifact_dict:
-            if "path" not in artifact_dict:
-                self.fail(
-                    "When using 'pathspec', you must also specify 'path'", param, ctx
-                )
-
-            # Return as pathspec format
-            return {
-                "pathspec": artifact_dict["pathspec"],
-                "path": artifact_dict["path"],
-            }
-        elif (
-            "flow" in artifact_dict
-            and "artifact" in artifact_dict
-            and "path" in artifact_dict
-        ):
-            # Return as flow/artifact format
-            result = {
-                "flow": artifact_dict["flow"],
-                "artifact": artifact_dict["artifact"],
-                "path": artifact_dict["path"],
-            }
-
-            # Add optional namespace if provided
-            if "namespace" in artifact_dict:
-                result["namespace"] = artifact_dict["namespace"]
-
-            return result
-        else:
-            self.fail(
-                "Invalid format. Must be either 'flow=X,artifact=Y,path=Z' or 'pathspec=X,path=Z'",
-                param,
-                ctx,
-            )
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "MOUNT-METAFLOW-ARTIFACT"
-
-
-class MountSecret(click.ParamType):
-    name = "MOUNT-SECRET"  # type: ignore
-
-    def convert(self, value, param, ctx):
-        """
-        Convert a string like "id=my_secret,path=/tmp/secret" to a dict.
-        """
-        secret_dict = {}
-        parts = value.split(",")
-
-        for part in parts:
-            if "=" not in part:
-                self.fail(
-                    f"Invalid format in part '{part}'. Expected 'key=value'", param, ctx
-                )
-
-            key, val = part.split("=", 1)
-            secret_dict[key.strip()] = val.strip()
-
-        # Validate required fields
-        if "id" in secret_dict and "path" in secret_dict:
-            return {"id": secret_dict["id"], "path": secret_dict["path"]}
-        else:
-            self.fail("Invalid format. Must be 'key=X,path=Y'", param, ctx)
-
-    def __str__(self):
-        return repr(self)
-
-    def __repr__(self):
-        return "MOUNT-SECRET"
-
-
 class CommaSeparatedList(click.ParamType):
     name = "COMMA-SEPARATED-LIST"  # type: ignore
 
@@ -209,11 +112,9 @@ class CommaSeparatedList(click.ParamType):
         return "COMMA-SEPARATED-LIST"
 
 
-KVPairType = KeyValuePair()
-MetaflowArtifactType = MountMetaflowArtifact()
-SecretMountType = MountSecret()
-CommaSeparatedListType = CommaSeparatedList()
-KVDictType = KeyValueDictPair()
+KVPairType = KeyValuePair()  # used for --tag and --env
+CommaSeparatedListType = CommaSeparatedList()  # used for --compute-pools
+KVDictType = KeyValueDictPair()  # only Used for the list/delete commands for tags
 
 
 class ColorTheme:
@@ -224,10 +125,13 @@ class ColorTheme:
     DEBUG_COLOR = "yellow"
 
     TL_HEADER_COLOR = "magenta"
-    ROW_COLOR = "bright_white"
+    # Use a color that is readable in both light and dark mode.
+    # "white" can be hard to see in light mode, so use "black" for rows.
+    # Alternatively, "reset" (default terminal color) is safest.
+    ROW_COLOR = "reset"
 
     INFO_KEY_COLOR = "green"
-    INFO_VALUE_COLOR = "bright_white"
+    INFO_VALUE_COLOR = "reset"
 
 
 NativeList = list
@@ -293,7 +197,6 @@ def _pre_create_debug(
     capsule: CapsuleDeployer,
     state_dir: str,
     options: Dict[str, Any],
-    cli_parsed_config: Dict[str, Any],
 ):
     if CAPSULE_DEBUG:
         os.makedirs(state_dir, exist_ok=True)
@@ -309,7 +212,6 @@ def _pre_create_debug(
                         "capsule_input": capsule.create_input(),  # This is the input that is passed to the capsule deploy API
                         "deploy_response": capsule._capsule_deploy_response,  # This is the response from the capsule deploy API
                         "cli_options": options,  # These are the actual options passing down to the CLI
-                        "cli_parsed_config": cli_parsed_config,  # This is the config object that is created after parsing options from the CLI
                     },
                     indent=2,
                     default=str,
@@ -404,25 +306,18 @@ def app(ctx):
     os.makedirs(ctx.obj.app_state_dir, exist_ok=True)
 
 
-def parse_commands(app_config: AppConfig, cli_command_input):
+def parse_cli_commands(cli_command_input):
     # There can be two modes:
     # 1. User passes command via `--` in the CLI
     # 2. User passes the `commands` key in the config.
+    # This function parses the command for mode 1.
     base_commands = []
     if len(cli_command_input) > 0:
-        # TODO: we can be a little more fancy here by allowing the user to just call
-        #       `outerbounds app deploy -- foo.py` and figure out if we need to stuff python
-        #       in front of the command or not. But for sake of dumb simplicity, we can just
-        #       assume what ever the user called on local needs to be called remotely, we can
-        #       just ask them to add the outerbounds command in front of it.
-        #       So the dev ex would be :
-        #       `python foo.py` -> `outerbounds app deploy -- python foo.py`
         if type(cli_command_input) == str:
             base_commands.append(cli_command_input)
         else:
             base_commands.append(shlex.join(cli_command_input))
-    elif app_config.get("commands", None) is not None:
-        base_commands.extend(app_config.get("commands"))
+
     return base_commands
 
 
@@ -466,194 +361,6 @@ def deployment_instance_options(func):
     return wrapper
 
 
-def common_deploy_options(func):
-    @click.option(
-        "--name",
-        type=str,
-        help="The name of the app to deploy.",
-    )
-    @click.option("--port", type=int, help="Port where the app is hosted.")
-    @click.option(
-        "--tag",
-        "tags",
-        multiple=True,
-        type=KVPairType,
-        help="The tags of the app to deploy. Format KEY=VALUE. Example --tag foo=bar --tag x=y",
-        default=None,
-    )
-    @click.option(
-        "--image",
-        type=str,
-        help="The Docker image to deploy with the App",
-        default=None,
-    )
-    @click.option(
-        "--cpu",
-        type=str,
-        help="CPU resource request and limit",
-        default=None,
-    )
-    @click.option(
-        "--memory",
-        type=str,
-        help="Memory resource request and limit",
-        default=None,
-    )
-    @click.option(
-        "--gpu",
-        type=str,
-        help="GPU resource request and limit",
-        default=None,
-    )
-    @click.option(
-        "--disk",
-        type=str,
-        help="Storage resource request and limit",
-        default=None,
-    )
-    @click.option(
-        "--health-check-enabled",
-        type=bool,
-        help="Enable health checks",
-        default=None,
-    )
-    @click.option(
-        "--health-check-path",
-        type=str,
-        help="Health check path",
-        default=None,
-    )
-    @click.option(
-        "--health-check-initial-delay",
-        type=int,
-        help="Initial delay seconds for health check",
-        default=None,
-    )
-    @click.option(
-        "--health-check-period",
-        type=int,
-        help="Period seconds for health check",
-        default=None,
-    )
-    @click.option(
-        "--compute-pools",
-        type=CommaSeparatedListType,
-        help="The compute pools to deploy the app to. Example: --compute-pools default,large",
-        default=None,
-    )
-    @click.option(
-        "--auth-type",
-        type=click.Choice(AuthType.enums()),
-        help="The type of authentication to use for the app.",
-        default=None,
-    )
-    @click.option(
-        "--public-access/--private-access",
-        "auth_public",
-        type=bool,
-        help="Whether the app is public or not.",
-        default=None,
-    )
-    @click.option(
-        "--no-deps",
-        is_flag=True,
-        help="Do not any dependencies. Directly used the image provided",
-        default=False,
-    )
-    @click.option(
-        "--min-replicas",
-        type=int,
-        help="Minimum number of replicas to deploy",
-        default=None,
-    )
-    @click.option(
-        "--max-replicas",
-        type=int,
-        help="Maximum number of replicas to deploy",
-        default=None,
-    )
-    @click.option(
-        "--description",
-        type=str,
-        help="The description of the app to deploy.",
-        default=None,
-    )
-    @click.option(
-        "--app-type",
-        type=str,
-        help="The type of app to deploy.",
-        default=None,
-    )
-    @click.option(
-        "--force-upgrade",
-        is_flag=True,
-        help="Force upgrade the app even if it is currently being upgraded.",
-        default=False,
-    )
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-def common_run_options(func):
-    """Common options for running and deploying apps."""
-
-    @click.option(
-        "--config-file",
-        type=str,
-        help="The config file to use for the App (YAML or JSON)",
-        default=None,
-    )
-    @click.option(
-        "--secret",
-        "secrets",
-        multiple=True,
-        type=str,
-        help="Secrets to deploy with the App",
-        default=None,
-    )
-    @click.option(
-        "--env",
-        "envs",
-        multiple=True,
-        type=KVPairType,
-        help="Environment variables to deploy with the App. Use format KEY=VALUE",
-        default=None,
-    )
-    @click.option(
-        "--package-src-path",
-        type=str,
-        help="The path to the source code to deploy with the App.",
-        default=None,
-    )
-    @click.option(
-        "--package-suffixes",
-        type=CommaSeparatedListType,
-        help="The suffixes of the source code to deploy with the App.",
-        default=None,
-    )
-    @click.option(
-        "--dep-from-requirements",
-        type=str,
-        help="Path to requirements.txt file for dependencies",
-        default=None,
-    )
-    @click.option(
-        "--dep-from-pyproject",
-        type=str,
-        help="Path to pyproject.toml file for dependencies",
-        default=None,
-    )
-    # TODO: [FIX ME]: Get better CLI abstraction for pypi/conda dependencies
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
 def _package_necessary_things(app_config: AppConfig, logger):
     # Packaging has a few things to be thought through:
     #   1. if `entrypoint_path` exists then should we package the directory
@@ -692,10 +399,14 @@ def _package_necessary_things(app_config: AppConfig, logger):
 
 
 @app.command(help="Deploy an app to the Outerbounds Platform.")
-@common_deploy_options
-@common_run_options
+@click.option(
+    "--config-file",
+    type=str,
+    help="The config file to use for the App (YAML or JSON)",
+    default=None,
+)
 @deployment_instance_options
-@experimental.wrapping_cli_options
+@auto_cli_options()
 @click.pass_context
 @click.argument("command", nargs=-1, type=click.UNPROCESSED, required=False)
 def deploy(
@@ -715,8 +426,10 @@ def deploy(
         raise AppConfigError("OB_CURRENT_PERIMETER is not set")
     _current_instance_debug_dir = None
     logger = partial(_logger, timestamp=True)
+
+    base_commands = parse_cli_commands(command)
+    options["commands"] = base_commands
     try:
-        _cli_parsed_config = build_config_from_options(options)
         # Create configuration
         if options["config_file"]:
             # Load from file
@@ -726,10 +439,10 @@ def deploy(
             app_config.update_from_cli_options(options)
         else:
             # Create from CLI options
-            app_config = AppConfig(_cli_parsed_config)
+            app_config = AppConfig.from_cli(options)
 
         # Validate the configuration
-        app_config.validate()
+        app_config.commit()
         logger(
             f"🚀 Deploying {app_config.get('name')} to the Outerbounds platform...",
             color=ColorTheme.INFO_COLOR,
@@ -754,23 +467,23 @@ def deploy(
         logger(
             "📦 Packaging Directory : %s" % app_config.get_state("packaging_directory"),
         )
-        # TODO: Construct the command needed to run the app
-        #       If we are constructing the directory with the src_path
-        #       then we need to add the command from the option otherwise
-        #       we use the command from the entrypoint path and whatever follows `--`
-        #       is the command to run.
-
-        # Set some defaults for the deploy command
-        app_config.set_deploy_defaults(packaging_directory)
 
-        if options.get("no_deps") == True:
+        if app_config.get("no_deps", False):
             # Setting this in the state will make it skip the fast-bakery step
             # of building an image.
             app_config.set_state("skip_dependencies", True)
         else:
             # Check if the user has set the dependencies in the app config
             dependencies = app_config.get("dependencies", {})
-            if len(dependencies) == 0:
+
+            if all(
+                [
+                    dependencies.get("from_pyproject_toml", None) is None,
+                    dependencies.get("from_requirements_file", None) is None,
+                    dependencies.get("pypi", None) is None,
+                    dependencies.get("conda", None) is None,
+                ]
+            ):
                 # The user has not set any dependencies, so we can sniff the packaging directory
                 # for a dependencies file.
                 requirements_file = os.path.join(
@@ -798,8 +511,6 @@ def deploy(
         # 2. TODO: validate that the compute pool specified in the app exists.
         # 3. Building Docker image if necessary (based on parameters)
         #   - We will bake images with fastbakery and pass it to the deploy command
-        # TODO: validation logic can be wrapped in try catch so that we can provide
-        #       better error messages.
         cache_dir = os.path.join(
             ctx.obj.app_state_dir, app_config.get("name", "default")
         )
@@ -808,11 +519,6 @@ def deploy(
             for m in msg:
                 logger(m, **kwargs)
 
-        deploy_validations(
-            app_config,
-            cache_dir=cache_dir,
-            logger=logger,
-        )
         image_spinner = None
         img_logger = _non_spinner_logger
         if not no_loader:
@@ -830,10 +536,6 @@ def deploy(
         if image_spinner:
             image_spinner.stop()
 
-        base_commands = parse_commands(app_config, command)
-
-        app_config.set_state("commands", base_commands)
-
         # TODO: Handle the case where packaging_directory is None
         # This would involve:
         # 1. Packaging the code:
@@ -889,7 +591,6 @@ def deploy(
             capsule,
             _current_instance_debug_dir,
             options,
-            _cli_parsed_config,
         )
 
         if len(currently_present_capsules) > 0:
@@ -1043,6 +744,14 @@ def list(ctx, project, branch, name, tags, format, auth_type):
         print_table(table_data, headers)
 
 
+@app.command()
+@auto_cli_options()
+@click.pass_context
+@click.argument("command", nargs=-1, type=click.UNPROCESSED, required=False)
+def deploy2(ctx, **kwargs):
+    pass
+
+
 @app.command(help="Delete an app/apps from the Outerbounds Platform.")
 @click.option("--name", type=str, help="Filter app to delete by name")
 @click.option("--id", "cap_id", type=str, help="Filter app to delete by id")
@@ -1118,47 +827,6 @@ def delete(ctx, name, cap_id, project, branch, tags, auto_approve):
             time.sleep(0.5 + random.random() * 2)  # delay to avoid rate limiting
 
 
-@app.command(help="Run an app locally (for testing).")
-@common_run_options
-@click.pass_context
-def run(ctx, **options):
-    """Run an app locally for testing."""
-    try:
-        # Create configuration
-        if options["config_file"]:
-            # Load from file
-            app_config = AppConfig.from_file(options["config_file"])
-
-            # Update with any CLI options using the unified method
-            app_config.update_from_cli_options(options)
-        else:
-            # Create from CLI options
-            config_dict = build_config_from_options(options)
-            app_config = AppConfig(config_dict)
-
-        # Validate the configuration
-        app_config.validate()
-
-        # Print the configuration
-        click.echo("Running App with configuration:")
-        click.echo(app_config.to_yaml())
-
-        # TODO: Implement local run logic
-        # This would involve:
-        # 1. Setting up the environment
-        # 2. Running the app locally
-        # 3. Reporting status
-
-        click.echo(f"App '{app_config.config['name']}' running locally!")
-
-    except AppConfigError as e:
-        click.echo(f"Error in app configuration: {e}", err=True)
-        ctx.exit(1)
-    except Exception as e:
-        click.echo(f"Error running app: {e}", err=True)
-        ctx.exit(1)
-
-
 @app.command(
     help="Get detailed information about an app from the Outerbounds Platform."
 )