ob-metaflow-extensions 1.4.32__tar.gz → 1.4.39__tar.gz

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob_metaflow_extensions
-Version: 1.4.32
+Version: 1.4.39
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/metaflow_extensions/outerbounds/plugins/aws/assume_role_decorator.py

@@ -25,10 +25,29 @@ class assume_role(FlowMutator):
         def end(self):
             from metaflow import get_aws_client
             client = get_aws_client("dynamodb")  # Automatically uses the role in the flow decorator
+
+    You can also filter which steps should use the role:
+    @assume_role(role_arn="arn:aws:iam::123456789012:role/my-iam-role", steps=["start", "process"])
+    class MyFlow(FlowSpec):
+        @step
+        def start(self):
+            # user code in this step will use the assumed role
+            pass
+
+        @step
+        def process(self):
+            # user code in this step will use the assumed role
+            pass
+
+        @step
+        def end(self):
+            # user code in this step will NOT use the assumed role
+            pass
     """
 
     def init(self, *args, **kwargs):
         self.role_arn = kwargs.get("role_arn", None)
+        self.steps = kwargs.get("steps", None)
 
         if self.role_arn is None:
             raise ValueError(
@@ -40,6 +59,13 @@ class assume_role(FlowMutator):
                 "`role_arn` must be a valid AWS IAM role ARN starting with 'arn:aws:iam::'"
             )
 
+        # Validate steps parameter
+        if self.steps is not None:
+            if not isinstance(self.steps, (list, tuple)):
+                raise ValueError("`steps` must be a list or tuple of step names")
+            if not all(isinstance(s, str) for s in self.steps):
+                raise ValueError("All step names in `steps` must be strings")
+
     def pre_mutate(self, mutable_flow: MutableFlow) -> None:
         """
         This method is called by Metaflow to apply the decorator to the flow.
@@ -49,6 +75,18 @@ class assume_role(FlowMutator):
         # Import environment decorator at runtime to avoid circular imports
         from metaflow import environment
 
+        # Validate that all specified steps exist in the flow
+        if self.steps is not None:
+            flow_step_names = {step_name for step_name, _ in mutable_flow.steps}
+            specified_steps = set(self.steps)
+            missing_steps = specified_steps - flow_step_names
+
+            if missing_steps:
+                raise ValueError(
+                    f"Step(s) {sorted(missing_steps)} specified in `steps` parameter "
+                    f"do not exist in the flow. Available steps: {sorted(flow_step_names)}"
+                )
+
         def _swap_environment_variables(step: MutableStep, role_arn: str) -> None:
             _step_has_env_set = True
             _env_kwargs = {OBP_ASSUME_ROLE_ARN_ENV_VAR: role_arn}
@@ -73,6 +111,8 @@ class assume_role(FlowMutator):
         def _setup_role_assumption(step: MutableStep) -> None:
             _swap_environment_variables(step, self.role_arn)
 
-        # Apply the role assumption setup to all steps in the flow
-        for _, step in mutable_flow.steps:
-            _setup_role_assumption(step)
+        # Apply the role assumption setup to all steps in the flow (or filtered steps)
+        for step_name, step in mutable_flow.steps:
+            # If steps filter is specified, only apply to those steps
+            if self.steps is None or step_name in self.steps:
+                _setup_role_assumption(step)

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py

@@ -83,22 +83,32 @@ def get_snowflake_token(user: str = "", role: str = "", integration: str = "") -
     return response.json()["token"]
 
 
-def connect(user: str = "", role: str = "", integration: str = "", **kwargs):
+def get_oauth_connection_params(
+    user: str = "", role: str = "", integration: str = "", **kwargs
+) -> Dict:
     """
-    Connect to snowflake using the token minted by Outerbounds
+    Get OAuth connection parameters for Snowflake authentication using Outerbounds integration.
+
+    This is a helper function that returns connection parameters dict that can be used
+    with both snowflake-connector-python and snowflake-snowpark-python.
+
     user: str
         The user name used to authenticate with snowflake
     role: str
-        The role to request when connect with snowflake
+        The role to request when connecting with snowflake
     integration: str
-        The name of the snowflake integration to use. If not set, an existing integration will be used provided that only one exists in the current perimeter. If integration is not set and more than one exists in the current perimeter, then we raise an exception.
+        The name of the snowflake integration to use. If not set, an existing integration
+        will be used provided that only one exists in the current perimeter.
     kwargs: dict
-        Additional arguments to pass to the python snowflake connector
+        Additional arguments to include in the connection parameters
+
+    Returns:
+        Dict with connection parameters including OAuth token
     """
     # ensure password is not set
     if "password" in kwargs:
         raise OuterboundsSnowflakeConnectorException(
-            "Password should not be set when using Outerbounds snowflake connector."
+            "Password should not be set when using Outerbounds OAuth authentication."
         )
 
     provisioner = SnowflakeIntegrationProvisioner(integration)
@@ -137,11 +147,31 @@ def connect(user: str = "", role: str = "", integration: str = "", **kwargs):
     kwargs["role"] = role
     kwargs["user"] = user
 
+    return kwargs
+
+
+def connect(user: str = "", role: str = "", integration: str = "", **kwargs):
+    """
+    Connect to snowflake using the token minted by Outerbounds
+    user: str
+        The user name used to authenticate with snowflake
+    role: str
+        The role to request when connect with snowflake
+    integration: str
+        The name of the snowflake integration to use. If not set, an existing integration will be used provided that only one exists in the current perimeter. If integration is not set and more than one exists in the current perimeter, then we raise an exception.
+    kwargs: dict
+        Additional arguments to pass to the python snowflake connector
+    """
+    # Get OAuth connection params using the helper
+    connection_params = get_oauth_connection_params(
+        user=user, role=role, integration=integration, **kwargs
+    )
+
     # connect to snowflake
     try:
         from snowflake.connector import connect
 
-        cn = connect(**kwargs)
+        cn = connect(**connection_params)
         return cn
     except ImportError as ie:
         raise OuterboundsSnowflakeConnectorException(

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark.py

@@ -1,9 +1,11 @@
 import os
+import re
 import shlex
 import atexit
 import json
 import math
 import time
+import hashlib
 
 from metaflow import util
 
@@ -57,21 +59,29 @@ class Snowpark(object):
         atexit.register(lambda: self.job.kill() if hasattr(self, "job") else None)
 
     def _job_name(self, user, flow_name, run_id, step_name, task_id, retry_count):
-        return "{user}-{flow_name}-{run_id}-{step_name}-{task_id}-{retry_count}".format(
-            user=user,
-            flow_name=flow_name,
-            run_id=str(run_id) if run_id is not None else "",
-            step_name=step_name,
-            task_id=str(task_id) if task_id is not None else "",
-            retry_count=str(retry_count) if retry_count is not None else "",
+        unique_str = (
+            "{user}-{flow_name}-{run_id}-{step_name}-{task_id}-{retry_count}".format(
+                user=user,
+                flow_name=flow_name,
+                run_id=str(run_id) if run_id is not None else "",
+                step_name=step_name,
+                task_id=str(task_id) if task_id is not None else "",
+                retry_count=str(retry_count) if retry_count is not None else "",
+            )
         )
+        unique_hash = hashlib.md5(unique_str.encode("utf-8")).hexdigest()[:8]
+        raw_prefix = f"{flow_name}-{step_name}"
+        safe_prefix = re.sub(r"[^a-z0-9]", "-", raw_prefix.lower())
+        safe_prefix = safe_prefix[:54]
+        safe_prefix = safe_prefix.lstrip("-")
+        return f"{safe_prefix}-{unique_hash}"
 
     def _command(self, environment, code_package_url, step_name, step_cmds, task_spec):
         mflog_expr = export_mflog_env_vars(
             datastore_type=self.datastore.TYPE,
             stdout_path=STDOUT_PATH,
             stderr_path=STDERR_PATH,
-            **task_spec
+            **task_spec,
         )
         init_cmds = environment.get_package_commands(
             code_package_url, self.datastore.TYPE

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_cli.py

@@ -66,6 +66,10 @@ def snowpark():
     "--schema",
     help="Schema for Snowpark Container Services.",
 )
+@click.option(
+    "--integration",
+    help="Outerbounds OAuth integration name for Snowpark Container Services. When set, uses OAuth authentication instead of password.",
+)
 @click.option(
     "--image",
     help="Docker image requirement for Snowpark Container Services. In name:version format.",
@@ -119,6 +123,7 @@ def step(
     database=None,
     warehouse=None,
     schema=None,
+    integration=None,
     image=None,
     stage=None,
     compute_pool=None,
@@ -235,6 +240,7 @@ def step(
                 "database": database,
                 "warehouse": warehouse,
                 "schema": schema,
+                "integration": integration,
             },
         )
         with ctx.obj.monitor.measure("metaflow.snowpark.launch_job"):

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_client.py

@@ -10,14 +10,15 @@ from metaflow.exception import MetaflowException
 class SnowparkClient(object):
     def __init__(
         self,
-        account: str,
-        user: str,
-        password: str,
-        role: str,
-        database: str,
-        warehouse: str,
-        schema: str,
+        account: str = None,
+        user: str = None,
+        password: str = None,
+        role: str = None,
+        database: str = None,
+        warehouse: str = None,
+        schema: str = None,
         autocommit: bool = True,
+        integration: str = None,
     ):
         try:
             from snowflake.core import Root
@@ -37,15 +38,38 @@ class SnowparkClient(object):
                 % sys.executable
             )
 
+        if integration:
+            # Use OAuth authentication via Outerbounds integration
+            from metaflow_extensions.outerbounds.plugins.snowflake.snowflake import (
+                get_oauth_connection_params,
+            )
+
+            self.connection_parameters = get_oauth_connection_params(
+                user=user or "",
+                role=role or "",
+                integration=integration,
+                schema=schema or "",
+                account=account,
+                warehouse=warehouse,
+                database=database,
+            )
+            self.connection_parameters["autocommit"] = autocommit
+        else:
+            # Password-based authentication
+            self.connection_parameters = {
+                "account": account,
+                "user": user,
+                "password": password,
+                "role": role,
+                "warehouse": warehouse,
+                "database": database,
+                "schema": schema,
+                "autocommit": autocommit,
+            }
+
+        # Remove None values from connection parameters
         self.connection_parameters = {
-            "account": account,
-            "user": user,
-            "password": password,
-            "role": role,
-            "warehouse": warehouse,
-            "database": database,
-            "schema": schema,
-            "autocommit": autocommit,
+            k: v for k, v in self.connection_parameters.items() if v is not None
         }
 
         try:

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_decorator.py

@@ -71,6 +71,7 @@ class SnowparkDecorator(StepDecorator):
         "cpu": None,
         "gpu": None,
         "memory": None,
+        "integration": None,  # Outerbounds OAuth integration name
     }
 
     package_url = None
@@ -80,12 +81,11 @@ class SnowparkDecorator(StepDecorator):
     def __init__(self, attributes=None, statically_defined=False):
         super(SnowparkDecorator, self).__init__(attributes, statically_defined)
 
+        # Set defaults from config (user can override via decorator or integration)
         if not self.attributes["account"]:
             self.attributes["account"] = SNOWPARK_ACCOUNT
         if not self.attributes["user"]:
             self.attributes["user"] = SNOWPARK_USER
-        if not self.attributes["password"]:
-            self.attributes["password"] = SNOWPARK_PASSWORD
         if not self.attributes["role"]:
             self.attributes["role"] = SNOWPARK_ROLE
         if not self.attributes["database"]:
@@ -94,6 +94,9 @@ class SnowparkDecorator(StepDecorator):
             self.attributes["warehouse"] = SNOWPARK_WAREHOUSE
         if not self.attributes["schema"]:
             self.attributes["schema"] = SNOWPARK_SCHEMA
+        # Only use password from config if not using integration (OAuth)
+        if not self.attributes["integration"] and not self.attributes["password"]:
+            self.attributes["password"] = SNOWPARK_PASSWORD
 
         # If no docker image is explicitly specified, impute a default image.
         if not self.attributes["image"]:

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py

@@ -12,9 +12,9 @@ from .snowpark_exceptions import SnowparkException
 mapping = str.maketrans("0123456789", "abcdefghij")
 
 
-# keep only alpha numeric characters and underscores..
+# keep only alpha numeric characters and dashes..
 def sanitize_name(job_name: str):
-    return "".join(char for char in job_name if char.isalnum() or char == "_")
+    return "".join(char for char in job_name if char.isalnum() or char == "-")
 
 
 # this is not a decorator since the exception imports need to be inside

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/ob_metaflow_extensions.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.4.32
+Version: 1.4.39
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

ob_metaflow_extensions-1.4.39/ob_metaflow_extensions.egg-info/requires.txt

@@ -0,0 +1,3 @@
+boto3
+kubernetes
+ob-metaflow==2.19.15.1

{ob_metaflow_extensions-1.4.32 → ob_metaflow_extensions-1.4.39}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
 from pathlib import Path
 
 
-version = "1.4.32"
+version = "1.4.39"
 this_directory = Path(__file__).parent
 long_description = (this_directory / "README.md").read_text()
 
@@ -19,6 +19,6 @@ setup(
     install_requires=[
         "boto3",
         "kubernetes",
-        "ob-metaflow == 2.19.8.1",
+        "ob-metaflow == 2.19.15.1",
     ],
 )

ob_metaflow_extensions-1.4.32/ob_metaflow_extensions.egg-info/requires.txt

@@ -1,3 +0,0 @@
-boto3
-kubernetes
-ob-metaflow==2.19.8.1