ob-metaflow-extensions 1.4.33__py2.py3-none-any.whl → 1.6.2__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow_extensions/outerbounds/plugins/__init__.py +8 -1
- metaflow_extensions/outerbounds/plugins/apps/core/__init__.py +8 -2
- metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py +6 -6
- metaflow_extensions/outerbounds/plugins/apps/core/app_config.py +1 -19
- metaflow_extensions/outerbounds/plugins/apps/core/app_deploy_decorator.py +333 -0
- metaflow_extensions/outerbounds/plugins/apps/core/capsule.py +150 -79
- metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py +4 -1
- metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py +4 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py +103 -5
- metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py +12 -1
- metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py +100 -6
- metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py +141 -2
- metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py +74 -37
- metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml +6 -6
- metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py +2 -2
- metaflow_extensions/outerbounds/plugins/apps/core/deployer.py +1102 -105
- metaflow_extensions/outerbounds/plugins/apps/core/exceptions.py +341 -0
- metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py +42 -6
- metaflow_extensions/outerbounds/plugins/aws/assume_role_decorator.py +43 -3
- metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py +10 -1
- metaflow_extensions/outerbounds/plugins/optuna/__init__.py +2 -1
- metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py +37 -7
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark.py +18 -8
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_cli.py +6 -0
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_client.py +39 -15
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_decorator.py +5 -2
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py +2 -2
- metaflow_extensions/outerbounds/remote_config.py +20 -7
- metaflow_extensions/outerbounds/toplevel/apps/__init__.py +9 -0
- metaflow_extensions/outerbounds/toplevel/apps/exceptions.py +11 -0
- metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py +1 -1
- metaflow_extensions/outerbounds/toplevel/ob_internal.py +1 -1
- {ob_metaflow_extensions-1.4.33.dist-info → ob_metaflow_extensions-1.6.2.dist-info}/METADATA +2 -2
- {ob_metaflow_extensions-1.4.33.dist-info → ob_metaflow_extensions-1.6.2.dist-info}/RECORD +36 -34
- metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py +0 -146
- metaflow_extensions/outerbounds/plugins/apps/core/app_cli.py +0 -1200
- {ob_metaflow_extensions-1.4.33.dist-info → ob_metaflow_extensions-1.6.2.dist-info}/WHEEL +0 -0
- {ob_metaflow_extensions-1.4.33.dist-info → ob_metaflow_extensions-1.6.2.dist-info}/top_level.txt +0 -0
|
@@ -66,6 +66,10 @@ def snowpark():
|
|
|
66
66
|
"--schema",
|
|
67
67
|
help="Schema for Snowpark Container Services.",
|
|
68
68
|
)
|
|
69
|
+
@click.option(
|
|
70
|
+
"--integration",
|
|
71
|
+
help="Outerbounds OAuth integration name for Snowpark Container Services. When set, uses OAuth authentication instead of password.",
|
|
72
|
+
)
|
|
69
73
|
@click.option(
|
|
70
74
|
"--image",
|
|
71
75
|
help="Docker image requirement for Snowpark Container Services. In name:version format.",
|
|
@@ -119,6 +123,7 @@ def step(
|
|
|
119
123
|
database=None,
|
|
120
124
|
warehouse=None,
|
|
121
125
|
schema=None,
|
|
126
|
+
integration=None,
|
|
122
127
|
image=None,
|
|
123
128
|
stage=None,
|
|
124
129
|
compute_pool=None,
|
|
@@ -235,6 +240,7 @@ def step(
|
|
|
235
240
|
"database": database,
|
|
236
241
|
"warehouse": warehouse,
|
|
237
242
|
"schema": schema,
|
|
243
|
+
"integration": integration,
|
|
238
244
|
},
|
|
239
245
|
)
|
|
240
246
|
with ctx.obj.monitor.measure("metaflow.snowpark.launch_job"):
|
|
@@ -10,14 +10,15 @@ from metaflow.exception import MetaflowException
|
|
|
10
10
|
class SnowparkClient(object):
|
|
11
11
|
def __init__(
|
|
12
12
|
self,
|
|
13
|
-
account: str,
|
|
14
|
-
user: str,
|
|
15
|
-
password: str,
|
|
16
|
-
role: str,
|
|
17
|
-
database: str,
|
|
18
|
-
warehouse: str,
|
|
19
|
-
schema: str,
|
|
13
|
+
account: str = None,
|
|
14
|
+
user: str = None,
|
|
15
|
+
password: str = None,
|
|
16
|
+
role: str = None,
|
|
17
|
+
database: str = None,
|
|
18
|
+
warehouse: str = None,
|
|
19
|
+
schema: str = None,
|
|
20
20
|
autocommit: bool = True,
|
|
21
|
+
integration: str = None,
|
|
21
22
|
):
|
|
22
23
|
try:
|
|
23
24
|
from snowflake.core import Root
|
|
@@ -37,15 +38,38 @@ class SnowparkClient(object):
|
|
|
37
38
|
% sys.executable
|
|
38
39
|
)
|
|
39
40
|
|
|
41
|
+
if integration:
|
|
42
|
+
# Use OAuth authentication via Outerbounds integration
|
|
43
|
+
from metaflow_extensions.outerbounds.plugins.snowflake.snowflake import (
|
|
44
|
+
get_oauth_connection_params,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
self.connection_parameters = get_oauth_connection_params(
|
|
48
|
+
user=user or "",
|
|
49
|
+
role=role or "",
|
|
50
|
+
integration=integration,
|
|
51
|
+
schema=schema or "",
|
|
52
|
+
account=account,
|
|
53
|
+
warehouse=warehouse,
|
|
54
|
+
database=database,
|
|
55
|
+
)
|
|
56
|
+
self.connection_parameters["autocommit"] = autocommit
|
|
57
|
+
else:
|
|
58
|
+
# Password-based authentication
|
|
59
|
+
self.connection_parameters = {
|
|
60
|
+
"account": account,
|
|
61
|
+
"user": user,
|
|
62
|
+
"password": password,
|
|
63
|
+
"role": role,
|
|
64
|
+
"warehouse": warehouse,
|
|
65
|
+
"database": database,
|
|
66
|
+
"schema": schema,
|
|
67
|
+
"autocommit": autocommit,
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
# Remove None values from connection parameters
|
|
40
71
|
self.connection_parameters = {
|
|
41
|
-
|
|
42
|
-
"user": user,
|
|
43
|
-
"password": password,
|
|
44
|
-
"role": role,
|
|
45
|
-
"warehouse": warehouse,
|
|
46
|
-
"database": database,
|
|
47
|
-
"schema": schema,
|
|
48
|
-
"autocommit": autocommit,
|
|
72
|
+
k: v for k, v in self.connection_parameters.items() if v is not None
|
|
49
73
|
}
|
|
50
74
|
|
|
51
75
|
try:
|
|
@@ -71,6 +71,7 @@ class SnowparkDecorator(StepDecorator):
|
|
|
71
71
|
"cpu": None,
|
|
72
72
|
"gpu": None,
|
|
73
73
|
"memory": None,
|
|
74
|
+
"integration": None, # Outerbounds OAuth integration name
|
|
74
75
|
}
|
|
75
76
|
|
|
76
77
|
package_url = None
|
|
@@ -80,12 +81,11 @@ class SnowparkDecorator(StepDecorator):
|
|
|
80
81
|
def __init__(self, attributes=None, statically_defined=False):
|
|
81
82
|
super(SnowparkDecorator, self).__init__(attributes, statically_defined)
|
|
82
83
|
|
|
84
|
+
# Set defaults from config (user can override via decorator or integration)
|
|
83
85
|
if not self.attributes["account"]:
|
|
84
86
|
self.attributes["account"] = SNOWPARK_ACCOUNT
|
|
85
87
|
if not self.attributes["user"]:
|
|
86
88
|
self.attributes["user"] = SNOWPARK_USER
|
|
87
|
-
if not self.attributes["password"]:
|
|
88
|
-
self.attributes["password"] = SNOWPARK_PASSWORD
|
|
89
89
|
if not self.attributes["role"]:
|
|
90
90
|
self.attributes["role"] = SNOWPARK_ROLE
|
|
91
91
|
if not self.attributes["database"]:
|
|
@@ -94,6 +94,9 @@ class SnowparkDecorator(StepDecorator):
|
|
|
94
94
|
self.attributes["warehouse"] = SNOWPARK_WAREHOUSE
|
|
95
95
|
if not self.attributes["schema"]:
|
|
96
96
|
self.attributes["schema"] = SNOWPARK_SCHEMA
|
|
97
|
+
# Only use password from config if not using integration (OAuth)
|
|
98
|
+
if not self.attributes["integration"] and not self.attributes["password"]:
|
|
99
|
+
self.attributes["password"] = SNOWPARK_PASSWORD
|
|
97
100
|
|
|
98
101
|
# If no docker image is explicitly specified, impute a default image.
|
|
99
102
|
if not self.attributes["image"]:
|
|
@@ -12,9 +12,9 @@ from .snowpark_exceptions import SnowparkException
|
|
|
12
12
|
mapping = str.maketrans("0123456789", "abcdefghij")
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
# keep only alpha numeric characters and
|
|
15
|
+
# keep only alpha numeric characters and dashes..
|
|
16
16
|
def sanitize_name(job_name: str):
|
|
17
|
-
return "".join(char for char in job_name if char.isalnum() or char == "
|
|
17
|
+
return "".join(char for char in job_name if char.isalnum() or char == "-")
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
# this is not a decorator since the exception imports need to be inside
|
|
@@ -59,11 +59,24 @@ def resolve_config_from_remote(remote_url: str, auth_token: str) -> Dict[str, st
|
|
|
59
59
|
response.raise_for_status()
|
|
60
60
|
data = response.json()
|
|
61
61
|
return data["config"]
|
|
62
|
-
except HTTPError:
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
62
|
+
except HTTPError as e:
|
|
63
|
+
if e.response.status_code >= 500:
|
|
64
|
+
raise OuterboundsConfigException(
|
|
65
|
+
"Error resolving outerbounds configuration [status:%s]. Please reach out to "
|
|
66
|
+
"the outerbounds support team to help resolve this issue."
|
|
67
|
+
% e.response.status_code
|
|
68
|
+
)
|
|
69
|
+
elif e.response.status_code == 403:
|
|
70
|
+
raise OuterboundsConfigException(
|
|
71
|
+
"Outerbounds token validity expired [status:%s]. Please re-run the `outerbounds configure` "
|
|
72
|
+
"command with a new magic string from the UI." % e.response.status_code
|
|
73
|
+
)
|
|
74
|
+
else:
|
|
75
|
+
raise OuterboundsConfigException(
|
|
76
|
+
"Failed to fetch the outerbounds configuration string [status:%s]. Please reach out to "
|
|
77
|
+
"the outerbounds support team to help resolve this issue."
|
|
78
|
+
% e.response.status_code
|
|
79
|
+
)
|
|
67
80
|
|
|
68
81
|
|
|
69
82
|
def init_config() -> Dict[str, str]:
|
|
@@ -87,8 +100,8 @@ def init_config() -> Dict[str, str]:
|
|
|
87
100
|
remote_config = json.loads(config_path.read_text())
|
|
88
101
|
except ValueError:
|
|
89
102
|
raise OuterboundsConfigException(
|
|
90
|
-
"Error decoding your metaflow config. Please run the `outerbounds configure`
|
|
91
|
-
|
|
103
|
+
"Error decoding your metaflow config. Please re-run the `outerbounds configure` "
|
|
104
|
+
"command with a new magic string from the UI."
|
|
92
105
|
)
|
|
93
106
|
|
|
94
107
|
perimeter_config_url = get_perimeter_config_url_if_set_in_ob_config()
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from ...plugins.apps.core.exceptions import (
|
|
2
|
+
AppDeploymentException,
|
|
3
|
+
AppCrashLoopException,
|
|
4
|
+
AppReadinessException,
|
|
5
|
+
AppConcurrentUpgradeException,
|
|
6
|
+
AppUpgradeInProgressException,
|
|
7
|
+
AppCreationFailedException,
|
|
8
|
+
AppNotFoundException,
|
|
9
|
+
AppDeletedDuringDeploymentException,
|
|
10
|
+
OuterboundsBackendUnhealthyException,
|
|
11
|
+
)
|
|
@@ -136,4 +136,4 @@ from ..plugins.snowflake import Snowflake
|
|
|
136
136
|
from ..plugins.checkpoint_datastores import nebius_checkpoints, coreweave_checkpoints
|
|
137
137
|
from ..plugins.aws import assume_role
|
|
138
138
|
from . import ob_internal
|
|
139
|
-
from .
|
|
139
|
+
from . import apps
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
from ..plugins.kubernetes.pod_killer import PodKiller
|
|
2
|
-
from ..plugins.fast_bakery.baker import bake_image
|
|
2
|
+
from ..plugins.fast_bakery.baker import bake_image as internal_bake_image
|
|
3
3
|
from ..plugins.apps import core as app_core
|
|
4
4
|
from ..plugins.apps.core import AppDeployer
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: ob-metaflow-extensions
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.6.2
|
|
4
4
|
Summary: Outerbounds Platform Extensions for Metaflow
|
|
5
5
|
Author: Outerbounds, Inc.
|
|
6
6
|
License: Commercial
|
|
7
7
|
Description-Content-Type: text/markdown
|
|
8
8
|
Requires-Dist: boto3
|
|
9
9
|
Requires-Dist: kubernetes
|
|
10
|
-
Requires-Dist: ob-metaflow (==2.19.
|
|
10
|
+
Requires-Dist: ob-metaflow (==2.19.15.3)
|
|
11
11
|
|
|
12
12
|
# Outerbounds platform package
|
|
13
13
|
|
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
metaflow_extensions/outerbounds/__init__.py,sha256=Gb8u06s9ClQsA_vzxmkCzuMnigPy7kKcDnLfb7eB-64,514
|
|
2
|
-
metaflow_extensions/outerbounds/remote_config.py,sha256=
|
|
2
|
+
metaflow_extensions/outerbounds/remote_config.py,sha256=bP5GhDvnjf5nuKUgHsaImstFxrylzJksBqBWaPEME1s,5458
|
|
3
3
|
metaflow_extensions/outerbounds/config/__init__.py,sha256=JsQGRuGFz28fQWjUvxUgR8EKBLGRdLUIk_buPLJplJY,1225
|
|
4
|
-
metaflow_extensions/outerbounds/plugins/__init__.py,sha256=
|
|
4
|
+
metaflow_extensions/outerbounds/plugins/__init__.py,sha256=5sxtyUdh2KoeXvpMRspqdyf50_2v9FnHUynYxuSkDvo,14189
|
|
5
5
|
metaflow_extensions/outerbounds/plugins/auth_server.py,sha256=_Q9_2EL0Xy77bCRphkwT1aSu8gQXRDOH-Z-RxTUO8N4,2202
|
|
6
6
|
metaflow_extensions/outerbounds/plugins/perimeters.py,sha256=QXh3SFP7GQbS-RAIxUOPbhPzQ7KDFVxZkTdKqFKgXjI,2697
|
|
7
7
|
metaflow_extensions/outerbounds/plugins/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
8
|
metaflow_extensions/outerbounds/plugins/apps/app_cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
-
metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py,sha256=WWFvyl0EDUbp_UwmWjXBqlLfPEZ-lq98NJ5F5skBDKU,4901
|
|
10
9
|
metaflow_extensions/outerbounds/plugins/apps/app_utils.py,sha256=sw9whU17lAzlD2K2kEDNjlk1Ib-2xE2UNhJkmzD8Qv8,8543
|
|
11
10
|
metaflow_extensions/outerbounds/plugins/apps/consts.py,sha256=iHsyqbUg9k-rgswCs1Jxf5QZIxR1V-peCDRjgr9kdBM,177
|
|
12
11
|
metaflow_extensions/outerbounds/plugins/apps/deploy_decorator.py,sha256=VkmiMdNYHhNdt-Qm9AVv7aE2LWFsIFEc16YcOYjwF6Q,8568
|
|
13
12
|
metaflow_extensions/outerbounds/plugins/apps/supervisord_utils.py,sha256=GQoN2gyPClcpR9cLldJmbCfqXnoAHxp8xUnY7vzaYtY,9026
|
|
14
|
-
metaflow_extensions/outerbounds/plugins/apps/core/__init__.py,sha256=
|
|
15
|
-
metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py,sha256=
|
|
16
|
-
metaflow_extensions/outerbounds/plugins/apps/core/
|
|
17
|
-
metaflow_extensions/outerbounds/plugins/apps/core/
|
|
13
|
+
metaflow_extensions/outerbounds/plugins/apps/core/__init__.py,sha256=2-tVXFdajOBzXaY70uHh7lqdZqSx_UmnQdpNRmszRNA,439
|
|
14
|
+
metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py,sha256=OtM129BVCP2UXUL_kUPlG9OX0LJfOd45-b42hxO6A60,20350
|
|
15
|
+
metaflow_extensions/outerbounds/plugins/apps/core/app_config.py,sha256=9NfQe7DEaz_cG_3E7nGwLdlLDb-azcN4RW3l-jJXYPI,3863
|
|
16
|
+
metaflow_extensions/outerbounds/plugins/apps/core/app_deploy_decorator.py,sha256=cvrbMY2EauFccU17oDJ3fnVsvMiyfa3jWQQlJToLpjk,10102
|
|
18
17
|
metaflow_extensions/outerbounds/plugins/apps/core/artifacts.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
-
metaflow_extensions/outerbounds/plugins/apps/core/capsule.py,sha256=
|
|
18
|
+
metaflow_extensions/outerbounds/plugins/apps/core/capsule.py,sha256=86e6TNg53J25D3lUkVmnIgs_L7kc2OONdz9_J5lf3oc,41406
|
|
20
19
|
metaflow_extensions/outerbounds/plugins/apps/core/click_importer.py,sha256=kgoPQmK_-8PSSTc3QMSaynCLQ5VWTkKFOC69FPURyXA,998
|
|
21
|
-
metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml,sha256=
|
|
22
|
-
metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py,sha256=
|
|
23
|
-
metaflow_extensions/outerbounds/plugins/apps/core/deployer.py,sha256=
|
|
24
|
-
metaflow_extensions/outerbounds/plugins/apps/core/
|
|
20
|
+
metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml,sha256=DoDD3kqlnqJMrEHEAnJd2ll9w9CT8Oq7dLlukAswHD8,10073
|
|
21
|
+
metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py,sha256=k6moZjlU8V1eSMHFY22NW4k5j4SKNYhpFiX_f68hjwc,4024
|
|
22
|
+
metaflow_extensions/outerbounds/plugins/apps/core/deployer.py,sha256=fmvJD-3qus2RpF6Q_I8SwwnwGCsVi1XgQuGL8YElc3A,42055
|
|
23
|
+
metaflow_extensions/outerbounds/plugins/apps/core/exceptions.py,sha256=T6f4WurJpxHR7rufS-3yC2c22T_a35aW1rAWXwmlEg4,11663
|
|
24
|
+
metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py,sha256=XXSwO72R4lsLKmtmlGz7TBuCX9_r1MYyQWHY2CF7WlA,4558
|
|
25
25
|
metaflow_extensions/outerbounds/plugins/apps/core/secrets.py,sha256=sgDiAmpSC8Y5xjlaOEp79F6m0S3x4RONf_vJ5PUAfu8,6127
|
|
26
26
|
metaflow_extensions/outerbounds/plugins/apps/core/utils.py,sha256=2M2zU8DhbAlJee8P0xKXINAku81PcUylS3sVCSb0TUs,7896
|
|
27
27
|
metaflow_extensions/outerbounds/plugins/apps/core/validations.py,sha256=Inr9AJDe-L3PMMMxcJPH1zulh9_SynqITb2BzGseLh4,471
|
|
@@ -31,17 +31,17 @@ metaflow_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.py,sh
|
|
|
31
31
|
metaflow_extensions/outerbounds/plugins/apps/core/code_package/__init__.py,sha256=8McF7pgx8ghvjRnazp2Qktlxi9yYwNiwESSQrk-2oW8,68
|
|
32
32
|
metaflow_extensions/outerbounds/plugins/apps/core/code_package/code_packager.py,sha256=QHlF94T7ZGSOJH5Xeg4mI_CJvCrbaikiIOTnkFgE83g,23228
|
|
33
33
|
metaflow_extensions/outerbounds/plugins/apps/core/code_package/examples.py,sha256=aF8qKIJxCVv_ugcShQjqUsXKKKMsm1oMkQIl8w3QKuw,4016
|
|
34
|
-
metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py,sha256=
|
|
35
|
-
metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py,sha256=
|
|
36
|
-
metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py,sha256=
|
|
37
|
-
metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py,sha256=
|
|
38
|
-
metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py,sha256=
|
|
39
|
-
metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py,sha256=
|
|
40
|
-
metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py,sha256=
|
|
34
|
+
metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py,sha256=iIHHyCuH7V3xbVvlS0uCJnboQx67NeA03y8kuKxyW1A,489
|
|
35
|
+
metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py,sha256=JGARTvKESPQbuSy3RdnIF4PHUDsT0W0liFB_jmuCark,5496
|
|
36
|
+
metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py,sha256=tMj3belC7yBYV6DHjsWfFq2MNsZuIXnIAeEO8HQEFf4,37768
|
|
37
|
+
metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py,sha256=gl3toISxzm297pehSBobdrFmdIpVCVgcYWJO_YYrvn0,9954
|
|
38
|
+
metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py,sha256=HYpc_RnvQmrP52mkn0G5HDNa6jUhc738Xrpklio1ytQ,8090
|
|
39
|
+
metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py,sha256=SCInjZFTubHNcbVgeXkLWuHir0X6DzwYqWR_LFDtejU,18138
|
|
40
|
+
metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py,sha256=KnLQiLR_oKdv2LumBaSF6-IMcwzSV5SRoC25p6iEe3s,39924
|
|
41
41
|
metaflow_extensions/outerbounds/plugins/apps/core/experimental/__init__.py,sha256=_OT0RsbQdYerT-fjG_vcu85ava6sHbCsGBpLcGcrlX8,3043
|
|
42
42
|
metaflow_extensions/outerbounds/plugins/aws/__init__.py,sha256=VBGdjNKeFLXGZuqh4jVk8cFtO1AWof73a6k_cnbAOYA,145
|
|
43
43
|
metaflow_extensions/outerbounds/plugins/aws/assume_role.py,sha256=mBewNlnSYsR2rFXFkX-DUH6ku01h2yOcMcLHoCL7eyI,161
|
|
44
|
-
metaflow_extensions/outerbounds/plugins/aws/assume_role_decorator.py,sha256=
|
|
44
|
+
metaflow_extensions/outerbounds/plugins/aws/assume_role_decorator.py,sha256=cZg2zUcyvehpjE-a12HTP6vpwPG2MAJuK5KSywBk8xs,4638
|
|
45
45
|
metaflow_extensions/outerbounds/plugins/card_utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
46
46
|
metaflow_extensions/outerbounds/plugins/card_utilities/async_cards.py,sha256=6rQhtZXK5DenXPfCRS1ul0jvLJYd48jrJAlnodID21w,4347
|
|
47
47
|
metaflow_extensions/outerbounds/plugins/card_utilities/extra_components.py,sha256=D8rgCaSc7PuLD0MHJjqsjN0g0PQMN1H-ySOJqi5uIOE,19111
|
|
@@ -51,7 +51,7 @@ metaflow_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.py,sha25
|
|
|
51
51
|
metaflow_extensions/outerbounds/plugins/checkpoint_datastores/external_chckpt.py,sha256=BRigyuyYYv5H1CxfifFJPPvgMdQeCNIT8Hm0j1xo-YA,3025
|
|
52
52
|
metaflow_extensions/outerbounds/plugins/checkpoint_datastores/nebius.py,sha256=xFGdSxlHcwZwgTSKj8efEv6ydCWYqRGrreGqXhf02ko,2321
|
|
53
53
|
metaflow_extensions/outerbounds/plugins/fast_bakery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
54
|
-
metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py,sha256=
|
|
54
|
+
metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py,sha256=0i3UKZqB3iQD3dMxVEVvro-e4wLTiISf-B4_0wfWS58,3968
|
|
55
55
|
metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py,sha256=vKN3Waz0SgbOALVTRcf_8MCGhxdRueRFZJ7JmWbRJRM,15487
|
|
56
56
|
metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery.py,sha256=9iqIBowKk95vUYvnMrFNWmmSuK-whKnqJwcCmEcRjTU,5727
|
|
57
57
|
metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_cli.py,sha256=kqFyu2bJSnc9_9aYfBpz5xK6L6luWFZK_NMuh8f1eVk,1494
|
|
@@ -83,7 +83,7 @@ metaflow_extensions/outerbounds/plugins/ollama/constants.py,sha256=hxkTpWEJp1pKH
|
|
|
83
83
|
metaflow_extensions/outerbounds/plugins/ollama/exceptions.py,sha256=8Ss296_MGZl1wXAoDNwpH-hsPe6iYLe90Ji1pczNocU,668
|
|
84
84
|
metaflow_extensions/outerbounds/plugins/ollama/ollama.py,sha256=C-6Hz8OxsJiB14AAxmunq3P4k7DrmVHsSOxE0xsP-nY,79780
|
|
85
85
|
metaflow_extensions/outerbounds/plugins/ollama/status_card.py,sha256=F5e4McDl28lhtjeUyInkl03bqjr1lgLxWoau8Q9xwBE,10994
|
|
86
|
-
metaflow_extensions/outerbounds/plugins/optuna/__init__.py,sha256=
|
|
86
|
+
metaflow_extensions/outerbounds/plugins/optuna/__init__.py,sha256=hCT62WFWB-xyCvqKXFCvLlxOMpZ6hZm3_aEa4cASrBg,1681
|
|
87
87
|
metaflow_extensions/outerbounds/plugins/profilers/deco_injector.py,sha256=oI_C3c64XBm7n88FILqHwn-Nnc5DeT_68I67lM9rXaI,2434
|
|
88
88
|
metaflow_extensions/outerbounds/plugins/profilers/gpu_profile_decorator.py,sha256=gDHQ2sMIp4NuZSzUspbSd8RGdFAoO5mgZAyFcZ2a51Y,2619
|
|
89
89
|
metaflow_extensions/outerbounds/plugins/profilers/simple_card_decorator.py,sha256=4W9tLGCmkFx-4XYLa1xF6qMiaWOBYYFx_RclZDKej30,3259
|
|
@@ -98,14 +98,14 @@ metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_manager.py,sha256=Grl7
|
|
|
98
98
|
metaflow_extensions/outerbounds/plugins/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
99
99
|
metaflow_extensions/outerbounds/plugins/secrets/secrets.py,sha256=3s98hO_twKkM22tKyDdcUjGQNfYpSXW_jLKISV9ju_U,8433
|
|
100
100
|
metaflow_extensions/outerbounds/plugins/snowflake/__init__.py,sha256=RG4ixt3jwqcK1_tt0QxLcUbNmf7wWAMnZhBx-ZMGgLk,114
|
|
101
|
-
metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py,sha256=
|
|
101
|
+
metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py,sha256=_ULhLrKZ4rOVcJFb_EmMGi7GFDN6Vtawz8o86mLBMq8,14756
|
|
102
102
|
metaflow_extensions/outerbounds/plugins/snowpark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
103
|
-
metaflow_extensions/outerbounds/plugins/snowpark/snowpark.py,sha256=
|
|
104
|
-
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_cli.py,sha256=
|
|
105
|
-
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_client.py,sha256=
|
|
106
|
-
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_decorator.py,sha256=
|
|
103
|
+
metaflow_extensions/outerbounds/plugins/snowpark/snowpark.py,sha256=tVpSOizbVjoXp-iMWuzJgE41hUstMrPC8O_eCKY9Mg4,11083
|
|
104
|
+
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_cli.py,sha256=aesXYZGoEF8UtAIvCDJNLTSc9pM62KuOqd9Za95HjZ4,8997
|
|
105
|
+
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_client.py,sha256=6U1sdXx0pdhNdkGba8DaE7GabQTc5fyJI3LpPsPQt0s,5463
|
|
106
|
+
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_decorator.py,sha256=EN-XQafl14uQ3oVGgrkNII5PCYq2lWOt3t2d-JGPeBk,10430
|
|
107
107
|
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_exceptions.py,sha256=FTfYlJu-sn9DkPOs2R1V1ChWb1vZthOgeq0BZdT1ucY,296
|
|
108
|
-
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py,sha256=
|
|
108
|
+
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py,sha256=KcbAYZdInAlS10IAZWdqtWAJXY4I6N2ebCTXzAmT72o,7070
|
|
109
109
|
metaflow_extensions/outerbounds/plugins/snowpark/snowpark_service_spec.py,sha256=AI_kcm1hZV3JRxJkookcH6twiGnAYjk9Dx-MeoYz60Y,8511
|
|
110
110
|
metaflow_extensions/outerbounds/plugins/tensorboard/__init__.py,sha256=9lUM4Cqi5RjrHBRfG6AQMRz8-R96eZC8Ih0KD2lv22Y,1858
|
|
111
111
|
metaflow_extensions/outerbounds/plugins/torchtune/__init__.py,sha256=Psj2ybj_E1qp5KK2inon9e4ZecaRxnPtW3ngcirbO2g,6094
|
|
@@ -117,9 +117,11 @@ metaflow_extensions/outerbounds/plugins/vllm/vllm_manager.py,sha256=sp_TX2SrImJG
|
|
|
117
117
|
metaflow_extensions/outerbounds/profilers/__init__.py,sha256=wa_jhnCBr82TBxoS0e8b6_6sLyZX0fdHicuGJZNTqKw,29
|
|
118
118
|
metaflow_extensions/outerbounds/profilers/gpu.py,sha256=3Er8uKQzfm_082uadg4yn_D4Y-iSCgzUfFmguYxZsz4,27485
|
|
119
119
|
metaflow_extensions/outerbounds/toplevel/__init__.py,sha256=qWUJSv_r5hXJ7jV_On4nEasKIfUCm6_UjkjXWA_A1Ts,90
|
|
120
|
-
metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py,sha256=
|
|
121
|
-
metaflow_extensions/outerbounds/toplevel/ob_internal.py,sha256=
|
|
120
|
+
metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py,sha256=9tTaDi36bcdl2U3lCu_ASLG3Orpm2ezEfY5ZWu8ggqs,4528
|
|
121
|
+
metaflow_extensions/outerbounds/toplevel/ob_internal.py,sha256=AS0rJ9_Cb_NAjQthirFVMfkQpe-1uS3lTPU1hP0DBIA,216
|
|
122
122
|
metaflow_extensions/outerbounds/toplevel/s3_proxy.py,sha256=zdqG7Z12cGuoYYCi2P4kqC3WsgL3xfdJGIb7ejecHH4,2862
|
|
123
|
+
metaflow_extensions/outerbounds/toplevel/apps/__init__.py,sha256=MRhSZWbOSz1jEKHzZU6OssOVqdDEYAL9CgFUXCBzdsY,164
|
|
124
|
+
metaflow_extensions/outerbounds/toplevel/apps/exceptions.py,sha256=qikhm0Gf4zw_LWcl9RT6zjcGj_RVeuXJ8FDsNePWmhE,341
|
|
123
125
|
metaflow_extensions/outerbounds/toplevel/plugins/azure/__init__.py,sha256=WUuhz2YQfI4fz7nIcipwwWq781eaoHEk7n4GAn1npDg,63
|
|
124
126
|
metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3uILlEZ6ntBLKeNyqn3If8nIXZFq_Apd7Dhco,70
|
|
125
127
|
metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
|
|
@@ -128,7 +130,7 @@ metaflow_extensions/outerbounds/toplevel/plugins/optuna/__init__.py,sha256=6D1wL
|
|
|
128
130
|
metaflow_extensions/outerbounds/toplevel/plugins/snowflake/__init__.py,sha256=LptpH-ziXHrednMYUjIaosS1SXD3sOtF_9_eRqd8SJw,50
|
|
129
131
|
metaflow_extensions/outerbounds/toplevel/plugins/torchtune/__init__.py,sha256=uTVkdSk3xZ7hEKYfdlyVteWj5KeDwaM1hU9WT-_YKfI,50
|
|
130
132
|
metaflow_extensions/outerbounds/toplevel/plugins/vllm/__init__.py,sha256=ekcgD3KVydf-a0xMI60P4uy6ePkSEoFHiGnDq1JM940,45
|
|
131
|
-
ob_metaflow_extensions-1.
|
|
132
|
-
ob_metaflow_extensions-1.
|
|
133
|
-
ob_metaflow_extensions-1.
|
|
134
|
-
ob_metaflow_extensions-1.
|
|
133
|
+
ob_metaflow_extensions-1.6.2.dist-info/METADATA,sha256=Z7NNOTL3Ca0wTEK8sveSeLkeHM-eZIiomhBchWw8Xvo,519
|
|
134
|
+
ob_metaflow_extensions-1.6.2.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
|
|
135
|
+
ob_metaflow_extensions-1.6.2.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
|
|
136
|
+
ob_metaflow_extensions-1.6.2.dist-info/RECORD,,
|
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
from metaflow.exception import MetaflowException
|
|
2
|
-
from metaflow.decorators import StepDecorator
|
|
3
|
-
from metaflow import current
|
|
4
|
-
from .core import AppDeployer, apps
|
|
5
|
-
from .core.perimeters import PerimeterExtractor
|
|
6
|
-
import os
|
|
7
|
-
import hashlib
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class AppDeployDecorator(StepDecorator):
|
|
11
|
-
|
|
12
|
-
"""
|
|
13
|
-
MF Add To Current
|
|
14
|
-
-----------------
|
|
15
|
-
apps -> metaflow_extensions.outerbounds.plugins.apps.core.apps
|
|
16
|
-
|
|
17
|
-
@@ Returns
|
|
18
|
-
----------
|
|
19
|
-
apps
|
|
20
|
-
The object carrying the Deployer class to deploy apps.
|
|
21
|
-
"""
|
|
22
|
-
|
|
23
|
-
name = "app_deploy"
|
|
24
|
-
defaults = {}
|
|
25
|
-
|
|
26
|
-
package_url = None
|
|
27
|
-
package_sha = None
|
|
28
|
-
|
|
29
|
-
MAX_ENTROPY = 6
|
|
30
|
-
MAX_NAME_LENGTH = 15 - MAX_ENTROPY - 1 # -1 for the hyphen
|
|
31
|
-
|
|
32
|
-
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
|
33
|
-
self.logger = logger
|
|
34
|
-
self.environment = environment
|
|
35
|
-
self.step = step
|
|
36
|
-
self.flow_datastore = flow_datastore
|
|
37
|
-
|
|
38
|
-
def _resolve_package_url_and_sha(self):
|
|
39
|
-
return os.environ.get("METAFLOW_CODE_URL", self.package_url), os.environ.get(
|
|
40
|
-
"METAFLOW_CODE_SHA", self.package_sha
|
|
41
|
-
)
|
|
42
|
-
|
|
43
|
-
def _extract_project_info(self):
|
|
44
|
-
project = current.get("project_name")
|
|
45
|
-
branch = current.get("branch_name")
|
|
46
|
-
is_production = current.get("is_production")
|
|
47
|
-
return project, branch, is_production
|
|
48
|
-
|
|
49
|
-
def _resolve_default_image(self, flow):
|
|
50
|
-
# TODO : Resolve the default image over here.
|
|
51
|
-
pass
|
|
52
|
-
|
|
53
|
-
def _resolve_default_name_prefix(self, flow, step_name):
|
|
54
|
-
# TODO: Only tweek MAX_NAME_LENGTH as backend support allows longer names.
|
|
55
|
-
base_prefix = (flow.name + "-" + step_name).lower()
|
|
56
|
-
if len(base_prefix) > self.MAX_NAME_LENGTH:
|
|
57
|
-
base_prefix = "mf-app"
|
|
58
|
-
return base_prefix
|
|
59
|
-
|
|
60
|
-
def task_pre_step(
|
|
61
|
-
self,
|
|
62
|
-
step_name,
|
|
63
|
-
task_datastore,
|
|
64
|
-
metadata,
|
|
65
|
-
run_id,
|
|
66
|
-
task_id,
|
|
67
|
-
flow,
|
|
68
|
-
graph,
|
|
69
|
-
retry_count,
|
|
70
|
-
max_user_code_retries,
|
|
71
|
-
ubf_context,
|
|
72
|
-
inputs,
|
|
73
|
-
):
|
|
74
|
-
perimeter, api_server = PerimeterExtractor.during_metaflow_execution()
|
|
75
|
-
package_url, package_sha = self._resolve_package_url_and_sha()
|
|
76
|
-
if package_url is None or package_sha is None:
|
|
77
|
-
raise MetaflowException(
|
|
78
|
-
"METAFLOW_CODE_URL or METAFLOW_CODE_SHA is not set. "
|
|
79
|
-
"Please set METAFLOW_CODE_URL and METAFLOW_CODE_SHA in your environment."
|
|
80
|
-
)
|
|
81
|
-
image = os.environ.get("FASTBAKERY_IMAGE", None)
|
|
82
|
-
|
|
83
|
-
# TODO [Apps] - This is temporary. Backend will support longer names in the future.
|
|
84
|
-
default_name = self._resolve_default_name_prefix(flow, step_name)
|
|
85
|
-
project, branch, is_production = self._extract_project_info()
|
|
86
|
-
project_info = {}
|
|
87
|
-
if project is not None:
|
|
88
|
-
project_info["metaflow/project"] = project
|
|
89
|
-
project_info["metaflow/branch"] = branch
|
|
90
|
-
project_info["metaflow/is_production"] = is_production
|
|
91
|
-
|
|
92
|
-
default_tags = {
|
|
93
|
-
"metaflow/flow_name": flow.name,
|
|
94
|
-
"metaflow/step_name": step_name,
|
|
95
|
-
"metaflow/run_id": run_id,
|
|
96
|
-
"metaflow/task_id": task_id,
|
|
97
|
-
"metaflow/retry_count": retry_count,
|
|
98
|
-
"metaflow/pathspec": current.pathspec,
|
|
99
|
-
**project_info,
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
AppDeployer._set_state(
|
|
103
|
-
perimeter,
|
|
104
|
-
api_server,
|
|
105
|
-
code_package_url=package_url,
|
|
106
|
-
code_package_key=package_sha,
|
|
107
|
-
name_prefix=default_name,
|
|
108
|
-
image=image,
|
|
109
|
-
max_entropy=self.MAX_ENTROPY,
|
|
110
|
-
default_tags=[{k: str(v)} for k, v in default_tags.items()],
|
|
111
|
-
)
|
|
112
|
-
current._update_env(
|
|
113
|
-
{
|
|
114
|
-
"apps": apps(),
|
|
115
|
-
}
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
def task_post_step(
|
|
119
|
-
self, step_name, flow, graph, retry_count, max_user_code_retries
|
|
120
|
-
):
|
|
121
|
-
pass
|
|
122
|
-
|
|
123
|
-
def runtime_init(self, flow, graph, package, run_id):
|
|
124
|
-
# Set some more internal state.
|
|
125
|
-
self.flow = flow
|
|
126
|
-
self.graph = graph
|
|
127
|
-
self.package = package
|
|
128
|
-
self.run_id = run_id
|
|
129
|
-
|
|
130
|
-
def runtime_task_created(
|
|
131
|
-
self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context
|
|
132
|
-
):
|
|
133
|
-
# To execute the Kubernetes job, the job container needs to have
|
|
134
|
-
# access to the code package. We store the package in the datastore
|
|
135
|
-
# which the pod is able to download as part of it's entrypoint.
|
|
136
|
-
if not is_cloned:
|
|
137
|
-
self._save_package_once(self.flow_datastore, self.package)
|
|
138
|
-
|
|
139
|
-
@classmethod
|
|
140
|
-
def _save_package_once(cls, flow_datastore, package):
|
|
141
|
-
if cls.package_url is None:
|
|
142
|
-
cls.package_url, cls.package_sha = flow_datastore.save_data(
|
|
143
|
-
[package.blob], len_hint=1
|
|
144
|
-
)[0]
|
|
145
|
-
os.environ["METAFLOW_CODE_URL"] = cls.package_url
|
|
146
|
-
os.environ["METAFLOW_CODE_SHA"] = cls.package_sha
|