ob-metaflow-extensions 1.4.33__py2.py3-none-any.whl → 1.6.2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. metaflow_extensions/outerbounds/plugins/__init__.py +8 -1
  2. metaflow_extensions/outerbounds/plugins/apps/core/__init__.py +8 -2
  3. metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py +6 -6
  4. metaflow_extensions/outerbounds/plugins/apps/core/app_config.py +1 -19
  5. metaflow_extensions/outerbounds/plugins/apps/core/app_deploy_decorator.py +333 -0
  6. metaflow_extensions/outerbounds/plugins/apps/core/capsule.py +150 -79
  7. metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py +4 -1
  8. metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py +4 -0
  9. metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py +103 -5
  10. metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py +12 -1
  11. metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py +100 -6
  12. metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py +141 -2
  13. metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py +74 -37
  14. metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml +6 -6
  15. metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py +2 -2
  16. metaflow_extensions/outerbounds/plugins/apps/core/deployer.py +1102 -105
  17. metaflow_extensions/outerbounds/plugins/apps/core/exceptions.py +341 -0
  18. metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py +42 -6
  19. metaflow_extensions/outerbounds/plugins/aws/assume_role_decorator.py +43 -3
  20. metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py +10 -1
  21. metaflow_extensions/outerbounds/plugins/optuna/__init__.py +2 -1
  22. metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py +37 -7
  23. metaflow_extensions/outerbounds/plugins/snowpark/snowpark.py +18 -8
  24. metaflow_extensions/outerbounds/plugins/snowpark/snowpark_cli.py +6 -0
  25. metaflow_extensions/outerbounds/plugins/snowpark/snowpark_client.py +39 -15
  26. metaflow_extensions/outerbounds/plugins/snowpark/snowpark_decorator.py +5 -2
  27. metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py +2 -2
  28. metaflow_extensions/outerbounds/remote_config.py +20 -7
  29. metaflow_extensions/outerbounds/toplevel/apps/__init__.py +9 -0
  30. metaflow_extensions/outerbounds/toplevel/apps/exceptions.py +11 -0
  31. metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py +1 -1
  32. metaflow_extensions/outerbounds/toplevel/ob_internal.py +1 -1
  33. {ob_metaflow_extensions-1.4.33.dist-info → ob_metaflow_extensions-1.6.2.dist-info}/METADATA +2 -2
  34. {ob_metaflow_extensions-1.4.33.dist-info → ob_metaflow_extensions-1.6.2.dist-info}/RECORD +36 -34
  35. metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py +0 -146
  36. metaflow_extensions/outerbounds/plugins/apps/core/app_cli.py +0 -1200
  37. {ob_metaflow_extensions-1.4.33.dist-info → ob_metaflow_extensions-1.6.2.dist-info}/WHEEL +0 -0
  38. {ob_metaflow_extensions-1.4.33.dist-info → ob_metaflow_extensions-1.6.2.dist-info}/top_level.txt +0 -0
@@ -66,6 +66,10 @@ def snowpark():
66
66
  "--schema",
67
67
  help="Schema for Snowpark Container Services.",
68
68
  )
69
+ @click.option(
70
+ "--integration",
71
+ help="Outerbounds OAuth integration name for Snowpark Container Services. When set, uses OAuth authentication instead of password.",
72
+ )
69
73
  @click.option(
70
74
  "--image",
71
75
  help="Docker image requirement for Snowpark Container Services. In name:version format.",
@@ -119,6 +123,7 @@ def step(
119
123
  database=None,
120
124
  warehouse=None,
121
125
  schema=None,
126
+ integration=None,
122
127
  image=None,
123
128
  stage=None,
124
129
  compute_pool=None,
@@ -235,6 +240,7 @@ def step(
235
240
  "database": database,
236
241
  "warehouse": warehouse,
237
242
  "schema": schema,
243
+ "integration": integration,
238
244
  },
239
245
  )
240
246
  with ctx.obj.monitor.measure("metaflow.snowpark.launch_job"):
@@ -10,14 +10,15 @@ from metaflow.exception import MetaflowException
10
10
  class SnowparkClient(object):
11
11
  def __init__(
12
12
  self,
13
- account: str,
14
- user: str,
15
- password: str,
16
- role: str,
17
- database: str,
18
- warehouse: str,
19
- schema: str,
13
+ account: str = None,
14
+ user: str = None,
15
+ password: str = None,
16
+ role: str = None,
17
+ database: str = None,
18
+ warehouse: str = None,
19
+ schema: str = None,
20
20
  autocommit: bool = True,
21
+ integration: str = None,
21
22
  ):
22
23
  try:
23
24
  from snowflake.core import Root
@@ -37,15 +38,38 @@ class SnowparkClient(object):
37
38
  % sys.executable
38
39
  )
39
40
 
41
+ if integration:
42
+ # Use OAuth authentication via Outerbounds integration
43
+ from metaflow_extensions.outerbounds.plugins.snowflake.snowflake import (
44
+ get_oauth_connection_params,
45
+ )
46
+
47
+ self.connection_parameters = get_oauth_connection_params(
48
+ user=user or "",
49
+ role=role or "",
50
+ integration=integration,
51
+ schema=schema or "",
52
+ account=account,
53
+ warehouse=warehouse,
54
+ database=database,
55
+ )
56
+ self.connection_parameters["autocommit"] = autocommit
57
+ else:
58
+ # Password-based authentication
59
+ self.connection_parameters = {
60
+ "account": account,
61
+ "user": user,
62
+ "password": password,
63
+ "role": role,
64
+ "warehouse": warehouse,
65
+ "database": database,
66
+ "schema": schema,
67
+ "autocommit": autocommit,
68
+ }
69
+
70
+ # Remove None values from connection parameters
40
71
  self.connection_parameters = {
41
- "account": account,
42
- "user": user,
43
- "password": password,
44
- "role": role,
45
- "warehouse": warehouse,
46
- "database": database,
47
- "schema": schema,
48
- "autocommit": autocommit,
72
+ k: v for k, v in self.connection_parameters.items() if v is not None
49
73
  }
50
74
 
51
75
  try:
@@ -71,6 +71,7 @@ class SnowparkDecorator(StepDecorator):
71
71
  "cpu": None,
72
72
  "gpu": None,
73
73
  "memory": None,
74
+ "integration": None, # Outerbounds OAuth integration name
74
75
  }
75
76
 
76
77
  package_url = None
@@ -80,12 +81,11 @@ class SnowparkDecorator(StepDecorator):
80
81
  def __init__(self, attributes=None, statically_defined=False):
81
82
  super(SnowparkDecorator, self).__init__(attributes, statically_defined)
82
83
 
84
+ # Set defaults from config (user can override via decorator or integration)
83
85
  if not self.attributes["account"]:
84
86
  self.attributes["account"] = SNOWPARK_ACCOUNT
85
87
  if not self.attributes["user"]:
86
88
  self.attributes["user"] = SNOWPARK_USER
87
- if not self.attributes["password"]:
88
- self.attributes["password"] = SNOWPARK_PASSWORD
89
89
  if not self.attributes["role"]:
90
90
  self.attributes["role"] = SNOWPARK_ROLE
91
91
  if not self.attributes["database"]:
@@ -94,6 +94,9 @@ class SnowparkDecorator(StepDecorator):
94
94
  self.attributes["warehouse"] = SNOWPARK_WAREHOUSE
95
95
  if not self.attributes["schema"]:
96
96
  self.attributes["schema"] = SNOWPARK_SCHEMA
97
+ # Only use password from config if not using integration (OAuth)
98
+ if not self.attributes["integration"] and not self.attributes["password"]:
99
+ self.attributes["password"] = SNOWPARK_PASSWORD
97
100
 
98
101
  # If no docker image is explicitly specified, impute a default image.
99
102
  if not self.attributes["image"]:
@@ -12,9 +12,9 @@ from .snowpark_exceptions import SnowparkException
12
12
  mapping = str.maketrans("0123456789", "abcdefghij")
13
13
 
14
14
 
15
- # keep only alpha numeric characters and underscores..
15
+ # keep only alpha numeric characters and dashes..
16
16
  def sanitize_name(job_name: str):
17
- return "".join(char for char in job_name if char.isalnum() or char == "_")
17
+ return "".join(char for char in job_name if char.isalnum() or char == "-")
18
18
 
19
19
 
20
20
  # this is not a decorator since the exception imports need to be inside
@@ -59,11 +59,24 @@ def resolve_config_from_remote(remote_url: str, auth_token: str) -> Dict[str, st
59
59
  response.raise_for_status()
60
60
  data = response.json()
61
61
  return data["config"]
62
- except HTTPError:
63
- raise OuterboundsConfigException(
64
- "Error fetching resolving configuration. Make sure you have run \
65
- `outerbounds configure` with the correct value"
66
- )
62
+ except HTTPError as e:
63
+ if e.response.status_code >= 500:
64
+ raise OuterboundsConfigException(
65
+ "Error resolving outerbounds configuration [status:%s]. Please reach out to "
66
+ "the outerbounds support team to help resolve this issue."
67
+ % e.response.status_code
68
+ )
69
+ elif e.response.status_code == 403:
70
+ raise OuterboundsConfigException(
71
+ "Outerbounds token validity expired [status:%s]. Please re-run the `outerbounds configure` "
72
+ "command with a new magic string from the UI." % e.response.status_code
73
+ )
74
+ else:
75
+ raise OuterboundsConfigException(
76
+ "Failed to fetch the outerbounds configuration string [status:%s]. Please reach out to "
77
+ "the outerbounds support team to help resolve this issue."
78
+ % e.response.status_code
79
+ )
67
80
 
68
81
 
69
82
  def init_config() -> Dict[str, str]:
@@ -87,8 +100,8 @@ def init_config() -> Dict[str, str]:
87
100
  remote_config = json.loads(config_path.read_text())
88
101
  except ValueError:
89
102
  raise OuterboundsConfigException(
90
- "Error decoding your metaflow config. Please run the `outerbounds configure` \
91
- command with the string provided in the Outerbounds dashboard"
103
+ "Error decoding your metaflow config. Please re-run the `outerbounds configure` "
104
+ "command with a new magic string from the UI."
92
105
  )
93
106
 
94
107
  perimeter_config_url = get_perimeter_config_url_if_set_in_ob_config()
@@ -0,0 +1,9 @@
1
+ from ...plugins.apps.core import (
2
+ AppDeployer,
3
+ DeployedApp,
4
+ bake_image,
5
+ BakedImage,
6
+ package_code,
7
+ PackagedCode,
8
+ )
9
+ from . import exceptions
@@ -0,0 +1,11 @@
1
+ from ...plugins.apps.core.exceptions import (
2
+ AppDeploymentException,
3
+ AppCrashLoopException,
4
+ AppReadinessException,
5
+ AppConcurrentUpgradeException,
6
+ AppUpgradeInProgressException,
7
+ AppCreationFailedException,
8
+ AppNotFoundException,
9
+ AppDeletedDuringDeploymentException,
10
+ OuterboundsBackendUnhealthyException,
11
+ )
@@ -136,4 +136,4 @@ from ..plugins.snowflake import Snowflake
136
136
  from ..plugins.checkpoint_datastores import nebius_checkpoints, coreweave_checkpoints
137
137
  from ..plugins.aws import assume_role
138
138
  from . import ob_internal
139
- from .ob_internal import AppDeployer
139
+ from . import apps
@@ -1,4 +1,4 @@
1
1
  from ..plugins.kubernetes.pod_killer import PodKiller
2
- from ..plugins.fast_bakery.baker import bake_image
2
+ from ..plugins.fast_bakery.baker import bake_image as internal_bake_image
3
3
  from ..plugins.apps import core as app_core
4
4
  from ..plugins.apps.core import AppDeployer
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ob-metaflow-extensions
3
- Version: 1.4.33
3
+ Version: 1.6.2
4
4
  Summary: Outerbounds Platform Extensions for Metaflow
5
5
  Author: Outerbounds, Inc.
6
6
  License: Commercial
7
7
  Description-Content-Type: text/markdown
8
8
  Requires-Dist: boto3
9
9
  Requires-Dist: kubernetes
10
- Requires-Dist: ob-metaflow (==2.19.9.1)
10
+ Requires-Dist: ob-metaflow (==2.19.15.3)
11
11
 
12
12
  # Outerbounds platform package
13
13
 
@@ -1,27 +1,27 @@
1
1
  metaflow_extensions/outerbounds/__init__.py,sha256=Gb8u06s9ClQsA_vzxmkCzuMnigPy7kKcDnLfb7eB-64,514
2
- metaflow_extensions/outerbounds/remote_config.py,sha256=NIKiq-hrThOll7RLw2JfSN7K5Wz0QtjtRb5AVlHZ1vA,4787
2
+ metaflow_extensions/outerbounds/remote_config.py,sha256=bP5GhDvnjf5nuKUgHsaImstFxrylzJksBqBWaPEME1s,5458
3
3
  metaflow_extensions/outerbounds/config/__init__.py,sha256=JsQGRuGFz28fQWjUvxUgR8EKBLGRdLUIk_buPLJplJY,1225
4
- metaflow_extensions/outerbounds/plugins/__init__.py,sha256=nrsobix6XPIn5Uyg6yTSOtNwr6yaV2-gGbU83hjnbIM,14038
4
+ metaflow_extensions/outerbounds/plugins/__init__.py,sha256=5sxtyUdh2KoeXvpMRspqdyf50_2v9FnHUynYxuSkDvo,14189
5
5
  metaflow_extensions/outerbounds/plugins/auth_server.py,sha256=_Q9_2EL0Xy77bCRphkwT1aSu8gQXRDOH-Z-RxTUO8N4,2202
6
6
  metaflow_extensions/outerbounds/plugins/perimeters.py,sha256=QXh3SFP7GQbS-RAIxUOPbhPzQ7KDFVxZkTdKqFKgXjI,2697
7
7
  metaflow_extensions/outerbounds/plugins/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
8
  metaflow_extensions/outerbounds/plugins/apps/app_cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
- metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py,sha256=WWFvyl0EDUbp_UwmWjXBqlLfPEZ-lq98NJ5F5skBDKU,4901
10
9
  metaflow_extensions/outerbounds/plugins/apps/app_utils.py,sha256=sw9whU17lAzlD2K2kEDNjlk1Ib-2xE2UNhJkmzD8Qv8,8543
11
10
  metaflow_extensions/outerbounds/plugins/apps/consts.py,sha256=iHsyqbUg9k-rgswCs1Jxf5QZIxR1V-peCDRjgr9kdBM,177
12
11
  metaflow_extensions/outerbounds/plugins/apps/deploy_decorator.py,sha256=VkmiMdNYHhNdt-Qm9AVv7aE2LWFsIFEc16YcOYjwF6Q,8568
13
12
  metaflow_extensions/outerbounds/plugins/apps/supervisord_utils.py,sha256=GQoN2gyPClcpR9cLldJmbCfqXnoAHxp8xUnY7vzaYtY,9026
14
- metaflow_extensions/outerbounds/plugins/apps/core/__init__.py,sha256=c6uCgKlgEkTmM9BVdAO-m3vZvUpK2KW_AZZ2236now4,237
15
- metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py,sha256=al907t2C86BPf4V1V03PLTJRJMOc8gdl1CxLLbklnDU,20281
16
- metaflow_extensions/outerbounds/plugins/apps/core/app_cli.py,sha256=rD7oimnkEm___K2Ttiwe12B-Xzqi76tk4_HyFQArBQs,42736
17
- metaflow_extensions/outerbounds/plugins/apps/core/app_config.py,sha256=MDOWrLuLK3CsbO6SRuidRmoB_g0SGrdC-8sTdE5kWLs,4226
13
+ metaflow_extensions/outerbounds/plugins/apps/core/__init__.py,sha256=2-tVXFdajOBzXaY70uHh7lqdZqSx_UmnQdpNRmszRNA,439
14
+ metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py,sha256=OtM129BVCP2UXUL_kUPlG9OX0LJfOd45-b42hxO6A60,20350
15
+ metaflow_extensions/outerbounds/plugins/apps/core/app_config.py,sha256=9NfQe7DEaz_cG_3E7nGwLdlLDb-azcN4RW3l-jJXYPI,3863
16
+ metaflow_extensions/outerbounds/plugins/apps/core/app_deploy_decorator.py,sha256=cvrbMY2EauFccU17oDJ3fnVsvMiyfa3jWQQlJToLpjk,10102
18
17
  metaflow_extensions/outerbounds/plugins/apps/core/artifacts.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
- metaflow_extensions/outerbounds/plugins/apps/core/capsule.py,sha256=MneFa7qna90Gg_GVP7t5jUkQ_VLwT0jGJmLL9toevyk,37063
18
+ metaflow_extensions/outerbounds/plugins/apps/core/capsule.py,sha256=86e6TNg53J25D3lUkVmnIgs_L7kc2OONdz9_J5lf3oc,41406
20
19
  metaflow_extensions/outerbounds/plugins/apps/core/click_importer.py,sha256=kgoPQmK_-8PSSTc3QMSaynCLQ5VWTkKFOC69FPURyXA,998
21
- metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml,sha256=vpvhpZOiPvqu-9mSoYk38vZ5wnFdeSA5DAPGVxFSoWA,10176
22
- metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py,sha256=JlWT9f27yzZeJPlqTQk134WDfQgOdyxC5iaw3pLlhqY,4006
23
- metaflow_extensions/outerbounds/plugins/apps/core/deployer.py,sha256=dNKlDu6n8SufEd5NKmsErl1RYhQXuEe_DgtA0mk7awg,9472
24
- metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py,sha256=ETlqTkHDvVaWusrbNncFf7pcGJarj0r-5qodC4gZWQM,3068
20
+ metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml,sha256=DoDD3kqlnqJMrEHEAnJd2ll9w9CT8Oq7dLlukAswHD8,10073
21
+ metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py,sha256=k6moZjlU8V1eSMHFY22NW4k5j4SKNYhpFiX_f68hjwc,4024
22
+ metaflow_extensions/outerbounds/plugins/apps/core/deployer.py,sha256=fmvJD-3qus2RpF6Q_I8SwwnwGCsVi1XgQuGL8YElc3A,42055
23
+ metaflow_extensions/outerbounds/plugins/apps/core/exceptions.py,sha256=T6f4WurJpxHR7rufS-3yC2c22T_a35aW1rAWXwmlEg4,11663
24
+ metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py,sha256=XXSwO72R4lsLKmtmlGz7TBuCX9_r1MYyQWHY2CF7WlA,4558
25
25
  metaflow_extensions/outerbounds/plugins/apps/core/secrets.py,sha256=sgDiAmpSC8Y5xjlaOEp79F6m0S3x4RONf_vJ5PUAfu8,6127
26
26
  metaflow_extensions/outerbounds/plugins/apps/core/utils.py,sha256=2M2zU8DhbAlJee8P0xKXINAku81PcUylS3sVCSb0TUs,7896
27
27
  metaflow_extensions/outerbounds/plugins/apps/core/validations.py,sha256=Inr9AJDe-L3PMMMxcJPH1zulh9_SynqITb2BzGseLh4,471
@@ -31,17 +31,17 @@ metaflow_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.py,sh
31
31
  metaflow_extensions/outerbounds/plugins/apps/core/code_package/__init__.py,sha256=8McF7pgx8ghvjRnazp2Qktlxi9yYwNiwESSQrk-2oW8,68
32
32
  metaflow_extensions/outerbounds/plugins/apps/core/code_package/code_packager.py,sha256=QHlF94T7ZGSOJH5Xeg4mI_CJvCrbaikiIOTnkFgE83g,23228
33
33
  metaflow_extensions/outerbounds/plugins/apps/core/code_package/examples.py,sha256=aF8qKIJxCVv_ugcShQjqUsXKKKMsm1oMkQIl8w3QKuw,4016
34
- metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py,sha256=ZgC9U4NFu7TNngUuUMzt69PmjMENXb6eUK2HCSroIDo,393
35
- metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py,sha256=0R0-wy7RxAMR9doVRvuluRYxAYgyjZXlTIkOeYGyz7M,5350
36
- metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py,sha256=bozzUR8rbfOnb5M532RZxB5QNvVgEC1gnVjfCvQ82Yk,34053
37
- metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py,sha256=vfmoSXGRocXryOQOGHbh-zIboEP3ty0BVucOza6uzXU,9614
38
- metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py,sha256=9FWdp_F_jBsoxxQf40sRcNgvba8RBYkn2Q5HjXHg_6Y,4733
39
- metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py,sha256=KiJ1eiwtBR5eWdBzWqvO6KlqJ2qzjJvl3w4c1uJ3g0Y,13419
40
- metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py,sha256=gqf8p2OYvUWtslEDPsvgZKl13RMhfKeIaq3bGrIjGqs,38990
34
+ metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py,sha256=iIHHyCuH7V3xbVvlS0uCJnboQx67NeA03y8kuKxyW1A,489
35
+ metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py,sha256=JGARTvKESPQbuSy3RdnIF4PHUDsT0W0liFB_jmuCark,5496
36
+ metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py,sha256=tMj3belC7yBYV6DHjsWfFq2MNsZuIXnIAeEO8HQEFf4,37768
37
+ metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py,sha256=gl3toISxzm297pehSBobdrFmdIpVCVgcYWJO_YYrvn0,9954
38
+ metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py,sha256=HYpc_RnvQmrP52mkn0G5HDNa6jUhc738Xrpklio1ytQ,8090
39
+ metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py,sha256=SCInjZFTubHNcbVgeXkLWuHir0X6DzwYqWR_LFDtejU,18138
40
+ metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py,sha256=KnLQiLR_oKdv2LumBaSF6-IMcwzSV5SRoC25p6iEe3s,39924
41
41
  metaflow_extensions/outerbounds/plugins/apps/core/experimental/__init__.py,sha256=_OT0RsbQdYerT-fjG_vcu85ava6sHbCsGBpLcGcrlX8,3043
42
42
  metaflow_extensions/outerbounds/plugins/aws/__init__.py,sha256=VBGdjNKeFLXGZuqh4jVk8cFtO1AWof73a6k_cnbAOYA,145
43
43
  metaflow_extensions/outerbounds/plugins/aws/assume_role.py,sha256=mBewNlnSYsR2rFXFkX-DUH6ku01h2yOcMcLHoCL7eyI,161
44
- metaflow_extensions/outerbounds/plugins/aws/assume_role_decorator.py,sha256=MrExIdwnQ4ZVKbrWqEQAIuZO4lWEy1rE0ihGUx7GMJA,2976
44
+ metaflow_extensions/outerbounds/plugins/aws/assume_role_decorator.py,sha256=cZg2zUcyvehpjE-a12HTP6vpwPG2MAJuK5KSywBk8xs,4638
45
45
  metaflow_extensions/outerbounds/plugins/card_utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
46
46
  metaflow_extensions/outerbounds/plugins/card_utilities/async_cards.py,sha256=6rQhtZXK5DenXPfCRS1ul0jvLJYd48jrJAlnodID21w,4347
47
47
  metaflow_extensions/outerbounds/plugins/card_utilities/extra_components.py,sha256=D8rgCaSc7PuLD0MHJjqsjN0g0PQMN1H-ySOJqi5uIOE,19111
@@ -51,7 +51,7 @@ metaflow_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.py,sha25
51
51
  metaflow_extensions/outerbounds/plugins/checkpoint_datastores/external_chckpt.py,sha256=BRigyuyYYv5H1CxfifFJPPvgMdQeCNIT8Hm0j1xo-YA,3025
52
52
  metaflow_extensions/outerbounds/plugins/checkpoint_datastores/nebius.py,sha256=xFGdSxlHcwZwgTSKj8efEv6ydCWYqRGrreGqXhf02ko,2321
53
53
  metaflow_extensions/outerbounds/plugins/fast_bakery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
54
- metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py,sha256=ShE5omFBr83wkvEhL_ptRFvDNMs6wefg4BjaafQjTcM,3602
54
+ metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py,sha256=0i3UKZqB3iQD3dMxVEVvro-e4wLTiISf-B4_0wfWS58,3968
55
55
  metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py,sha256=vKN3Waz0SgbOALVTRcf_8MCGhxdRueRFZJ7JmWbRJRM,15487
56
56
  metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery.py,sha256=9iqIBowKk95vUYvnMrFNWmmSuK-whKnqJwcCmEcRjTU,5727
57
57
  metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_cli.py,sha256=kqFyu2bJSnc9_9aYfBpz5xK6L6luWFZK_NMuh8f1eVk,1494
@@ -83,7 +83,7 @@ metaflow_extensions/outerbounds/plugins/ollama/constants.py,sha256=hxkTpWEJp1pKH
83
83
  metaflow_extensions/outerbounds/plugins/ollama/exceptions.py,sha256=8Ss296_MGZl1wXAoDNwpH-hsPe6iYLe90Ji1pczNocU,668
84
84
  metaflow_extensions/outerbounds/plugins/ollama/ollama.py,sha256=C-6Hz8OxsJiB14AAxmunq3P4k7DrmVHsSOxE0xsP-nY,79780
85
85
  metaflow_extensions/outerbounds/plugins/ollama/status_card.py,sha256=F5e4McDl28lhtjeUyInkl03bqjr1lgLxWoau8Q9xwBE,10994
86
- metaflow_extensions/outerbounds/plugins/optuna/__init__.py,sha256=iFNDFNtWOfqwXi8bhMoItM8it-bATETbx8PGK66UbHI,1615
86
+ metaflow_extensions/outerbounds/plugins/optuna/__init__.py,sha256=hCT62WFWB-xyCvqKXFCvLlxOMpZ6hZm3_aEa4cASrBg,1681
87
87
  metaflow_extensions/outerbounds/plugins/profilers/deco_injector.py,sha256=oI_C3c64XBm7n88FILqHwn-Nnc5DeT_68I67lM9rXaI,2434
88
88
  metaflow_extensions/outerbounds/plugins/profilers/gpu_profile_decorator.py,sha256=gDHQ2sMIp4NuZSzUspbSd8RGdFAoO5mgZAyFcZ2a51Y,2619
89
89
  metaflow_extensions/outerbounds/plugins/profilers/simple_card_decorator.py,sha256=4W9tLGCmkFx-4XYLa1xF6qMiaWOBYYFx_RclZDKej30,3259
@@ -98,14 +98,14 @@ metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_manager.py,sha256=Grl7
98
98
  metaflow_extensions/outerbounds/plugins/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
99
99
  metaflow_extensions/outerbounds/plugins/secrets/secrets.py,sha256=3s98hO_twKkM22tKyDdcUjGQNfYpSXW_jLKISV9ju_U,8433
100
100
  metaflow_extensions/outerbounds/plugins/snowflake/__init__.py,sha256=RG4ixt3jwqcK1_tt0QxLcUbNmf7wWAMnZhBx-ZMGgLk,114
101
- metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py,sha256=zoWSHM4CJSfUmJSP-_i4zREWyQOW4USBlgjhQnEhlTE,13669
101
+ metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py,sha256=_ULhLrKZ4rOVcJFb_EmMGi7GFDN6Vtawz8o86mLBMq8,14756
102
102
  metaflow_extensions/outerbounds/plugins/snowpark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
103
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark.py,sha256=0R8aFN9MpgWraqiaI6ZF82YpLdFJ1f-3z_-BPRpZfxM,10674
104
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_cli.py,sha256=ezJ2Jr8JJ48SvTH0ET7pRUGaBuq3XTSfNxJVgXhZsnY,8756
105
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_client.py,sha256=JWA0n3imeMP3HoDw9_9Ge9UPMAwY_TnPKaETHDN8qjA,4460
106
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_decorator.py,sha256=SMND-1BJoqMkYFfFa3B_OT6Xu6HH07degq3qXWOrJ2w,10167
103
+ metaflow_extensions/outerbounds/plugins/snowpark/snowpark.py,sha256=tVpSOizbVjoXp-iMWuzJgE41hUstMrPC8O_eCKY9Mg4,11083
104
+ metaflow_extensions/outerbounds/plugins/snowpark/snowpark_cli.py,sha256=aesXYZGoEF8UtAIvCDJNLTSc9pM62KuOqd9Za95HjZ4,8997
105
+ metaflow_extensions/outerbounds/plugins/snowpark/snowpark_client.py,sha256=6U1sdXx0pdhNdkGba8DaE7GabQTc5fyJI3LpPsPQt0s,5463
106
+ metaflow_extensions/outerbounds/plugins/snowpark/snowpark_decorator.py,sha256=EN-XQafl14uQ3oVGgrkNII5PCYq2lWOt3t2d-JGPeBk,10430
107
107
  metaflow_extensions/outerbounds/plugins/snowpark/snowpark_exceptions.py,sha256=FTfYlJu-sn9DkPOs2R1V1ChWb1vZthOgeq0BZdT1ucY,296
108
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py,sha256=k0OLbsExQOcLbU3_qBd-p2-HCgzcWqXUHxj6L5b6FOk,7075
108
+ metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py,sha256=KcbAYZdInAlS10IAZWdqtWAJXY4I6N2ebCTXzAmT72o,7070
109
109
  metaflow_extensions/outerbounds/plugins/snowpark/snowpark_service_spec.py,sha256=AI_kcm1hZV3JRxJkookcH6twiGnAYjk9Dx-MeoYz60Y,8511
110
110
  metaflow_extensions/outerbounds/plugins/tensorboard/__init__.py,sha256=9lUM4Cqi5RjrHBRfG6AQMRz8-R96eZC8Ih0KD2lv22Y,1858
111
111
  metaflow_extensions/outerbounds/plugins/torchtune/__init__.py,sha256=Psj2ybj_E1qp5KK2inon9e4ZecaRxnPtW3ngcirbO2g,6094
@@ -117,9 +117,11 @@ metaflow_extensions/outerbounds/plugins/vllm/vllm_manager.py,sha256=sp_TX2SrImJG
117
117
  metaflow_extensions/outerbounds/profilers/__init__.py,sha256=wa_jhnCBr82TBxoS0e8b6_6sLyZX0fdHicuGJZNTqKw,29
118
118
  metaflow_extensions/outerbounds/profilers/gpu.py,sha256=3Er8uKQzfm_082uadg4yn_D4Y-iSCgzUfFmguYxZsz4,27485
119
119
  metaflow_extensions/outerbounds/toplevel/__init__.py,sha256=qWUJSv_r5hXJ7jV_On4nEasKIfUCm6_UjkjXWA_A1Ts,90
120
- metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py,sha256=EQbWEatFfsJah7kEiUVrCC2rNTj2UeISJB9N7gtGitc,4546
121
- metaflow_extensions/outerbounds/toplevel/ob_internal.py,sha256=DXCaAtLzlE-bFIiVWEv-iV2JKIWsoSGaUeH4jIQZ9gs,193
120
+ metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py,sha256=9tTaDi36bcdl2U3lCu_ASLG3Orpm2ezEfY5ZWu8ggqs,4528
121
+ metaflow_extensions/outerbounds/toplevel/ob_internal.py,sha256=AS0rJ9_Cb_NAjQthirFVMfkQpe-1uS3lTPU1hP0DBIA,216
122
122
  metaflow_extensions/outerbounds/toplevel/s3_proxy.py,sha256=zdqG7Z12cGuoYYCi2P4kqC3WsgL3xfdJGIb7ejecHH4,2862
123
+ metaflow_extensions/outerbounds/toplevel/apps/__init__.py,sha256=MRhSZWbOSz1jEKHzZU6OssOVqdDEYAL9CgFUXCBzdsY,164
124
+ metaflow_extensions/outerbounds/toplevel/apps/exceptions.py,sha256=qikhm0Gf4zw_LWcl9RT6zjcGj_RVeuXJ8FDsNePWmhE,341
123
125
  metaflow_extensions/outerbounds/toplevel/plugins/azure/__init__.py,sha256=WUuhz2YQfI4fz7nIcipwwWq781eaoHEk7n4GAn1npDg,63
124
126
  metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3uILlEZ6ntBLKeNyqn3If8nIXZFq_Apd7Dhco,70
125
127
  metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
@@ -128,7 +130,7 @@ metaflow_extensions/outerbounds/toplevel/plugins/optuna/__init__.py,sha256=6D1wL
128
130
  metaflow_extensions/outerbounds/toplevel/plugins/snowflake/__init__.py,sha256=LptpH-ziXHrednMYUjIaosS1SXD3sOtF_9_eRqd8SJw,50
129
131
  metaflow_extensions/outerbounds/toplevel/plugins/torchtune/__init__.py,sha256=uTVkdSk3xZ7hEKYfdlyVteWj5KeDwaM1hU9WT-_YKfI,50
130
132
  metaflow_extensions/outerbounds/toplevel/plugins/vllm/__init__.py,sha256=ekcgD3KVydf-a0xMI60P4uy6ePkSEoFHiGnDq1JM940,45
131
- ob_metaflow_extensions-1.4.33.dist-info/METADATA,sha256=YBNcBQmObfPB4tnDJUHmPvpBTn03Wt6gj3CGo7Sgk_U,519
132
- ob_metaflow_extensions-1.4.33.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
133
- ob_metaflow_extensions-1.4.33.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
134
- ob_metaflow_extensions-1.4.33.dist-info/RECORD,,
133
+ ob_metaflow_extensions-1.6.2.dist-info/METADATA,sha256=Z7NNOTL3Ca0wTEK8sveSeLkeHM-eZIiomhBchWw8Xvo,519
134
+ ob_metaflow_extensions-1.6.2.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
135
+ ob_metaflow_extensions-1.6.2.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
136
+ ob_metaflow_extensions-1.6.2.dist-info/RECORD,,
@@ -1,146 +0,0 @@
1
- from metaflow.exception import MetaflowException
2
- from metaflow.decorators import StepDecorator
3
- from metaflow import current
4
- from .core import AppDeployer, apps
5
- from .core.perimeters import PerimeterExtractor
6
- import os
7
- import hashlib
8
-
9
-
10
- class AppDeployDecorator(StepDecorator):
11
-
12
- """
13
- MF Add To Current
14
- -----------------
15
- apps -> metaflow_extensions.outerbounds.plugins.apps.core.apps
16
-
17
- @@ Returns
18
- ----------
19
- apps
20
- The object carrying the Deployer class to deploy apps.
21
- """
22
-
23
- name = "app_deploy"
24
- defaults = {}
25
-
26
- package_url = None
27
- package_sha = None
28
-
29
- MAX_ENTROPY = 6
30
- MAX_NAME_LENGTH = 15 - MAX_ENTROPY - 1 # -1 for the hyphen
31
-
32
- def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
33
- self.logger = logger
34
- self.environment = environment
35
- self.step = step
36
- self.flow_datastore = flow_datastore
37
-
38
- def _resolve_package_url_and_sha(self):
39
- return os.environ.get("METAFLOW_CODE_URL", self.package_url), os.environ.get(
40
- "METAFLOW_CODE_SHA", self.package_sha
41
- )
42
-
43
- def _extract_project_info(self):
44
- project = current.get("project_name")
45
- branch = current.get("branch_name")
46
- is_production = current.get("is_production")
47
- return project, branch, is_production
48
-
49
- def _resolve_default_image(self, flow):
50
- # TODO : Resolve the default image over here.
51
- pass
52
-
53
- def _resolve_default_name_prefix(self, flow, step_name):
54
- # TODO: Only tweek MAX_NAME_LENGTH as backend support allows longer names.
55
- base_prefix = (flow.name + "-" + step_name).lower()
56
- if len(base_prefix) > self.MAX_NAME_LENGTH:
57
- base_prefix = "mf-app"
58
- return base_prefix
59
-
60
- def task_pre_step(
61
- self,
62
- step_name,
63
- task_datastore,
64
- metadata,
65
- run_id,
66
- task_id,
67
- flow,
68
- graph,
69
- retry_count,
70
- max_user_code_retries,
71
- ubf_context,
72
- inputs,
73
- ):
74
- perimeter, api_server = PerimeterExtractor.during_metaflow_execution()
75
- package_url, package_sha = self._resolve_package_url_and_sha()
76
- if package_url is None or package_sha is None:
77
- raise MetaflowException(
78
- "METAFLOW_CODE_URL or METAFLOW_CODE_SHA is not set. "
79
- "Please set METAFLOW_CODE_URL and METAFLOW_CODE_SHA in your environment."
80
- )
81
- image = os.environ.get("FASTBAKERY_IMAGE", None)
82
-
83
- # TODO [Apps] - This is temporary. Backend will support longer names in the future.
84
- default_name = self._resolve_default_name_prefix(flow, step_name)
85
- project, branch, is_production = self._extract_project_info()
86
- project_info = {}
87
- if project is not None:
88
- project_info["metaflow/project"] = project
89
- project_info["metaflow/branch"] = branch
90
- project_info["metaflow/is_production"] = is_production
91
-
92
- default_tags = {
93
- "metaflow/flow_name": flow.name,
94
- "metaflow/step_name": step_name,
95
- "metaflow/run_id": run_id,
96
- "metaflow/task_id": task_id,
97
- "metaflow/retry_count": retry_count,
98
- "metaflow/pathspec": current.pathspec,
99
- **project_info,
100
- }
101
-
102
- AppDeployer._set_state(
103
- perimeter,
104
- api_server,
105
- code_package_url=package_url,
106
- code_package_key=package_sha,
107
- name_prefix=default_name,
108
- image=image,
109
- max_entropy=self.MAX_ENTROPY,
110
- default_tags=[{k: str(v)} for k, v in default_tags.items()],
111
- )
112
- current._update_env(
113
- {
114
- "apps": apps(),
115
- }
116
- )
117
-
118
- def task_post_step(
119
- self, step_name, flow, graph, retry_count, max_user_code_retries
120
- ):
121
- pass
122
-
123
- def runtime_init(self, flow, graph, package, run_id):
124
- # Set some more internal state.
125
- self.flow = flow
126
- self.graph = graph
127
- self.package = package
128
- self.run_id = run_id
129
-
130
- def runtime_task_created(
131
- self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context
132
- ):
133
- # To execute the Kubernetes job, the job container needs to have
134
- # access to the code package. We store the package in the datastore
135
- # which the pod is able to download as part of it's entrypoint.
136
- if not is_cloned:
137
- self._save_package_once(self.flow_datastore, self.package)
138
-
139
- @classmethod
140
- def _save_package_once(cls, flow_datastore, package):
141
- if cls.package_url is None:
142
- cls.package_url, cls.package_sha = flow_datastore.save_data(
143
- [package.blob], len_hint=1
144
- )[0]
145
- os.environ["METAFLOW_CODE_URL"] = cls.package_url
146
- os.environ["METAFLOW_CODE_SHA"] = cls.package_sha