ob-metaflow-extensions 1.1.142__py2.py3-none-any.whl → 1.4.33__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow_extensions/outerbounds/__init__.py +1 -1
- metaflow_extensions/outerbounds/plugins/__init__.py +26 -5
- metaflow_extensions/outerbounds/plugins/apps/app_cli.py +0 -0
- metaflow_extensions/outerbounds/plugins/apps/app_deploy_decorator.py +146 -0
- metaflow_extensions/outerbounds/plugins/apps/core/__init__.py +10 -0
- metaflow_extensions/outerbounds/plugins/apps/core/_state_machine.py +506 -0
- metaflow_extensions/outerbounds/plugins/apps/core/_vendor/__init__.py +0 -0
- metaflow_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.py +4 -0
- metaflow_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.py +478 -0
- metaflow_extensions/outerbounds/plugins/apps/core/app_cli.py +1200 -0
- metaflow_extensions/outerbounds/plugins/apps/core/app_config.py +146 -0
- metaflow_extensions/outerbounds/plugins/apps/core/artifacts.py +0 -0
- metaflow_extensions/outerbounds/plugins/apps/core/capsule.py +958 -0
- metaflow_extensions/outerbounds/plugins/apps/core/click_importer.py +24 -0
- metaflow_extensions/outerbounds/plugins/apps/core/code_package/__init__.py +3 -0
- metaflow_extensions/outerbounds/plugins/apps/core/code_package/code_packager.py +618 -0
- metaflow_extensions/outerbounds/plugins/apps/core/code_package/examples.py +125 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/__init__.py +12 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/cli_generator.py +161 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/config_utils.py +868 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/schema_export.py +288 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/typed_configs.py +139 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/typed_init_generator.py +398 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config/unified_config.py +1088 -0
- metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml +337 -0
- metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py +115 -0
- metaflow_extensions/outerbounds/plugins/apps/core/deployer.py +303 -0
- metaflow_extensions/outerbounds/plugins/apps/core/experimental/__init__.py +89 -0
- metaflow_extensions/outerbounds/plugins/apps/core/perimeters.py +87 -0
- metaflow_extensions/outerbounds/plugins/apps/core/secrets.py +164 -0
- metaflow_extensions/outerbounds/plugins/apps/core/utils.py +233 -0
- metaflow_extensions/outerbounds/plugins/apps/core/validations.py +17 -0
- metaflow_extensions/outerbounds/plugins/aws/__init__.py +4 -0
- metaflow_extensions/outerbounds/plugins/aws/assume_role.py +3 -0
- metaflow_extensions/outerbounds/plugins/aws/assume_role_decorator.py +78 -0
- metaflow_extensions/outerbounds/plugins/checkpoint_datastores/__init__.py +2 -0
- metaflow_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.py +71 -0
- metaflow_extensions/outerbounds/plugins/checkpoint_datastores/external_chckpt.py +85 -0
- metaflow_extensions/outerbounds/plugins/checkpoint_datastores/nebius.py +73 -0
- metaflow_extensions/outerbounds/plugins/fast_bakery/baker.py +110 -0
- metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py +17 -3
- metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery.py +1 -0
- metaflow_extensions/outerbounds/plugins/kubernetes/kubernetes_client.py +18 -44
- metaflow_extensions/outerbounds/plugins/kubernetes/pod_killer.py +374 -0
- metaflow_extensions/outerbounds/plugins/nim/card.py +1 -6
- metaflow_extensions/outerbounds/plugins/nim/{__init__.py → nim_decorator.py} +13 -49
- metaflow_extensions/outerbounds/plugins/nim/nim_manager.py +294 -233
- metaflow_extensions/outerbounds/plugins/nim/utils.py +36 -0
- metaflow_extensions/outerbounds/plugins/nvcf/constants.py +2 -2
- metaflow_extensions/outerbounds/plugins/nvcf/nvcf.py +100 -19
- metaflow_extensions/outerbounds/plugins/nvcf/nvcf_decorator.py +6 -1
- metaflow_extensions/outerbounds/plugins/nvct/__init__.py +0 -0
- metaflow_extensions/outerbounds/plugins/nvct/exceptions.py +71 -0
- metaflow_extensions/outerbounds/plugins/nvct/nvct.py +131 -0
- metaflow_extensions/outerbounds/plugins/nvct/nvct_cli.py +289 -0
- metaflow_extensions/outerbounds/plugins/nvct/nvct_decorator.py +286 -0
- metaflow_extensions/outerbounds/plugins/nvct/nvct_runner.py +218 -0
- metaflow_extensions/outerbounds/plugins/nvct/utils.py +29 -0
- metaflow_extensions/outerbounds/plugins/ollama/__init__.py +171 -16
- metaflow_extensions/outerbounds/plugins/ollama/constants.py +1 -0
- metaflow_extensions/outerbounds/plugins/ollama/exceptions.py +22 -0
- metaflow_extensions/outerbounds/plugins/ollama/ollama.py +1710 -114
- metaflow_extensions/outerbounds/plugins/ollama/status_card.py +292 -0
- metaflow_extensions/outerbounds/plugins/optuna/__init__.py +48 -0
- metaflow_extensions/outerbounds/plugins/profilers/simple_card_decorator.py +96 -0
- metaflow_extensions/outerbounds/plugins/s3_proxy/__init__.py +7 -0
- metaflow_extensions/outerbounds/plugins/s3_proxy/binary_caller.py +132 -0
- metaflow_extensions/outerbounds/plugins/s3_proxy/constants.py +11 -0
- metaflow_extensions/outerbounds/plugins/s3_proxy/exceptions.py +13 -0
- metaflow_extensions/outerbounds/plugins/s3_proxy/proxy_bootstrap.py +59 -0
- metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_api.py +93 -0
- metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_decorator.py +250 -0
- metaflow_extensions/outerbounds/plugins/s3_proxy/s3_proxy_manager.py +225 -0
- metaflow_extensions/outerbounds/plugins/secrets/secrets.py +38 -2
- metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py +44 -4
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_client.py +6 -3
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_decorator.py +13 -7
- metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py +8 -2
- metaflow_extensions/outerbounds/plugins/torchtune/__init__.py +163 -0
- metaflow_extensions/outerbounds/plugins/vllm/__init__.py +255 -0
- metaflow_extensions/outerbounds/plugins/vllm/constants.py +1 -0
- metaflow_extensions/outerbounds/plugins/vllm/exceptions.py +1 -0
- metaflow_extensions/outerbounds/plugins/vllm/status_card.py +352 -0
- metaflow_extensions/outerbounds/plugins/vllm/vllm_manager.py +621 -0
- metaflow_extensions/outerbounds/remote_config.py +27 -3
- metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py +87 -2
- metaflow_extensions/outerbounds/toplevel/ob_internal.py +4 -0
- metaflow_extensions/outerbounds/toplevel/plugins/optuna/__init__.py +1 -0
- metaflow_extensions/outerbounds/toplevel/plugins/torchtune/__init__.py +1 -0
- metaflow_extensions/outerbounds/toplevel/plugins/vllm/__init__.py +1 -0
- metaflow_extensions/outerbounds/toplevel/s3_proxy.py +88 -0
- {ob_metaflow_extensions-1.1.142.dist-info → ob_metaflow_extensions-1.4.33.dist-info}/METADATA +2 -2
- ob_metaflow_extensions-1.4.33.dist-info/RECORD +134 -0
- metaflow_extensions/outerbounds/plugins/nim/utilities.py +0 -5
- ob_metaflow_extensions-1.1.142.dist-info/RECORD +0 -64
- {ob_metaflow_extensions-1.1.142.dist-info → ob_metaflow_extensions-1.4.33.dist-info}/WHEEL +0 -0
- {ob_metaflow_extensions-1.1.142.dist-info → ob_metaflow_extensions-1.4.33.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from typing import Dict, Any
|
|
5
|
+
from .config import (
|
|
6
|
+
CoreConfig,
|
|
7
|
+
MergingNotAllowedFieldsException,
|
|
8
|
+
ConfigValidationFailedException,
|
|
9
|
+
RequiredFieldMissingException,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
CODE_PACKAGE_PREFIX = "mf.obp-apps"
|
|
13
|
+
|
|
14
|
+
CAPSULE_DEBUG = os.environ.get("OUTERBOUNDS_CAPSULE_DEBUG", False)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class classproperty(property):
|
|
18
|
+
def __get__(self, owner_self, owner_cls):
|
|
19
|
+
return self.fget(owner_cls)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AppConfigError(Exception):
|
|
23
|
+
"""Exception raised when app configuration is invalid."""
|
|
24
|
+
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _try_loading_yaml(file):
|
|
29
|
+
try:
|
|
30
|
+
import yaml
|
|
31
|
+
|
|
32
|
+
return yaml.safe_load(file)
|
|
33
|
+
except ImportError:
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
from outerbounds._vendor import yaml
|
|
38
|
+
|
|
39
|
+
return yaml.safe_load(file)
|
|
40
|
+
except ImportError:
|
|
41
|
+
pass
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class AuthType:
|
|
46
|
+
BROWSER = "Browser"
|
|
47
|
+
API = "API"
|
|
48
|
+
BROWSER_AND_API = "BrowserAndApi"
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def enums(cls):
|
|
52
|
+
return [cls.BROWSER, cls.API, cls.BROWSER_AND_API]
|
|
53
|
+
|
|
54
|
+
@classproperty
|
|
55
|
+
def default(cls):
|
|
56
|
+
return cls.BROWSER
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class AppConfig:
|
|
60
|
+
"""Class representing an Outerbounds App configuration."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, core_config: CoreConfig):
|
|
63
|
+
"""Initialize configuration from a dictionary."""
|
|
64
|
+
self._core_config = core_config
|
|
65
|
+
self._final_state: Dict[str, Any] = {}
|
|
66
|
+
self.config = {}
|
|
67
|
+
|
|
68
|
+
def set_state(self, key, value):
|
|
69
|
+
self._final_state[key] = value
|
|
70
|
+
return self
|
|
71
|
+
|
|
72
|
+
def get_state(self, key, default=None):
|
|
73
|
+
return self._final_state.get(key, self.get(key, default))
|
|
74
|
+
|
|
75
|
+
def dump_state(self):
|
|
76
|
+
x = {k: v for k, v in self.config.items()}
|
|
77
|
+
for k, v in self._final_state.items():
|
|
78
|
+
x[k] = v
|
|
79
|
+
return x
|
|
80
|
+
|
|
81
|
+
def commit(self):
|
|
82
|
+
try:
|
|
83
|
+
self._core_config.commit()
|
|
84
|
+
self.config = self._core_config.to_dict()
|
|
85
|
+
except RequiredFieldMissingException as e:
|
|
86
|
+
raise AppConfigError(
|
|
87
|
+
"The configuration is missing the following required fields: %s. \n\tException: %s"
|
|
88
|
+
% (e.field_name, e.message)
|
|
89
|
+
)
|
|
90
|
+
except ConfigValidationFailedException as e:
|
|
91
|
+
raise AppConfigError(
|
|
92
|
+
"The configuration is invalid. \n\n\tException: %s" % (e.message)
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def get(self, key: str, default: Any = None) -> Any:
|
|
96
|
+
"""Get a configuration value by key."""
|
|
97
|
+
config_value = self.config.get(key, default)
|
|
98
|
+
if config_value is None:
|
|
99
|
+
return default
|
|
100
|
+
return config_value
|
|
101
|
+
|
|
102
|
+
def to_json(self):
|
|
103
|
+
return json.dumps(self.config, indent=2)
|
|
104
|
+
|
|
105
|
+
def to_yaml(self):
|
|
106
|
+
return self.to_json()
|
|
107
|
+
|
|
108
|
+
def to_dict(self):
|
|
109
|
+
return self.config
|
|
110
|
+
|
|
111
|
+
@classmethod
|
|
112
|
+
def from_file(cls, file_path: str) -> "AppConfig":
|
|
113
|
+
"""Create a configuration from a file."""
|
|
114
|
+
if not os.path.exists(file_path):
|
|
115
|
+
raise AppConfigError(f"Configuration file '{file_path}' does not exist.")
|
|
116
|
+
|
|
117
|
+
with open(file_path, "r") as f:
|
|
118
|
+
try:
|
|
119
|
+
config_dict = _try_loading_yaml(f)
|
|
120
|
+
if config_dict is None:
|
|
121
|
+
config_dict = json.load(f)
|
|
122
|
+
except json.JSONDecodeError as e:
|
|
123
|
+
raise AppConfigError(
|
|
124
|
+
"The PyYAML package is not available as a dependency and JSON parsing of the configuration file also failed %s: \n%s"
|
|
125
|
+
% (file_path, str(e))
|
|
126
|
+
)
|
|
127
|
+
except Exception as e:
|
|
128
|
+
raise AppConfigError(f"Failed to parse configuration file: {e}")
|
|
129
|
+
|
|
130
|
+
return cls(CoreConfig.from_dict(config_dict))
|
|
131
|
+
|
|
132
|
+
@classmethod
|
|
133
|
+
def from_cli(cls, options: Dict[str, Any]):
|
|
134
|
+
return cls(CoreConfig.from_cli(options))
|
|
135
|
+
|
|
136
|
+
def update_from_cli_options(self, options):
|
|
137
|
+
cli_options_config = CoreConfig.from_cli(options)
|
|
138
|
+
try:
|
|
139
|
+
self._core_config = CoreConfig.merge_configs(
|
|
140
|
+
self._core_config, cli_options_config
|
|
141
|
+
)
|
|
142
|
+
except MergingNotAllowedFieldsException as e:
|
|
143
|
+
raise AppConfigError(
|
|
144
|
+
"CLI Overrides are not allowed for the following fields: %s. \n\tException: %s"
|
|
145
|
+
% (e.field_name, e.message)
|
|
146
|
+
)
|
|
File without changes
|