ob-metaflow 2.15.15.1__py2.py3-none-any.whl → 2.15.18.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ob-metaflow might be problematic. Click here for more details.
- metaflow/cli.py +14 -2
- metaflow/cli_components/run_cmds.py +1 -1
- metaflow/cmd/develop/stub_generator.py +30 -16
- metaflow/flowspec.py +16 -4
- metaflow/metaflow_config.py +2 -2
- metaflow/plugins/argo/argo_workflows.py +7 -0
- metaflow/plugins/argo/argo_workflows_deployer_objects.py +6 -49
- metaflow/plugins/aws/aws_client.py +6 -0
- metaflow/plugins/cards/card_datastore.py +8 -36
- metaflow/plugins/datatools/s3/s3op.py +1 -1
- metaflow/plugins/kubernetes/kubernetes.py +4 -0
- metaflow/plugins/kubernetes/kubernetes_cli.py +8 -0
- metaflow/plugins/kubernetes/kubernetes_decorator.py +11 -0
- metaflow/plugins/kubernetes/kubernetes_job.py +4 -2
- metaflow/plugins/kubernetes/kubernetes_jobsets.py +5 -2
- metaflow/plugins/metadata_providers/service.py +12 -8
- metaflow/plugins/pypi/conda_environment.py +8 -4
- metaflow/plugins/pypi/micromamba.py +9 -1
- metaflow/runner/deployer.py +49 -0
- metaflow/user_configs/config_decorators.py +1 -1
- metaflow/user_configs/config_options.py +6 -2
- metaflow/version.py +1 -1
- {ob_metaflow-2.15.15.1.dist-info → ob_metaflow-2.15.18.1.dist-info}/METADATA +2 -2
- {ob_metaflow-2.15.15.1.dist-info → ob_metaflow-2.15.18.1.dist-info}/RECORD +31 -31
- {ob_metaflow-2.15.15.1.data → ob_metaflow-2.15.18.1.data}/data/share/metaflow/devtools/Makefile +0 -0
- {ob_metaflow-2.15.15.1.data → ob_metaflow-2.15.18.1.data}/data/share/metaflow/devtools/Tiltfile +0 -0
- {ob_metaflow-2.15.15.1.data → ob_metaflow-2.15.18.1.data}/data/share/metaflow/devtools/pick_services.sh +0 -0
- {ob_metaflow-2.15.15.1.dist-info → ob_metaflow-2.15.18.1.dist-info}/WHEEL +0 -0
- {ob_metaflow-2.15.15.1.dist-info → ob_metaflow-2.15.18.1.dist-info}/entry_points.txt +0 -0
- {ob_metaflow-2.15.15.1.dist-info → ob_metaflow-2.15.18.1.dist-info}/licenses/LICENSE +0 -0
- {ob_metaflow-2.15.15.1.dist-info → ob_metaflow-2.15.18.1.dist-info}/top_level.txt +0 -0
metaflow/cli.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import functools
|
|
2
2
|
import inspect
|
|
3
|
+
import os
|
|
3
4
|
import sys
|
|
4
5
|
import traceback
|
|
5
6
|
from datetime import datetime
|
|
@@ -242,6 +243,14 @@ def version(obj):
|
|
|
242
243
|
type=click.Choice(["local"] + [m.TYPE for m in ENVIRONMENTS]),
|
|
243
244
|
help="Execution environment type",
|
|
244
245
|
)
|
|
246
|
+
@click.option(
|
|
247
|
+
"--force-rebuild-environments/--no-force-rebuild-environments",
|
|
248
|
+
is_flag=True,
|
|
249
|
+
default=False,
|
|
250
|
+
hidden=True,
|
|
251
|
+
type=bool,
|
|
252
|
+
help="Explicitly rebuild the execution environments",
|
|
253
|
+
)
|
|
245
254
|
# See comment for --quiet
|
|
246
255
|
@click.option(
|
|
247
256
|
"--datastore",
|
|
@@ -300,6 +309,7 @@ def start(
|
|
|
300
309
|
quiet=False,
|
|
301
310
|
metadata=None,
|
|
302
311
|
environment=None,
|
|
312
|
+
force_rebuild_environments=False,
|
|
303
313
|
datastore=None,
|
|
304
314
|
datastore_root=None,
|
|
305
315
|
decospecs=None,
|
|
@@ -435,6 +445,8 @@ def start(
|
|
|
435
445
|
ctx.obj.environment = [
|
|
436
446
|
e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == environment
|
|
437
447
|
][0](ctx.obj.flow)
|
|
448
|
+
# set force rebuild flag for environments that support it.
|
|
449
|
+
ctx.obj.environment._force_rebuild = force_rebuild_environments
|
|
438
450
|
ctx.obj.environment.validate_environment(ctx.obj.logger, datastore)
|
|
439
451
|
|
|
440
452
|
ctx.obj.event_logger = LOGGING_SIDECARS[event_logger](
|
|
@@ -496,7 +508,7 @@ def start(
|
|
|
496
508
|
ctx.obj.echo,
|
|
497
509
|
ctx.obj.flow_datastore,
|
|
498
510
|
{
|
|
499
|
-
k: ConfigValue(v)
|
|
511
|
+
k: ConfigValue(v) if v is not None else None
|
|
500
512
|
for k, v in ctx.obj.flow.__class__._flow_state.get(
|
|
501
513
|
_FlowState.CONFIGS, {}
|
|
502
514
|
).items()
|
|
@@ -524,7 +536,7 @@ def start(
|
|
|
524
536
|
decorators._attach_decorators(ctx.obj.flow, all_decospecs)
|
|
525
537
|
decorators._init(ctx.obj.flow)
|
|
526
538
|
# Regenerate graph if we attached more decorators
|
|
527
|
-
ctx.obj.flow.__class__.
|
|
539
|
+
ctx.obj.flow.__class__._init_graph()
|
|
528
540
|
ctx.obj.graph = ctx.obj.flow._graph
|
|
529
541
|
|
|
530
542
|
decorators._init_step_decorators(
|
|
@@ -45,7 +45,7 @@ def before_run(obj, tags, decospecs):
|
|
|
45
45
|
decorators._attach_decorators(obj.flow, all_decospecs)
|
|
46
46
|
decorators._init(obj.flow)
|
|
47
47
|
# Regenerate graph if we attached more decorators
|
|
48
|
-
obj.flow.__class__.
|
|
48
|
+
obj.flow.__class__._init_graph()
|
|
49
49
|
obj.graph = obj.flow._graph
|
|
50
50
|
|
|
51
51
|
obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint)
|
|
@@ -488,9 +488,6 @@ class StubGenerator:
|
|
|
488
488
|
self._imports.add(name)
|
|
489
489
|
|
|
490
490
|
def _add_to_typing_check(name, is_module=False):
|
|
491
|
-
# if name != self._current_module_name:
|
|
492
|
-
# self._typing_imports.add(name)
|
|
493
|
-
#
|
|
494
491
|
if name == "None":
|
|
495
492
|
return
|
|
496
493
|
if is_module:
|
|
@@ -504,6 +501,24 @@ class StubGenerator:
|
|
|
504
501
|
# the current file
|
|
505
502
|
self._typing_imports.add(splits[0])
|
|
506
503
|
|
|
504
|
+
def _format_qualified_class_name(cls: type) -> str:
|
|
505
|
+
"""Helper to format a class with its qualified module name"""
|
|
506
|
+
# Special case for NoneType - return None
|
|
507
|
+
if cls.__name__ == "NoneType":
|
|
508
|
+
return "None"
|
|
509
|
+
|
|
510
|
+
module = inspect.getmodule(cls)
|
|
511
|
+
if (
|
|
512
|
+
module
|
|
513
|
+
and module.__name__ != "builtins"
|
|
514
|
+
and module.__name__ != "__main__"
|
|
515
|
+
):
|
|
516
|
+
module_name = self._get_module_name_alias(module.__name__)
|
|
517
|
+
_add_to_typing_check(module_name, is_module=True)
|
|
518
|
+
return f"{module_name}.{cls.__name__}"
|
|
519
|
+
else:
|
|
520
|
+
return cls.__name__
|
|
521
|
+
|
|
507
522
|
if isinstance(element, str):
|
|
508
523
|
# Special case for self referential things (particularly in a class)
|
|
509
524
|
if element == self._current_name:
|
|
@@ -557,19 +572,15 @@ class StubGenerator:
|
|
|
557
572
|
return element.__name__
|
|
558
573
|
elif isinstance(element, type(Ellipsis)):
|
|
559
574
|
return "..."
|
|
560
|
-
# elif (
|
|
561
|
-
# isinstance(element, typing._GenericAlias)
|
|
562
|
-
# and hasattr(element, "_name")
|
|
563
|
-
# and element._name in ("List", "Tuple", "Dict", "Set")
|
|
564
|
-
# ):
|
|
565
|
-
# # 3.7 has these as _GenericAlias but they don't behave like the ones in 3.10
|
|
566
|
-
# _add_to_import("typing")
|
|
567
|
-
# return str(element)
|
|
568
575
|
elif isinstance(element, typing._GenericAlias):
|
|
569
576
|
# We need to check things recursively in __args__ if it exists
|
|
570
577
|
args_str = []
|
|
571
578
|
for arg in getattr(element, "__args__", []):
|
|
572
|
-
|
|
579
|
+
# Special handling for class objects in type arguments
|
|
580
|
+
if isinstance(arg, type):
|
|
581
|
+
args_str.append(_format_qualified_class_name(arg))
|
|
582
|
+
else:
|
|
583
|
+
args_str.append(self._get_element_name_with_module(arg))
|
|
573
584
|
|
|
574
585
|
_add_to_import("typing")
|
|
575
586
|
if element._name:
|
|
@@ -584,12 +595,15 @@ class StubGenerator:
|
|
|
584
595
|
args_str = [call_args, args_str[-1]]
|
|
585
596
|
return "typing.%s[%s]" % (element._name, ", ".join(args_str))
|
|
586
597
|
else:
|
|
587
|
-
|
|
598
|
+
# Handle the case where we have a generic type without a _name
|
|
599
|
+
origin = element.__origin__
|
|
600
|
+
if isinstance(origin, type):
|
|
601
|
+
origin_str = _format_qualified_class_name(origin)
|
|
602
|
+
else:
|
|
603
|
+
origin_str = str(origin)
|
|
604
|
+
return "%s[%s]" % (origin_str, ", ".join(args_str))
|
|
588
605
|
elif isinstance(element, ForwardRef):
|
|
589
606
|
f_arg = self._get_module_name_alias(element.__forward_arg__)
|
|
590
|
-
# if f_arg in ("Run", "Task"): # HACK -- forward references in current.py
|
|
591
|
-
# _add_to_import("metaflow")
|
|
592
|
-
# f_arg = "metaflow.%s" % f_arg
|
|
593
607
|
_add_to_typing_check(f_arg)
|
|
594
608
|
return '"%s"' % f_arg
|
|
595
609
|
elif inspect.getmodule(element) == inspect.getmodule(typing):
|
metaflow/flowspec.py
CHANGED
|
@@ -87,6 +87,9 @@ class FlowSpecMeta(type):
|
|
|
87
87
|
if name == "FlowSpec":
|
|
88
88
|
return
|
|
89
89
|
|
|
90
|
+
cls._init_attrs()
|
|
91
|
+
|
|
92
|
+
def _init_attrs(cls):
|
|
90
93
|
from .decorators import (
|
|
91
94
|
DuplicateFlowDecoratorException,
|
|
92
95
|
) # Prevent circular import
|
|
@@ -103,6 +106,12 @@ class FlowSpecMeta(type):
|
|
|
103
106
|
# Keys are _FlowState enum values
|
|
104
107
|
cls._flow_state = {}
|
|
105
108
|
|
|
109
|
+
# Keep track if configs have been processed -- this is particularly applicable
|
|
110
|
+
# for the Runner/Deployer where calling multiple APIs on the same flow could
|
|
111
|
+
# cause the configs to be processed multiple times. For a given flow, once
|
|
112
|
+
# the configs have been processed, we do not process them again.
|
|
113
|
+
cls._configs_processed = False
|
|
114
|
+
|
|
106
115
|
# We inherit stuff from our parent classes as well -- we need to be careful
|
|
107
116
|
# in terms of the order; we will follow the MRO with the following rules:
|
|
108
117
|
# - decorators (cls._flow_decorators) will cause an error if they do not
|
|
@@ -127,10 +136,9 @@ class FlowSpecMeta(type):
|
|
|
127
136
|
cls._flow_state.setdefault(_FlowState.CONFIG_DECORATORS, []).extend(
|
|
128
137
|
base_configs
|
|
129
138
|
)
|
|
139
|
+
cls._init_graph()
|
|
130
140
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def _init_attrs(cls):
|
|
141
|
+
def _init_graph(cls):
|
|
134
142
|
# Graph and steps are specific to the class -- store here so we can access
|
|
135
143
|
# in class method _process_config_decorators
|
|
136
144
|
cls._graph = FlowGraph(cls)
|
|
@@ -225,6 +233,10 @@ class FlowSpec(metaclass=FlowSpecMeta):
|
|
|
225
233
|
@classmethod
|
|
226
234
|
def _process_config_decorators(cls, config_options, process_configs=True):
|
|
227
235
|
|
|
236
|
+
if cls._configs_processed:
|
|
237
|
+
return None
|
|
238
|
+
cls._configs_processed = True
|
|
239
|
+
|
|
228
240
|
# Fast path for no user configurations
|
|
229
241
|
if not process_configs or (
|
|
230
242
|
not cls._flow_state.get(_FlowState.CONFIG_DECORATORS)
|
|
@@ -325,7 +337,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
|
|
|
325
337
|
parameters.replace_flow_context(cls)
|
|
326
338
|
|
|
327
339
|
# Re-calculate class level attributes after modifying the class
|
|
328
|
-
cls.
|
|
340
|
+
cls._init_graph()
|
|
329
341
|
return cls
|
|
330
342
|
|
|
331
343
|
def _set_constants(self, graph, kwargs, config_options):
|
metaflow/metaflow_config.py
CHANGED
|
@@ -214,8 +214,6 @@ CARD_GSROOT = from_conf(
|
|
|
214
214
|
)
|
|
215
215
|
CARD_NO_WARNING = from_conf("CARD_NO_WARNING", False)
|
|
216
216
|
|
|
217
|
-
SKIP_CARD_DUALWRITE = from_conf("SKIP_CARD_DUALWRITE", False)
|
|
218
|
-
|
|
219
217
|
RUNTIME_CARD_RENDER_INTERVAL = from_conf("RUNTIME_CARD_RENDER_INTERVAL", 60)
|
|
220
218
|
|
|
221
219
|
# Azure storage account URL
|
|
@@ -371,6 +369,8 @@ KUBERNETES_CONTAINER_IMAGE = from_conf(
|
|
|
371
369
|
)
|
|
372
370
|
# Image pull policy for container images
|
|
373
371
|
KUBERNETES_IMAGE_PULL_POLICY = from_conf("KUBERNETES_IMAGE_PULL_POLICY", None)
|
|
372
|
+
# Image pull secrets for container images
|
|
373
|
+
KUBERNETES_IMAGE_PULL_SECRETS = from_conf("KUBERNETES_IMAGE_PULL_SECRETS", "")
|
|
374
374
|
# Default container registry for K8S
|
|
375
375
|
KUBERNETES_CONTAINER_REGISTRY = from_conf(
|
|
376
376
|
"KUBERNETES_CONTAINER_REGISTRY", DEFAULT_CONTAINER_REGISTRY
|
|
@@ -2021,6 +2021,7 @@ class ArgoWorkflows(object):
|
|
|
2021
2021
|
namespace=resources["namespace"],
|
|
2022
2022
|
image=resources["image"],
|
|
2023
2023
|
image_pull_policy=resources["image_pull_policy"],
|
|
2024
|
+
image_pull_secrets=resources["image_pull_secrets"],
|
|
2024
2025
|
service_account=resources["service_account"],
|
|
2025
2026
|
secrets=(
|
|
2026
2027
|
[
|
|
@@ -2209,6 +2210,8 @@ class ArgoWorkflows(object):
|
|
|
2209
2210
|
.node_selectors(resources.get("node_selector"))
|
|
2210
2211
|
# Set tolerations
|
|
2211
2212
|
.tolerations(resources.get("tolerations"))
|
|
2213
|
+
# Set image pull secrets
|
|
2214
|
+
.image_pull_secrets(resources.get("image_pull_secrets"))
|
|
2212
2215
|
# Set container
|
|
2213
2216
|
.container(
|
|
2214
2217
|
# TODO: Unify the logic with kubernetes.py
|
|
@@ -3785,6 +3788,10 @@ class Template(object):
|
|
|
3785
3788
|
self.payload["tolerations"] = tolerations
|
|
3786
3789
|
return self
|
|
3787
3790
|
|
|
3791
|
+
def image_pull_secrets(self, image_pull_secrets):
|
|
3792
|
+
self.payload["image_pull_secrets"] = image_pull_secrets
|
|
3793
|
+
return self
|
|
3794
|
+
|
|
3788
3795
|
def to_json(self):
|
|
3789
3796
|
return self.payload
|
|
3790
3797
|
|
|
@@ -9,59 +9,16 @@ from metaflow.exception import MetaflowException
|
|
|
9
9
|
from metaflow.plugins.argo.argo_client import ArgoClient
|
|
10
10
|
from metaflow.metaflow_config import KUBERNETES_NAMESPACE
|
|
11
11
|
from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
|
|
12
|
-
from metaflow.runner.deployer import
|
|
12
|
+
from metaflow.runner.deployer import (
|
|
13
|
+
Deployer,
|
|
14
|
+
DeployedFlow,
|
|
15
|
+
TriggeredRun,
|
|
16
|
+
generate_fake_flow_file_contents,
|
|
17
|
+
)
|
|
13
18
|
|
|
14
19
|
from metaflow.runner.utils import get_lower_level_group, handle_timeout, temporary_fifo
|
|
15
20
|
|
|
16
21
|
|
|
17
|
-
def generate_fake_flow_file_contents(
|
|
18
|
-
flow_name: str, param_info: dict, project_name: Optional[str] = None
|
|
19
|
-
):
|
|
20
|
-
params_code = ""
|
|
21
|
-
for _, param_details in param_info.items():
|
|
22
|
-
param_python_var_name = param_details["python_var_name"]
|
|
23
|
-
param_name = param_details["name"]
|
|
24
|
-
param_type = param_details["type"]
|
|
25
|
-
param_help = param_details["description"]
|
|
26
|
-
param_required = param_details["is_required"]
|
|
27
|
-
|
|
28
|
-
if param_type == "JSON":
|
|
29
|
-
params_code += (
|
|
30
|
-
f" {param_python_var_name} = Parameter('{param_name}', "
|
|
31
|
-
f"type=JSONType, help='''{param_help}''', required={param_required})\n"
|
|
32
|
-
)
|
|
33
|
-
elif param_type == "FilePath":
|
|
34
|
-
is_text = param_details.get("is_text", True)
|
|
35
|
-
encoding = param_details.get("encoding", "utf-8")
|
|
36
|
-
params_code += (
|
|
37
|
-
f" {param_python_var_name} = IncludeFile('{param_name}', "
|
|
38
|
-
f"is_text={is_text}, encoding='{encoding}', help='''{param_help}''', "
|
|
39
|
-
f"required={param_required})\n"
|
|
40
|
-
)
|
|
41
|
-
else:
|
|
42
|
-
params_code += (
|
|
43
|
-
f" {param_python_var_name} = Parameter('{param_name}', "
|
|
44
|
-
f"type={param_type}, help='''{param_help}''', required={param_required})\n"
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
project_decorator = f"@project(name='{project_name}')\n" if project_name else ""
|
|
48
|
-
|
|
49
|
-
contents = f"""\
|
|
50
|
-
from metaflow import FlowSpec, Parameter, IncludeFile, JSONType, step, project
|
|
51
|
-
{project_decorator}class {flow_name}(FlowSpec):
|
|
52
|
-
{params_code}
|
|
53
|
-
@step
|
|
54
|
-
def start(self):
|
|
55
|
-
self.next(self.end)
|
|
56
|
-
@step
|
|
57
|
-
def end(self):
|
|
58
|
-
pass
|
|
59
|
-
if __name__ == '__main__':
|
|
60
|
-
{flow_name}()
|
|
61
|
-
"""
|
|
62
|
-
return contents
|
|
63
|
-
|
|
64
|
-
|
|
65
22
|
class ArgoWorkflowsTriggeredRun(TriggeredRun):
|
|
66
23
|
"""
|
|
67
24
|
A class representing a triggered Argo Workflow execution.
|
|
@@ -35,6 +35,12 @@ class Boto3ClientProvider(object):
|
|
|
35
35
|
"Could not import module 'boto3'. Install boto3 first."
|
|
36
36
|
)
|
|
37
37
|
|
|
38
|
+
# Convert dictionary config to Config object if needed
|
|
39
|
+
if "config" in client_params and not isinstance(
|
|
40
|
+
client_params["config"], Config
|
|
41
|
+
):
|
|
42
|
+
client_params["config"] = Config(**client_params["config"])
|
|
43
|
+
|
|
38
44
|
if module == "s3" and (
|
|
39
45
|
"config" not in client_params or client_params["config"].retries is None
|
|
40
46
|
):
|
|
@@ -16,7 +16,6 @@ from metaflow.metaflow_config import (
|
|
|
16
16
|
CARD_SUFFIX,
|
|
17
17
|
CARD_AZUREROOT,
|
|
18
18
|
CARD_GSROOT,
|
|
19
|
-
SKIP_CARD_DUALWRITE,
|
|
20
19
|
)
|
|
21
20
|
import metaflow.metaflow_config as metaflow_config
|
|
22
21
|
|
|
@@ -231,23 +230,6 @@ class CardDatastore(object):
|
|
|
231
230
|
|
|
232
231
|
def save_card(self, uuid, card_type, card_html, card_id=None, overwrite=True):
|
|
233
232
|
card_file_name = card_type
|
|
234
|
-
# TEMPORARY_WORKAROUND: FIXME (LATER) : Fix the duplication of below block in a few months.
|
|
235
|
-
# Check file blame to understand the age of this temporary workaround.
|
|
236
|
-
|
|
237
|
-
# This function will end up saving cards at two locations.
|
|
238
|
-
# Thereby doubling the number of cards. (Which is a temporary fix)
|
|
239
|
-
# Why do this ? :
|
|
240
|
-
# When cards were introduced there was an assumption made about task-ids being unique.
|
|
241
|
-
# This assumption was incorrect.
|
|
242
|
-
# Only the pathspec needs to be unique but there is no such guarantees about task-ids.
|
|
243
|
-
# When task-ids are non-unique, card read would result in finding incorrect cards.
|
|
244
|
-
# This happens because cards were stored based on task-ids.
|
|
245
|
-
# If we immediately switch from storing based on task-ids to a step-name abstraction folder,
|
|
246
|
-
# then card reading will crash for many users.
|
|
247
|
-
# It would especially happen for users who are accessing cards created by a newer
|
|
248
|
-
# MF client from an older version of MF client.
|
|
249
|
-
# It will also easily end up breaking the metaflow-ui (which maybe using a client from an older version).
|
|
250
|
-
# Hence, we are writing cards to both paths so that we can introduce breaking changes later in the future.
|
|
251
233
|
card_path_with_steps = self.get_card_location(
|
|
252
234
|
self._get_card_write_path(),
|
|
253
235
|
card_file_name,
|
|
@@ -255,24 +237,10 @@ class CardDatastore(object):
|
|
|
255
237
|
card_id=card_id,
|
|
256
238
|
suffix=CardNameSuffix.CARD,
|
|
257
239
|
)
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
)
|
|
263
|
-
else:
|
|
264
|
-
card_path_without_steps = self.get_card_location(
|
|
265
|
-
self._get_card_read_path(with_steps=False),
|
|
266
|
-
card_file_name,
|
|
267
|
-
uuid,
|
|
268
|
-
card_id=card_id,
|
|
269
|
-
suffix=CardNameSuffix.CARD,
|
|
270
|
-
)
|
|
271
|
-
for cp in [card_path_with_steps, card_path_without_steps]:
|
|
272
|
-
self._backend.save_bytes(
|
|
273
|
-
[(cp, BytesIO(bytes(card_html, "utf-8")))], overwrite=overwrite
|
|
274
|
-
)
|
|
275
|
-
|
|
240
|
+
self._backend.save_bytes(
|
|
241
|
+
[(card_path_with_steps, BytesIO(bytes(card_html, "utf-8")))],
|
|
242
|
+
overwrite=overwrite,
|
|
243
|
+
)
|
|
276
244
|
return self.info_from_path(card_path_with_steps, suffix=CardNameSuffix.CARD)
|
|
277
245
|
|
|
278
246
|
def _list_card_paths(self, card_type=None, card_hash=None, card_id=None):
|
|
@@ -283,6 +251,10 @@ class CardDatastore(object):
|
|
|
283
251
|
)
|
|
284
252
|
|
|
285
253
|
if len(card_paths_with_steps) == 0:
|
|
254
|
+
# The listing logic is reading the cards with steps and without steps
|
|
255
|
+
# because earlier versions of clients (ones that wrote cards before June 2022),
|
|
256
|
+
# would have written cards without steps. So as a fallback we will try to check for the
|
|
257
|
+
# cards without steps.
|
|
286
258
|
card_paths_without_steps = self._backend.list_content(
|
|
287
259
|
[self._get_card_read_path(with_steps=False)]
|
|
288
260
|
)
|
|
@@ -131,7 +131,7 @@ def normalize_client_error(err):
|
|
|
131
131
|
except ValueError:
|
|
132
132
|
if error_code in ("AccessDenied", "AllAccessDisabled", "InvalidAccessKeyId"):
|
|
133
133
|
return 403
|
|
134
|
-
if error_code
|
|
134
|
+
if error_code in ("NoSuchKey", "NoSuchBucket"):
|
|
135
135
|
return 404
|
|
136
136
|
if error_code == "InvalidRange":
|
|
137
137
|
return 416
|
|
@@ -170,6 +170,7 @@ class Kubernetes(object):
|
|
|
170
170
|
code_package_ds,
|
|
171
171
|
docker_image,
|
|
172
172
|
docker_image_pull_policy,
|
|
173
|
+
image_pull_secrets=None,
|
|
173
174
|
step_cli=None,
|
|
174
175
|
service_account=None,
|
|
175
176
|
secrets=None,
|
|
@@ -206,6 +207,7 @@ class Kubernetes(object):
|
|
|
206
207
|
node_selector=node_selector,
|
|
207
208
|
image=docker_image,
|
|
208
209
|
image_pull_policy=docker_image_pull_policy,
|
|
210
|
+
image_pull_secrets=image_pull_secrets,
|
|
209
211
|
cpu=cpu,
|
|
210
212
|
memory=memory,
|
|
211
213
|
disk=disk,
|
|
@@ -483,6 +485,7 @@ class Kubernetes(object):
|
|
|
483
485
|
step_cli,
|
|
484
486
|
docker_image,
|
|
485
487
|
docker_image_pull_policy,
|
|
488
|
+
image_pull_secrets=None,
|
|
486
489
|
service_account=None,
|
|
487
490
|
secrets=None,
|
|
488
491
|
node_selector=None,
|
|
@@ -529,6 +532,7 @@ class Kubernetes(object):
|
|
|
529
532
|
),
|
|
530
533
|
image=docker_image,
|
|
531
534
|
image_pull_policy=docker_image_pull_policy,
|
|
535
|
+
image_pull_secrets=image_pull_secrets,
|
|
532
536
|
cpu=cpu,
|
|
533
537
|
memory=memory,
|
|
534
538
|
disk=disk,
|
|
@@ -53,6 +53,12 @@ def kubernetes():
|
|
|
53
53
|
default=None,
|
|
54
54
|
help="Optional Docker Image Pull Policy for Kubernetes pod.",
|
|
55
55
|
)
|
|
56
|
+
@click.option(
|
|
57
|
+
"--image-pull-secrets",
|
|
58
|
+
default=None,
|
|
59
|
+
type=JSONTypeClass(),
|
|
60
|
+
multiple=False,
|
|
61
|
+
)
|
|
56
62
|
@click.option(
|
|
57
63
|
"--service-account",
|
|
58
64
|
help="IRSA requirement for Kubernetes pod.",
|
|
@@ -160,6 +166,7 @@ def step(
|
|
|
160
166
|
executable=None,
|
|
161
167
|
image=None,
|
|
162
168
|
image_pull_policy=None,
|
|
169
|
+
image_pull_secrets=None,
|
|
163
170
|
service_account=None,
|
|
164
171
|
secrets=None,
|
|
165
172
|
node_selector=None,
|
|
@@ -303,6 +310,7 @@ def step(
|
|
|
303
310
|
step_cli=step_cli,
|
|
304
311
|
docker_image=image,
|
|
305
312
|
docker_image_pull_policy=image_pull_policy,
|
|
313
|
+
image_pull_secrets=image_pull_secrets,
|
|
306
314
|
service_account=service_account,
|
|
307
315
|
secrets=secrets,
|
|
308
316
|
node_selector=node_selector,
|
|
@@ -18,6 +18,7 @@ from metaflow.metaflow_config import (
|
|
|
18
18
|
KUBERNETES_FETCH_EC2_METADATA,
|
|
19
19
|
KUBERNETES_GPU_VENDOR,
|
|
20
20
|
KUBERNETES_IMAGE_PULL_POLICY,
|
|
21
|
+
KUBERNETES_IMAGE_PULL_SECRETS,
|
|
21
22
|
KUBERNETES_MEMORY,
|
|
22
23
|
KUBERNETES_LABELS,
|
|
23
24
|
KUBERNETES_ANNOTATIONS,
|
|
@@ -74,6 +75,10 @@ class KubernetesDecorator(StepDecorator):
|
|
|
74
75
|
not, a default Docker image mapping to the current version of Python is used.
|
|
75
76
|
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
|
76
77
|
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
|
78
|
+
image_pull_secrets: List[str], default []
|
|
79
|
+
The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
|
|
80
|
+
Kubernetes image pull secrets to use when pulling container images
|
|
81
|
+
in Kubernetes.
|
|
77
82
|
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
|
78
83
|
Kubernetes service account to use when launching pod in Kubernetes.
|
|
79
84
|
secrets : List[str], optional, default None
|
|
@@ -141,6 +146,7 @@ class KubernetesDecorator(StepDecorator):
|
|
|
141
146
|
"disk": "10240",
|
|
142
147
|
"image": None,
|
|
143
148
|
"image_pull_policy": None,
|
|
149
|
+
"image_pull_secrets": None, # e.g., ["regcred"]
|
|
144
150
|
"service_account": None,
|
|
145
151
|
"secrets": None, # e.g., mysecret
|
|
146
152
|
"node_selector": None, # e.g., kubernetes.io/os=linux
|
|
@@ -194,6 +200,10 @@ class KubernetesDecorator(StepDecorator):
|
|
|
194
200
|
)
|
|
195
201
|
if not self.attributes["image_pull_policy"] and KUBERNETES_IMAGE_PULL_POLICY:
|
|
196
202
|
self.attributes["image_pull_policy"] = KUBERNETES_IMAGE_PULL_POLICY
|
|
203
|
+
if not self.attributes["image_pull_secrets"] and KUBERNETES_IMAGE_PULL_SECRETS:
|
|
204
|
+
self.attributes["image_pull_secrets"] = json.loads(
|
|
205
|
+
KUBERNETES_IMAGE_PULL_SECRETS
|
|
206
|
+
)
|
|
197
207
|
|
|
198
208
|
if isinstance(self.attributes["node_selector"], str):
|
|
199
209
|
self.attributes["node_selector"] = parse_kube_keyvalue_list(
|
|
@@ -494,6 +504,7 @@ class KubernetesDecorator(StepDecorator):
|
|
|
494
504
|
for key, val in v.items()
|
|
495
505
|
]
|
|
496
506
|
elif k in [
|
|
507
|
+
"image_pull_secrets",
|
|
497
508
|
"tolerations",
|
|
498
509
|
"persistent_volume_claims",
|
|
499
510
|
"labels",
|
|
@@ -235,8 +235,10 @@ class KubernetesJob(object):
|
|
|
235
235
|
)
|
|
236
236
|
],
|
|
237
237
|
node_selector=self._kwargs.get("node_selector"),
|
|
238
|
-
|
|
239
|
-
|
|
238
|
+
image_pull_secrets=[
|
|
239
|
+
client.V1LocalObjectReference(secret)
|
|
240
|
+
for secret in self._kwargs.get("image_pull_secrets") or []
|
|
241
|
+
],
|
|
240
242
|
# TODO (savin): Support preemption policies
|
|
241
243
|
# preemption_policy=?,
|
|
242
244
|
#
|
|
@@ -718,8 +718,11 @@ class JobSetSpec(object):
|
|
|
718
718
|
)
|
|
719
719
|
],
|
|
720
720
|
node_selector=self._kwargs.get("node_selector"),
|
|
721
|
-
|
|
722
|
-
|
|
721
|
+
image_pull_secrets=[
|
|
722
|
+
client.V1LocalObjectReference(secret)
|
|
723
|
+
for secret in self._kwargs.get("image_pull_secrets")
|
|
724
|
+
or []
|
|
725
|
+
],
|
|
723
726
|
# TODO (savin): Support preemption policies
|
|
724
727
|
# preemption_policy=?,
|
|
725
728
|
#
|
|
@@ -72,14 +72,18 @@ class ServiceMetadataProvider(MetadataProvider):
|
|
|
72
72
|
@classmethod
|
|
73
73
|
def compute_info(cls, val):
|
|
74
74
|
v = val.rstrip("/")
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
75
|
+
for i in range(SERVICE_RETRY_COUNT):
|
|
76
|
+
try:
|
|
77
|
+
resp = cls._session.get(
|
|
78
|
+
os.path.join(v, "ping"), headers=SERVICE_HEADERS.copy()
|
|
79
|
+
)
|
|
80
|
+
resp.raise_for_status()
|
|
81
|
+
except: # noqa E722
|
|
82
|
+
time.sleep(2 ** (i - 1))
|
|
83
|
+
else:
|
|
84
|
+
return v
|
|
85
|
+
|
|
86
|
+
raise ValueError("Metaflow service [%s] unreachable." % v)
|
|
83
87
|
|
|
84
88
|
@classmethod
|
|
85
89
|
def default_info(cls):
|
|
@@ -32,6 +32,7 @@ class CondaEnvironmentException(MetaflowException):
|
|
|
32
32
|
class CondaEnvironment(MetaflowEnvironment):
|
|
33
33
|
TYPE = "conda"
|
|
34
34
|
_filecache = None
|
|
35
|
+
_force_rebuild = False
|
|
35
36
|
|
|
36
37
|
def __init__(self, flow):
|
|
37
38
|
self.flow = flow
|
|
@@ -71,7 +72,7 @@ class CondaEnvironment(MetaflowEnvironment):
|
|
|
71
72
|
self.logger = make_thread_safe(logger)
|
|
72
73
|
|
|
73
74
|
# TODO: Wire up logging
|
|
74
|
-
micromamba = Micromamba(self.logger)
|
|
75
|
+
micromamba = Micromamba(self.logger, self._force_rebuild)
|
|
75
76
|
self.solvers = {"conda": micromamba, "pypi": Pip(micromamba, self.logger)}
|
|
76
77
|
|
|
77
78
|
def init_environment(self, echo, only_steps=None):
|
|
@@ -107,7 +108,10 @@ class CondaEnvironment(MetaflowEnvironment):
|
|
|
107
108
|
return (
|
|
108
109
|
id_,
|
|
109
110
|
(
|
|
110
|
-
|
|
111
|
+
(
|
|
112
|
+
not self._force_rebuild
|
|
113
|
+
and self.read_from_environment_manifest([id_, platform, type_])
|
|
114
|
+
)
|
|
111
115
|
or self.write_to_environment_manifest(
|
|
112
116
|
[id_, platform, type_],
|
|
113
117
|
self.solvers[type_].solve(id_, **environment),
|
|
@@ -153,7 +157,7 @@ class CondaEnvironment(MetaflowEnvironment):
|
|
|
153
157
|
_meta = copy.deepcopy(local_packages)
|
|
154
158
|
for id_, packages, _, _ in results:
|
|
155
159
|
for package in packages:
|
|
156
|
-
if package.get("path"):
|
|
160
|
+
if package.get("path") and not self._force_rebuild:
|
|
157
161
|
# Cache only those packages that manifest is unaware of
|
|
158
162
|
local_packages.pop(package["url"], None)
|
|
159
163
|
else:
|
|
@@ -186,7 +190,7 @@ class CondaEnvironment(MetaflowEnvironment):
|
|
|
186
190
|
storage.save_bytes(
|
|
187
191
|
list_of_path_and_filehandle,
|
|
188
192
|
len_hint=len(list_of_path_and_filehandle),
|
|
189
|
-
|
|
193
|
+
overwrite=self._force_rebuild,
|
|
190
194
|
)
|
|
191
195
|
for id_, packages, _, platform in results:
|
|
192
196
|
if id_ in dirty:
|
|
@@ -2,6 +2,7 @@ import functools
|
|
|
2
2
|
import json
|
|
3
3
|
import os
|
|
4
4
|
import re
|
|
5
|
+
import shutil
|
|
5
6
|
import subprocess
|
|
6
7
|
import tempfile
|
|
7
8
|
import time
|
|
@@ -30,7 +31,7 @@ _double_equal_match = re.compile("==(?=[<=>!~])")
|
|
|
30
31
|
|
|
31
32
|
|
|
32
33
|
class Micromamba(object):
|
|
33
|
-
def __init__(self, logger=None):
|
|
34
|
+
def __init__(self, logger=None, force_rebuild=False):
|
|
34
35
|
# micromamba is a tiny version of the mamba package manager and comes with
|
|
35
36
|
# metaflow specific performance enhancements.
|
|
36
37
|
|
|
@@ -60,6 +61,8 @@ class Micromamba(object):
|
|
|
60
61
|
# which causes a race condition in case micromamba needs to be installed first.
|
|
61
62
|
self.install_mutex = Lock()
|
|
62
63
|
|
|
64
|
+
self.force_rebuild = force_rebuild
|
|
65
|
+
|
|
63
66
|
@property
|
|
64
67
|
def bin(self) -> str:
|
|
65
68
|
"Defer installing Micromamba until when the binary path is actually requested"
|
|
@@ -152,6 +155,11 @@ class Micromamba(object):
|
|
|
152
155
|
keyword="metaflow", # indicates metaflow generated environment
|
|
153
156
|
id=id_,
|
|
154
157
|
)
|
|
158
|
+
# If we are forcing a rebuild of the environment, we make sure to remove existing files beforehand.
|
|
159
|
+
# This is to ensure that no irrelevant packages get bundled relative to the resolved environment.
|
|
160
|
+
# NOTE: download always happens before create, so we want to do the cleanup here instead.
|
|
161
|
+
if self.force_rebuild:
|
|
162
|
+
shutil.rmtree(self.path_to_environment(id_, platform), ignore_errors=True)
|
|
155
163
|
|
|
156
164
|
# cheap check
|
|
157
165
|
if os.path.exists(f"{prefix}/fake.done"):
|
metaflow/runner/deployer.py
CHANGED
|
@@ -7,6 +7,55 @@ from typing import ClassVar, Dict, Optional, TYPE_CHECKING
|
|
|
7
7
|
from metaflow.exception import MetaflowNotFound
|
|
8
8
|
from metaflow.metaflow_config import DEFAULT_FROM_DEPLOYMENT_IMPL
|
|
9
9
|
|
|
10
|
+
|
|
11
|
+
def generate_fake_flow_file_contents(
|
|
12
|
+
flow_name: str, param_info: dict, project_name: Optional[str] = None
|
|
13
|
+
):
|
|
14
|
+
params_code = ""
|
|
15
|
+
for _, param_details in param_info.items():
|
|
16
|
+
param_python_var_name = param_details["python_var_name"]
|
|
17
|
+
param_name = param_details["name"]
|
|
18
|
+
param_type = param_details["type"]
|
|
19
|
+
param_help = param_details["description"]
|
|
20
|
+
param_required = param_details["is_required"]
|
|
21
|
+
|
|
22
|
+
if param_type == "JSON":
|
|
23
|
+
params_code += (
|
|
24
|
+
f" {param_python_var_name} = Parameter('{param_name}', "
|
|
25
|
+
f"type=JSONType, help='''{param_help}''', required={param_required})\n"
|
|
26
|
+
)
|
|
27
|
+
elif param_type == "FilePath":
|
|
28
|
+
is_text = param_details.get("is_text", True)
|
|
29
|
+
encoding = param_details.get("encoding", "utf-8")
|
|
30
|
+
params_code += (
|
|
31
|
+
f" {param_python_var_name} = IncludeFile('{param_name}', "
|
|
32
|
+
f"is_text={is_text}, encoding='{encoding}', help='''{param_help}''', "
|
|
33
|
+
f"required={param_required})\n"
|
|
34
|
+
)
|
|
35
|
+
else:
|
|
36
|
+
params_code += (
|
|
37
|
+
f" {param_python_var_name} = Parameter('{param_name}', "
|
|
38
|
+
f"type={param_type}, help='''{param_help}''', required={param_required})\n"
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
project_decorator = f"@project(name='{project_name}')\n" if project_name else ""
|
|
42
|
+
|
|
43
|
+
contents = f"""\
|
|
44
|
+
from metaflow import FlowSpec, Parameter, IncludeFile, JSONType, step, project
|
|
45
|
+
{project_decorator}class {flow_name}(FlowSpec):
|
|
46
|
+
{params_code}
|
|
47
|
+
@step
|
|
48
|
+
def start(self):
|
|
49
|
+
self.next(self.end)
|
|
50
|
+
@step
|
|
51
|
+
def end(self):
|
|
52
|
+
pass
|
|
53
|
+
if __name__ == '__main__':
|
|
54
|
+
{flow_name}()
|
|
55
|
+
"""
|
|
56
|
+
return contents
|
|
57
|
+
|
|
58
|
+
|
|
10
59
|
if TYPE_CHECKING:
|
|
11
60
|
import metaflow
|
|
12
61
|
import metaflow.runner.deployer_impl
|
|
@@ -200,7 +200,7 @@ class MutableFlow:
|
|
|
200
200
|
for name, value in self._flow_cls._flow_state.get(
|
|
201
201
|
_FlowState.CONFIGS, {}
|
|
202
202
|
).items():
|
|
203
|
-
yield name, ConfigValue(value)
|
|
203
|
+
yield name, ConfigValue(value) if value is not None else None
|
|
204
204
|
|
|
205
205
|
@property
|
|
206
206
|
def parameters(self) -> Generator[Tuple[str, Any], None, None]:
|
|
@@ -352,7 +352,9 @@ class ConfigInput:
|
|
|
352
352
|
return None
|
|
353
353
|
raise exc from e
|
|
354
354
|
flow_cls._flow_state[_FlowState.CONFIGS][name] = read_value
|
|
355
|
-
to_return[name] =
|
|
355
|
+
to_return[name] = (
|
|
356
|
+
ConfigValue(read_value) if read_value is not None else None
|
|
357
|
+
)
|
|
356
358
|
else:
|
|
357
359
|
if self._parsers[name]:
|
|
358
360
|
read_value = self._call_parser(self._parsers[name], val)
|
|
@@ -367,7 +369,9 @@ class ConfigInput:
|
|
|
367
369
|
continue
|
|
368
370
|
# TODO: Support YAML
|
|
369
371
|
flow_cls._flow_state[_FlowState.CONFIGS][name] = read_value
|
|
370
|
-
to_return[name] =
|
|
372
|
+
to_return[name] = (
|
|
373
|
+
ConfigValue(read_value) if read_value is not None else None
|
|
374
|
+
)
|
|
371
375
|
|
|
372
376
|
reqs = missing_configs.intersection(self._req_configs)
|
|
373
377
|
for missing in reqs:
|
metaflow/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
metaflow_version = "2.15.
|
|
1
|
+
metaflow_version = "2.15.18.1"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ob-metaflow
|
|
3
|
-
Version: 2.15.
|
|
3
|
+
Version: 2.15.18.1
|
|
4
4
|
Summary: Metaflow: More AI and ML, Less Engineering
|
|
5
5
|
Author: Netflix, Outerbounds & the Metaflow Community
|
|
6
6
|
Author-email: help@outerbounds.co
|
|
@@ -12,7 +12,7 @@ Requires-Dist: boto3
|
|
|
12
12
|
Requires-Dist: pylint
|
|
13
13
|
Requires-Dist: kubernetes
|
|
14
14
|
Provides-Extra: stubs
|
|
15
|
-
Requires-Dist: metaflow-stubs==2.15.
|
|
15
|
+
Requires-Dist: metaflow-stubs==2.15.18.1; extra == "stubs"
|
|
16
16
|
Dynamic: author
|
|
17
17
|
Dynamic: author-email
|
|
18
18
|
Dynamic: description
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
|
|
2
2
|
metaflow/__init__.py,sha256=YE4ezkE7_Hi5K_EI5WAamSUmnzNi7BZqftou1UxuZfM,6306
|
|
3
3
|
metaflow/cards.py,sha256=IbRmredvmFEU0V6JL7DR8wCESwVmmZJubr6x24bo7U4,442
|
|
4
|
-
metaflow/cli.py,sha256=
|
|
4
|
+
metaflow/cli.py,sha256=ype7221pgVp3sFXTT4YpLmEx0yLahYD9UuHizLCbIT0,21714
|
|
5
5
|
metaflow/cli_args.py,sha256=hDsdWdRmfXYifVGq6b6FDfgoWxtIG2nr_lU6EBV0Pnk,3584
|
|
6
6
|
metaflow/clone_util.py,sha256=LSuVbFpPUh92UW32DBcnZbL0FFw-4w3CLa0tpEbCkzk,2066
|
|
7
7
|
metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
|
|
@@ -10,13 +10,13 @@ metaflow/decorators.py,sha256=qiAV-NOXt09I53y01Vm7EPZtJ4tROlt2Jsij7nVWOY4,24147
|
|
|
10
10
|
metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
|
|
11
11
|
metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
|
|
12
12
|
metaflow/exception.py,sha256=_m9ZBJM0cooHRslDqfxCPQmkChqaTh6fGxp7HvISnYI,5161
|
|
13
|
-
metaflow/flowspec.py,sha256=
|
|
13
|
+
metaflow/flowspec.py,sha256=FEC3sstpV3uHgqhUWQhNytjuqKls0BZ_NJTc3igZR4M,36471
|
|
14
14
|
metaflow/graph.py,sha256=cdpnWr85aEj_rRn-7EjbndWjr_i8Dt3P7-oPUW0NNpI,12393
|
|
15
15
|
metaflow/includefile.py,sha256=RtISGl1V48qjkJBakUZ9yPpHV102h7pOIFiKP8PLHpc,20927
|
|
16
16
|
metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
|
|
17
17
|
metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
|
|
18
18
|
metaflow/lint.py,sha256=x4p6tnRzYqNNniCGXyrUW0WuYfTUgnaOMRivxvnxask,11661
|
|
19
|
-
metaflow/metaflow_config.py,sha256=
|
|
19
|
+
metaflow/metaflow_config.py,sha256=FMUxvHs01zIoypoh_cioZg76f2IF8BwFzFA2Nrm5fzQ,24147
|
|
20
20
|
metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
|
|
21
21
|
metaflow/metaflow_current.py,sha256=pfkXmkyHeMJhxIs6HBJNBEaBDpcl5kz9Wx5mW6F_3qo,7164
|
|
22
22
|
metaflow/metaflow_environment.py,sha256=CWG90qpfz9iJ6hHhFlAmMVNALn2v_5eTVk3mFbQR4Pw,8379
|
|
@@ -37,7 +37,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
|
|
37
37
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
|
38
38
|
metaflow/util.py,sha256=MCXCjcGwpuR7y9euBf1GTNRHPtlh6fCpdPMEtbULefw,14554
|
|
39
39
|
metaflow/vendor.py,sha256=EDZokNMrx1PU07jNMiWFMFtC7TL03pMXZ1kKn13k-2g,5139
|
|
40
|
-
metaflow/version.py,sha256=
|
|
40
|
+
metaflow/version.py,sha256=PFf5JEF-aj-NoF1lVpGihsDxW6lVrkJgaDQ5Lp4Q6MM,31
|
|
41
41
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
|
42
42
|
metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
|
|
43
43
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
|
@@ -136,7 +136,7 @@ metaflow/_vendor/v3_7/typeguard/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
|
|
|
136
136
|
metaflow/cli_components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
137
137
|
metaflow/cli_components/dump_cmd.py,sha256=SZEX51BWNd1o3H2uHDkYA8KRvou5X8g5rTwpdu5vnNQ,2704
|
|
138
138
|
metaflow/cli_components/init_cmd.py,sha256=Er-BO59UEUV1HIsg81bRtZWT2D2IZNMp93l-AoZLCls,1519
|
|
139
|
-
metaflow/cli_components/run_cmds.py,sha256=
|
|
139
|
+
metaflow/cli_components/run_cmds.py,sha256=A0knVmimskHxcSBAxherL3TLcYao7UHHyXoyUl-n3CY,11304
|
|
140
140
|
metaflow/cli_components/step_cmd.py,sha256=zGJgTv7wxrv34nWDi__CHaC2eS6kItR95EdVGJX803w,4766
|
|
141
141
|
metaflow/cli_components/utils.py,sha256=gpoDociadjnJD7MuiJup_MDR02ZJjjleejr0jPBu29c,6057
|
|
142
142
|
metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
|
|
@@ -150,7 +150,7 @@ metaflow/cmd/tutorials_cmd.py,sha256=8FdlKkicTOhCIDKcBR5b0Oz6giDvS-EMY3o9skIrRqw
|
|
|
150
150
|
metaflow/cmd/util.py,sha256=jS_0rUjOnGGzPT65fzRLdGjrYAOOLA4jU2S0HJLV0oc,406
|
|
151
151
|
metaflow/cmd/code/__init__.py,sha256=VO4dNM9M9LHYy5nTgEiJvCV1RBl8lpDlYGJm6GIcaBA,7413
|
|
152
152
|
metaflow/cmd/develop/__init__.py,sha256=p1Sy8yU1MEKSrH5ttOWOZvNcI1qYu6J6jghdTHwPgOw,689
|
|
153
|
-
metaflow/cmd/develop/stub_generator.py,sha256=
|
|
153
|
+
metaflow/cmd/develop/stub_generator.py,sha256=8Ap5rCC2lKOq4KQJ_LTuhQGj2xXYJ_Nnc1h842PsGBs,65929
|
|
154
154
|
metaflow/cmd/develop/stubs.py,sha256=J8DtpaN3okiPeeAqhxuUe9wQDlW4YYmZgKsVZLlVt0k,11181
|
|
155
155
|
metaflow/datastore/__init__.py,sha256=VxP6ddJt3rwiCkpiSfAhyVkUCOe1pgZZsytVEJzFmSQ,155
|
|
156
156
|
metaflow/datastore/content_addressed_store.py,sha256=6T7tNqL29kpmecyMLHF35RhoSBOb-OZcExnsB65AvnI,7641
|
|
@@ -206,16 +206,16 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
|
|
|
206
206
|
metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
207
207
|
metaflow/plugins/argo/argo_client.py,sha256=A1kI9rjVjCadDsBscZ2Wk8xRBI6GNgWV6SU7TyrdfrI,16530
|
|
208
208
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
|
209
|
-
metaflow/plugins/argo/argo_workflows.py,sha256=
|
|
209
|
+
metaflow/plugins/argo/argo_workflows.py,sha256=mu17m3Vi3dQkom8MOO1ERNS-t8h8oFZSfuEYgt99XmY,188741
|
|
210
210
|
metaflow/plugins/argo/argo_workflows_cli.py,sha256=1bdG8BQgwyfHooBidyY1Aw52uwjCxq2Edt0bO7qezWM,38375
|
|
211
211
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=ogCSBmwsC2C3eusydrgjuAJd4qK18f1sI4jJwA4Fd-o,7800
|
|
212
212
|
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
|
|
213
|
-
metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=
|
|
213
|
+
metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=7OiapcIM_r-aBkuIobhofgLC5NRJHC-p9bvBmxvhqoM,12500
|
|
214
214
|
metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
|
|
215
215
|
metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
|
|
216
216
|
metaflow/plugins/argo/jobset_input_paths.py,sha256=-h0E_e0w6FMiBUod9Rf_XOSCtZv_C0exacw4q1SfIfg,501
|
|
217
217
|
metaflow/plugins/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
218
|
-
metaflow/plugins/aws/aws_client.py,sha256=
|
|
218
|
+
metaflow/plugins/aws/aws_client.py,sha256=BTiLMXa1agjja-N73oWinaOZHs-lGPbfKJG8CqdRgaU,4287
|
|
219
219
|
metaflow/plugins/aws/aws_utils.py,sha256=U7fSBcn3NDJ95DKUbqoALN5jaHQSUzREpWTuxHI_ado,8268
|
|
220
220
|
metaflow/plugins/aws/batch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
221
221
|
metaflow/plugins/aws/batch/batch.py,sha256=e9ssahWM18GnipPK2sqYB-ztx9w7Eoo7YtWyEtufYxs,17787
|
|
@@ -248,7 +248,7 @@ metaflow/plugins/cards/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
|
|
|
248
248
|
metaflow/plugins/cards/card_cli.py,sha256=XyiU5f6i_xjFHPyowWGs9mKQm9fm3dK6id-IKsoe9cY,35177
|
|
249
249
|
metaflow/plugins/cards/card_client.py,sha256=30dFBoC3axc261GeV7QCIs_V1OHhRtS31S0wEWsjw90,9501
|
|
250
250
|
metaflow/plugins/cards/card_creator.py,sha256=vRz1EUFa4xQ1fUIWzqyACViC6D7KGFaq5XlLIEssXTI,7741
|
|
251
|
-
metaflow/plugins/cards/card_datastore.py,sha256=
|
|
251
|
+
metaflow/plugins/cards/card_datastore.py,sha256=bPLjBFWNlAgIjONAb0IjYXMmp9vMYlh3EYhRjAlDA0U,12714
|
|
252
252
|
metaflow/plugins/cards/card_decorator.py,sha256=nU-uZ4_ubU7bCsSb6uT61YnN8ruo43_FzqZ24MsZpKg,12070
|
|
253
253
|
metaflow/plugins/cards/card_resolver.py,sha256=bjyujYpGUFbLJNwXNGHlHhL4f-gVdVKebl7XW1vWDtE,717
|
|
254
254
|
metaflow/plugins/cards/card_server.py,sha256=DHv0RcepaPULWbkDahiEMrU5A281Cfb0DvHLUYd8JsU,11772
|
|
@@ -279,7 +279,7 @@ metaflow/plugins/datatools/__init__.py,sha256=ge4L16OBQLy2J_MMvoHg3lMfdm-MluQgRW
|
|
|
279
279
|
metaflow/plugins/datatools/local.py,sha256=FJvMOBcjdyhSPHmdLocBSiIT0rmKkKBmsaclxH75x08,4233
|
|
280
280
|
metaflow/plugins/datatools/s3/__init__.py,sha256=14tr9fPjN3ULW5IOfKHeG7Uhjmgm7LMtQHfz1SFv-h8,248
|
|
281
281
|
metaflow/plugins/datatools/s3/s3.py,sha256=3xrWD6pXoVRpuAQHyLGVya79UIE0S8AqAqe2prg4yMw,67182
|
|
282
|
-
metaflow/plugins/datatools/s3/s3op.py,sha256=
|
|
282
|
+
metaflow/plugins/datatools/s3/s3op.py,sha256=nDFgc5m0ud-uK4D2_vyweGpGoMjESxu3TlX86zA1MdQ,47561
|
|
283
283
|
metaflow/plugins/datatools/s3/s3tail.py,sha256=boQjQGQMI-bvTqcMP2y7uSlSYLcvWOy7J3ZUaF78NAA,2597
|
|
284
284
|
metaflow/plugins/datatools/s3/s3util.py,sha256=FgRgaVmEq7-i2dV7q8XK5w5PfFt-xJjZa8WrK8IJfdI,3769
|
|
285
285
|
metaflow/plugins/env_escape/__init__.py,sha256=tGNUZnmPvk52eNs__VK443b3CZ7ogEFTT-s9_n_HF8Q,8837
|
|
@@ -313,22 +313,22 @@ metaflow/plugins/gcp/gs_utils.py,sha256=ZmIGFse1qYyvAVrwga23PQUzF6dXEDLLsZ2F-YRm
|
|
|
313
313
|
metaflow/plugins/gcp/includefile_support.py,sha256=OQO0IVWv4ObboL0VqEZwcDOyj9ORLdur66JToxQ84vU,3887
|
|
314
314
|
metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
315
315
|
metaflow/plugins/kubernetes/kube_utils.py,sha256=jdFMGbEmIow-oli26v31W9CmbZXigx06b3D_xIobpk0,4140
|
|
316
|
-
metaflow/plugins/kubernetes/kubernetes.py,sha256=
|
|
317
|
-
metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=
|
|
316
|
+
metaflow/plugins/kubernetes/kubernetes.py,sha256=8vR0TDQPVt789biONCZPIwJ1bQAcR0sRdPst1OPQEm0,30760
|
|
317
|
+
metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=SsZi5-Ky5_ocuRK3PeBxuLqUqXL1qDZIw1zI3mvQt9Q,14296
|
|
318
318
|
metaflow/plugins/kubernetes/kubernetes_client.py,sha256=tuvXP-QKpdeSmzVolB2R_TaacOr5DIb0j642eKcjsiM,6491
|
|
319
|
-
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=
|
|
320
|
-
metaflow/plugins/kubernetes/kubernetes_job.py,sha256=
|
|
321
|
-
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=
|
|
319
|
+
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=duUZ4w2PIdpyt10sgOmK95vB1_6WvQh3JLlaYHF5pg4,32811
|
|
320
|
+
metaflow/plugins/kubernetes/kubernetes_job.py,sha256=xhucNJR7EpM-XMsfY0nt-BASbjo_T4vL_-tmQ_xHL1U,33284
|
|
321
|
+
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=ZZU5vsBe67NmGuxgXw6clf7kKRST7867AW6_t3fCD5g,43065
|
|
322
322
|
metaflow/plugins/kubernetes/spot_metadata_cli.py,sha256=an0nWCxgflmqIPBCBrlb4m3DereDFFJBLt-KKhqcHc8,1670
|
|
323
323
|
metaflow/plugins/kubernetes/spot_monitor_sidecar.py,sha256=zrWU-smQwPnL6MBHmzTxWyEA00R6iKKQbhhy50xFwQ8,3832
|
|
324
324
|
metaflow/plugins/metadata_providers/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
325
325
|
metaflow/plugins/metadata_providers/local.py,sha256=Z0CXaGZJbAkj4II3WspJi-uCCtShH64yaXZQ5i9Ym7g,24390
|
|
326
|
-
metaflow/plugins/metadata_providers/service.py,sha256=
|
|
326
|
+
metaflow/plugins/metadata_providers/service.py,sha256=5UlK0R5M9_nq2J6MgJgCZwqAC3bEsofFbglq1K4p4QI,22942
|
|
327
327
|
metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
|
|
328
328
|
metaflow/plugins/pypi/bootstrap.py,sha256=NaQboUVNJc90gmUa69-cK_T3G7piHNNsUSaN96IJoic,14745
|
|
329
329
|
metaflow/plugins/pypi/conda_decorator.py,sha256=N0HGiaS1mRsa6qT4eYzu2C3DHtas22QIXowW4vEl44M,15961
|
|
330
|
-
metaflow/plugins/pypi/conda_environment.py,sha256=
|
|
331
|
-
metaflow/plugins/pypi/micromamba.py,sha256=
|
|
330
|
+
metaflow/plugins/pypi/conda_environment.py,sha256=GdpeHIih9z1Qyn3IY9_pNE4j8Ft4_uh2Z0B-LUbMglA,24980
|
|
331
|
+
metaflow/plugins/pypi/micromamba.py,sha256=UltfY8NmLphfZ-AbpaMFIdxIeOXLdTYDrMrabvPrYVU,17352
|
|
332
332
|
metaflow/plugins/pypi/parsers.py,sha256=gpOOG2Ph95wI73MWCAi7XjpK0gYhv5k5YIGBs73QPuE,8556
|
|
333
333
|
metaflow/plugins/pypi/pip.py,sha256=WhPyA18RoVT40sqbZRJdCEij4euL9PZwLfm1SrAKqmU,14333
|
|
334
334
|
metaflow/plugins/pypi/pypi_decorator.py,sha256=ybNgo-T5Z_0W2KNuED0pdjyI0qygZ4a1MXAzKqdHt_E,7250
|
|
@@ -342,7 +342,7 @@ metaflow/plugins/uv/bootstrap.py,sha256=PJP-lrFsLUUfiXCNse2Uy5uI3b8-X3-ji3Uwf30I
|
|
|
342
342
|
metaflow/plugins/uv/uv_environment.py,sha256=6hUaWrTgqHyzS6igGQUXpS6jb_GHv3Wq9ZrWhZgEals,2587
|
|
343
343
|
metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
344
344
|
metaflow/runner/click_api.py,sha256=u9Y1bu9vN13ektBs-L8lg2oxb5oYvGe1dPJjV2shMAw,23727
|
|
345
|
-
metaflow/runner/deployer.py,sha256=
|
|
345
|
+
metaflow/runner/deployer.py,sha256=U-hwf4gVzwUlXgnkfTW3y1daGXvo5eP4HTQZwb-vS0g,11058
|
|
346
346
|
metaflow/runner/deployer_impl.py,sha256=zTING0_fwP44JcGo69DuNrVut5KqdBVzYOM7MYTZgIY,7049
|
|
347
347
|
metaflow/runner/metaflow_runner.py,sha256=BaDAp-3tCPwJRy_R8hLTAu6L0qjjuQmjeo-L1inhr_c,17498
|
|
348
348
|
metaflow/runner/nbdeploy.py,sha256=Sp5w-6nCZwjHaRBHWxi8udya-RYnJOB76KNLjB4L7Gs,4166
|
|
@@ -386,15 +386,15 @@ metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuV
|
|
|
386
386
|
metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
|
|
387
387
|
metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
|
|
388
388
|
metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
389
|
-
metaflow/user_configs/config_decorators.py,sha256=
|
|
390
|
-
metaflow/user_configs/config_options.py,sha256=
|
|
389
|
+
metaflow/user_configs/config_decorators.py,sha256=n4OTbfuaXejPVLHnXXgVlNJNcrSvWrVOdXCMcqvQlik,20512
|
|
390
|
+
metaflow/user_configs/config_options.py,sha256=yyaiKLB4QWlR5AaTg9I7GoBv-OlffOFdGPe1BL85Llc,21241
|
|
391
391
|
metaflow/user_configs/config_parameters.py,sha256=Eyiqcz4YV_z4algDHAh2gaejGFgIdHk8Vix9AUdPSh0,20989
|
|
392
|
-
ob_metaflow-2.15.
|
|
393
|
-
ob_metaflow-2.15.
|
|
394
|
-
ob_metaflow-2.15.
|
|
395
|
-
ob_metaflow-2.15.
|
|
396
|
-
ob_metaflow-2.15.
|
|
397
|
-
ob_metaflow-2.15.
|
|
398
|
-
ob_metaflow-2.15.
|
|
399
|
-
ob_metaflow-2.15.
|
|
400
|
-
ob_metaflow-2.15.
|
|
392
|
+
ob_metaflow-2.15.18.1.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
|
|
393
|
+
ob_metaflow-2.15.18.1.data/data/share/metaflow/devtools/Tiltfile,sha256=P5_rn_F3xYLN1_cEAQ9mNeS22HG2rb8beKIz2RIK6fU,20634
|
|
394
|
+
ob_metaflow-2.15.18.1.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
|
|
395
|
+
ob_metaflow-2.15.18.1.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
|
396
|
+
ob_metaflow-2.15.18.1.dist-info/METADATA,sha256=JdoPmLwnh901695I1cfx4o1_hbqagh4tFZsh2uTAu0o,5937
|
|
397
|
+
ob_metaflow-2.15.18.1.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
|
|
398
|
+
ob_metaflow-2.15.18.1.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
|
|
399
|
+
ob_metaflow-2.15.18.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
|
400
|
+
ob_metaflow-2.15.18.1.dist-info/RECORD,,
|
{ob_metaflow-2.15.15.1.data → ob_metaflow-2.15.18.1.data}/data/share/metaflow/devtools/Makefile
RENAMED
|
File without changes
|
{ob_metaflow-2.15.15.1.data → ob_metaflow-2.15.18.1.data}/data/share/metaflow/devtools/Tiltfile
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|