metaflow 2.16.8__py2.py3-none-any.whl → 2.17.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/plugins/argo/argo_workflows.py +1 -0
- metaflow/plugins/argo/argo_workflows_cli.py +347 -83
- metaflow/plugins/cards/card_modules/basic.py +1 -0
- metaflow/version.py +1 -1
- {metaflow-2.16.8.dist-info → metaflow-2.17.0.dist-info}/METADATA +2 -2
- {metaflow-2.16.8.dist-info → metaflow-2.17.0.dist-info}/RECORD +13 -13
- {metaflow-2.16.8.data → metaflow-2.17.0.data}/data/share/metaflow/devtools/Makefile +0 -0
- {metaflow-2.16.8.data → metaflow-2.17.0.data}/data/share/metaflow/devtools/Tiltfile +0 -0
- {metaflow-2.16.8.data → metaflow-2.17.0.data}/data/share/metaflow/devtools/pick_services.sh +0 -0
- {metaflow-2.16.8.dist-info → metaflow-2.17.0.dist-info}/WHEEL +0 -0
- {metaflow-2.16.8.dist-info → metaflow-2.17.0.dist-info}/entry_points.txt +0 -0
- {metaflow-2.16.8.dist-info → metaflow-2.17.0.dist-info}/licenses/LICENSE +0 -0
- {metaflow-2.16.8.dist-info → metaflow-2.17.0.dist-info}/top_level.txt +0 -0
@@ -33,9 +33,11 @@ from metaflow.plugins.kubernetes.kubernetes_decorator import KubernetesDecorator
|
|
33
33
|
from metaflow.tagging_util import validate_tags
|
34
34
|
from metaflow.util import get_username, to_bytes, to_unicode, version_parse
|
35
35
|
|
36
|
-
from .argo_workflows import ArgoWorkflows
|
36
|
+
from .argo_workflows import ArgoWorkflows, ArgoWorkflowsException
|
37
37
|
|
38
|
-
|
38
|
+
NEW_ARGO_NAMELENGTH_METAFLOW_VERSION = "2.17"
|
39
|
+
|
40
|
+
VALID_NAME = re.compile(r"^[a-z]([a-z0-9\.\-]*[a-z0-9])?$")
|
39
41
|
|
40
42
|
unsupported_decorators = {
|
41
43
|
"snowpark": "Step *%s* is marked for execution on Snowpark with Argo Workflows which isn't currently supported.",
|
@@ -86,7 +88,16 @@ def argo_workflows(obj, name=None):
|
|
86
88
|
obj.workflow_name,
|
87
89
|
obj.token_prefix,
|
88
90
|
obj.is_project,
|
89
|
-
|
91
|
+
obj._is_workflow_name_modified,
|
92
|
+
obj._exception_on_create, # exception_on_create is used to prevent deploying new flows with too long names via --name
|
93
|
+
) = resolve_workflow_name_v2(obj, name)
|
94
|
+
# Backward compatibility for Metaflow versions <=2.16 because of
|
95
|
+
# change in name length restrictions in Argo Workflows from 253 to 52
|
96
|
+
# characters.
|
97
|
+
(
|
98
|
+
obj._v1_workflow_name,
|
99
|
+
obj._v1_is_workflow_name_modified,
|
100
|
+
) = resolve_workflow_name_v1(obj, name)
|
90
101
|
|
91
102
|
|
92
103
|
@argo_workflows.command(help="Deploy a new version of this workflow to Argo Workflows.")
|
@@ -240,6 +251,11 @@ def create(
|
|
240
251
|
deployer_attribute_file=None,
|
241
252
|
enable_error_msg_capture=False,
|
242
253
|
):
|
254
|
+
# check if we are supposed to block deploying the flow due to name length constraints.
|
255
|
+
if obj._exception_on_create is not None:
|
256
|
+
raise obj._exception_on_create
|
257
|
+
|
258
|
+
# TODO: Remove this once we have a proper validator system in place
|
243
259
|
for node in obj.graph:
|
244
260
|
for decorator, error_message in unsupported_decorators.items():
|
245
261
|
if any([d.name == decorator for d in node.decorators]):
|
@@ -258,7 +274,7 @@ def create(
|
|
258
274
|
f,
|
259
275
|
)
|
260
276
|
|
261
|
-
obj.echo("Deploying *%s* to Argo Workflows..." % obj.
|
277
|
+
obj.echo("Deploying *%s* to Argo Workflows..." % obj.flow.name, bold=True)
|
262
278
|
|
263
279
|
if SERVICE_VERSION_CHECK:
|
264
280
|
# TODO: Consider dispelling with this check since it's been 2 years since the
|
@@ -305,7 +321,7 @@ def create(
|
|
305
321
|
flow.deploy()
|
306
322
|
obj.echo(
|
307
323
|
"Workflow *{workflow_name}* "
|
308
|
-
"for flow *{name}*
|
324
|
+
"for flow *{name}* deployed to "
|
309
325
|
"Argo Workflows successfully.\n".format(
|
310
326
|
workflow_name=obj.workflow_name, name=current.flow_name
|
311
327
|
),
|
@@ -314,8 +330,40 @@ def create(
|
|
314
330
|
if obj._is_workflow_name_modified:
|
315
331
|
obj.echo(
|
316
332
|
"Note that the flow was deployed with a modified name "
|
317
|
-
"due to Kubernetes naming conventions
|
318
|
-
"original flow name is stored in the workflow
|
333
|
+
"due to Kubernetes naming conventions on Argo Workflows. The "
|
334
|
+
"original flow name is stored in the workflow annotations.\n",
|
335
|
+
wrap=True,
|
336
|
+
)
|
337
|
+
|
338
|
+
if obj.workflow_name != obj._v1_workflow_name:
|
339
|
+
# Delete the old workflow if it exists
|
340
|
+
try:
|
341
|
+
ArgoWorkflows.delete(obj._v1_workflow_name)
|
342
|
+
obj.echo("Important!", bold=True, nl=False)
|
343
|
+
obj.echo(
|
344
|
+
" To comply with new naming restrictions on Argo "
|
345
|
+
"Workflows, this deployment replaced the previously "
|
346
|
+
"deployed workflow {v1_workflow_name}.\n".format(
|
347
|
+
v1_workflow_name=obj._v1_workflow_name
|
348
|
+
),
|
349
|
+
wrap=True,
|
350
|
+
)
|
351
|
+
except ArgoWorkflowsException as e:
|
352
|
+
# TODO: Catch a more specific exception
|
353
|
+
pass
|
354
|
+
|
355
|
+
obj.echo("Warning! ", bold=True, nl=False)
|
356
|
+
obj.echo(
|
357
|
+
"Due to new naming restrictions on Argo Workflows, "
|
358
|
+
"re-deploying this flow with older versions of Metaflow (<{version}) "
|
359
|
+
"will result in the flow being deployed with a different name -\n"
|
360
|
+
"*{v1_workflow_name}* without replacing the version you just deployed. "
|
361
|
+
"This may result in duplicate executions of this flow. To avoid this issue, "
|
362
|
+
"always deploy this flow using Metaflow ≥{version} or specify the flow name with --name.".format(
|
363
|
+
v1_workflow_name=obj._v1_workflow_name,
|
364
|
+
version=NEW_ARGO_NAMELENGTH_METAFLOW_VERSION,
|
365
|
+
),
|
366
|
+
wrap=True,
|
319
367
|
)
|
320
368
|
|
321
369
|
if ARGO_WORKFLOWS_UI_URL:
|
@@ -395,9 +443,108 @@ def check_metadata_service_version(obj):
|
|
395
443
|
)
|
396
444
|
|
397
445
|
|
398
|
-
|
446
|
+
# Argo Workflows has a few restrictions on workflow names:
|
447
|
+
# - Argo Workflow Template names can't be longer than 253 characters since
|
448
|
+
# they follow DNS Subdomain name restrictions.
|
449
|
+
# - Argo Workflows stores workflow template names as a label in the workflow
|
450
|
+
# template metadata - workflows.argoproj.io/workflow-template, which follows
|
451
|
+
# RFC 1123, which is a strict subset of DNS Subdomain names and allows for
|
452
|
+
# 63 characters.
|
453
|
+
# - Argo Workflows appends a unix timestamp to the workflow name when the workflow
|
454
|
+
# is created (-1243856725) from a workflow template deployed as a cron workflow template
|
455
|
+
# reducing the number of characters available to 52.
|
456
|
+
# - TODO: Check naming restrictions for Argo Events.
|
457
|
+
|
458
|
+
# In summary -
|
459
|
+
# - We truncate the workflow name to 45 characters to leave enough room for future
|
460
|
+
# enhancements to the Argo Workflows integration.
|
461
|
+
# - We remove any underscores since Argo Workflows doesn't allow them.
|
462
|
+
# - We convert the name to lower case.
|
463
|
+
# - We remove + and @ as not allowed characters, which can be part of the
|
464
|
+
# project branch due to using email addresses as user names.
|
465
|
+
# - We append a hash of the workflow name to the end to make it unique.
|
466
|
+
|
467
|
+
# A complication here is that in previous versions of Metaflow (=<2.16), the limit was a
|
468
|
+
# rather lax 253 characters - so we have two issues to contend with:
|
469
|
+
# 1. Replacing any equivalent flows deployed using previous versions of Metaflow which
|
470
|
+
# adds a bit of complexity to the business logic.
|
471
|
+
# 2. Breaking Metaflow users who have multiple versions of Metaflow floating in their
|
472
|
+
# organization. Imagine a scenario, where metaflow-v1 (253 chars) deploys the same
|
473
|
+
# flow which was previously deployed using the new metaflow-v2 (45 chars) - the user
|
474
|
+
# will end up with two workflows templates instead of one since metaflow-v1 has no
|
475
|
+
# awareness of the new name truncation logic introduced by metaflow-v2. Unfortunately,
|
476
|
+
# there is no way to avoid this scenario - so we will do our best to message to the
|
477
|
+
# user to not use an older version of Metaflow to redeploy affected flows.
|
478
|
+
# ------------------------------------------------------------------------------------------
|
479
|
+
# | metaflow-v1 (253 chars) | metaflow-v2 (45 chars) | Result |
|
480
|
+
# ------------------------------------------------------------------------------------------
|
481
|
+
# | workflow_name_modified = True | workflow_name_modified = False | Not possible |
|
482
|
+
# ------------------------------------------------------------------------------------------
|
483
|
+
# | workflow_name_modified = False | workflow_name_modified = True | Messaging needed |
|
484
|
+
# ------------------------------------------------------------------------------------------
|
485
|
+
# | workflow_name_modified = False | workflow_name_modified = False | No message needed |
|
486
|
+
# ------------------------------------------------------------------------------------------
|
487
|
+
# | workflow_name_modified = True | workflow_name_modified = True | Messaging needed |
|
488
|
+
# ------------------------------------------------------------------------------------------
|
489
|
+
|
490
|
+
|
491
|
+
def resolve_workflow_name_v1(obj, name):
|
492
|
+
# models the workflow_name calculation logic in Metaflow versions =<2.16
|
493
|
+
# important!! - should stay static including any future bugs
|
399
494
|
project = current.get("project_name")
|
400
|
-
|
495
|
+
is_workflow_name_modified = False
|
496
|
+
if project:
|
497
|
+
if name:
|
498
|
+
return None, False # not possible in versions =<2.16
|
499
|
+
workflow_name = current.project_flow_name
|
500
|
+
if len(workflow_name) > 253:
|
501
|
+
name_hash = to_unicode(
|
502
|
+
base64.b32encode(sha1(to_bytes(workflow_name)).digest())
|
503
|
+
)[:8].lower()
|
504
|
+
workflow_name = "%s-%s" % (workflow_name[:242], name_hash)
|
505
|
+
is_workflow_name_modified = True
|
506
|
+
if not VALID_NAME.search(workflow_name):
|
507
|
+
workflow_name = (
|
508
|
+
re.compile(r"^[^A-Za-z0-9]+")
|
509
|
+
.sub("", workflow_name)
|
510
|
+
.replace("_", "")
|
511
|
+
.replace("@", "")
|
512
|
+
.replace("+", "")
|
513
|
+
.lower()
|
514
|
+
)
|
515
|
+
is_workflow_name_modified = True
|
516
|
+
else:
|
517
|
+
if name and not VALID_NAME.search(name):
|
518
|
+
return None, False # not possible in versions =<2.16
|
519
|
+
workflow_name = name if name else current.flow_name
|
520
|
+
if len(workflow_name) > 253:
|
521
|
+
return None, False # not possible in versions =<2.16
|
522
|
+
if not VALID_NAME.search(workflow_name):
|
523
|
+
# Note - since the original name sanitization was a surjective
|
524
|
+
# mapping, using it here is a bug, but we leave this in
|
525
|
+
# place since the usage of v1_workflow_name is to generate
|
526
|
+
# historical workflow names, so we need to replicate all
|
527
|
+
# the bugs too :'(
|
528
|
+
|
529
|
+
workflow_name = (
|
530
|
+
re.compile(r"^[^A-Za-z0-9]+")
|
531
|
+
.sub("", workflow_name)
|
532
|
+
.replace("_", "")
|
533
|
+
.replace("@", "")
|
534
|
+
.replace("+", "")
|
535
|
+
.lower()
|
536
|
+
)
|
537
|
+
is_workflow_name_modified = True
|
538
|
+
return workflow_name, is_workflow_name_modified
|
539
|
+
|
540
|
+
|
541
|
+
def resolve_workflow_name_v2(obj, name):
|
542
|
+
# current logic for imputing workflow_name
|
543
|
+
limit = 45
|
544
|
+
project = current.get("project_name")
|
545
|
+
is_workflow_name_modified = False
|
546
|
+
exception_on_create = None
|
547
|
+
|
401
548
|
if project:
|
402
549
|
if name:
|
403
550
|
raise MetaflowException(
|
@@ -410,48 +557,86 @@ def resolve_workflow_name(obj, name):
|
|
410
557
|
% to_unicode(base64.b32encode(sha1(project_branch).digest()))[:16]
|
411
558
|
)
|
412
559
|
is_project = True
|
413
|
-
|
414
|
-
|
415
|
-
# Workflows doesn't (DNS Subdomain names as defined in RFC 1123) - so we will
|
416
|
-
# remove any underscores as well as convert the name to lower case.
|
417
|
-
# Also remove + and @ as not allowed characters, which can be part of the
|
418
|
-
# project branch due to using email addresses as user names.
|
419
|
-
if len(workflow_name) > 253:
|
560
|
+
|
561
|
+
if len(workflow_name) > limit:
|
420
562
|
name_hash = to_unicode(
|
421
563
|
base64.b32encode(sha1(to_bytes(workflow_name)).digest())
|
422
|
-
)[:
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
564
|
+
)[:5].lower()
|
565
|
+
|
566
|
+
# Generate a meaningful short name
|
567
|
+
project_name = project
|
568
|
+
branch_name = current.branch_name
|
569
|
+
flow_name = current.flow_name
|
570
|
+
parts = [project_name, branch_name, flow_name]
|
571
|
+
max_name_len = limit - 6
|
572
|
+
min_each = 7
|
573
|
+
total_len = sum(len(p) for p in parts)
|
574
|
+
remaining = max_name_len - 3 * min_each
|
575
|
+
extras = [int(remaining * len(p) / total_len) for p in parts]
|
576
|
+
while sum(extras) < remaining:
|
577
|
+
extras[extras.index(min(extras))] += 1
|
578
|
+
budgets = [min_each + e for e in extras]
|
579
|
+
proj_budget = budgets[0]
|
580
|
+
if len(project_name) <= proj_budget:
|
581
|
+
proj_str = project_name
|
582
|
+
else:
|
583
|
+
h = proj_budget // 2
|
584
|
+
t = proj_budget - h
|
585
|
+
proj_str = project_name[:h] + project_name[-t:]
|
586
|
+
branch_budget = budgets[1]
|
587
|
+
branch_str = branch_name[:branch_budget]
|
588
|
+
flow_budget = budgets[2]
|
589
|
+
if len(flow_name) <= flow_budget:
|
590
|
+
flow_str = flow_name
|
591
|
+
else:
|
592
|
+
h = flow_budget // 2
|
593
|
+
t = flow_budget - h
|
594
|
+
flow_str = flow_name[:h] + flow_name[-t:]
|
595
|
+
descriptive_name = sanitize_for_argo(
|
596
|
+
"%s.%s.%s" % (proj_str, branch_str, flow_str)
|
597
|
+
)
|
598
|
+
workflow_name = "%s-%s" % (descriptive_name, name_hash)
|
599
|
+
is_workflow_name_modified = True
|
428
600
|
else:
|
429
601
|
if name and not VALID_NAME.search(name):
|
430
602
|
raise MetaflowException(
|
431
603
|
"Name '%s' contains invalid characters. The "
|
432
604
|
"name must consist of lower case alphanumeric characters, '-' or '.'"
|
433
|
-
", and must start
|
605
|
+
", and must start with an alphabetic character, "
|
606
|
+
"and end with an alphanumeric character." % name
|
434
607
|
)
|
435
|
-
|
436
608
|
workflow_name = name if name else current.flow_name
|
437
609
|
token_prefix = workflow_name
|
438
610
|
is_project = False
|
439
611
|
|
440
|
-
if len(workflow_name) >
|
441
|
-
|
442
|
-
|
612
|
+
if len(workflow_name) > limit:
|
613
|
+
# NOTE: We could have opted for truncating names specified by --name and flow_name
|
614
|
+
# as well, but chose to error instead due to the expectation that users would
|
615
|
+
# be intentionally explicit in their naming, and truncating these would lose
|
616
|
+
# information they intended to encode in the deployment.
|
617
|
+
exception_on_create = ArgoWorkflowsNameTooLong(
|
618
|
+
"The full name of the workflow:\n*%s*\nis longer than %s "
|
443
619
|
"characters.\n\n"
|
444
620
|
"To deploy this workflow to Argo Workflows, please "
|
445
621
|
"assign a shorter name\nusing the option\n"
|
446
|
-
"*argo-workflows --name <name> create*." %
|
622
|
+
"*argo-workflows --name <name> create*." % (name, limit)
|
447
623
|
)
|
448
|
-
raise ArgoWorkflowsNameTooLong(msg)
|
449
624
|
|
450
|
-
|
451
|
-
|
452
|
-
|
625
|
+
if not VALID_NAME.search(workflow_name):
|
626
|
+
# NOTE: Even though sanitize_for_argo is surjective which can result in collisions,
|
627
|
+
# we still use it here since production tokens guard against name collisions
|
628
|
+
# and if we made it injective, metaflow 2.17 will result in every deployed
|
629
|
+
# flow's name changing, significantly increasing the blast radius of the change.
|
630
|
+
workflow_name = sanitize_for_argo(workflow_name)
|
631
|
+
is_workflow_name_modified = True
|
453
632
|
|
454
|
-
return
|
633
|
+
return (
|
634
|
+
workflow_name,
|
635
|
+
token_prefix.lower(),
|
636
|
+
is_project,
|
637
|
+
is_workflow_name_modified,
|
638
|
+
exception_on_create,
|
639
|
+
)
|
455
640
|
|
456
641
|
|
457
642
|
def make_flow(
|
@@ -701,7 +886,28 @@ def trigger(obj, run_id_file=None, deployer_attribute_file=None, **kwargs):
|
|
701
886
|
if kwargs.get(param.name.replace("-", "_").lower()) is not None
|
702
887
|
}
|
703
888
|
|
704
|
-
|
889
|
+
workflow_name_to_deploy = obj.workflow_name
|
890
|
+
# For users that upgraded the client but did not redeploy their flow,
|
891
|
+
# we fallback to old workflow names in case of a conflict.
|
892
|
+
if obj.workflow_name != obj._v1_workflow_name:
|
893
|
+
# use the old name only if there exists a deployment.
|
894
|
+
if ArgoWorkflows.get_existing_deployment(obj._v1_workflow_name):
|
895
|
+
obj.echo("Warning! ", bold=True, nl=False)
|
896
|
+
obj.echo(
|
897
|
+
"Found a deployment of this flow with an old style name, defaulted to triggering *%s*."
|
898
|
+
% obj._v1_workflow_name,
|
899
|
+
wrap=True,
|
900
|
+
)
|
901
|
+
obj.echo(
|
902
|
+
"Due to new naming restrictions on Argo Workflows, "
|
903
|
+
"this flow will have a shorter name with newer versions of Metaflow (>=%s) "
|
904
|
+
"which will allow it to be triggered through Argo UI as well. "
|
905
|
+
% NEW_ARGO_NAMELENGTH_METAFLOW_VERSION,
|
906
|
+
wrap=True,
|
907
|
+
)
|
908
|
+
obj.echo("re-deploy your flow in order to get rid of this message.")
|
909
|
+
workflow_name_to_deploy = obj._v1_workflow_name
|
910
|
+
response = ArgoWorkflows.trigger(workflow_name_to_deploy, params)
|
705
911
|
run_id = "argo-" + response["metadata"]["name"]
|
706
912
|
|
707
913
|
if run_id_file:
|
@@ -712,7 +918,7 @@ def trigger(obj, run_id_file=None, deployer_attribute_file=None, **kwargs):
|
|
712
918
|
with open(deployer_attribute_file, "w") as f:
|
713
919
|
json.dump(
|
714
920
|
{
|
715
|
-
"name":
|
921
|
+
"name": workflow_name_to_deploy,
|
716
922
|
"metadata": obj.metadata.metadata_str(),
|
717
923
|
"pathspec": "/".join((obj.flow.name, run_id)),
|
718
924
|
},
|
@@ -721,7 +927,7 @@ def trigger(obj, run_id_file=None, deployer_attribute_file=None, **kwargs):
|
|
721
927
|
|
722
928
|
obj.echo(
|
723
929
|
"Workflow *{name}* triggered on Argo Workflows "
|
724
|
-
"(run-id *{run_id}*).".format(name=
|
930
|
+
"(run-id *{run_id}*).".format(name=workflow_name_to_deploy, run_id=run_id),
|
725
931
|
bold=True,
|
726
932
|
)
|
727
933
|
|
@@ -763,26 +969,57 @@ def delete(obj, authorize=None):
|
|
763
969
|
"about production tokens."
|
764
970
|
)
|
765
971
|
|
766
|
-
|
767
|
-
|
972
|
+
# Cases and expected behaviours:
|
973
|
+
# old name exists, new name does not exist -> delete old and do not fail on missing new
|
974
|
+
# old name exists, new name exists -> delete both
|
975
|
+
# old name does not exist, new name exists -> only try to delete new
|
976
|
+
# old name does not exist, new name does not exist -> keep previous behaviour where missing deployment raises error for the new name.
|
977
|
+
def _delete(workflow_name):
|
978
|
+
validate_token(workflow_name, obj.token_prefix, authorize, _token_instructions)
|
979
|
+
obj.echo("Deleting workflow *{name}*...".format(name=workflow_name), bold=True)
|
980
|
+
|
981
|
+
schedule_deleted, sensor_deleted, workflow_deleted = ArgoWorkflows.delete(
|
982
|
+
workflow_name
|
983
|
+
)
|
768
984
|
|
769
|
-
|
770
|
-
|
771
|
-
|
985
|
+
if schedule_deleted:
|
986
|
+
obj.echo(
|
987
|
+
"Deleting cronworkflow *{name}*...".format(name=workflow_name),
|
988
|
+
bold=True,
|
989
|
+
)
|
772
990
|
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
991
|
+
if sensor_deleted:
|
992
|
+
obj.echo(
|
993
|
+
"Deleting sensor *{name}*...".format(name=workflow_name),
|
994
|
+
bold=True,
|
995
|
+
)
|
996
|
+
return workflow_deleted
|
997
|
+
|
998
|
+
workflows_deleted = False
|
999
|
+
cleanup_old_name = False
|
1000
|
+
if obj.workflow_name != obj._v1_workflow_name:
|
1001
|
+
# Only add the old name if there exists a deployment with such name.
|
1002
|
+
# This is due to the way validate_token is tied to an existing deployment.
|
1003
|
+
if ArgoWorkflows.get_existing_deployment(obj._v1_workflow_name) is not None:
|
1004
|
+
cleanup_old_name = True
|
1005
|
+
obj.echo(
|
1006
|
+
"This flow has been deployed with another name in the past due to a limitation with Argo Workflows. "
|
1007
|
+
"Will also delete the older deployment.",
|
1008
|
+
wrap=True,
|
1009
|
+
)
|
1010
|
+
_delete(obj._v1_workflow_name)
|
1011
|
+
workflows_deleted = True
|
778
1012
|
|
779
|
-
|
780
|
-
|
781
|
-
|
782
|
-
|
783
|
-
|
1013
|
+
# Always try to delete the current name.
|
1014
|
+
# Do not raise exception if we deleted old name before this.
|
1015
|
+
try:
|
1016
|
+
_delete(obj.workflow_name)
|
1017
|
+
workflows_deleted = True
|
1018
|
+
except ArgoWorkflowsException:
|
1019
|
+
if not cleanup_old_name:
|
1020
|
+
raise
|
784
1021
|
|
785
|
-
if
|
1022
|
+
if workflows_deleted:
|
786
1023
|
obj.echo(
|
787
1024
|
"Deleting Kubernetes resources may take a while. "
|
788
1025
|
"Deploying the flow again to Argo Workflows while the delete is in-flight will fail."
|
@@ -821,17 +1058,21 @@ def suspend(obj, run_id, authorize=None):
|
|
821
1058
|
"about production tokens."
|
822
1059
|
)
|
823
1060
|
|
824
|
-
|
825
|
-
obj.workflow_name, obj.token_prefix, authorize, run_id, _token_instructions
|
826
|
-
)
|
1061
|
+
workflows = _get_existing_workflow_names(obj)
|
827
1062
|
|
828
|
-
|
829
|
-
|
1063
|
+
for workflow_name in workflows:
|
1064
|
+
validate_run_id(
|
1065
|
+
workflow_name, obj.token_prefix, authorize, run_id, _token_instructions
|
1066
|
+
)
|
1067
|
+
|
1068
|
+
# Trim prefix from run_id
|
1069
|
+
name = run_id[5:]
|
830
1070
|
|
831
|
-
|
1071
|
+
workflow_suspended = ArgoWorkflows.suspend(name)
|
832
1072
|
|
833
|
-
|
834
|
-
|
1073
|
+
if workflow_suspended:
|
1074
|
+
obj.echo("Suspended execution of *%s*" % run_id)
|
1075
|
+
break # no need to try out all workflow_names if we found the running one.
|
835
1076
|
|
836
1077
|
|
837
1078
|
@argo_workflows.command(help="Unsuspend flow execution on Argo Workflows.")
|
@@ -865,17 +1106,21 @@ def unsuspend(obj, run_id, authorize=None):
|
|
865
1106
|
"about production tokens."
|
866
1107
|
)
|
867
1108
|
|
868
|
-
|
869
|
-
obj.workflow_name, obj.token_prefix, authorize, run_id, _token_instructions
|
870
|
-
)
|
1109
|
+
workflows = _get_existing_workflow_names(obj)
|
871
1110
|
|
872
|
-
|
873
|
-
|
1111
|
+
for workflow_name in workflows:
|
1112
|
+
validate_run_id(
|
1113
|
+
workflow_name, obj.token_prefix, authorize, run_id, _token_instructions
|
1114
|
+
)
|
1115
|
+
|
1116
|
+
# Trim prefix from run_id
|
1117
|
+
name = run_id[5:]
|
874
1118
|
|
875
|
-
|
1119
|
+
workflow_suspended = ArgoWorkflows.unsuspend(name)
|
876
1120
|
|
877
|
-
|
878
|
-
|
1121
|
+
if workflow_suspended:
|
1122
|
+
obj.echo("Unsuspended execution of *%s*" % run_id)
|
1123
|
+
break # no need to try all workflow_names if we found one.
|
879
1124
|
|
880
1125
|
|
881
1126
|
def validate_token(name, token_prefix, authorize, instructions_fn=None):
|
@@ -983,22 +1228,26 @@ def terminate(obj, run_id, authorize=None):
|
|
983
1228
|
"about production tokens."
|
984
1229
|
)
|
985
1230
|
|
986
|
-
|
987
|
-
obj.workflow_name, obj.token_prefix, authorize, run_id, _token_instructions
|
988
|
-
)
|
1231
|
+
workflows = _get_existing_workflow_names(obj)
|
989
1232
|
|
990
|
-
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
|
995
|
-
|
996
|
-
|
997
|
-
|
1233
|
+
for workflow_name in workflows:
|
1234
|
+
validate_run_id(
|
1235
|
+
workflow_name, obj.token_prefix, authorize, run_id, _token_instructions
|
1236
|
+
)
|
1237
|
+
|
1238
|
+
# Trim prefix from run_id
|
1239
|
+
name = run_id[5:]
|
1240
|
+
obj.echo(
|
1241
|
+
"Terminating run *{run_id}* for {flow_name} ...".format(
|
1242
|
+
run_id=run_id, flow_name=obj.flow.name
|
1243
|
+
),
|
1244
|
+
bold=True,
|
1245
|
+
)
|
998
1246
|
|
999
|
-
|
1000
|
-
|
1001
|
-
|
1247
|
+
terminated = ArgoWorkflows.terminate(obj.flow.name, name)
|
1248
|
+
if terminated:
|
1249
|
+
obj.echo("\nRun terminated.")
|
1250
|
+
break # no need to try all workflow_names if we found the running one.
|
1002
1251
|
|
1003
1252
|
|
1004
1253
|
@argo_workflows.command(help="List Argo Workflow templates for the flow.")
|
@@ -1101,9 +1350,24 @@ def validate_run_id(
|
|
1101
1350
|
return True
|
1102
1351
|
|
1103
1352
|
|
1353
|
+
def _get_existing_workflow_names(obj):
|
1354
|
+
"""
|
1355
|
+
Construct a list of the current workflow name and possible existing deployments of old workflow names
|
1356
|
+
"""
|
1357
|
+
workflows = [obj.workflow_name]
|
1358
|
+
if obj.workflow_name != obj._v1_workflow_name:
|
1359
|
+
# Only add the old name if there exists a deployment with such name.
|
1360
|
+
# This is due to the way validate_token is tied to an existing deployment.
|
1361
|
+
if ArgoWorkflows.get_existing_deployment(obj._v1_workflow_name) is not None:
|
1362
|
+
workflows.append(obj._v1_workflow_name)
|
1363
|
+
|
1364
|
+
return workflows
|
1365
|
+
|
1366
|
+
|
1104
1367
|
def sanitize_for_argo(text):
|
1105
1368
|
"""
|
1106
|
-
Sanitizes a string so it does not contain characters that are not permitted in
|
1369
|
+
Sanitizes a string so it does not contain characters that are not permitted in
|
1370
|
+
Argo Workflow resource names.
|
1107
1371
|
"""
|
1108
1372
|
return (
|
1109
1373
|
re.compile(r"^[^A-Za-z0-9]+")
|
@@ -418,6 +418,7 @@ class TaskInfoComponent(MetaflowCardComponent):
|
|
418
418
|
"Task Finished On": task_data_dict["finished_at"],
|
419
419
|
# Remove Microseconds from timedelta
|
420
420
|
"Tags": ", ".join(tags),
|
421
|
+
"Attempt": self._task.current_attempt,
|
421
422
|
}
|
422
423
|
if not self.runtime:
|
423
424
|
task_metadata_dict["Task Duration"] = str(
|
metaflow/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
metaflow_version = "2.
|
1
|
+
metaflow_version = "2.17.0"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: metaflow
|
3
|
-
Version: 2.
|
3
|
+
Version: 2.17.0
|
4
4
|
Summary: Metaflow: More AI and ML, Less Engineering
|
5
5
|
Author: Metaflow Developers
|
6
6
|
Author-email: help@metaflow.org
|
@@ -26,7 +26,7 @@ License-File: LICENSE
|
|
26
26
|
Requires-Dist: requests
|
27
27
|
Requires-Dist: boto3
|
28
28
|
Provides-Extra: stubs
|
29
|
-
Requires-Dist: metaflow-stubs==2.
|
29
|
+
Requires-Dist: metaflow-stubs==2.17.0; extra == "stubs"
|
30
30
|
Dynamic: author
|
31
31
|
Dynamic: author-email
|
32
32
|
Dynamic: classifier
|
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
|
36
36
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
37
37
|
metaflow/util.py,sha256=g2SOU_CRzJLgDM_UGF9QDMANMAIHAsDRXE6S76_YzsY,14594
|
38
38
|
metaflow/vendor.py,sha256=A82CGHfStZGDP5pQ5XzRjFkbN1ZC-vFmghXIrzMDDNg,5868
|
39
|
-
metaflow/version.py,sha256=
|
39
|
+
metaflow/version.py,sha256=XFgMunwxLtn9LRW_fsHkzorB2mBbwIUjTsgDGX1FNQo,28
|
40
40
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
41
41
|
metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
|
42
42
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
@@ -230,8 +230,8 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
|
|
230
230
|
metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
231
231
|
metaflow/plugins/argo/argo_client.py,sha256=jLz0FjCTBvFLZt-8lZcMQhDcInhgEcGdPrU2Gvh67zA,17080
|
232
232
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
233
|
-
metaflow/plugins/argo/argo_workflows.py,sha256=
|
234
|
-
metaflow/plugins/argo/argo_workflows_cli.py,sha256=
|
233
|
+
metaflow/plugins/argo/argo_workflows.py,sha256=lnyS5xXBDTwdFdnjA0F8C2VfTuINscoG-ZWCeJ7uer0,189138
|
234
|
+
metaflow/plugins/argo/argo_workflows_cli.py,sha256=L5KwcT6Vd4HqAXFCPGmHUONgM2eOCTbvxdoIs6CQchw,51877
|
235
235
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=ogCSBmwsC2C3eusydrgjuAJd4qK18f1sI4jJwA4Fd-o,7800
|
236
236
|
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
|
237
237
|
metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=ydBE-lP42eNKvep36nQdUBPS3rQQErvoA7rCgyp5M6I,14949
|
@@ -282,7 +282,7 @@ metaflow/plugins/cards/exception.py,sha256=2UqlNb-Kxpg6cuLu2sBEIPTIElwlVBsSpeCgD
|
|
282
282
|
metaflow/plugins/cards/metadata.py,sha256=tACaw7_XNAICZ4A25celIbgxUF0CxHh7BBpFzKrMLTo,487
|
283
283
|
metaflow/plugins/cards/card_modules/__init__.py,sha256=WI2IAsFiKGyqPrHtO9S9-MbyVtUTgWJNL4xjJaBErRo,3437
|
284
284
|
metaflow/plugins/cards/card_modules/base.html,sha256=Y208ZKIZqEWWUcoBFTLTdWKAG0C8xH5lmyCRSjaN2FY,21004
|
285
|
-
metaflow/plugins/cards/card_modules/basic.py,sha256=
|
285
|
+
metaflow/plugins/cards/card_modules/basic.py,sha256=vFWijqHnil-d337qRztfekgySSkZJjCNLUbhU-u1Csw,25405
|
286
286
|
metaflow/plugins/cards/card_modules/bundle.css,sha256=ms2wOKftlPM_i6bC_4BkrmqCOj8mYw9OFvRCJF9FSV4,11981
|
287
287
|
metaflow/plugins/cards/card_modules/card.py,sha256=6sbqP5mwf7QWvQvX2N_bC78H9ixuI5sQ8612Q5islys,4627
|
288
288
|
metaflow/plugins/cards/card_modules/components.py,sha256=Y-Yo7UgSOyTB3zs-wDfUqUKl0PI_BzeY1QGcy2fqE2M,26752
|
@@ -426,12 +426,12 @@ metaflow/user_decorators/mutable_flow.py,sha256=icF7XFCS5FdlW3OEL68ZbQOtTPhLsUyc
|
|
426
426
|
metaflow/user_decorators/mutable_step.py,sha256=-BY0UDXf_RCAEnC5JlLzEXGdiw1KD9oSrSxS_SWaB9Y,16791
|
427
427
|
metaflow/user_decorators/user_flow_decorator.py,sha256=2yDwZq9QGv9W-7kEuKwa8o4ZkTvuHJ5ESz7VVrGViAI,9890
|
428
428
|
metaflow/user_decorators/user_step_decorator.py,sha256=JYNGXONWCpzwn-_bF5WiAkof4Ii9tRS4xdK8ojSxG6M,26007
|
429
|
-
metaflow-2.
|
430
|
-
metaflow-2.
|
431
|
-
metaflow-2.
|
432
|
-
metaflow-2.
|
433
|
-
metaflow-2.
|
434
|
-
metaflow-2.
|
435
|
-
metaflow-2.
|
436
|
-
metaflow-2.
|
437
|
-
metaflow-2.
|
429
|
+
metaflow-2.17.0.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
|
430
|
+
metaflow-2.17.0.data/data/share/metaflow/devtools/Tiltfile,sha256=I55XTG4RBnrMfDcYRtREXqqS8T9bF8agkZq0DlvdFLk,21404
|
431
|
+
metaflow-2.17.0.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
|
432
|
+
metaflow-2.17.0.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
433
|
+
metaflow-2.17.0.dist-info/METADATA,sha256=KtUYktrM1mNhRT9fVjx5DLXa9e3Dz6SIsKpMmOfaey0,6740
|
434
|
+
metaflow-2.17.0.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
|
435
|
+
metaflow-2.17.0.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
|
436
|
+
metaflow-2.17.0.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
437
|
+
metaflow-2.17.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|