ob-metaflow-stubs 6.0.3.104__py2.py3-none-any.whl → 6.0.3.106__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +853 -515
  2. metaflow-stubs/cards.pyi +211 -5
  3. metaflow-stubs/cli.pyi +22 -2
  4. metaflow-stubs/client/__init__.pyi +128 -3
  5. metaflow-stubs/client/core.pyi +226 -6
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +5 -2
  8. metaflow-stubs/events.pyi +20 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +70 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +156 -3
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +8 -2
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +22 -19
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +68 -2
  21. metaflow-stubs/plugins/__init__.pyi +14 -3
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +44 -2
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +44 -2
  29. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +32 -4
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
  36. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +102 -2
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
  52. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  59. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/cards/card_cli.pyi +63 -5
  61. metaflow-stubs/plugins/cards/card_client.pyi +34 -3
  62. metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +41 -2
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +35 -2
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +55 -2
  74. metaflow-stubs/plugins/cards/exception.pyi +8 -2
  75. metaflow-stubs/plugins/catch_decorator.pyi +19 -2
  76. metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
  77. metaflow-stubs/plugins/datatools/local.pyi +16 -2
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +83 -6
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  82. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  83. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  84. metaflow-stubs/plugins/environment_decorator.pyi +10 -2
  85. metaflow-stubs/plugins/events_decorator.pyi +106 -2
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +24 -3
  88. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +11 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +99 -2
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
  101. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  102. metaflow-stubs/plugins/package_cli.pyi +2 -2
  103. metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
  104. metaflow-stubs/plugins/perimeters.pyi +2 -2
  105. metaflow-stubs/plugins/project_decorator.pyi +59 -2
  106. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  111. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  112. metaflow-stubs/plugins/resources_decorator.pyi +33 -2
  113. metaflow-stubs/plugins/retry_decorator.pyi +21 -2
  114. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
  117. metaflow-stubs/plugins/storage_executor.pyi +6 -2
  118. metaflow-stubs/plugins/tag_cli.pyi +37 -6
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +10 -3
  120. metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
  121. metaflow-stubs/procpoll.pyi +2 -2
  122. metaflow-stubs/profilers/__init__.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +70 -131
  126. metaflow-stubs/runner/metaflow_runner.pyi +118 -10
  127. metaflow-stubs/runner/nbdeploy.pyi +66 -2
  128. metaflow-stubs/runner/nbrun.pyi +79 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +16 -4
  130. metaflow-stubs/runner/utils.pyi +32 -2
  131. metaflow-stubs/system/__init__.pyi +3 -3
  132. metaflow-stubs/system/system_logger.pyi +3 -3
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.106.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.104.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.374159 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.933043 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -21,11 +21,115 @@ class MetaflowException(Exception, metaclass=type):
21
21
  ...
22
22
 
23
23
  class TriggerDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
24
+ """
25
+ Specifies the event(s) that this flow depends on.
26
+
27
+ ```
28
+ @trigger(event='foo')
29
+ ```
30
+ or
31
+ ```
32
+ @trigger(events=['foo', 'bar'])
33
+ ```
34
+
35
+ Additionally, you can specify the parameter mappings
36
+ to map event payload to Metaflow parameters for the flow.
37
+ ```
38
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
39
+ ```
40
+ or
41
+ ```
42
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
43
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
44
+ ```
45
+
46
+ 'parameters' can also be a list of strings and tuples like so:
47
+ ```
48
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
49
+ ```
50
+ This is equivalent to:
51
+ ```
52
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
53
+ ```
54
+
55
+ Parameters
56
+ ----------
57
+ event : Union[str, Dict[str, Any]], optional, default None
58
+ Event dependency for this flow.
59
+ events : List[Union[str, Dict[str, Any]]], default []
60
+ Events dependency for this flow.
61
+ options : Dict[str, Any], default {}
62
+ Backend-specific configuration for tuning eventing behavior.
63
+
64
+ MF Add To Current
65
+ -----------------
66
+ trigger -> metaflow.events.Trigger
67
+ Returns `Trigger` if the current run is triggered by an event
68
+
69
+ @@ Returns
70
+ -------
71
+ Trigger
72
+ `Trigger` if triggered by an event
73
+ """
24
74
  def flow_init(self, flow_name, graph, environment, flow_datastore, metadata, logger, echo, options):
25
75
  ...
26
76
  ...
27
77
 
28
78
  class TriggerOnFinishDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
79
+ """
80
+ Specifies the flow(s) that this flow depends on.
81
+
82
+ ```
83
+ @trigger_on_finish(flow='FooFlow')
84
+ ```
85
+ or
86
+ ```
87
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
88
+ ```
89
+ This decorator respects the @project decorator and triggers the flow
90
+ when upstream runs within the same namespace complete successfully
91
+
92
+ Additionally, you can specify project aware upstream flow dependencies
93
+ by specifying the fully qualified project_flow_name.
94
+ ```
95
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
96
+ ```
97
+ or
98
+ ```
99
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
100
+ ```
101
+
102
+ You can also specify just the project or project branch (other values will be
103
+ inferred from the current project or project branch):
104
+ ```
105
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
106
+ ```
107
+
108
+ Note that `branch` is typically one of:
109
+ - `prod`
110
+ - `user.bob`
111
+ - `test.my_experiment`
112
+ - `prod.staging`
113
+
114
+ Parameters
115
+ ----------
116
+ flow : Union[str, Dict[str, str]], optional, default None
117
+ Upstream flow dependency for this flow.
118
+ flows : List[Union[str, Dict[str, str]]], default []
119
+ Upstream flow dependencies for this flow.
120
+ options : Dict[str, Any], default {}
121
+ Backend-specific configuration for tuning eventing behavior.
122
+
123
+ MF Add To Current
124
+ -----------------
125
+ trigger -> metaflow.events.Trigger
126
+ Returns `Trigger` if the current run is triggered by an event
127
+
128
+ @@ Returns
129
+ -------
130
+ Trigger
131
+ `Trigger` if triggered by an event
132
+ """
29
133
  def flow_init(self, flow_name, graph, environment, flow_datastore, metadata, logger, echo, options):
30
134
  ...
31
135
  def get_top_level_options(self):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.373293 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.930581 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,20 +1,41 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.406194 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.964748 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.plugins.parallel_decorator
12
11
  import metaflow.decorators
12
+ import metaflow.plugins.parallel_decorator
13
13
  import metaflow.metaflow_current
14
14
 
15
15
  current: metaflow.metaflow_current.Current
16
16
 
17
17
  class ParallelDecorator(metaflow.decorators.StepDecorator, metaclass=type):
18
+ """
19
+ MF Add To Current
20
+ -----------------
21
+ parallel -> metaflow.metaflow_current.Parallel
22
+
23
+ @@ Returns
24
+ -------
25
+ Parallel
26
+ `namedtuple` with the following fields:
27
+ - main_ip : str
28
+ The IP address of the control task.
29
+ - num_nodes : int
30
+ The total number of tasks created by @parallel
31
+ - node_index : int
32
+ The index of the current task in all the @parallel tasks.
33
+ - control_task_id : Optional[str]
34
+ The task ID of the control task. Available to all tasks.
35
+
36
+ is_parallel -> bool
37
+ True if the current step is a @parallel step.
38
+ """
18
39
  def __init__(self, attributes = None, statically_defined = False):
19
40
  ...
20
41
  def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.370508 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.928954 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.386147 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.955820 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,8 +9,8 @@ from __future__ import annotations
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
11
  import metaflow.plugins.secrets
12
- import abc
13
12
  import metaflow.exception
13
+ import abc
14
14
 
15
15
  class MetaflowException(Exception, metaclass=type):
16
16
  def __init__(self, msg = "", lineno = None):
@@ -34,15 +34,27 @@ def get_credentials(scopes, *args, **kwargs):
34
34
  GCP_SECRET_MANAGER_PREFIX: None
35
35
 
36
36
  class MetaflowGcpSecretsManagerBadResponse(metaflow.exception.MetaflowException, metaclass=type):
37
+ """
38
+ Raised when the response from GCP Secrets Manager is not valid in some way
39
+ """
37
40
  ...
38
41
 
39
42
  class MetaflowGcpSecretsManagerDuplicateKey(metaflow.exception.MetaflowException, metaclass=type):
43
+ """
44
+ Raised when the response from GCP Secrets Manager contains duplicate keys
45
+ """
40
46
  ...
41
47
 
42
48
  class MetaflowGcpSecretsManagerJSONParseError(metaflow.exception.MetaflowException, metaclass=type):
49
+ """
50
+ Raised when the SecretString response from GCP Secrets Manager is not valid JSON
51
+ """
43
52
  ...
44
53
 
45
54
  class MetaflowGcpSecretsManagerNotJSONObject(metaflow.exception.MetaflowException, metaclass=type):
55
+ """
56
+ Raised when the SecretString response from GCP Secrets Manager is not valid JSON dictionary
57
+ """
46
58
  ...
47
59
 
48
60
  class GcpSecretManagerSecretsProvider(metaflow.plugins.secrets.SecretsProvider, metaclass=abc.ABCMeta):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.385463 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.955088 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.385284 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.954905 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.385739 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.955370 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.368474 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.926734 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.373739 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.932190 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.409750 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.969325 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.407292 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.966818 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -88,6 +88,15 @@ UBF_CONTROL: str
88
88
 
89
89
  UBF_TASK: str
90
90
 
91
+ def init_config() -> typing.Dict[str, str]:
92
+ """
93
+ OSS Metaflow reads the config file on every step initialization. This is because OSS assumes config files change
94
+ relatively infrequently. We want to avoid config values changing between flow steps. Our solution to prevent this
95
+ is to read a config once and cache it on an environment variable. Environment variables carry over between steps
96
+ because steps are executed in subprocesses (local) or environments which expect environment variables to be set.
97
+ """
98
+ ...
99
+
91
100
  BASH_SAVE_LOGS: str
92
101
 
93
102
  class KubernetesClient(object, metaclass=type):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.410888 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.970487 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -86,6 +86,69 @@ def parse_kube_keyvalue_list(items: typing.List[str], requires_both: bool = True
86
86
  ...
87
87
 
88
88
  class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
89
+ """
90
+ Specifies that this step should execute on Kubernetes.
91
+
92
+ Parameters
93
+ ----------
94
+ cpu : int, default 1
95
+ Number of CPUs required for this step. If `@resources` is
96
+ also present, the maximum value from all decorators is used.
97
+ memory : int, default 4096
98
+ Memory size (in MB) required for this step. If
99
+ `@resources` is also present, the maximum value from all decorators is
100
+ used.
101
+ disk : int, default 10240
102
+ Disk size (in MB) required for this step. If
103
+ `@resources` is also present, the maximum value from all decorators is
104
+ used.
105
+ image : str, optional, default None
106
+ Docker image to use when launching on Kubernetes. If not specified, and
107
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
108
+ not, a default Docker image mapping to the current version of Python is used.
109
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
110
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
111
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
112
+ Kubernetes service account to use when launching pod in Kubernetes.
113
+ secrets : List[str], optional, default None
114
+ Kubernetes secrets to use when launching pod in Kubernetes. These
115
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
116
+ in Metaflow configuration.
117
+ node_selector: Union[Dict[str,str], str], optional, default None
118
+ Kubernetes node selector(s) to apply to the pod running the task.
119
+ Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
120
+ or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
121
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
122
+ Kubernetes namespace to use when launching pod in Kubernetes.
123
+ gpu : int, optional, default None
124
+ Number of GPUs required for this step. A value of zero implies that
125
+ the scheduled node should not have GPUs.
126
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
127
+ The vendor of the GPUs to be used for this step.
128
+ tolerations : List[str], default []
129
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
130
+ Kubernetes tolerations to use when launching pod in Kubernetes.
131
+ use_tmpfs : bool, default False
132
+ This enables an explicit tmpfs mount for this step.
133
+ tmpfs_tempdir : bool, default True
134
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
135
+ tmpfs_size : int, optional, default: None
136
+ The value for the size (in MiB) of the tmpfs mount for this step.
137
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
138
+ memory allocated for this step.
139
+ tmpfs_path : str, optional, default /metaflow_temp
140
+ Path to tmpfs mount for this step.
141
+ persistent_volume_claims : Dict[str, str], optional, default None
142
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
143
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
144
+ shared_memory: int, optional
145
+ Shared memory size (in MiB) required for this step
146
+ port: int, optional
147
+ Port number to specify in the Kubernetes job object
148
+ compute_pool : str, optional, default None
149
+ Compute pool to be used for for this step.
150
+ If not specified, any accessible compute pool within the perimeter is used.
151
+ """
89
152
  def __init__(self, attributes = None, statically_defined = False):
90
153
  ...
91
154
  def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.327505 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.884879 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.408174 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.967778 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -22,6 +22,9 @@ class MetaflowException(Exception, metaclass=type):
22
22
  ...
23
23
 
24
24
  class MetaDatum(tuple, metaclass=type):
25
+ """
26
+ MetaDatum(field, value, type, tags)
27
+ """
25
28
  @staticmethod
26
29
  def __new__(_cls, field, value, type, tags):
27
30
  """
@@ -76,6 +79,37 @@ KUBERNETES_SHARED_MEMORY: None
76
79
  KUBERNETES_TOLERATIONS: str
77
80
 
78
81
  class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
82
+ """
83
+ Specifies the resources needed when executing this step.
84
+
85
+ Use `@resources` to specify the resource requirements
86
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
87
+
88
+ You can choose the compute layer on the command line by executing e.g.
89
+ ```
90
+ python myflow.py run --with batch
91
+ ```
92
+ or
93
+ ```
94
+ python myflow.py run --with kubernetes
95
+ ```
96
+ which executes the flow on the desired system using the
97
+ requirements specified in `@resources`.
98
+
99
+ Parameters
100
+ ----------
101
+ cpu : int, default 1
102
+ Number of CPUs required for this step.
103
+ gpu : int, default 0
104
+ Number of GPUs required for this step.
105
+ disk : int, optional, default None
106
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
107
+ memory : int, default 4096
108
+ Memory size (in MB) required for this step.
109
+ shared_memory : int, optional, default None
110
+ The value for the size (in MiB) of the /dev/shm volume for this step.
111
+ This parameter maps to the `--shm-size` option in Docker.
112
+ """
79
113
  ...
80
114
 
81
115
  def get_run_time_limit_for_task(step_decos):
@@ -151,6 +185,69 @@ MAX_MEMORY_PER_TASK: None
151
185
  MAX_CPU_PER_TASK: None
152
186
 
153
187
  class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
188
+ """
189
+ Specifies that this step should execute on Kubernetes.
190
+
191
+ Parameters
192
+ ----------
193
+ cpu : int, default 1
194
+ Number of CPUs required for this step. If `@resources` is
195
+ also present, the maximum value from all decorators is used.
196
+ memory : int, default 4096
197
+ Memory size (in MB) required for this step. If
198
+ `@resources` is also present, the maximum value from all decorators is
199
+ used.
200
+ disk : int, default 10240
201
+ Disk size (in MB) required for this step. If
202
+ `@resources` is also present, the maximum value from all decorators is
203
+ used.
204
+ image : str, optional, default None
205
+ Docker image to use when launching on Kubernetes. If not specified, and
206
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
207
+ not, a default Docker image mapping to the current version of Python is used.
208
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
209
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
210
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
211
+ Kubernetes service account to use when launching pod in Kubernetes.
212
+ secrets : List[str], optional, default None
213
+ Kubernetes secrets to use when launching pod in Kubernetes. These
214
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
215
+ in Metaflow configuration.
216
+ node_selector: Union[Dict[str,str], str], optional, default None
217
+ Kubernetes node selector(s) to apply to the pod running the task.
218
+ Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
219
+ or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
220
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
221
+ Kubernetes namespace to use when launching pod in Kubernetes.
222
+ gpu : int, optional, default None
223
+ Number of GPUs required for this step. A value of zero implies that
224
+ the scheduled node should not have GPUs.
225
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
226
+ The vendor of the GPUs to be used for this step.
227
+ tolerations : List[str], default []
228
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
229
+ Kubernetes tolerations to use when launching pod in Kubernetes.
230
+ use_tmpfs : bool, default False
231
+ This enables an explicit tmpfs mount for this step.
232
+ tmpfs_tempdir : bool, default True
233
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
234
+ tmpfs_size : int, optional, default: None
235
+ The value for the size (in MiB) of the tmpfs mount for this step.
236
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
237
+ memory allocated for this step.
238
+ tmpfs_path : str, optional, default /metaflow_temp
239
+ Path to tmpfs mount for this step.
240
+ persistent_volume_claims : Dict[str, str], optional, default None
241
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
242
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
243
+ shared_memory: int, optional
244
+ Shared memory size (in MiB) required for this step
245
+ port: int, optional
246
+ Port number to specify in the Kubernetes job object
247
+ compute_pool : str, optional, default None
248
+ Compute pool to be used for for this step.
249
+ If not specified, any accessible compute pool within the perimeter is used.
250
+ """
154
251
  def __init__(self, attributes = None, statically_defined = False):
155
252
  ...
156
253
  def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.409567 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.969138 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -30,6 +30,9 @@ def k8s_retry(deadline_seconds = 60, max_backoff = 32):
30
30
  ...
31
31
 
32
32
  class JobsetStatus(tuple, metaclass=type):
33
+ """
34
+ JobsetStatus(control_pod_failed, control_exit_code, control_pod_status, control_started, control_completed, worker_pods_failed, workers_are_suspended, workers_have_started, all_jobs_are_suspended, jobset_finished, jobset_failed, status_unknown, jobset_was_terminated, some_jobs_are_running)
35
+ """
33
36
  @staticmethod
34
37
  def __new__(_cls, control_pod_failed, control_exit_code, control_pod_status, control_started, control_completed, worker_pods_failed, workers_are_suspended, workers_have_started, all_jobs_are_suspended, jobset_finished, jobset_failed, status_unknown, jobset_was_terminated, some_jobs_are_running):
35
38
  """
@@ -1,15 +1,15 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.376746 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.935357 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.exception
12
11
  import metaflow.plugins.logs_cli
12
+ import metaflow.exception
13
13
  import metaflow._vendor.click.core
14
14
 
15
15
  LOGGER_TIMESTAMP: str
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.376349 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.934962 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations