ob-metaflow-stubs 6.0.3.117__py2.py3-none-any.whl → 6.0.3.118__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (175) hide show
  1. metaflow-stubs/__init__.pyi +319 -2739
  2. metaflow-stubs/cards.pyi +19 -473
  3. metaflow-stubs/cli.pyi +17 -81
  4. metaflow-stubs/client/__init__.pyi +19 -1113
  5. metaflow-stubs/client/core.pyi +18 -158
  6. metaflow-stubs/client/filecache.pyi +8 -12
  7. metaflow-stubs/clone_util.pyi +6 -26
  8. metaflow-stubs/events.pyi +6 -5
  9. metaflow-stubs/exception.pyi +8 -6
  10. metaflow-stubs/flowspec.pyi +22 -106
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +16 -564
  13. metaflow-stubs/info_file.pyi +6 -5
  14. metaflow-stubs/metadata_provider/__init__.pyi +16 -0
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +34 -0
  16. metaflow-stubs/{metadata → metadata_provider}/metadata.pyi +10 -22
  17. metaflow-stubs/metadata_provider/util.pyi +19 -0
  18. metaflow-stubs/metaflow_config.pyi +8 -13
  19. metaflow-stubs/metaflow_current.pyi +134 -133
  20. metaflow-stubs/mf_extensions/__init__.pyi +6 -0
  21. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +6 -0
  22. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +6 -0
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +6 -0
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +6 -0
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +129 -0
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +26 -0
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +156 -0
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +25 -0
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +111 -0
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +6 -0
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +188 -0
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +30 -0
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +28 -0
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +19 -0
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +115 -0
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +40 -0
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +6 -0
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +71 -0
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +56 -0
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +67 -0
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +49 -0
  42. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +6 -0
  43. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +6 -0
  44. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +130 -0
  45. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +42 -0
  46. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +6 -0
  47. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +17 -0
  48. metaflow-stubs/mflog/__init__.pyi +6 -0
  49. metaflow-stubs/mflog/mflog.pyi +52 -5
  50. metaflow-stubs/multicore_utils.pyi +6 -5
  51. metaflow-stubs/parameters.pyi +13 -23
  52. metaflow-stubs/plugins/__init__.pyi +52 -165
  53. metaflow-stubs/plugins/airflow/__init__.pyi +9 -5
  54. metaflow-stubs/plugins/airflow/airflow_utils.pyi +7 -6
  55. metaflow-stubs/plugins/airflow/exception.pyi +7 -11
  56. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +10 -97
  57. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +9 -30
  58. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +9 -40
  59. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +9 -40
  60. metaflow-stubs/plugins/argo/__init__.pyi +12 -5
  61. metaflow-stubs/plugins/argo/argo_client.pyi +8 -26
  62. metaflow-stubs/plugins/argo/argo_events.pyi +7 -11
  63. metaflow-stubs/plugins/argo/argo_workflows.pyi +18 -130
  64. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +22 -460
  65. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +12 -404
  66. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +63 -448
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +165 -0
  68. metaflow-stubs/plugins/aws/__init__.pyi +11 -5
  69. metaflow-stubs/plugins/aws/aws_client.pyi +6 -5
  70. metaflow-stubs/plugins/aws/aws_utils.pyi +6 -11
  71. metaflow-stubs/plugins/aws/batch/__init__.pyi +9 -5
  72. metaflow-stubs/plugins/aws/batch/batch.pyi +10 -55
  73. metaflow-stubs/plugins/aws/batch/batch_client.pyi +7 -11
  74. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +15 -140
  75. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +7 -5
  76. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +10 -21
  77. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +9 -5
  78. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +6 -5
  79. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +6 -5
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +7 -5
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +11 -65
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +6 -5
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +52 -292
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +127 -0
  85. metaflow-stubs/plugins/azure/__init__.pyi +12 -7
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +6 -5
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +7 -11
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +11 -24
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +11 -29
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +8 -23
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +7 -17
  92. metaflow-stubs/plugins/cards/__init__.pyi +15 -5
  93. metaflow-stubs/plugins/cards/card_cli.pyi +22 -491
  94. metaflow-stubs/plugins/cards/card_client.pyi +13 -75
  95. metaflow-stubs/plugins/cards/card_creator.pyi +7 -10
  96. metaflow-stubs/plugins/cards/card_datastore.pyi +10 -18
  97. metaflow-stubs/plugins/cards/card_decorator.pyi +10 -126
  98. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +13 -81
  99. metaflow-stubs/plugins/cards/card_modules/basic.pyi +13 -96
  100. metaflow-stubs/plugins/cards/card_modules/card.pyi +6 -5
  101. metaflow-stubs/plugins/cards/card_modules/components.pyi +24 -107
  102. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +6 -5
  103. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +6 -12
  104. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +11 -88
  105. metaflow-stubs/plugins/cards/card_resolver.pyi +6 -49
  106. metaflow-stubs/plugins/cards/component_serializer.pyi +13 -63
  107. metaflow-stubs/plugins/cards/exception.pyi +7 -11
  108. metaflow-stubs/plugins/catch_decorator.pyi +10 -30
  109. metaflow-stubs/plugins/datatools/__init__.pyi +13 -392
  110. metaflow-stubs/plugins/datatools/local.pyi +7 -11
  111. metaflow-stubs/plugins/datatools/s3/__init__.pyi +19 -653
  112. metaflow-stubs/plugins/datatools/s3/s3.pyi +16 -264
  113. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +7 -10
  114. metaflow-stubs/plugins/datatools/s3/s3util.pyi +6 -11
  115. metaflow-stubs/plugins/debug_logger.pyi +7 -5
  116. metaflow-stubs/plugins/debug_monitor.pyi +7 -5
  117. metaflow-stubs/plugins/environment_decorator.pyi +7 -5
  118. metaflow-stubs/plugins/events_decorator.pyi +8 -14
  119. metaflow-stubs/plugins/frameworks/__init__.pyi +7 -5
  120. metaflow-stubs/plugins/frameworks/pytorch.pyi +8 -45
  121. metaflow-stubs/plugins/gcp/__init__.pyi +11 -7
  122. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +11 -24
  123. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +7 -11
  124. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +6 -5
  125. metaflow-stubs/plugins/gcp/gs_utils.pyi +8 -20
  126. metaflow-stubs/plugins/gcp/includefile_support.pyi +7 -17
  127. metaflow-stubs/plugins/kubernetes/__init__.pyi +12 -5
  128. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +6 -10
  129. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +10 -38
  130. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +16 -155
  131. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +7 -11
  132. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +19 -142
  133. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +7 -11
  134. metaflow-stubs/plugins/logs_cli.pyi +10 -9
  135. metaflow-stubs/plugins/package_cli.pyi +7 -5
  136. metaflow-stubs/plugins/parallel_decorator.pyi +11 -59
  137. metaflow-stubs/plugins/perimeters.pyi +6 -11
  138. metaflow-stubs/plugins/project_decorator.pyi +8 -14
  139. metaflow-stubs/plugins/pypi/__init__.pyi +12 -11
  140. metaflow-stubs/plugins/pypi/conda_decorator.pyi +8 -27
  141. metaflow-stubs/plugins/pypi/conda_environment.pyi +14 -20
  142. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +7 -5
  143. metaflow-stubs/plugins/pypi/pypi_environment.pyi +7 -39
  144. metaflow-stubs/plugins/pypi/utils.pyi +7 -11
  145. metaflow-stubs/plugins/resources_decorator.pyi +7 -5
  146. metaflow-stubs/plugins/retry_decorator.pyi +7 -11
  147. metaflow-stubs/plugins/secrets/__init__.pyi +9 -5
  148. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +8 -13
  149. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +7 -11
  150. metaflow-stubs/plugins/storage_executor.pyi +6 -11
  151. metaflow-stubs/plugins/tag_cli.pyi +14 -396
  152. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +8 -33
  153. metaflow-stubs/plugins/timeout_decorator.pyi +8 -12
  154. metaflow-stubs/procpoll.pyi +7 -5
  155. metaflow-stubs/profilers/__init__.pyi +7 -11
  156. metaflow-stubs/pylint_wrapper.pyi +7 -11
  157. metaflow-stubs/runner/__init__.pyi +13 -5
  158. metaflow-stubs/runner/deployer.pyi +99 -223
  159. metaflow-stubs/runner/deployer_impl.pyi +87 -0
  160. metaflow-stubs/runner/metaflow_runner.pyi +24 -508
  161. metaflow-stubs/runner/nbdeploy.pyi +16 -60
  162. metaflow-stubs/runner/nbrun.pyi +11 -148
  163. metaflow-stubs/runner/subprocess_manager.pyi +9 -10
  164. metaflow-stubs/runner/utils.pyi +44 -9
  165. metaflow-stubs/system/__init__.pyi +9 -107
  166. metaflow-stubs/system/system_logger.pyi +6 -15
  167. metaflow-stubs/system/system_monitor.pyi +6 -16
  168. metaflow-stubs/tagging_util.pyi +6 -10
  169. metaflow-stubs/tuple_util.pyi +6 -5
  170. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/METADATA +1 -1
  171. ob_metaflow_stubs-6.0.3.118.dist-info/RECORD +174 -0
  172. metaflow-stubs/metadata/util.pyi +0 -18
  173. ob_metaflow_stubs-6.0.3.117.dist-info/RECORD +0 -140
  174. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/WHEEL +0 -0
  175. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/top_level.txt +0 -0
@@ -1,12 +1,13 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.400258 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.879684 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
 
10
+
10
11
  class GcpDefaultClientProvider(object, metaclass=type):
11
12
  @staticmethod
12
13
  def get_gs_storage_client(*args, **kwargs):
@@ -1,27 +1,15 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.400725 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.880030 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
- if typing.TYPE_CHECKING:
11
- import metaflow.exception
12
9
 
13
- class MetaflowException(Exception, metaclass=type):
14
- def __init__(self, msg = "", lineno = None):
15
- ...
16
- def __str__(self):
17
- ...
18
- ...
19
-
20
- class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
21
- ...
22
-
23
- class MetaflowGSPackageError(metaflow.exception.MetaflowException, metaclass=type):
24
- ...
10
+ from ...exception import MetaflowException as MetaflowException
11
+ from ...exception import MetaflowInternalError as MetaflowInternalError
12
+ from .gs_exceptions import MetaflowGSPackageError as MetaflowGSPackageError
25
13
 
26
14
  def parse_gs_full_path(gs_uri):
27
15
  ...
@@ -1,24 +1,14 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.381696 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.880471 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
- if typing.TYPE_CHECKING:
11
- import metaflow.exception
12
9
 
13
- class MetaflowException(Exception, metaclass=type):
14
- def __init__(self, msg = "", lineno = None):
15
- ...
16
- def __str__(self):
17
- ...
18
- ...
19
-
20
- class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
21
- ...
10
+ from ...exception import MetaflowException as MetaflowException
11
+ from ...exception import MetaflowInternalError as MetaflowInternalError
22
12
 
23
13
  class GS(object, metaclass=type):
24
14
  @classmethod
@@ -1,9 +1,16 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.387750 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.864188 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
 
10
+ from . import kubernetes_client as kubernetes_client
11
+ from . import kubernetes as kubernetes
12
+ from . import kubernetes_decorator as kubernetes_decorator
13
+ from . import kube_utils as kube_utils
14
+ from . import kubernetes_cli as kubernetes_cli
15
+ from . import kubernetes_jobsets as kubernetes_jobsets
16
+
@@ -1,17 +1,13 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.425146 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.893342 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
- if typing.TYPE_CHECKING:
11
- import metaflow.exception
12
9
 
13
- class CommandException(metaflow.exception.MetaflowException, metaclass=type):
14
- ...
10
+ from ...exception import CommandException as CommandException
15
11
 
16
12
  def parse_cli_options(flow_name, run_id, user, my_runs, echo):
17
13
  ...
@@ -1,24 +1,20 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.422776 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.892127 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
- import metaflow.metaflow_current
12
12
  import metaflow.exception
13
13
 
14
- current: metaflow.metaflow_current.Current
15
-
16
- class MetaflowException(Exception, metaclass=type):
17
- def __init__(self, msg = "", lineno = None):
18
- ...
19
- def __str__(self):
20
- ...
21
- ...
14
+ from ...metaflow_current import current as current
15
+ from ...exception import MetaflowException as MetaflowException
16
+ from ...mf_extensions.outerbounds.remote_config import init_config as init_config
17
+ from .kubernetes_client import KubernetesClient as KubernetesClient
22
18
 
23
19
  ARGO_EVENTS_EVENT: None
24
20
 
@@ -88,32 +84,8 @@ UBF_CONTROL: str
88
84
 
89
85
  UBF_TASK: str
90
86
 
91
- def init_config() -> typing.Dict[str, str]:
92
- """
93
- OSS Metaflow reads the config file on every step initialization. This is because OSS assumes config files change
94
- relatively infrequently. We want to avoid config values changing between flow steps. Our solution to prevent this
95
- is to read a config once and cache it on an environment variable. Environment variables carry over between steps
96
- because steps are executed in subprocesses (local) or environments which expect environment variables to be set.
97
- """
98
- ...
99
-
100
87
  BASH_SAVE_LOGS: str
101
88
 
102
- class KubernetesClient(object, metaclass=type):
103
- def __init__(self):
104
- ...
105
- def get(self):
106
- ...
107
- def list(self, flow_name, run_id, user):
108
- ...
109
- def kill_pods(self, flow_name, run_id, user, echo):
110
- ...
111
- def job(self, **kwargs):
112
- ...
113
- def jobset(self, **kwargs):
114
- ...
115
- ...
116
-
117
89
  LOGS_DIR: str
118
90
 
119
91
  STDOUT_FILE: str
@@ -1,56 +1,26 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.426349 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.893841 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
- if typing.TYPE_CHECKING:
11
- import metaflow.decorators
12
- import metaflow.exception
13
- import metaflow._vendor.click.types
14
9
 
15
- def parse_cli_options(flow_name, run_id, user, my_runs, echo):
16
- ...
17
-
18
- class KubernetesClient(object, metaclass=type):
19
- def __init__(self):
20
- ...
21
- def get(self):
22
- ...
23
- def list(self, flow_name, run_id, user):
24
- ...
25
- def kill_pods(self, flow_name, run_id, user, echo):
26
- ...
27
- def job(self, **kwargs):
28
- ...
29
- def jobset(self, **kwargs):
30
- ...
31
- ...
32
-
33
- class JSONTypeClass(metaflow._vendor.click.types.ParamType, metaclass=type):
34
- def convert(self, value, param, ctx):
35
- ...
36
- def __str__(self):
37
- ...
38
- def __repr__(self):
39
- ...
40
- ...
10
+ from .kube_utils import parse_cli_options as parse_cli_options
11
+ from .kubernetes_client import KubernetesClient as KubernetesClient
12
+ from ...parameters import JSONTypeClass as JSONTypeClass
13
+ from ..._vendor import click as click
14
+ from ...exception import MetaflowException as MetaflowException
15
+ from ...metadata_provider.util import sync_local_metadata_from_datastore as sync_local_metadata_from_datastore
16
+ from .kubernetes import Kubernetes as Kubernetes
17
+ from .kubernetes import KubernetesException as KubernetesException
18
+ from .kubernetes import KubernetesKilledException as KubernetesKilledException
19
+ from .kubernetes import parse_kube_keyvalue_list as parse_kube_keyvalue_list
20
+ from .kubernetes_decorator import KubernetesDecorator as KubernetesDecorator
41
21
 
42
22
  METAFLOW_EXIT_DISALLOW_RETRY: int
43
23
 
44
- class MetaflowException(Exception, metaclass=type):
45
- def __init__(self, msg = "", lineno = None):
46
- ...
47
- def __str__(self):
48
- ...
49
- ...
50
-
51
- def sync_local_metadata_from_datastore(metadata_local_dir, task_ds):
52
- ...
53
-
54
24
  DATASTORE_LOCAL_DIR: str
55
25
 
56
26
  KUBERNETES_LABELS: str
@@ -61,112 +31,3 @@ UBF_CONTROL: str
61
31
 
62
32
  UBF_TASK: str
63
33
 
64
- class Kubernetes(object, metaclass=type):
65
- def __init__(self, datastore, metadata, environment):
66
- ...
67
- def launch_job(self, **kwargs):
68
- ...
69
- def create_jobset(self, flow_name, run_id, step_name, task_id, attempt, user, code_package_sha, code_package_url, code_package_ds, docker_image, docker_image_pull_policy, step_cli = None, service_account = None, secrets = None, node_selector = None, namespace = None, cpu = None, gpu = None, gpu_vendor = None, disk = None, memory = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, run_time_limit = None, env = None, persistent_volume_claims = None, tolerations = None, labels = None, shared_memory = None, port = None, num_parallel = None):
70
- ...
71
- def create_job_object(self, flow_name, run_id, step_name, task_id, attempt, user, code_package_sha, code_package_url, code_package_ds, step_cli, docker_image, docker_image_pull_policy, service_account = None, secrets = None, node_selector = None, namespace = None, cpu = None, gpu = None, gpu_vendor = None, disk = None, memory = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, run_time_limit = None, env = None, persistent_volume_claims = None, tolerations = None, labels = None, shared_memory = None, port = None, name_pattern = None):
72
- ...
73
- def create_k8sjob(self, job):
74
- ...
75
- def wait(self, stdout_location, stderr_location, echo = None):
76
- ...
77
- ...
78
-
79
- class KubernetesException(metaflow.exception.MetaflowException, metaclass=type):
80
- ...
81
-
82
- class KubernetesKilledException(metaflow.exception.MetaflowException, metaclass=type):
83
- ...
84
-
85
- def parse_kube_keyvalue_list(items: typing.List[str], requires_both: bool = True):
86
- ...
87
-
88
- class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
89
- """
90
- Specifies that this step should execute on Kubernetes.
91
-
92
- Parameters
93
- ----------
94
- cpu : int, default 1
95
- Number of CPUs required for this step. If `@resources` is
96
- also present, the maximum value from all decorators is used.
97
- memory : int, default 4096
98
- Memory size (in MB) required for this step. If
99
- `@resources` is also present, the maximum value from all decorators is
100
- used.
101
- disk : int, default 10240
102
- Disk size (in MB) required for this step. If
103
- `@resources` is also present, the maximum value from all decorators is
104
- used.
105
- image : str, optional, default None
106
- Docker image to use when launching on Kubernetes. If not specified, and
107
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
108
- not, a default Docker image mapping to the current version of Python is used.
109
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
110
- If given, the imagePullPolicy to be applied to the Docker image of the step.
111
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
112
- Kubernetes service account to use when launching pod in Kubernetes.
113
- secrets : List[str], optional, default None
114
- Kubernetes secrets to use when launching pod in Kubernetes. These
115
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
116
- in Metaflow configuration.
117
- node_selector: Union[Dict[str,str], str], optional, default None
118
- Kubernetes node selector(s) to apply to the pod running the task.
119
- Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
120
- or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
121
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
122
- Kubernetes namespace to use when launching pod in Kubernetes.
123
- gpu : int, optional, default None
124
- Number of GPUs required for this step. A value of zero implies that
125
- the scheduled node should not have GPUs.
126
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
127
- The vendor of the GPUs to be used for this step.
128
- tolerations : List[str], default []
129
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
130
- Kubernetes tolerations to use when launching pod in Kubernetes.
131
- use_tmpfs : bool, default False
132
- This enables an explicit tmpfs mount for this step.
133
- tmpfs_tempdir : bool, default True
134
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
135
- tmpfs_size : int, optional, default: None
136
- The value for the size (in MiB) of the tmpfs mount for this step.
137
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
138
- memory allocated for this step.
139
- tmpfs_path : str, optional, default /metaflow_temp
140
- Path to tmpfs mount for this step.
141
- persistent_volume_claims : Dict[str, str], optional, default None
142
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
143
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
144
- shared_memory: int, optional
145
- Shared memory size (in MiB) required for this step
146
- port: int, optional
147
- Port number to specify in the Kubernetes job object
148
- compute_pool : str, optional, default None
149
- Compute pool to be used for for this step.
150
- If not specified, any accessible compute pool within the perimeter is used.
151
- hostname_resolution_timeout: int, default 10 * 60
152
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
153
- Only applicable when @parallel is used.
154
- """
155
- def __init__(self, attributes = None, statically_defined = False):
156
- ...
157
- def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
158
- ...
159
- def package_init(self, flow, step_name, environment):
160
- ...
161
- def runtime_init(self, flow, graph, package, run_id):
162
- ...
163
- def runtime_task_created(self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context):
164
- ...
165
- def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
166
- ...
167
- def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_retries, ubf_context, inputs):
168
- ...
169
- def task_finished(self, step_name, flow, graph, is_task_ok, retry_count, max_retries):
170
- ...
171
- ...
172
-
@@ -1,21 +1,17 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.341392 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.829816 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
12
  import metaflow.exception
12
13
 
13
- class MetaflowException(Exception, metaclass=type):
14
- def __init__(self, msg = "", lineno = None):
15
- ...
16
- def __str__(self):
17
- ...
18
- ...
14
+ from ...exception import MetaflowException as MetaflowException
19
15
 
20
16
  KUBERNETES_NAMESPACE: str
21
17
 
@@ -1,50 +1,26 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.423731 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.893002 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
12
  import metaflow.decorators
12
- import metaflow.metaflow_current
13
- import metaflow.exception
14
13
 
15
- current: metaflow.metaflow_current.Current
16
-
17
- class MetaflowException(Exception, metaclass=type):
18
- def __init__(self, msg = "", lineno = None):
19
- ...
20
- def __str__(self):
21
- ...
22
- ...
23
-
24
- class MetaDatum(tuple, metaclass=type):
25
- """
26
- MetaDatum(field, value, type, tags)
27
- """
28
- @staticmethod
29
- def __new__(_cls, field, value, type, tags):
30
- """
31
- Create new instance of MetaDatum(field, value, type, tags)
32
- """
33
- ...
34
- def __repr__(self):
35
- """
36
- Return a nicely formatted representation string
37
- """
38
- ...
39
- def __getnewargs__(self):
40
- """
41
- Return self as a plain tuple. Used by copy and pickle.
42
- """
43
- ...
44
- ...
45
-
46
- def sync_local_metadata_to_datastore(metadata_local_dir, task_ds):
47
- ...
14
+ from ...metaflow_current import current as current
15
+ from ...exception import MetaflowException as MetaflowException
16
+ from ...metadata_provider.metadata import MetaDatum as MetaDatum
17
+ from ...metadata_provider.util import sync_local_metadata_to_datastore as sync_local_metadata_to_datastore
18
+ from ..resources_decorator import ResourcesDecorator as ResourcesDecorator
19
+ from ..timeout_decorator import get_run_time_limit_for_task as get_run_time_limit_for_task
20
+ from ..aws.aws_utils import get_docker_registry as get_docker_registry
21
+ from ..aws.aws_utils import get_ec2_instance_metadata as get_ec2_instance_metadata
22
+ from .kubernetes import KubernetesException as KubernetesException
23
+ from .kubernetes import parse_kube_keyvalue_list as parse_kube_keyvalue_list
48
24
 
49
25
  DATASTORE_LOCAL_DIR: str
50
26
 
@@ -78,108 +54,8 @@ KUBERNETES_SHARED_MEMORY: None
78
54
 
79
55
  KUBERNETES_TOLERATIONS: str
80
56
 
81
- class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
82
- """
83
- Specifies the resources needed when executing this step.
84
-
85
- Use `@resources` to specify the resource requirements
86
- independently of the specific compute layer (`@batch`, `@kubernetes`).
87
-
88
- You can choose the compute layer on the command line by executing e.g.
89
- ```
90
- python myflow.py run --with batch
91
- ```
92
- or
93
- ```
94
- python myflow.py run --with kubernetes
95
- ```
96
- which executes the flow on the desired system using the
97
- requirements specified in `@resources`.
98
-
99
- Parameters
100
- ----------
101
- cpu : int, default 1
102
- Number of CPUs required for this step.
103
- gpu : int, optional, default None
104
- Number of GPUs required for this step.
105
- disk : int, optional, default None
106
- Disk size (in MB) required for this step. Only applies on Kubernetes.
107
- memory : int, default 4096
108
- Memory size (in MB) required for this step.
109
- shared_memory : int, optional, default None
110
- The value for the size (in MiB) of the /dev/shm volume for this step.
111
- This parameter maps to the `--shm-size` option in Docker.
112
- """
113
- ...
114
-
115
- def get_run_time_limit_for_task(step_decos):
116
- ...
117
-
118
57
  UBF_CONTROL: str
119
58
 
120
- def get_docker_registry(image_uri):
121
- """
122
- Explanation:
123
- (.+?(?:[:.].+?)\/)? - [GROUP 0] REGISTRY
124
- .+? - A registry must start with at least one character
125
- (?:[:.].+?)\/ - A registry must have ":" or "." and end with "/"
126
- ? - Make a registry optional
127
- (.*?) - [GROUP 1] REPOSITORY
128
- .*? - Get repository name until separator
129
- (?:[@:])? - SEPARATOR
130
- ?: - Don't capture separator
131
- [@:] - The separator must be either "@" or ":"
132
- ? - The separator is optional
133
- ((?<=[@:]).*)? - [GROUP 2] TAG / DIGEST
134
- (?<=[@:]) - A tag / digest must be preceded by "@" or ":"
135
- .* - Capture rest of tag / digest
136
- ? - A tag / digest is optional
137
- Examples:
138
- image
139
- - None
140
- - image
141
- - None
142
- example/image
143
- - None
144
- - example/image
145
- - None
146
- example/image:tag
147
- - None
148
- - example/image
149
- - tag
150
- example.domain.com/example/image:tag
151
- - example.domain.com/
152
- - example/image
153
- - tag
154
- 123.123.123.123:123/example/image:tag
155
- - 123.123.123.123:123/
156
- - example/image
157
- - tag
158
- example.domain.com/example/image@sha256:45b23dee0
159
- - example.domain.com/
160
- - example/image
161
- - sha256:45b23dee0
162
- """
163
- ...
164
-
165
- def get_ec2_instance_metadata():
166
- """
167
- Fetches the EC2 instance metadata through AWS instance metadata service
168
-
169
- Returns either an empty dictionary, or one with the keys
170
- - ec2-instance-id
171
- - ec2-instance-type
172
- - ec2-region
173
- - ec2-availability-zone
174
- """
175
- ...
176
-
177
- class KubernetesException(metaflow.exception.MetaflowException, metaclass=type):
178
- ...
179
-
180
- def parse_kube_keyvalue_list(items: typing.List[str], requires_both: bool = True):
181
- ...
182
-
183
59
  MAX_MEMORY_PER_TASK: None
184
60
 
185
61
  MAX_CPU_PER_TASK: None
@@ -215,8 +91,9 @@ class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
215
91
  in Metaflow configuration.
216
92
  node_selector: Union[Dict[str,str], str], optional, default None
217
93
  Kubernetes node selector(s) to apply to the pod running the task.
218
- Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
219
- or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
94
+ Can be passed in as a comma separated string of values e.g.
95
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
96
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
220
97
  namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
221
98
  Kubernetes namespace to use when launching pod in Kubernetes.
222
99
  gpu : int, optional, default None
@@ -1,21 +1,17 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.424958 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.895224 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
12
  import metaflow.exception
12
13
 
13
- class MetaflowException(Exception, metaclass=type):
14
- def __init__(self, msg = "", lineno = None):
15
- ...
16
- def __str__(self):
17
- ...
18
- ...
14
+ from ...exception import MetaflowException as MetaflowException
19
15
 
20
16
  KUBERNETES_JOBSET_GROUP: str
21
17