ob-metaflow-stubs 6.0.3.117__py2.py3-none-any.whl → 6.0.3.119__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (175) hide show
  1. metaflow-stubs/__init__.pyi +318 -2738
  2. metaflow-stubs/cards.pyi +19 -473
  3. metaflow-stubs/cli.pyi +17 -81
  4. metaflow-stubs/client/__init__.pyi +19 -1113
  5. metaflow-stubs/client/core.pyi +18 -158
  6. metaflow-stubs/client/filecache.pyi +8 -12
  7. metaflow-stubs/clone_util.pyi +6 -26
  8. metaflow-stubs/events.pyi +7 -6
  9. metaflow-stubs/exception.pyi +8 -6
  10. metaflow-stubs/flowspec.pyi +20 -104
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +16 -564
  13. metaflow-stubs/info_file.pyi +6 -5
  14. metaflow-stubs/metadata_provider/__init__.pyi +16 -0
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +34 -0
  16. metaflow-stubs/{metadata → metadata_provider}/metadata.pyi +10 -22
  17. metaflow-stubs/metadata_provider/util.pyi +19 -0
  18. metaflow-stubs/metaflow_config.pyi +8 -13
  19. metaflow-stubs/metaflow_current.pyi +45 -44
  20. metaflow-stubs/mf_extensions/__init__.pyi +6 -0
  21. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +6 -0
  22. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +6 -0
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +6 -0
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +6 -0
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +129 -0
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +26 -0
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +156 -0
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +25 -0
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +111 -0
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +6 -0
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +188 -0
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +30 -0
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +28 -0
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +19 -0
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +115 -0
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +40 -0
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +6 -0
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +71 -0
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +56 -0
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +67 -0
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +49 -0
  42. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +6 -0
  43. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +6 -0
  44. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +130 -0
  45. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +42 -0
  46. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +6 -0
  47. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +17 -0
  48. metaflow-stubs/mflog/__init__.pyi +6 -0
  49. metaflow-stubs/mflog/mflog.pyi +52 -5
  50. metaflow-stubs/multicore_utils.pyi +6 -5
  51. metaflow-stubs/parameters.pyi +13 -23
  52. metaflow-stubs/plugins/__init__.pyi +52 -165
  53. metaflow-stubs/plugins/airflow/__init__.pyi +9 -5
  54. metaflow-stubs/plugins/airflow/airflow_utils.pyi +7 -6
  55. metaflow-stubs/plugins/airflow/exception.pyi +7 -11
  56. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +10 -97
  57. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +9 -30
  58. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +9 -40
  59. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +9 -40
  60. metaflow-stubs/plugins/argo/__init__.pyi +12 -5
  61. metaflow-stubs/plugins/argo/argo_client.pyi +8 -26
  62. metaflow-stubs/plugins/argo/argo_events.pyi +7 -11
  63. metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -129
  64. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +22 -460
  65. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +11 -403
  66. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +63 -448
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +165 -0
  68. metaflow-stubs/plugins/aws/__init__.pyi +11 -5
  69. metaflow-stubs/plugins/aws/aws_client.pyi +6 -5
  70. metaflow-stubs/plugins/aws/aws_utils.pyi +6 -11
  71. metaflow-stubs/plugins/aws/batch/__init__.pyi +9 -5
  72. metaflow-stubs/plugins/aws/batch/batch.pyi +10 -55
  73. metaflow-stubs/plugins/aws/batch/batch_client.pyi +7 -11
  74. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +15 -140
  75. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +7 -5
  76. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +10 -21
  77. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +9 -5
  78. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +6 -5
  79. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +6 -5
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +7 -5
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +11 -65
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +6 -5
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +52 -292
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +127 -0
  85. metaflow-stubs/plugins/azure/__init__.pyi +12 -7
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +6 -5
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +7 -11
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +11 -24
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +11 -29
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +8 -23
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +7 -17
  92. metaflow-stubs/plugins/cards/__init__.pyi +15 -5
  93. metaflow-stubs/plugins/cards/card_cli.pyi +22 -491
  94. metaflow-stubs/plugins/cards/card_client.pyi +14 -76
  95. metaflow-stubs/plugins/cards/card_creator.pyi +7 -10
  96. metaflow-stubs/plugins/cards/card_datastore.pyi +10 -18
  97. metaflow-stubs/plugins/cards/card_decorator.pyi +10 -126
  98. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +13 -81
  99. metaflow-stubs/plugins/cards/card_modules/basic.pyi +13 -96
  100. metaflow-stubs/plugins/cards/card_modules/card.pyi +6 -5
  101. metaflow-stubs/plugins/cards/card_modules/components.pyi +24 -107
  102. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +6 -5
  103. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +6 -12
  104. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +11 -88
  105. metaflow-stubs/plugins/cards/card_resolver.pyi +6 -49
  106. metaflow-stubs/plugins/cards/component_serializer.pyi +13 -63
  107. metaflow-stubs/plugins/cards/exception.pyi +7 -11
  108. metaflow-stubs/plugins/catch_decorator.pyi +10 -30
  109. metaflow-stubs/plugins/datatools/__init__.pyi +13 -392
  110. metaflow-stubs/plugins/datatools/local.pyi +7 -11
  111. metaflow-stubs/plugins/datatools/s3/__init__.pyi +19 -653
  112. metaflow-stubs/plugins/datatools/s3/s3.pyi +15 -263
  113. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +7 -10
  114. metaflow-stubs/plugins/datatools/s3/s3util.pyi +6 -11
  115. metaflow-stubs/plugins/debug_logger.pyi +7 -5
  116. metaflow-stubs/plugins/debug_monitor.pyi +7 -5
  117. metaflow-stubs/plugins/environment_decorator.pyi +7 -5
  118. metaflow-stubs/plugins/events_decorator.pyi +8 -14
  119. metaflow-stubs/plugins/frameworks/__init__.pyi +7 -5
  120. metaflow-stubs/plugins/frameworks/pytorch.pyi +8 -45
  121. metaflow-stubs/plugins/gcp/__init__.pyi +11 -7
  122. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +11 -24
  123. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +7 -11
  124. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +6 -5
  125. metaflow-stubs/plugins/gcp/gs_utils.pyi +8 -20
  126. metaflow-stubs/plugins/gcp/includefile_support.pyi +7 -17
  127. metaflow-stubs/plugins/kubernetes/__init__.pyi +12 -5
  128. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +6 -10
  129. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +10 -38
  130. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +16 -155
  131. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +7 -11
  132. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +19 -142
  133. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +7 -11
  134. metaflow-stubs/plugins/logs_cli.pyi +10 -9
  135. metaflow-stubs/plugins/package_cli.pyi +7 -5
  136. metaflow-stubs/plugins/parallel_decorator.pyi +11 -59
  137. metaflow-stubs/plugins/perimeters.pyi +6 -11
  138. metaflow-stubs/plugins/project_decorator.pyi +8 -14
  139. metaflow-stubs/plugins/pypi/__init__.pyi +12 -11
  140. metaflow-stubs/plugins/pypi/conda_decorator.pyi +8 -27
  141. metaflow-stubs/plugins/pypi/conda_environment.pyi +14 -20
  142. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +7 -5
  143. metaflow-stubs/plugins/pypi/pypi_environment.pyi +7 -39
  144. metaflow-stubs/plugins/pypi/utils.pyi +7 -11
  145. metaflow-stubs/plugins/resources_decorator.pyi +7 -5
  146. metaflow-stubs/plugins/retry_decorator.pyi +7 -11
  147. metaflow-stubs/plugins/secrets/__init__.pyi +9 -5
  148. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +8 -13
  149. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +7 -11
  150. metaflow-stubs/plugins/storage_executor.pyi +6 -11
  151. metaflow-stubs/plugins/tag_cli.pyi +14 -396
  152. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +9 -34
  153. metaflow-stubs/plugins/timeout_decorator.pyi +8 -12
  154. metaflow-stubs/procpoll.pyi +7 -5
  155. metaflow-stubs/profilers/__init__.pyi +7 -11
  156. metaflow-stubs/pylint_wrapper.pyi +7 -11
  157. metaflow-stubs/runner/__init__.pyi +13 -5
  158. metaflow-stubs/runner/deployer.pyi +99 -223
  159. metaflow-stubs/runner/deployer_impl.pyi +87 -0
  160. metaflow-stubs/runner/metaflow_runner.pyi +24 -508
  161. metaflow-stubs/runner/nbdeploy.pyi +16 -60
  162. metaflow-stubs/runner/nbrun.pyi +11 -148
  163. metaflow-stubs/runner/subprocess_manager.pyi +9 -10
  164. metaflow-stubs/runner/utils.pyi +44 -9
  165. metaflow-stubs/system/__init__.pyi +9 -107
  166. metaflow-stubs/system/system_logger.pyi +7 -16
  167. metaflow-stubs/system/system_monitor.pyi +6 -16
  168. metaflow-stubs/tagging_util.pyi +6 -10
  169. metaflow-stubs/tuple_util.pyi +6 -5
  170. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.119.dist-info}/METADATA +1 -1
  171. ob_metaflow_stubs-6.0.3.119.dist-info/RECORD +174 -0
  172. metaflow-stubs/metadata/util.pyi +0 -18
  173. ob_metaflow_stubs-6.0.3.117.dist-info/RECORD +0 -140
  174. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.119.dist-info}/WHEEL +0 -0
  175. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.119.dist-info}/top_level.txt +0 -0
@@ -1,31 +1,19 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.394809 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.818374 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
+ import metaflow
9
10
  import typing
10
11
  if typing.TYPE_CHECKING:
11
- import metaflow.metadata.metadata
12
- import metaflow.exception
12
+ import metaflow.metadata_provider.metadata
13
13
 
14
- class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
15
- ...
16
-
17
- class MetaflowTaggingError(metaflow.exception.MetaflowException, metaclass=type):
18
- ...
19
-
20
- def validate_tag(tag):
21
- """
22
- - Tag must be either of bytes-type or unicode-type.
23
- - If tag is of bytes-type, it must be UTF-8 decodable
24
- - If tag is of unicode-type, it must be UTF-8 encodable
25
- - Tag may not be empty string.
26
- - Tag cannot be too long (500 chars)
27
- """
28
- ...
14
+ from ..exception import MetaflowInternalError as MetaflowInternalError
15
+ from ..exception import MetaflowTaggingError as MetaflowTaggingError
16
+ from ..tagging_util import validate_tag as validate_tag
29
17
 
30
18
  class DataArtifact(tuple, metaclass=type):
31
19
  """
@@ -0,0 +1,19 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.818617 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+
10
+
11
+ def copy_tree(src, dst, update = False):
12
+ ...
13
+
14
+ def sync_local_metadata_to_datastore(metadata_local_dir, task_ds):
15
+ ...
16
+
17
+ def sync_local_metadata_from_datastore(metadata_local_dir, task_ds):
18
+ ...
19
+
@@ -1,18 +1,13 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.343920 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.799422 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
 
10
- class MetaflowException(Exception, metaclass=type):
11
- def __init__(self, msg = "", lineno = None):
12
- ...
13
- def __str__(self):
14
- ...
15
- ...
10
+ from .exception import MetaflowException as MetaflowException
16
11
 
17
12
  DATASTORE_LOCAL_DIR: str
18
13
 
@@ -40,6 +35,8 @@ DEFAULT_SECRETS_BACKEND_TYPE: None
40
35
 
41
36
  DEFAULT_SECRETS_ROLE: None
42
37
 
38
+ DEFAULT_FROM_DEPLOYMENT_IMPL: str
39
+
43
40
  USER: None
44
41
 
45
42
  DATASTORE_SYSROOT_LOCAL: None
@@ -240,8 +237,6 @@ AIRFLOW_KUBERNETES_KUBECONFIG_FILE: None
240
237
 
241
238
  AIRFLOW_KUBERNETES_KUBECONFIG_CONTEXT: None
242
239
 
243
- FROM_DEPLOYMENT_IMPL: str
244
-
245
240
  CONDA_PACKAGE_S3ROOT: None
246
241
 
247
242
  CONDA_PACKAGE_AZUREROOT: None
@@ -1,21 +1,22 @@
1
- ##################################################################################
2
- # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1) #
4
- # Generated on 2024-11-13T19:05:29.440634 #
5
- ##################################################################################
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.899445 #
5
+ ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator
13
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core
14
11
  import metaflow.plugins.cards.component_serializer
15
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
12
+ import metaflow
13
+ import typing
14
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core
16
15
  import metaflow.events
17
16
  import metaflow.metaflow_current
18
- import metaflow
17
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator
18
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
19
+
19
20
 
20
21
  TYPE_CHECKING: bool
21
22
 
@@ -224,10 +225,10 @@ class Current(object, metaclass=type):
224
225
  """
225
226
  ...
226
227
  @property
227
- def graph(_):
228
+ def graph(self):
228
229
  ...
229
230
  @property
230
- def model(self) -> "metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core.ModelSerializer":
231
+ def model(self) -> "metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core.ModelSerializer":
231
232
  """
232
233
  (only in the presence of the @model decorator)
233
234
 
@@ -306,36 +307,7 @@ class Current(object, metaclass=type):
306
307
  """
307
308
  ...
308
309
  @property
309
- def parallel(self) -> "metaflow.metaflow_current.Parallel":
310
- """
311
- (only in the presence of the @parallel decorator)
312
-
313
- Returns a namedtuple with relevant information about the parallel task.
314
-
315
- Returns
316
- -------
317
- Parallel
318
- `namedtuple` with the following fields:
319
- - main_ip (`str`)
320
- The IP address of the control task.
321
- - num_nodes (`int`)
322
- The total number of tasks created by @parallel
323
- - node_index (`int`)
324
- The index of the current task in all the @parallel tasks.
325
- - control_task_id (`Optional[str]`)
326
- The task ID of the control task. Available to all tasks.
327
- """
328
- ...
329
- @property
330
- def is_parallel(self) -> bool:
331
- """
332
- (only in the presence of the @parallel decorator)
333
-
334
- True if the current step is a @parallel step.
335
- """
336
- ...
337
- @property
338
- def checkpoint(self) -> "metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator.CurrentCheckpointer":
310
+ def checkpoint(self) -> "metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator.CurrentCheckpointer":
339
311
  """
340
312
  (only in the presence of the @checkpoint decorator)
341
313
 
@@ -396,7 +368,7 @@ class Current(object, metaclass=type):
396
368
  """
397
369
  ...
398
370
  @property
399
- def huggingface_hub(self) -> "metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator.HuggingfaceRegistry":
371
+ def huggingface_hub(self) -> "metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator.HuggingfaceRegistry":
400
372
  """
401
373
  (only in the presence of the @huggingface_hub decorator)
402
374
 
@@ -465,9 +437,38 @@ class Current(object, metaclass=type):
465
437
  """
466
438
  ...
467
439
  @property
440
+ def parallel(self) -> "metaflow.metaflow_current.Parallel":
441
+ """
442
+ (only in the presence of the @parallel decorator)
443
+
444
+ Returns a namedtuple with relevant information about the parallel task.
445
+
446
+ Returns
447
+ -------
448
+ Parallel
449
+ `namedtuple` with the following fields:
450
+ - main_ip (`str`)
451
+ The IP address of the control task.
452
+ - num_nodes (`int`)
453
+ The total number of tasks created by @parallel
454
+ - node_index (`int`)
455
+ The index of the current task in all the @parallel tasks.
456
+ - control_task_id (`Optional[str]`)
457
+ The task ID of the control task. Available to all tasks.
458
+ """
459
+ ...
460
+ @property
461
+ def is_parallel(self) -> bool:
462
+ """
463
+ (only in the presence of the @parallel decorator)
464
+
465
+ True if the current step is a @parallel step.
466
+ """
467
+ ...
468
+ @property
468
469
  def trigger(self) -> "metaflow.events.Trigger":
469
470
  """
470
- (only in the presence of the @trigger_on_finish, or @trigger decorators)
471
+ (only in the presence of the @trigger, or @trigger_on_finish decorators)
471
472
 
472
473
  Returns `Trigger` if the current run is triggered by an event
473
474
 
@@ -0,0 +1,6 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.804706 #
5
+ ######################################################################################################
6
+
@@ -0,0 +1,6 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.813004 #
5
+ ######################################################################################################
6
+
@@ -0,0 +1,6 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.813069 #
5
+ ######################################################################################################
6
+
@@ -0,0 +1,6 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.813128 #
5
+ ######################################################################################################
6
+
@@ -0,0 +1,6 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.813185 #
5
+ ######################################################################################################
6
+
@@ -0,0 +1,129 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.876821 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core
13
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
14
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
15
+ import metaflow.datastore.datastore_storage
16
+
17
+ from ..exceptions import KeyNotCompatibleWithObjectException as KeyNotCompatibleWithObjectException
18
+ from ..utils.identity_utils import pathspec_hash as pathspec_hash
19
+ from ..utils.general import replace_start_and_end_slash as replace_start_and_end_slash
20
+ from ..datastore.core import allow_safe as allow_safe
21
+ from ..datastore.core import DatastoreInterface as DatastoreInterface
22
+ from ..datastore.core import ObjectStorage as ObjectStorage
23
+ from ..datastore.core import STORAGE_FORMATS as STORAGE_FORMATS
24
+ from ..datastore.exceptions import DatastoreReadInitException as DatastoreReadInitException
25
+ from ..datastore.exceptions import DatastoreWriteInitException as DatastoreWriteInitException
26
+ from ..datastore.exceptions import DatastoreNotReadyException as DatastoreNotReadyException
27
+ from ..datastructures import CheckpointArtifact as CheckpointArtifact
28
+ from ..datastore.utils import safe_serialize as safe_serialize
29
+
30
+ DEFAULT_NAME: str
31
+
32
+ CHECKPOINTS_STORAGE_PREFIX: str
33
+
34
+ DEFAULT_STORAGE_FORMAT: str
35
+
36
+ ARTIFACT_STORE_NAME: str
37
+
38
+ METADATA_STORE_NAME: str
39
+
40
+ ARTIFACT_METADATA_STORE_NAME: str
41
+
42
+ class CheckpointsPathComponents(tuple, metaclass=type):
43
+ """
44
+ CheckpointsPathComponents(flow_name, step_name, scope, task_identifier, pathspec_hash, attempt, name, version_id, is_metadata, key_name, root_prefix)
45
+ """
46
+ @staticmethod
47
+ def __new__(_cls, flow_name, step_name, scope, task_identifier, pathspec_hash, attempt, name, version_id, is_metadata, key_name, root_prefix):
48
+ """
49
+ Create new instance of CheckpointsPathComponents(flow_name, step_name, scope, task_identifier, pathspec_hash, attempt, name, version_id, is_metadata, key_name, root_prefix)
50
+ """
51
+ ...
52
+ def __repr__(self):
53
+ """
54
+ Return a nicely formatted representation string
55
+ """
56
+ ...
57
+ def __getnewargs__(self):
58
+ """
59
+ Return self as a plain tuple. Used by copy and pickle.
60
+ """
61
+ ...
62
+ ...
63
+
64
+ def decompose_key_artifact_metadata_store(key) -> CheckpointsPathComponents:
65
+ ...
66
+
67
+ def decompose_key_artifact_store(key) -> CheckpointsPathComponents:
68
+ """
69
+ Convert Key into Path Components.
70
+ PATH COMPONENTS: mf.checkpoints/artifacts/<flow_name>/<step_name>/<scope>/<task_identifier>/<pathspec_hash>.<attempt>.<name>.<version_id>
71
+ """
72
+ ...
73
+
74
+ def decompose_key_metadata_store(key) -> CheckpointsPathComponents:
75
+ """
76
+ Convert Key into Path Components.
77
+ PATH COMPONENTS: mf.checkpoints/artifacts/<flow_name>/<step_name>/<scope>/<task_identifier>/<pathspec_hash>.<attempt>.<name>.<version_id>
78
+ """
79
+ ...
80
+
81
+ class CheckpointDatastore(metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.DatastoreInterface, metaclass=type):
82
+ """
83
+ Consisits of 3 main components:
84
+ - `artifact_store`: This is where the checkpoint artifacts are stored.
85
+ - This key to the checkpoint in this store becomes the "key for the checkpoint"
86
+ - `metadata_store`: This is where the metadata of the checkpoint is stored it based on the currently executing task (path structure resembles that of a metaflow pathspec).
87
+ - This store helps retrieve information about all the checkpoints stored for a task during the execution.
88
+ - `artifact_metadatastore`: This is similar to the metadata store but holds a pathstructure similar to the artifact store.
89
+ - this store helps reverse lookup the Checkpoint metadata object from the checkpoint key.
90
+ """
91
+ @property
92
+ def metadata_ready(self):
93
+ ...
94
+ @property
95
+ def artifact_ready(self):
96
+ ...
97
+ def set_root_prefix(self, root_prefix):
98
+ ...
99
+ @classmethod
100
+ def init_read_store(cls, storage_backend: metaflow.datastore.datastore_storage.DataStoreStorage, pathspec = None, checkpoint_key = None):
101
+ """
102
+ This will initialize the datastore for reading.
103
+
104
+ - If there is only the pathspec that's provided then it can mean the user is doing a list operations
105
+ - if only the checkpoint_key is provided then it can mean the user is trying to load a specific checkpoint
106
+ """
107
+ ...
108
+ def create_key_name(self, *args):
109
+ ...
110
+ @classmethod
111
+ def init_write_store(cls, storage_backend: metaflow.datastore.datastore_storage.DataStoreStorage, pathspec, scope, task_identifier):
112
+ ...
113
+ def save(self, local_path: str, attempt, version_id, name = 'mfchckpt', metadata = {}, set_latest = True, storage_format = 'files') -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact:
114
+ ...
115
+ def latest(*args, **kwargs):
116
+ ...
117
+ def load(self, local_path, version_id, attempt, name, storage_format = 'files'):
118
+ ...
119
+ def load_metadata(self, attempt, version_id, name = 'mfchckpt') -> dict:
120
+ ...
121
+ def list(self, name = None, attempt = None, within_task = True):
122
+ ...
123
+ @classmethod
124
+ def decompose_key(cls, key) -> CheckpointsPathComponents:
125
+ ...
126
+ def __init__(self, artifact_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, metadata_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, artifact_metadatastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage):
127
+ ...
128
+ ...
129
+
@@ -0,0 +1,26 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.835340 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import typing
10
+ if typing.TYPE_CHECKING:
11
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
12
+
13
+ from .core import Checkpointer as Checkpointer
14
+ from .core import WriteResolver as WriteResolver
15
+ from .core import ReadResolver as ReadResolver
16
+ from ..datastructures import CheckpointArtifact as CheckpointArtifact
17
+
18
+ TYPE_CHECKING: bool
19
+
20
+ CHECKPOINT_UID_ENV_VAR_NAME: str
21
+
22
+ DEFAULT_NAME: str
23
+
24
+ def load_checkpoint(checkpoint: typing.Union[metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact, dict, str], local_path: str):
25
+ ...
26
+
@@ -0,0 +1,156 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.865978 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import typing
10
+ if typing.TYPE_CHECKING:
11
+ import metaflow
12
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
13
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
14
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.core
15
+
16
+ from ......metaflow_current import current as current
17
+ from ......exception import MetaflowException as MetaflowException
18
+ from .exceptions import CheckpointNotAvailableException as CheckpointNotAvailableException
19
+ from .exceptions import CheckpointException as CheckpointException
20
+ from ..utils import flowspec_utils as flowspec_utils
21
+ from .checkpoint_storage import CheckpointDatastore as CheckpointDatastore
22
+ from ..datastructures import CheckpointArtifact as CheckpointArtifact
23
+ from ..datastore.task_utils import init_datastorage_object as init_datastorage_object
24
+ from ..datastore.task_utils import resolve_storage_backend as resolve_task_storage_backend
25
+
26
+ TYPE_CHECKING: bool
27
+
28
+ CHECKPOINT_TAG_PREFIX: str
29
+
30
+ MAX_HASH_LEN: int
31
+
32
+ DEFAULT_NAME: str
33
+
34
+ CHECKPOINT_UID_ENV_VAR_NAME: str
35
+
36
+ DEFAULT_STORAGE_FORMAT: str
37
+
38
+ class Checkpointer(object, metaclass=type):
39
+ @property
40
+ def current_version(self):
41
+ ...
42
+ def set_root_prefix(self, root_prefix):
43
+ ...
44
+ def __init__(self, datastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore, attempt: int = 0):
45
+ ...
46
+ def load_metadata(self, version_id: int = None, name = 'mfchckpt') -> dict:
47
+ ...
48
+ def save(self, path: str, metadata = {}, latest = True, name = 'mfchckpt', storage_format = 'files') -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact:
49
+ ...
50
+ ...
51
+
52
+ class CheckpointLoadPolicy(object, metaclass=type):
53
+ @classmethod
54
+ def fresh(cls, datastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore, flow: "metaflow.FlowSpec") -> typing.Optional[metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact]:
55
+ """
56
+ ```python
57
+ @checkpoint(load_policy="fresh")
58
+ ```
59
+ While in `fresh` mode, we want to load the "latest" checkpoint from
60
+ what ever task is executing at the current memoment.
61
+
62
+ The behavior is will be such that 1st attempt of any task will not load
63
+ any checkpoint and there after it will load the checkpoint from the previous
64
+ attempt of the task (ala the lastest checkpoint within the task).
65
+ """
66
+ ...
67
+ @classmethod
68
+ def eager(cls, datastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore, flow: "metaflow.FlowSpec") -> typing.Optional[metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact]:
69
+ """
70
+ ```python
71
+ @checkpoint(load_policy="eager")
72
+ ```
73
+ While in `eager` mode, we want to load the "latest" checkpoint ever
74
+ written at the "scope" level based on the kind of task that is executing.
75
+
76
+ Setting this mode helps "checkpoints leak across executions" for the same task
77
+ there by allowing a way to reboot the state when new executions start.
78
+ """
79
+ ...
80
+ ...
81
+
82
+ class ScopeResolver(object, metaclass=type):
83
+ @classmethod
84
+ def from_namespace(cls):
85
+ ...
86
+ @classmethod
87
+ def from_tags(cls, tags):
88
+ ...
89
+ ...
90
+
91
+ class ReadResolver(object, metaclass=type):
92
+ """
93
+ Responsible for instantiating the `CheckpointDatastore` during read operations
94
+ based on different context's.
95
+ """
96
+ @classmethod
97
+ def from_key(cls, checkpoint_key):
98
+ """
99
+ """
100
+ ...
101
+ @classmethod
102
+ def from_key_and_pathspec(cls, pathspec, checkpoint_key):
103
+ """
104
+ """
105
+ ...
106
+ @classmethod
107
+ def from_key_and_run(cls, run, checkpoint_key) -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore:
108
+ """
109
+ """
110
+ ...
111
+ ...
112
+
113
+ class WriteResolver(object, metaclass=type):
114
+ """
115
+ Responsible for instantiating the `CheckpointDatastore` and the subsequent
116
+ `_checkpointer_uid` which can instantiate the `Checkpointer` object outside
117
+ of the metaflow context.
118
+ """
119
+ @classmethod
120
+ def can_resolve_from_envionment(cls):
121
+ ...
122
+ @classmethod
123
+ def decompose_checkpoint_id(cls, checkpoint_id):
124
+ ...
125
+ @classmethod
126
+ def resolver_info(cls, flow, run, step, taskid, taskidf, scope, attempt):
127
+ ...
128
+ @classmethod
129
+ def construct_checkpoint_id(cls, resolver_info: ResolverInfo):
130
+ ...
131
+ @classmethod
132
+ def from_environment(cls):
133
+ ...
134
+ @classmethod
135
+ def from_run(cls, run: "metaflow.FlowSpec", scope: str, task_identifier: typing.Optional[str] = None, gang_scheduled_task = False):
136
+ """
137
+ The task-identifier gets computed in the Metaflow main process with the
138
+ i.e. in the decorator. The pathspec we choose to write the metadata store
139
+ depends on if the task is being gang scheduled or not.
140
+ """
141
+ ...
142
+ ...
143
+
144
+ class CheckpointReferenceResolver(object, metaclass=type):
145
+ """
146
+ Resolve the Metaflow checkpoint object based on the flow artifact reference
147
+ or key; Used for lineage derivation.
148
+ """
149
+ @classmethod
150
+ def from_key(cls, flow, checkpoint_key):
151
+ """
152
+ Used by lineage derivation
153
+ """
154
+ ...
155
+ ...
156
+
@@ -0,0 +1,25 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-21T22:12:20.835614 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.exception
13
+
14
+ from ......exception import MetaflowException as MetaflowException
15
+
16
+ class CheckpointNotAvailableException(metaflow.exception.MetaflowException, metaclass=type):
17
+ def __init__(self, message):
18
+ ...
19
+ ...
20
+
21
+ class CheckpointException(metaflow.exception.MetaflowException, metaclass=type):
22
+ def __init__(self, message):
23
+ ...
24
+ ...
25
+