ob-metaflow-stubs 6.0.3.104__py2.py3-none-any.whl → 6.0.3.106__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +853 -515
  2. metaflow-stubs/cards.pyi +211 -5
  3. metaflow-stubs/cli.pyi +22 -2
  4. metaflow-stubs/client/__init__.pyi +128 -3
  5. metaflow-stubs/client/core.pyi +226 -6
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +5 -2
  8. metaflow-stubs/events.pyi +20 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +70 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +156 -3
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +8 -2
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +22 -19
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +68 -2
  21. metaflow-stubs/plugins/__init__.pyi +14 -3
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +44 -2
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +44 -2
  29. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +32 -4
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
  36. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +102 -2
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
  52. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  59. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/cards/card_cli.pyi +63 -5
  61. metaflow-stubs/plugins/cards/card_client.pyi +34 -3
  62. metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +41 -2
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +35 -2
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +55 -2
  74. metaflow-stubs/plugins/cards/exception.pyi +8 -2
  75. metaflow-stubs/plugins/catch_decorator.pyi +19 -2
  76. metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
  77. metaflow-stubs/plugins/datatools/local.pyi +16 -2
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +83 -6
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  82. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  83. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  84. metaflow-stubs/plugins/environment_decorator.pyi +10 -2
  85. metaflow-stubs/plugins/events_decorator.pyi +106 -2
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +24 -3
  88. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +11 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +99 -2
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
  101. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  102. metaflow-stubs/plugins/package_cli.pyi +2 -2
  103. metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
  104. metaflow-stubs/plugins/perimeters.pyi +2 -2
  105. metaflow-stubs/plugins/project_decorator.pyi +59 -2
  106. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
  111. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  112. metaflow-stubs/plugins/resources_decorator.pyi +33 -2
  113. metaflow-stubs/plugins/retry_decorator.pyi +21 -2
  114. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
  117. metaflow-stubs/plugins/storage_executor.pyi +6 -2
  118. metaflow-stubs/plugins/tag_cli.pyi +37 -6
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +10 -3
  120. metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
  121. metaflow-stubs/procpoll.pyi +2 -2
  122. metaflow-stubs/profilers/__init__.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +70 -131
  126. metaflow-stubs/runner/metaflow_runner.pyi +118 -10
  127. metaflow-stubs/runner/nbdeploy.pyi +66 -2
  128. metaflow-stubs/runner/nbrun.pyi +79 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +16 -4
  130. metaflow-stubs/runner/utils.pyi +32 -2
  131. metaflow-stubs/system/__init__.pyi +3 -3
  132. metaflow-stubs/system/system_logger.pyi +3 -3
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.106.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.104.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.380829 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.939404 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.387895 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.957670 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -83,6 +83,13 @@ class WarningComponent(metaflow.plugins.cards.card_modules.basic.ErrorComponent,
83
83
  ...
84
84
 
85
85
  class ComponentStore(object, metaclass=type):
86
+ """
87
+ The `ComponentStore` object helps store the components for a single card in memory.
88
+ This class has combination of a array/dictionary like interfaces to access/change the stored components.
89
+
90
+ It exposes the `append` /`extend` methods (like an array) to add components.
91
+ It also exposes the `__getitem__`/`__setitem__` methods (like a dictionary) to access the components by their Ids.
92
+ """
86
93
  def __init__(self, logger, card_type = None, components = None, user_set_id = None):
87
94
  ...
88
95
  @property
@@ -118,6 +125,33 @@ class ComponentStore(object, metaclass=type):
118
125
  ...
119
126
 
120
127
  class CardComponentManager(object, metaclass=type):
128
+ """
129
+ This class manages the card's state for a single card.
130
+ - It uses the `ComponentStore` to manage the storage of the components
131
+ - It exposes methods to add, remove and access the components.
132
+ - It exposes a `refresh` method that will allow refreshing a card with new data
133
+ for realtime(ish) updates.
134
+ - The `CardComponentCollector` exposes convenience methods similar to this class for a default
135
+ editable card. These methods include :
136
+ - `append`
137
+ - `extend`
138
+ - `clear`
139
+ - `refresh`
140
+ - `components`
141
+ - `__iter__`
142
+
143
+ ## Usage Patterns :
144
+
145
+ ```python
146
+ current.card["mycardid"].append(component, id="comp123")
147
+ current.card["mycardid"].extend([component])
148
+ current.card["mycardid"].refresh(data) # refreshes the card with new data
149
+ current.card["mycardid"].components["comp123"] # returns the component with id "comp123"
150
+ current.card["mycardid"].components["comp123"].update()
151
+ current.card["mycardid"].components.clear() # Wipe all the components
152
+ del current.card["mycardid"].components["mycomponentid"] # Delete a component
153
+ ```
154
+ """
121
155
  def __init__(self, card_uuid, decorator_attributes, card_creator, components = None, logger = None, no_warnings = False, user_set_card_id = None, runtime_card = False, card_options = None, refresh_interval = 5):
122
156
  ...
123
157
  def append(self, component, id = None):
@@ -136,6 +170,25 @@ class CardComponentManager(object, metaclass=type):
136
170
  ...
137
171
 
138
172
  class CardComponentCollector(object, metaclass=type):
173
+ """
174
+ This class helps collect `MetaflowCardComponent`s during runtime execution
175
+
176
+ ### Usage with `current`
177
+ `current.card` is of type `CardComponentCollector`
178
+
179
+ ### Main Usage TLDR
180
+ - [x] `current.card.append` customizes the default editable card.
181
+ - [x] Only one card can be default editable in a step.
182
+ - [x] The card class must have `ALLOW_USER_COMPONENTS=True` to be considered default editable.
183
+ - [x] Classes with `ALLOW_USER_COMPONENTS=False` are never default editable.
184
+ - [x] The user can specify an `id` argument to a card, in which case the card is editable through `current.card[id].append`.
185
+ - [x] A card with an id can be also default editable, if there are no other cards that are eligible to be default editable.
186
+ - [x] If multiple default-editable cards exist but only one card doesn't have an id, the card without an id is considered to be default editable.
187
+ - [x] If we can't resolve a single default editable card through the above rules, `current.card`.append calls show a warning but the call doesn't fail.
188
+ - [x] A card that is not default editable can be still edited through:
189
+ - [x] its `current.card['myid']`
190
+ - [x] by looking it up by its type, e.g. `current.card.get(type='pytorch')`.
191
+ """
139
192
  def __init__(self, logger = None, card_creator = None):
140
193
  ...
141
194
  @staticmethod
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.382111 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.940837 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -20,6 +20,9 @@ class MetaflowException(Exception, metaclass=type):
20
20
  TYPE_CHECK_REGEX: str
21
21
 
22
22
  class CardClassFoundException(metaflow.exception.MetaflowException, metaclass=type):
23
+ """
24
+ This exception is raised with MetaflowCard class is not present for a particular card type.
25
+ """
23
26
  def __init__(self, card_name):
24
27
  ...
25
28
  ...
@@ -30,6 +33,9 @@ class TypeRequiredException(metaflow.exception.MetaflowException, metaclass=type
30
33
  ...
31
34
 
32
35
  class CardNotPresentException(metaflow.exception.MetaflowException, metaclass=type):
36
+ """
37
+ This exception is raised with a card is not present in the datastore.
38
+ """
33
39
  def __init__(self, pathspec, card_type = None, card_hash = None, card_id = None):
34
40
  ...
35
41
  ...
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.371619 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.930084 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -44,6 +44,23 @@ class FailureHandledByCatch(metaflow.exception.MetaflowException, metaclass=type
44
44
  ...
45
45
 
46
46
  class CatchDecorator(metaflow.decorators.StepDecorator, metaclass=type):
47
+ """
48
+ Specifies that the step will success under all circumstances.
49
+
50
+ The decorator will create an optional artifact, specified by `var`, which
51
+ contains the exception raised. You can use it to detect the presence
52
+ of errors, indicating that all happy-path artifacts produced by the step
53
+ are missing.
54
+
55
+ Parameters
56
+ ----------
57
+ var : str, optional, default None
58
+ Name of the artifact in which to store the caught exception.
59
+ If not specified, the exception is not stored.
60
+ print_exception : bool, default True
61
+ Determines whether or not the exception is printed to
62
+ stdout when caught.
63
+ """
47
64
  def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
48
65
  ...
49
66
  def task_exception(self, exception, step, flow, graph, retry_count, max_user_code_retries):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.370357 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.928799 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,8 +9,8 @@ from __future__ import annotations
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
11
  import metaflow.exception
12
- import metaflow.plugins.datatools.s3.s3
13
12
  import io
13
+ import metaflow.plugins.datatools.s3.s3
14
14
 
15
15
  def read_in_chunks(dst, src, src_sz, max_chunk_size):
16
16
  ...
@@ -22,6 +22,13 @@ class MetaflowLocalURLException(metaflow.exception.MetaflowException, metaclass=
22
22
  ...
23
23
 
24
24
  class Local(object, metaclass=type):
25
+ """
26
+ This class allows you to access the local filesystem in a way similar to the S3 datatools
27
+ client. It is a stripped down version for now and only implements the functionality needed
28
+ for this use case.
29
+
30
+ In the future, we may want to allow it to be used in a way similar to the S3() client.
31
+ """
25
32
  @classmethod
26
33
  def get_root_from_config(cls, echo, create_on_absent = True):
27
34
  ...
@@ -47,6 +54,59 @@ class MetaflowS3Exception(metaflow.exception.MetaflowException, metaclass=type):
47
54
  ...
48
55
 
49
56
  class S3(object, metaclass=type):
57
+ """
58
+ The Metaflow S3 client.
59
+
60
+ This object manages the connection to S3 and a temporary diretory that is used
61
+ to download objects. Note that in most cases when the data fits in memory, no local
62
+ disk IO is needed as operations are cached by the operating system, which makes
63
+ operations fast as long as there is enough memory available.
64
+
65
+ The easiest way is to use this object as a context manager:
66
+ ```
67
+ with S3() as s3:
68
+ data = [obj.blob for obj in s3.get_many(urls)]
69
+ print(data)
70
+ ```
71
+ The context manager takes care of creating and deleting a temporary directory
72
+ automatically. Without a context manager, you must call `.close()` to delete
73
+ the directory explicitly:
74
+ ```
75
+ s3 = S3()
76
+ data = [obj.blob for obj in s3.get_many(urls)]
77
+ s3.close()
78
+ ```
79
+ You can customize the location of the temporary directory with `tmproot`. It
80
+ defaults to the current working directory.
81
+
82
+ To make it easier to deal with object locations, the client can be initialized
83
+ with an S3 path prefix. There are three ways to handle locations:
84
+
85
+ 1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
86
+ initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
87
+ with the current run ID. This mode makes it easy to version data based
88
+ on the run ID consistently. You can use the `bucket` and `prefix` to
89
+ override parts of `DATATOOLS_S3ROOT`.
90
+
91
+ 2. Specify an S3 prefix explicitly with `s3root`,
92
+ e.g. `S3(s3root='s3://mybucket/some/path')`.
93
+
94
+ 3. Specify nothing, i.e. `S3()`, in which case all operations require
95
+ a full S3 url prefixed with `s3://`.
96
+
97
+ Parameters
98
+ ----------
99
+ tmproot : str, default: '.'
100
+ Where to store the temporary directory.
101
+ bucket : str, optional
102
+ Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
103
+ prefix : str, optional
104
+ Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
105
+ run : FlowSpec or Run, optional
106
+ Derive path prefix from the current or a past run ID, e.g. S3(run=self).
107
+ s3root : str, optional
108
+ If `run` is not specified, use this as the S3 prefix.
109
+ """
50
110
  @classmethod
51
111
  def get_root_from_config(cls, echo, create_on_absent = True):
52
112
  ...
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.364370 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.922519 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -28,6 +28,13 @@ class MetaflowLocalNotFound(metaflow.exception.MetaflowException, metaclass=type
28
28
  ...
29
29
 
30
30
  class LocalObject(object, metaclass=type):
31
+ """
32
+ This object represents a local object. It is a very thin wrapper
33
+ to allow it to be used in the same way as the S3Object (only as needed
34
+ in the IncludeFile use case)
35
+
36
+ Get or list calls return one or more of LocalObjects.
37
+ """
31
38
  def __init__(self, url, path):
32
39
  ...
33
40
  @property
@@ -59,6 +66,13 @@ class LocalObject(object, metaclass=type):
59
66
  ...
60
67
 
61
68
  class Local(object, metaclass=type):
69
+ """
70
+ This class allows you to access the local filesystem in a way similar to the S3 datatools
71
+ client. It is a stripped down version for now and only implements the functionality needed
72
+ for this use case.
73
+
74
+ In the future, we may want to allow it to be used in a way similar to the S3() client.
75
+ """
62
76
  @classmethod
63
77
  def get_root_from_config(cls, echo, create_on_absent = True):
64
78
  ...
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.385010 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.954629 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,10 +9,13 @@ from __future__ import annotations
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
11
  import metaflow.exception
12
- import metaflow.plugins.datatools.s3.s3
13
12
  import io
13
+ import metaflow.plugins.datatools.s3.s3
14
14
 
15
15
  class RangeInfo(tuple, metaclass=type):
16
+ """
17
+ RangeInfo(total_size, request_offset, request_length)
18
+ """
16
19
  @staticmethod
17
20
  def __new__(_cls, total_size: int, request_offset: int = 0, request_length: int = -1):
18
21
  """
@@ -34,6 +37,59 @@ class RangeInfo(tuple, metaclass=type):
34
37
  ...
35
38
 
36
39
  class S3(object, metaclass=type):
40
+ """
41
+ The Metaflow S3 client.
42
+
43
+ This object manages the connection to S3 and a temporary diretory that is used
44
+ to download objects. Note that in most cases when the data fits in memory, no local
45
+ disk IO is needed as operations are cached by the operating system, which makes
46
+ operations fast as long as there is enough memory available.
47
+
48
+ The easiest way is to use this object as a context manager:
49
+ ```
50
+ with S3() as s3:
51
+ data = [obj.blob for obj in s3.get_many(urls)]
52
+ print(data)
53
+ ```
54
+ The context manager takes care of creating and deleting a temporary directory
55
+ automatically. Without a context manager, you must call `.close()` to delete
56
+ the directory explicitly:
57
+ ```
58
+ s3 = S3()
59
+ data = [obj.blob for obj in s3.get_many(urls)]
60
+ s3.close()
61
+ ```
62
+ You can customize the location of the temporary directory with `tmproot`. It
63
+ defaults to the current working directory.
64
+
65
+ To make it easier to deal with object locations, the client can be initialized
66
+ with an S3 path prefix. There are three ways to handle locations:
67
+
68
+ 1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
69
+ initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
70
+ with the current run ID. This mode makes it easy to version data based
71
+ on the run ID consistently. You can use the `bucket` and `prefix` to
72
+ override parts of `DATATOOLS_S3ROOT`.
73
+
74
+ 2. Specify an S3 prefix explicitly with `s3root`,
75
+ e.g. `S3(s3root='s3://mybucket/some/path')`.
76
+
77
+ 3. Specify nothing, i.e. `S3()`, in which case all operations require
78
+ a full S3 url prefixed with `s3://`.
79
+
80
+ Parameters
81
+ ----------
82
+ tmproot : str, default: '.'
83
+ Where to store the temporary directory.
84
+ bucket : str, optional
85
+ Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
86
+ prefix : str, optional
87
+ Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
88
+ run : FlowSpec or Run, optional
89
+ Derive path prefix from the current or a past run ID, e.g. S3(run=self).
90
+ s3root : str, optional
91
+ If `run` is not specified, use this as the S3 prefix.
92
+ """
37
93
  @classmethod
38
94
  def get_root_from_config(cls, echo, create_on_absent = True):
39
95
  ...
@@ -325,6 +381,9 @@ class S3(object, metaclass=type):
325
381
  ...
326
382
 
327
383
  class S3GetObject(tuple, metaclass=type):
384
+ """
385
+ S3GetObject(key, offset, length)
386
+ """
328
387
  @staticmethod
329
388
  def __new__(_cls, key: str, offset: int, length: int):
330
389
  """
@@ -346,6 +405,13 @@ class S3GetObject(tuple, metaclass=type):
346
405
  ...
347
406
 
348
407
  class S3Object(object, metaclass=type):
408
+ """
409
+ This object represents a path or an object in S3,
410
+ with an optional local copy.
411
+
412
+ `S3Object`s are not instantiated directly, but they are returned
413
+ by many methods of the `S3` client.
414
+ """
349
415
  def __init__(self, prefix: str, url: str, path: str, size: typing.Optional[int] = None, content_type: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None, range_info: typing.Optional[metaflow.plugins.datatools.s3.s3.RangeInfo] = None, last_modified: typing.Optional[int] = None, encryption: typing.Optional[str] = None):
350
416
  ...
351
417
  @property
@@ -549,6 +615,9 @@ class S3Object(object, metaclass=type):
549
615
  ...
550
616
 
551
617
  class S3PutObject(tuple, metaclass=type):
618
+ """
619
+ S3PutObject(key, value, path, content_type, encryption, metadata)
620
+ """
552
621
  @staticmethod
553
622
  def __new__(_cls, key: str, value: typing.Union[io.RawIOBase, io.BufferedIOBase, str, bytes, None] = None, path: typing.Optional[str] = None, content_type: typing.Optional[str] = None, encryption: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None):
554
623
  """
@@ -1,24 +1,32 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.367443 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.925677 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
+ import metaflow.exception
11
12
  import metaflow.metaflow_current
12
- import io
13
- import metaflow.datastore.inputs
14
- import metaflow.flowspec
15
13
  import typing
16
- import metaflow.exception
17
14
  import metaflow.plugins.datatools.s3.s3
15
+ import metaflow.datastore.inputs
16
+ import metaflow.flowspec
17
+ import io
18
18
 
19
19
  TYPE_CHECKING: bool
20
20
 
21
21
  class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
22
+ """
23
+ Main class from which all Flows should inherit.
24
+
25
+ Attributes
26
+ ----------
27
+ index
28
+ input
29
+ """
22
30
  def __init__(self, use_cli = True):
23
31
  """
24
32
  Construct a FlowSpec
@@ -290,6 +298,9 @@ def ensure_unicode(x):
290
298
  ...
291
299
 
292
300
  class S3GetObject(tuple, metaclass=type):
301
+ """
302
+ S3GetObject(key, offset, length)
303
+ """
293
304
  @staticmethod
294
305
  def __new__(_cls, key: str, offset: int, length: int):
295
306
  """
@@ -311,6 +322,9 @@ class S3GetObject(tuple, metaclass=type):
311
322
  ...
312
323
 
313
324
  class S3PutObject(tuple, metaclass=type):
325
+ """
326
+ S3PutObject(key, value, path, content_type, encryption, metadata)
327
+ """
314
328
  @staticmethod
315
329
  def __new__(_cls, key: str, value: typing.Union[io.RawIOBase, io.BufferedIOBase, str, bytes, None] = None, path: typing.Optional[str] = None, content_type: typing.Optional[str] = None, encryption: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None):
316
330
  """
@@ -332,6 +346,9 @@ class S3PutObject(tuple, metaclass=type):
332
346
  ...
333
347
 
334
348
  class RangeInfo(tuple, metaclass=type):
349
+ """
350
+ RangeInfo(total_size, request_offset, request_length)
351
+ """
335
352
  @staticmethod
336
353
  def __new__(_cls, total_size: int, request_offset: int = 0, request_length: int = -1):
337
354
  """
@@ -371,6 +388,13 @@ class MetaflowS3InvalidRange(metaflow.exception.MetaflowException, metaclass=typ
371
388
  ...
372
389
 
373
390
  class S3Object(object, metaclass=type):
391
+ """
392
+ This object represents a path or an object in S3,
393
+ with an optional local copy.
394
+
395
+ `S3Object`s are not instantiated directly, but they are returned
396
+ by many methods of the `S3` client.
397
+ """
374
398
  def __init__(self, prefix: str, url: str, path: str, size: typing.Optional[int] = None, content_type: typing.Optional[str] = None, metadata: typing.Optional[typing.Dict[str, str]] = None, range_info: typing.Optional[RangeInfo] = None, last_modified: typing.Optional[int] = None, encryption: typing.Optional[str] = None):
375
399
  ...
376
400
  @property
@@ -587,6 +611,59 @@ class S3Client(object, metaclass=type):
587
611
  ...
588
612
 
589
613
  class S3(object, metaclass=type):
614
+ """
615
+ The Metaflow S3 client.
616
+
617
+ This object manages the connection to S3 and a temporary diretory that is used
618
+ to download objects. Note that in most cases when the data fits in memory, no local
619
+ disk IO is needed as operations are cached by the operating system, which makes
620
+ operations fast as long as there is enough memory available.
621
+
622
+ The easiest way is to use this object as a context manager:
623
+ ```
624
+ with S3() as s3:
625
+ data = [obj.blob for obj in s3.get_many(urls)]
626
+ print(data)
627
+ ```
628
+ The context manager takes care of creating and deleting a temporary directory
629
+ automatically. Without a context manager, you must call `.close()` to delete
630
+ the directory explicitly:
631
+ ```
632
+ s3 = S3()
633
+ data = [obj.blob for obj in s3.get_many(urls)]
634
+ s3.close()
635
+ ```
636
+ You can customize the location of the temporary directory with `tmproot`. It
637
+ defaults to the current working directory.
638
+
639
+ To make it easier to deal with object locations, the client can be initialized
640
+ with an S3 path prefix. There are three ways to handle locations:
641
+
642
+ 1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
643
+ initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
644
+ with the current run ID. This mode makes it easy to version data based
645
+ on the run ID consistently. You can use the `bucket` and `prefix` to
646
+ override parts of `DATATOOLS_S3ROOT`.
647
+
648
+ 2. Specify an S3 prefix explicitly with `s3root`,
649
+ e.g. `S3(s3root='s3://mybucket/some/path')`.
650
+
651
+ 3. Specify nothing, i.e. `S3()`, in which case all operations require
652
+ a full S3 url prefixed with `s3://`.
653
+
654
+ Parameters
655
+ ----------
656
+ tmproot : str, default: '.'
657
+ Where to store the temporary directory.
658
+ bucket : str, optional
659
+ Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
660
+ prefix : str, optional
661
+ Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
662
+ run : FlowSpec or Run, optional
663
+ Derive path prefix from the current or a past run ID, e.g. S3(run=self).
664
+ s3root : str, optional
665
+ If `run` is not specified, use this as the S3 prefix.
666
+ """
590
667
  @classmethod
591
668
  def get_root_from_config(cls, echo, create_on_absent = True):
592
669
  ...
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.414319 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.974266 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.383012 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.941772 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.374820 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.933395 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.375034 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.933634 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.372585 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T19:10:58.930468 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -11,6 +11,14 @@ if typing.TYPE_CHECKING:
11
11
  import metaflow.decorators
12
12
 
13
13
  class EnvironmentDecorator(metaflow.decorators.StepDecorator, metaclass=type):
14
+ """
15
+ Specifies environment variables to be set prior to the execution of a step.
16
+
17
+ Parameters
18
+ ----------
19
+ vars : Dict[str, str], default {}
20
+ Dictionary of environment variables to set.
21
+ """
14
22
  def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
15
23
  ...
16
24
  ...