ob-metaflow-stubs 6.0.3.104__py2.py3-none-any.whl → 6.0.3.105__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. metaflow-stubs/__init__.pyi +888 -550
  2. metaflow-stubs/cards.pyi +210 -4
  3. metaflow-stubs/cli.pyi +22 -2
  4. metaflow-stubs/client/__init__.pyi +128 -3
  5. metaflow-stubs/client/core.pyi +226 -6
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +5 -2
  8. metaflow-stubs/events.pyi +21 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +71 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +158 -5
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata/metadata.pyi +9 -3
  15. metaflow-stubs/metadata/util.pyi +2 -2
  16. metaflow-stubs/metaflow_config.pyi +2 -2
  17. metaflow-stubs/metaflow_current.pyi +69 -66
  18. metaflow-stubs/mflog/mflog.pyi +2 -2
  19. metaflow-stubs/multicore_utils.pyi +2 -2
  20. metaflow-stubs/parameters.pyi +69 -3
  21. metaflow-stubs/plugins/__init__.pyi +14 -3
  22. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
  26. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +6 -3
  27. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +46 -4
  28. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +46 -4
  29. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
  32. metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +97 -6
  34. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +35 -7
  35. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
  36. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +103 -3
  43. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
  45. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
  52. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
  56. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  57. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  58. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  59. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/cards/card_cli.pyi +62 -4
  61. metaflow-stubs/plugins/cards/card_client.pyi +34 -3
  62. metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
  63. metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
  64. metaflow-stubs/plugins/cards/card_decorator.pyi +53 -3
  65. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
  66. metaflow-stubs/plugins/cards/card_modules/basic.pyi +42 -3
  67. metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
  68. metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
  69. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
  70. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +36 -3
  72. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  73. metaflow-stubs/plugins/cards/component_serializer.pyi +56 -3
  74. metaflow-stubs/plugins/cards/exception.pyi +8 -2
  75. metaflow-stubs/plugins/catch_decorator.pyi +20 -3
  76. metaflow-stubs/plugins/datatools/__init__.pyi +64 -4
  77. metaflow-stubs/plugins/datatools/local.pyi +16 -2
  78. metaflow-stubs/plugins/datatools/s3/__init__.pyi +73 -4
  79. metaflow-stubs/plugins/datatools/s3/s3.pyi +82 -5
  80. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  82. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  83. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  84. metaflow-stubs/plugins/environment_decorator.pyi +10 -2
  85. metaflow-stubs/plugins/events_decorator.pyi +107 -3
  86. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/frameworks/pytorch.pyi +25 -4
  88. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
  90. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +11 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +67 -4
  98. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +100 -3
  100. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
  101. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  102. metaflow-stubs/plugins/package_cli.pyi +2 -2
  103. metaflow-stubs/plugins/parallel_decorator.pyi +30 -3
  104. metaflow-stubs/plugins/perimeters.pyi +2 -2
  105. metaflow-stubs/plugins/project_decorator.pyi +60 -3
  106. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
  108. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  109. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
  110. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  111. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  112. metaflow-stubs/plugins/resources_decorator.pyi +33 -2
  113. metaflow-stubs/plugins/retry_decorator.pyi +21 -2
  114. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  116. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
  117. metaflow-stubs/plugins/storage_executor.pyi +6 -2
  118. metaflow-stubs/plugins/tag_cli.pyi +35 -4
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +10 -3
  120. metaflow-stubs/plugins/timeout_decorator.pyi +24 -3
  121. metaflow-stubs/procpoll.pyi +2 -2
  122. metaflow-stubs/profilers/__init__.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +70 -131
  126. metaflow-stubs/runner/metaflow_runner.pyi +117 -9
  127. metaflow-stubs/runner/nbdeploy.pyi +66 -2
  128. metaflow-stubs/runner/nbrun.pyi +79 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +16 -4
  130. metaflow-stubs/runner/utils.pyi +32 -2
  131. metaflow-stubs/system/__init__.pyi +3 -3
  132. metaflow-stubs/system/system_logger.pyi +2 -2
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/METADATA +1 -1
  137. ob_metaflow_stubs-6.0.3.105.dist-info/RECORD +140 -0
  138. ob_metaflow_stubs-6.0.3.104.dist-info/RECORD +0 -140
  139. {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/WHEEL +0 -0
  140. {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/top_level.txt +0 -0
@@ -1,18 +1,18 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.343342 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.641159 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow._vendor.click.types
12
- import metaflow.parameters
13
- import io
14
11
  import typing
12
+ import io
13
+ import metaflow.parameters
15
14
  import metaflow.plugins.datatools.s3.s3
15
+ import metaflow._vendor.click.types
16
16
 
17
17
  class MetaflowException(Exception, metaclass=type):
18
18
  def __init__(self, msg = "", lineno = None):
@@ -22,6 +22,18 @@ class MetaflowException(Exception, metaclass=type):
22
22
  ...
23
23
 
24
24
  class DelayedEvaluationParameter(object, metaclass=type):
25
+ """
26
+ This is a very simple wrapper to allow parameter "conversion" to be delayed until
27
+ the `_set_constants` function in FlowSpec. Typically, parameters are converted
28
+ by click when the command line option is processed. For some parameters, like
29
+ IncludeFile, this is too early as it would mean we would trigger the upload
30
+ of the file too early. If a parameter converts to a DelayedEvaluationParameter
31
+ object through the usual click mechanisms, `_set_constants` knows to invoke the
32
+ __call__ method on that DelayedEvaluationParameter; in that case, the __call__
33
+ method is invoked without any parameter. The return_str parameter will be used
34
+ by schedulers when they need to convert DelayedEvaluationParameters to a
35
+ string to store them
36
+ """
25
37
  def __init__(self, name, field, fun):
26
38
  ...
27
39
  def __call__(self, return_str = False):
@@ -29,6 +41,14 @@ class DelayedEvaluationParameter(object, metaclass=type):
29
41
  ...
30
42
 
31
43
  class DeployTimeField(object, metaclass=type):
44
+ """
45
+ This a wrapper object for a user-defined function that is called
46
+ at deploy time to populate fields in a Parameter. The wrapper
47
+ is needed to make Click show the actual value returned by the
48
+ function instead of a function pointer in its help text. Also, this
49
+ object curries the context argument for the function, and pretty
50
+ prints any exceptions that occur during evaluation.
51
+ """
32
52
  def __init__(self, parameter_name, parameter_type, field, fun, return_str = True, print_representation = None):
33
53
  ...
34
54
  def __call__(self, deploy_time = False):
@@ -43,6 +63,49 @@ class DeployTimeField(object, metaclass=type):
43
63
  ...
44
64
 
45
65
  class Parameter(object, metaclass=type):
66
+ """
67
+ Defines a parameter for a flow.
68
+
69
+ Parameters must be instantiated as class variables in flow classes, e.g.
70
+ ```
71
+ class MyFlow(FlowSpec):
72
+ param = Parameter('myparam')
73
+ ```
74
+ in this case, the parameter is specified on the command line as
75
+ ```
76
+ python myflow.py run --myparam=5
77
+ ```
78
+ and its value is accessible through a read-only artifact like this:
79
+ ```
80
+ print(self.param == 5)
81
+ ```
82
+ Note that the user-visible parameter name, `myparam` above, can be
83
+ different from the artifact name, `param` above.
84
+
85
+ The parameter value is converted to a Python type based on the `type`
86
+ argument or to match the type of `default`, if it is set.
87
+
88
+ Parameters
89
+ ----------
90
+ name : str
91
+ User-visible parameter name.
92
+ default : str or float or int or bool or `JSONType` or a function.
93
+ Default value for the parameter. Use a special `JSONType` class to
94
+ indicate that the value must be a valid JSON object. A function
95
+ implies that the parameter corresponds to a *deploy-time parameter*.
96
+ The type of the default value is used as the parameter `type`.
97
+ type : Type, default None
98
+ If `default` is not specified, define the parameter type. Specify
99
+ one of `str`, `float`, `int`, `bool`, or `JSONType`. If None, defaults
100
+ to the type of `default` or `str` if none specified.
101
+ help : str, optional
102
+ Help text to show in `run --help`.
103
+ required : bool, default False
104
+ Require that the user specified a value for the parameter.
105
+ `required=True` implies that the `default` is not used.
106
+ show_default : bool, default True
107
+ If True, show the default value in the help text.
108
+ """
46
109
  def __init__(self, name: str, default: typing.Union[str, float, int, bool, typing.Dict[str, typing.Any], typing.Callable[[], typing.Union[str, float, int, bool, typing.Dict[str, typing.Any]]], None] = None, type: typing.Union[typing.Type[str], typing.Type[float], typing.Type[int], typing.Type[bool], metaflow.parameters.JSONTypeClass, None] = None, help: typing.Optional[str] = None, required: bool = False, show_default: bool = True, **kwargs: typing.Dict[str, typing.Any]):
47
110
  ...
48
111
  def __repr__(self):
@@ -61,6 +124,9 @@ class Parameter(object, metaclass=type):
61
124
  ...
62
125
 
63
126
  class ParameterContext(tuple, metaclass=type):
127
+ """
128
+ ParameterContext(flow_name, user_name, parameter_name, logger, ds_type)
129
+ """
64
130
  @staticmethod
65
131
  def __new__(_cls, flow_name: str, user_name: str, parameter_name: str, logger: typing.Callable[..., None], ds_type: str):
66
132
  """
@@ -82,6 +148,13 @@ class ParameterContext(tuple, metaclass=type):
82
148
  ...
83
149
 
84
150
  class Local(object, metaclass=type):
151
+ """
152
+ This class allows you to access the local filesystem in a way similar to the S3 datatools
153
+ client. It is a stripped down version for now and only implements the functionality needed
154
+ for this use case.
155
+
156
+ In the future, we may want to allow it to be used in a way similar to the S3() client.
157
+ """
85
158
  @classmethod
86
159
  def get_root_from_config(cls, echo, create_on_absent = True):
87
160
  ...
@@ -104,6 +177,59 @@ class Local(object, metaclass=type):
104
177
  ...
105
178
 
106
179
  class S3(object, metaclass=type):
180
+ """
181
+ The Metaflow S3 client.
182
+
183
+ This object manages the connection to S3 and a temporary diretory that is used
184
+ to download objects. Note that in most cases when the data fits in memory, no local
185
+ disk IO is needed as operations are cached by the operating system, which makes
186
+ operations fast as long as there is enough memory available.
187
+
188
+ The easiest way is to use this object as a context manager:
189
+ ```
190
+ with S3() as s3:
191
+ data = [obj.blob for obj in s3.get_many(urls)]
192
+ print(data)
193
+ ```
194
+ The context manager takes care of creating and deleting a temporary directory
195
+ automatically. Without a context manager, you must call `.close()` to delete
196
+ the directory explicitly:
197
+ ```
198
+ s3 = S3()
199
+ data = [obj.blob for obj in s3.get_many(urls)]
200
+ s3.close()
201
+ ```
202
+ You can customize the location of the temporary directory with `tmproot`. It
203
+ defaults to the current working directory.
204
+
205
+ To make it easier to deal with object locations, the client can be initialized
206
+ with an S3 path prefix. There are three ways to handle locations:
207
+
208
+ 1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
209
+ initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
210
+ with the current run ID. This mode makes it easy to version data based
211
+ on the run ID consistently. You can use the `bucket` and `prefix` to
212
+ override parts of `DATATOOLS_S3ROOT`.
213
+
214
+ 2. Specify an S3 prefix explicitly with `s3root`,
215
+ e.g. `S3(s3root='s3://mybucket/some/path')`.
216
+
217
+ 3. Specify nothing, i.e. `S3()`, in which case all operations require
218
+ a full S3 url prefixed with `s3://`.
219
+
220
+ Parameters
221
+ ----------
222
+ tmproot : str, default: '.'
223
+ Where to store the temporary directory.
224
+ bucket : str, optional
225
+ Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
226
+ prefix : str, optional
227
+ Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
228
+ run : FlowSpec or Run, optional
229
+ Derive path prefix from the current or a past run ID, e.g. S3(run=self).
230
+ s3root : str, optional
231
+ If `run` is not specified, use this as the S3 prefix.
232
+ """
107
233
  @classmethod
108
234
  def get_root_from_config(cls, echo, create_on_absent = True):
109
235
  ...
@@ -469,6 +595,33 @@ class FilePathClass(metaflow._vendor.click.types.ParamType, metaclass=type):
469
595
  ...
470
596
 
471
597
  class IncludeFile(metaflow.parameters.Parameter, metaclass=type):
598
+ """
599
+ Includes a local file as a parameter for the flow.
600
+
601
+ `IncludeFile` behaves like `Parameter` except that it reads its value from a file instead of
602
+ the command line. The user provides a path to a file on the command line. The file contents
603
+ are saved as a read-only artifact which is available in all steps of the flow.
604
+
605
+ Parameters
606
+ ----------
607
+ name : str
608
+ User-visible parameter name.
609
+ default : Union[str, Callable[ParameterContext, str]]
610
+ Default path to a local file. A function
611
+ implies that the parameter corresponds to a *deploy-time parameter*.
612
+ is_text : bool, default True
613
+ Convert the file contents to a string using the provided `encoding`.
614
+ If False, the artifact is stored in `bytes`.
615
+ encoding : str, optional, default 'utf-8'
616
+ Use this encoding to decode the file contexts if `is_text=True`.
617
+ required : bool, default False
618
+ Require that the user specified a value for the parameter.
619
+ `required=True` implies that the `default` is not used.
620
+ help : str, optional
621
+ Help text to show in `run --help`.
622
+ show_default : bool, default True
623
+ If True, show the default value in the help text.
624
+ """
472
625
  def __init__(self, name: str, required: bool = False, is_text: bool = True, encoding: str = "utf-8", help: typing.Optional[str] = None, **kwargs: typing.Dict[str, str]):
473
626
  ...
474
627
  def load_parameter(self, v):
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.328193 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.625444 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,15 +1,15 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.379898 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.678535 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.exception
12
11
  import metaflow.metadata.metadata
12
+ import metaflow.exception
13
13
 
14
14
  class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
15
15
  ...
@@ -28,6 +28,9 @@ def validate_tag(tag):
28
28
  ...
29
29
 
30
30
  class DataArtifact(tuple, metaclass=type):
31
+ """
32
+ DataArtifact(name, ds_type, ds_root, url, type, sha)
33
+ """
31
34
  @staticmethod
32
35
  def __new__(_cls, name, ds_type, ds_root, url, type, sha):
33
36
  """
@@ -47,6 +50,9 @@ class DataArtifact(tuple, metaclass=type):
47
50
  ...
48
51
 
49
52
  class MetaDatum(tuple, metaclass=type):
53
+ """
54
+ MetaDatum(field, value, type, tags)
55
+ """
50
56
  @staticmethod
51
57
  def __new__(_cls, field, value, type, tags):
52
58
  """
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.421987 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.713890 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.329970 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.627320 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,26 +1,29 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.424519 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.725041 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core
12
- import metaflow.events
13
11
  import metaflow.metaflow_current
12
+ import typing
14
13
  import metaflow
14
+ import metaflow.events
15
+ import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core
15
16
  import metaflow.plugins.cards.component_serializer
16
17
  import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
17
- import typing
18
18
 
19
19
  TYPE_CHECKING: bool
20
20
 
21
21
  TEMPDIR: str
22
22
 
23
23
  class Parallel(tuple, metaclass=type):
24
+ """
25
+ Parallel(main_ip, num_nodes, node_index, control_task_id)
26
+ """
24
27
  @staticmethod
25
28
  def __new__(_cls, main_ip, num_nodes, node_index, control_task_id):
26
29
  """
@@ -223,67 +226,6 @@ class Current(object, metaclass=type):
223
226
  def graph(_):
224
227
  ...
225
228
  @property
226
- def model(self) -> "metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core.ModelSerializer":
227
- """
228
- (only in the presence of the @model decorator)
229
-
230
- The object used for loading / saving models.
231
- `current.model` exposes a `save` method to save models and a `load` method to load models.
232
- `current.model.loaded` exposes the paths to the models loaded via the `load` argument in the @model decorator
233
- or models loaded via `current.model.load`.
234
-
235
- Usage (Saving a model):
236
- -------
237
-
238
- ```
239
- @model
240
- @step
241
- def train(self):
242
- # current.model.save returns a dictionary reference to the model saved
243
- self.my_model = current.model.save(
244
- path_to_my_model,
245
- label="my_model",
246
- metadata={
247
- "epochs": 10,
248
- "batch-size": 32,
249
- "learning-rate": 0.001,
250
- }
251
- )
252
- self.next(self.test)
253
-
254
- @model(load="my_model")
255
- @step
256
- def test(self):
257
- # `current.model.loaded` returns a dictionary of the loaded models
258
- # where the key is the name of the artifact and the value is the path to the model
259
- print(os.listdir(current.model.loaded["my_model"]))
260
- self.next(self.end)
261
- ```
262
-
263
- Usage (Loading models):
264
- -------
265
-
266
- ```
267
- @step
268
- def train(self):
269
- # current.model.load returns the path to the model loaded
270
- checkpoint_path = current.model.load(
271
- self.checkpoint_key,
272
- )
273
- model_path = current.model.load(
274
- self.model,
275
- )
276
- self.next(self.test)
277
- ```
278
-
279
-
280
- Returns
281
- -------
282
- ModelSerializer
283
- The object used for loading / saving models.
284
- """
285
- ...
286
- @property
287
229
  def card(self) -> "metaflow.plugins.cards.component_serializer.CardComponentCollector":
288
230
  """
289
231
  (only in the presence of the @card decorator)
@@ -363,6 +305,67 @@ class Current(object, metaclass=type):
363
305
  """
364
306
  ...
365
307
  @property
308
+ def model(self) -> "metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core.ModelSerializer":
309
+ """
310
+ (only in the presence of the @model decorator)
311
+
312
+ The object used for loading / saving models.
313
+ `current.model` exposes a `save` method to save models and a `load` method to load models.
314
+ `current.model.loaded` exposes the paths to the models loaded via the `load` argument in the @model decorator
315
+ or models loaded via `current.model.load`.
316
+
317
+ Usage (Saving a model):
318
+ -------
319
+
320
+ ```
321
+ @model
322
+ @step
323
+ def train(self):
324
+ # current.model.save returns a dictionary reference to the model saved
325
+ self.my_model = current.model.save(
326
+ path_to_my_model,
327
+ label="my_model",
328
+ metadata={
329
+ "epochs": 10,
330
+ "batch-size": 32,
331
+ "learning-rate": 0.001,
332
+ }
333
+ )
334
+ self.next(self.test)
335
+
336
+ @model(load="my_model")
337
+ @step
338
+ def test(self):
339
+ # `current.model.loaded` returns a dictionary of the loaded models
340
+ # where the key is the name of the artifact and the value is the path to the model
341
+ print(os.listdir(current.model.loaded["my_model"]))
342
+ self.next(self.end)
343
+ ```
344
+
345
+ Usage (Loading models):
346
+ -------
347
+
348
+ ```
349
+ @step
350
+ def train(self):
351
+ # current.model.load returns the path to the model loaded
352
+ checkpoint_path = current.model.load(
353
+ self.checkpoint_key,
354
+ )
355
+ model_path = current.model.load(
356
+ self.model,
357
+ )
358
+ self.next(self.test)
359
+ ```
360
+
361
+
362
+ Returns
363
+ -------
364
+ ModelSerializer
365
+ The object used for loading / saving models.
366
+ """
367
+ ...
368
+ @property
366
369
  def parallel(self) -> "metaflow.metaflow_current.Parallel":
367
370
  """
368
371
  (only in the presence of the @parallel decorator)
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.380198 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.678833 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.330476 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.627727 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.331622 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.628937 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -10,8 +10,8 @@ import typing
10
10
  if typing.TYPE_CHECKING:
11
11
  import typing
12
12
  import metaflow.exception
13
- import metaflow._vendor.click.types
14
13
  import metaflow.parameters
14
+ import metaflow._vendor.click.types
15
15
 
16
16
  class ParameterFieldFailed(metaflow.exception.MetaflowException, metaclass=type):
17
17
  def __init__(self, name, field):
@@ -31,6 +31,9 @@ class MetaflowException(Exception, metaclass=type):
31
31
  ...
32
32
 
33
33
  class ParameterContext(tuple, metaclass=type):
34
+ """
35
+ ParameterContext(flow_name, user_name, parameter_name, logger, ds_type)
36
+ """
34
37
  @staticmethod
35
38
  def __new__(_cls, flow_name: str, user_name: str, parameter_name: str, logger: typing.Callable[..., None], ds_type: str):
36
39
  """
@@ -71,6 +74,14 @@ class JSONTypeClass(metaflow._vendor.click.types.ParamType, metaclass=type):
71
74
  ...
72
75
 
73
76
  class DeployTimeField(object, metaclass=type):
77
+ """
78
+ This a wrapper object for a user-defined function that is called
79
+ at deploy time to populate fields in a Parameter. The wrapper
80
+ is needed to make Click show the actual value returned by the
81
+ function instead of a function pointer in its help text. Also, this
82
+ object curries the context argument for the function, and pretty
83
+ prints any exceptions that occur during evaluation.
84
+ """
74
85
  def __init__(self, parameter_name, parameter_type, field, fun, return_str = True, print_representation = None):
75
86
  ...
76
87
  def __call__(self, deploy_time = False):
@@ -91,6 +102,18 @@ def set_parameter_context(flow_name, echo, datastore):
91
102
  ...
92
103
 
93
104
  class DelayedEvaluationParameter(object, metaclass=type):
105
+ """
106
+ This is a very simple wrapper to allow parameter "conversion" to be delayed until
107
+ the `_set_constants` function in FlowSpec. Typically, parameters are converted
108
+ by click when the command line option is processed. For some parameters, like
109
+ IncludeFile, this is too early as it would mean we would trigger the upload
110
+ of the file too early. If a parameter converts to a DelayedEvaluationParameter
111
+ object through the usual click mechanisms, `_set_constants` knows to invoke the
112
+ __call__ method on that DelayedEvaluationParameter; in that case, the __call__
113
+ method is invoked without any parameter. The return_str parameter will be used
114
+ by schedulers when they need to convert DelayedEvaluationParameters to a
115
+ string to store them
116
+ """
94
117
  def __init__(self, name, field, fun):
95
118
  ...
96
119
  def __call__(self, return_str = False):
@@ -98,6 +121,49 @@ class DelayedEvaluationParameter(object, metaclass=type):
98
121
  ...
99
122
 
100
123
  class Parameter(object, metaclass=type):
124
+ """
125
+ Defines a parameter for a flow.
126
+
127
+ Parameters must be instantiated as class variables in flow classes, e.g.
128
+ ```
129
+ class MyFlow(FlowSpec):
130
+ param = Parameter('myparam')
131
+ ```
132
+ in this case, the parameter is specified on the command line as
133
+ ```
134
+ python myflow.py run --myparam=5
135
+ ```
136
+ and its value is accessible through a read-only artifact like this:
137
+ ```
138
+ print(self.param == 5)
139
+ ```
140
+ Note that the user-visible parameter name, `myparam` above, can be
141
+ different from the artifact name, `param` above.
142
+
143
+ The parameter value is converted to a Python type based on the `type`
144
+ argument or to match the type of `default`, if it is set.
145
+
146
+ Parameters
147
+ ----------
148
+ name : str
149
+ User-visible parameter name.
150
+ default : str or float or int or bool or `JSONType` or a function.
151
+ Default value for the parameter. Use a special `JSONType` class to
152
+ indicate that the value must be a valid JSON object. A function
153
+ implies that the parameter corresponds to a *deploy-time parameter*.
154
+ The type of the default value is used as the parameter `type`.
155
+ type : Type, default None
156
+ If `default` is not specified, define the parameter type. Specify
157
+ one of `str`, `float`, `int`, `bool`, or `JSONType`. If None, defaults
158
+ to the type of `default` or `str` if none specified.
159
+ help : str, optional
160
+ Help text to show in `run --help`.
161
+ required : bool, default False
162
+ Require that the user specified a value for the parameter.
163
+ `required=True` implies that the `default` is not used.
164
+ show_default : bool, default True
165
+ If True, show the default value in the help text.
166
+ """
101
167
  def __init__(self, name: str, default: typing.Union[str, float, int, bool, typing.Dict[str, typing.Any], typing.Callable[[], typing.Union[str, float, int, bool, typing.Dict[str, typing.Any]]], None] = None, type: typing.Union[typing.Type[str], typing.Type[float], typing.Type[int], typing.Type[bool], JSONTypeClass, None] = None, help: typing.Optional[str] = None, required: bool = False, show_default: bool = True, **kwargs: typing.Dict[str, typing.Any]):
102
168
  ...
103
169
  def __repr__(self):
@@ -1,19 +1,23 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.345014 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.642860 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.plugins.cards.card_modules.card
12
11
  import metaflow.unbounded_foreach
12
+ import metaflow.plugins.cards.card_modules.card
13
13
 
14
14
  CLIS_DESC: list
15
15
 
16
16
  class InternalTestUnboundedForeachInput(metaflow.unbounded_foreach.UnboundedForeachInput, metaclass=type):
17
+ """
18
+ Test class that wraps around values (any iterator) and simulates an
19
+ unbounded-foreach instead of a bounded foreach.
20
+ """
17
21
  def __init__(self, iterable):
18
22
  ...
19
23
  def __iter__(self):
@@ -194,6 +198,9 @@ class TestTimeoutCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, met
194
198
  ...
195
199
 
196
200
  class TestRefreshCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, metaclass=type):
201
+ """
202
+ This card takes no components and helps test the `current.card.refresh(data)` interface.
203
+ """
197
204
  def render(self, task) -> str:
198
205
  ...
199
206
  def render_runtime(self, task, data):
@@ -203,6 +210,10 @@ class TestRefreshCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, met
203
210
  ...
204
211
 
205
212
  class TestRefreshComponentCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, metaclass=type):
213
+ """
214
+ This card takes components and helps test the `current.card.components["A"].update()`
215
+ interface
216
+ """
206
217
  def __init__(self, options = {}, components = [], graph = None):
207
218
  ...
208
219
  def render(self, task) -> str:
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.373842 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.672699 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.22.1+obcheckpoint(0.0.11);ob(v1) #
4
- # Generated on 2024-09-20T21:46:07.412087 #
3
+ # MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
4
+ # Generated on 2024-10-04T10:13:10.711434 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations