wandb 0.13.10__py3-none-any.whl → 0.14.0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (228) hide show
  1. wandb/__init__.py +2 -3
  2. wandb/apis/__init__.py +1 -3
  3. wandb/apis/importers/__init__.py +4 -0
  4. wandb/apis/importers/base.py +312 -0
  5. wandb/apis/importers/mlflow.py +113 -0
  6. wandb/apis/internal.py +29 -2
  7. wandb/apis/normalize.py +6 -5
  8. wandb/apis/public.py +163 -180
  9. wandb/apis/reports/_templates.py +6 -12
  10. wandb/apis/reports/report.py +1 -1
  11. wandb/apis/reports/runset.py +1 -3
  12. wandb/apis/reports/util.py +12 -10
  13. wandb/beta/workflows.py +57 -34
  14. wandb/catboost/__init__.py +1 -2
  15. wandb/cli/cli.py +215 -133
  16. wandb/data_types.py +63 -56
  17. wandb/docker/__init__.py +78 -16
  18. wandb/docker/auth.py +21 -22
  19. wandb/env.py +0 -1
  20. wandb/errors/__init__.py +8 -116
  21. wandb/errors/term.py +1 -1
  22. wandb/fastai/__init__.py +1 -2
  23. wandb/filesync/dir_watcher.py +8 -5
  24. wandb/filesync/step_prepare.py +76 -75
  25. wandb/filesync/step_upload.py +1 -2
  26. wandb/integration/catboost/__init__.py +1 -3
  27. wandb/integration/catboost/catboost.py +8 -14
  28. wandb/integration/fastai/__init__.py +7 -13
  29. wandb/integration/gym/__init__.py +35 -4
  30. wandb/integration/keras/__init__.py +3 -3
  31. wandb/integration/keras/callbacks/metrics_logger.py +9 -8
  32. wandb/integration/keras/callbacks/model_checkpoint.py +9 -9
  33. wandb/integration/keras/callbacks/tables_builder.py +31 -19
  34. wandb/integration/kfp/kfp_patch.py +20 -17
  35. wandb/integration/kfp/wandb_logging.py +1 -2
  36. wandb/integration/lightgbm/__init__.py +21 -19
  37. wandb/integration/prodigy/prodigy.py +6 -7
  38. wandb/integration/sacred/__init__.py +9 -12
  39. wandb/integration/sagemaker/__init__.py +1 -3
  40. wandb/integration/sagemaker/auth.py +0 -1
  41. wandb/integration/sagemaker/config.py +1 -1
  42. wandb/integration/sagemaker/resources.py +1 -1
  43. wandb/integration/sb3/sb3.py +8 -4
  44. wandb/integration/tensorboard/__init__.py +1 -3
  45. wandb/integration/tensorboard/log.py +8 -8
  46. wandb/integration/tensorboard/monkeypatch.py +11 -9
  47. wandb/integration/tensorflow/__init__.py +1 -3
  48. wandb/integration/xgboost/__init__.py +4 -6
  49. wandb/integration/yolov8/__init__.py +7 -0
  50. wandb/integration/yolov8/yolov8.py +250 -0
  51. wandb/jupyter.py +31 -35
  52. wandb/lightgbm/__init__.py +1 -2
  53. wandb/old/settings.py +2 -2
  54. wandb/plot/bar.py +1 -2
  55. wandb/plot/confusion_matrix.py +1 -3
  56. wandb/plot/histogram.py +1 -2
  57. wandb/plot/line.py +1 -2
  58. wandb/plot/line_series.py +4 -4
  59. wandb/plot/pr_curve.py +17 -20
  60. wandb/plot/roc_curve.py +1 -3
  61. wandb/plot/scatter.py +1 -2
  62. wandb/proto/v3/wandb_server_pb2.py +85 -39
  63. wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
  64. wandb/proto/v4/wandb_server_pb2.py +51 -39
  65. wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
  66. wandb/sdk/__init__.py +1 -3
  67. wandb/sdk/backend/backend.py +1 -1
  68. wandb/sdk/data_types/_dtypes.py +38 -30
  69. wandb/sdk/data_types/base_types/json_metadata.py +1 -3
  70. wandb/sdk/data_types/base_types/media.py +17 -17
  71. wandb/sdk/data_types/base_types/wb_value.py +33 -26
  72. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +91 -125
  73. wandb/sdk/data_types/helper_types/classes.py +1 -1
  74. wandb/sdk/data_types/helper_types/image_mask.py +12 -12
  75. wandb/sdk/data_types/histogram.py +5 -4
  76. wandb/sdk/data_types/html.py +1 -2
  77. wandb/sdk/data_types/image.py +11 -11
  78. wandb/sdk/data_types/molecule.py +3 -6
  79. wandb/sdk/data_types/object_3d.py +1 -2
  80. wandb/sdk/data_types/plotly.py +1 -2
  81. wandb/sdk/data_types/saved_model.py +10 -8
  82. wandb/sdk/data_types/video.py +1 -1
  83. wandb/sdk/integration_utils/data_logging.py +5 -5
  84. wandb/sdk/interface/artifacts.py +288 -266
  85. wandb/sdk/interface/interface.py +2 -3
  86. wandb/sdk/interface/interface_grpc.py +1 -1
  87. wandb/sdk/interface/interface_queue.py +1 -1
  88. wandb/sdk/interface/interface_relay.py +1 -1
  89. wandb/sdk/interface/interface_shared.py +1 -2
  90. wandb/sdk/interface/interface_sock.py +1 -1
  91. wandb/sdk/interface/message_future.py +1 -1
  92. wandb/sdk/interface/message_future_poll.py +1 -1
  93. wandb/sdk/interface/router.py +1 -1
  94. wandb/sdk/interface/router_queue.py +1 -1
  95. wandb/sdk/interface/router_relay.py +1 -1
  96. wandb/sdk/interface/router_sock.py +1 -1
  97. wandb/sdk/interface/summary_record.py +1 -1
  98. wandb/sdk/internal/artifacts.py +1 -1
  99. wandb/sdk/internal/datastore.py +2 -3
  100. wandb/sdk/internal/file_pusher.py +5 -3
  101. wandb/sdk/internal/file_stream.py +22 -19
  102. wandb/sdk/internal/handler.py +5 -4
  103. wandb/sdk/internal/internal.py +1 -1
  104. wandb/sdk/internal/internal_api.py +115 -55
  105. wandb/sdk/internal/job_builder.py +1 -3
  106. wandb/sdk/internal/profiler.py +1 -1
  107. wandb/sdk/internal/progress.py +4 -6
  108. wandb/sdk/internal/sample.py +1 -3
  109. wandb/sdk/internal/sender.py +28 -16
  110. wandb/sdk/internal/settings_static.py +5 -5
  111. wandb/sdk/internal/system/assets/__init__.py +1 -0
  112. wandb/sdk/internal/system/assets/cpu.py +3 -9
  113. wandb/sdk/internal/system/assets/disk.py +2 -4
  114. wandb/sdk/internal/system/assets/gpu.py +6 -18
  115. wandb/sdk/internal/system/assets/gpu_apple.py +2 -4
  116. wandb/sdk/internal/system/assets/interfaces.py +50 -22
  117. wandb/sdk/internal/system/assets/ipu.py +1 -3
  118. wandb/sdk/internal/system/assets/memory.py +7 -13
  119. wandb/sdk/internal/system/assets/network.py +4 -8
  120. wandb/sdk/internal/system/assets/open_metrics.py +283 -0
  121. wandb/sdk/internal/system/assets/tpu.py +1 -4
  122. wandb/sdk/internal/system/assets/trainium.py +26 -14
  123. wandb/sdk/internal/system/system_info.py +2 -3
  124. wandb/sdk/internal/system/system_monitor.py +52 -20
  125. wandb/sdk/internal/tb_watcher.py +12 -13
  126. wandb/sdk/launch/_project_spec.py +54 -65
  127. wandb/sdk/launch/agent/agent.py +374 -90
  128. wandb/sdk/launch/builder/abstract.py +61 -7
  129. wandb/sdk/launch/builder/build.py +81 -110
  130. wandb/sdk/launch/builder/docker_builder.py +181 -0
  131. wandb/sdk/launch/builder/kaniko_builder.py +419 -0
  132. wandb/sdk/launch/builder/noop.py +31 -12
  133. wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +70 -20
  134. wandb/sdk/launch/environment/abstract.py +28 -0
  135. wandb/sdk/launch/environment/aws_environment.py +276 -0
  136. wandb/sdk/launch/environment/gcp_environment.py +271 -0
  137. wandb/sdk/launch/environment/local_environment.py +65 -0
  138. wandb/sdk/launch/github_reference.py +3 -8
  139. wandb/sdk/launch/launch.py +38 -29
  140. wandb/sdk/launch/launch_add.py +6 -8
  141. wandb/sdk/launch/loader.py +230 -0
  142. wandb/sdk/launch/registry/abstract.py +54 -0
  143. wandb/sdk/launch/registry/elastic_container_registry.py +163 -0
  144. wandb/sdk/launch/registry/google_artifact_registry.py +203 -0
  145. wandb/sdk/launch/registry/local_registry.py +62 -0
  146. wandb/sdk/launch/runner/abstract.py +1 -16
  147. wandb/sdk/launch/runner/{kubernetes.py → kubernetes_runner.py} +83 -95
  148. wandb/sdk/launch/runner/local_container.py +46 -22
  149. wandb/sdk/launch/runner/local_process.py +1 -4
  150. wandb/sdk/launch/runner/{aws.py → sagemaker_runner.py} +53 -212
  151. wandb/sdk/launch/runner/{gcp_vertex.py → vertex_runner.py} +38 -55
  152. wandb/sdk/launch/sweeps/__init__.py +3 -2
  153. wandb/sdk/launch/sweeps/scheduler.py +132 -39
  154. wandb/sdk/launch/sweeps/scheduler_sweep.py +80 -89
  155. wandb/sdk/launch/utils.py +101 -30
  156. wandb/sdk/launch/wandb_reference.py +2 -7
  157. wandb/sdk/lib/_settings_toposort_generate.py +166 -0
  158. wandb/sdk/lib/_settings_toposort_generated.py +201 -0
  159. wandb/sdk/lib/apikey.py +2 -4
  160. wandb/sdk/lib/config_util.py +4 -1
  161. wandb/sdk/lib/console.py +1 -3
  162. wandb/sdk/lib/deprecate.py +3 -3
  163. wandb/sdk/lib/file_stream_utils.py +7 -5
  164. wandb/sdk/lib/filenames.py +1 -1
  165. wandb/sdk/lib/filesystem.py +61 -5
  166. wandb/sdk/lib/git.py +1 -3
  167. wandb/sdk/lib/import_hooks.py +4 -7
  168. wandb/sdk/lib/ipython.py +8 -5
  169. wandb/sdk/lib/lazyloader.py +1 -3
  170. wandb/sdk/lib/mailbox.py +14 -4
  171. wandb/sdk/lib/proto_util.py +10 -5
  172. wandb/sdk/lib/redirect.py +15 -22
  173. wandb/sdk/lib/reporting.py +1 -3
  174. wandb/sdk/lib/retry.py +4 -5
  175. wandb/sdk/lib/runid.py +1 -3
  176. wandb/sdk/lib/server.py +15 -9
  177. wandb/sdk/lib/sock_client.py +1 -1
  178. wandb/sdk/lib/sparkline.py +1 -1
  179. wandb/sdk/lib/wburls.py +1 -1
  180. wandb/sdk/service/port_file.py +1 -2
  181. wandb/sdk/service/service.py +36 -13
  182. wandb/sdk/service/service_base.py +12 -1
  183. wandb/sdk/verify/verify.py +5 -7
  184. wandb/sdk/wandb_artifacts.py +142 -177
  185. wandb/sdk/wandb_config.py +5 -8
  186. wandb/sdk/wandb_helper.py +1 -1
  187. wandb/sdk/wandb_init.py +24 -13
  188. wandb/sdk/wandb_login.py +9 -9
  189. wandb/sdk/wandb_manager.py +39 -4
  190. wandb/sdk/wandb_metric.py +2 -6
  191. wandb/sdk/wandb_require.py +4 -15
  192. wandb/sdk/wandb_require_helpers.py +1 -9
  193. wandb/sdk/wandb_run.py +95 -141
  194. wandb/sdk/wandb_save.py +1 -3
  195. wandb/sdk/wandb_settings.py +149 -54
  196. wandb/sdk/wandb_setup.py +66 -46
  197. wandb/sdk/wandb_summary.py +13 -10
  198. wandb/sdk/wandb_sweep.py +6 -7
  199. wandb/sdk/wandb_watch.py +1 -1
  200. wandb/sklearn/calculate/confusion_matrix.py +1 -1
  201. wandb/sklearn/calculate/learning_curve.py +1 -1
  202. wandb/sklearn/calculate/summary_metrics.py +1 -3
  203. wandb/sklearn/plot/__init__.py +1 -1
  204. wandb/sklearn/plot/classifier.py +27 -18
  205. wandb/sklearn/plot/clusterer.py +4 -5
  206. wandb/sklearn/plot/regressor.py +4 -4
  207. wandb/sklearn/plot/shared.py +2 -2
  208. wandb/sync/__init__.py +1 -3
  209. wandb/sync/sync.py +4 -5
  210. wandb/testing/relay.py +11 -10
  211. wandb/trigger.py +1 -1
  212. wandb/util.py +106 -81
  213. wandb/viz.py +4 -4
  214. wandb/wandb_agent.py +50 -50
  215. wandb/wandb_controller.py +2 -3
  216. wandb/wandb_run.py +1 -2
  217. wandb/wandb_torch.py +1 -1
  218. wandb/xgboost/__init__.py +1 -2
  219. {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/METADATA +6 -2
  220. {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/RECORD +224 -209
  221. {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/WHEEL +1 -1
  222. wandb/sdk/launch/builder/docker.py +0 -80
  223. wandb/sdk/launch/builder/kaniko.py +0 -393
  224. wandb/sdk/launch/builder/loader.py +0 -32
  225. wandb/sdk/launch/runner/loader.py +0 -50
  226. {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/LICENSE +0 -0
  227. {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/entry_points.txt +0 -0
  228. {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/top_level.txt +0 -0
@@ -21,8 +21,10 @@ patch_tf_keras()
21
21
 
22
22
 
23
23
  class WandbMetricsLogger(callbacks.Callback):
24
- """`WandbMetricsLogger` automatically logs the `logs` dictionary
25
- that callback methods take as argument to wandb.
24
+ """Logger that sends system metrics to W&B.
25
+
26
+ `WandbMetricsLogger` automatically logs the `logs` dictionary that callback methods
27
+ take as argument to wandb.
26
28
 
27
29
  This callback automatically logs the following to a W&B run page:
28
30
  * system (CPU/GPU/TPU) metrics,
@@ -30,12 +32,11 @@ class WandbMetricsLogger(callbacks.Callback):
30
32
  * learning rate (both for a fixed value or a learning rate scheduler)
31
33
 
32
34
  Notes:
33
- If you resume training by passing `initial_epoch` to `model.fit` and
34
- you are using a learning rate scheduler, make sure to pass
35
- `initial_global_step` to `WandbMetricsLogger`. The `initial_global_step`
36
- is `step_size * initial_step`, where `step_size` is number of training
37
- steps per epoch. `step_size` can be calculated as the product of the
38
- cardinality of the training dataset and the batch size.
35
+ If you resume training by passing `initial_epoch` to `model.fit` and you are using a
36
+ learning rate scheduler, make sure to pass `initial_global_step` to
37
+ `WandbMetricsLogger`. The `initial_global_step` is `step_size * initial_step`, where
38
+ `step_size` is number of training steps per epoch. `step_size` can be calculated as
39
+ the product of the cardinality of the training dataset and the batch size.
39
40
 
40
41
  Arguments:
41
42
  log_freq ("epoch", "batch", or int): if "epoch", logs metrics
@@ -25,17 +25,17 @@ patch_tf_keras()
25
25
 
26
26
 
27
27
  class WandbModelCheckpoint(callbacks.ModelCheckpoint):
28
- """`WandbModelCheckpoint` periodically saves a Keras model or model weights
29
- and uploads it to W&B as a `wandb.Artifact`.
28
+ """A checkpoint that periodically saves a Keras model or model weights.
30
29
 
31
- Since this callback is subclassed from `tf.keras.callbacks.ModelCheckpoint`,
32
- the checkpointing logic is taken care of by the parent callback. You can learn
33
- more here:
34
- https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ModelCheckpoint
30
+ Saves weights are uploaded to W&B as a `wandb.Artifact`.
35
31
 
36
- This callback is to be used in conjunction with training using `model.fit()`
37
- to save a model or weights (in a checkpoint file) at some interval. The
38
- model checkpoints will be logged as W&B Artifacts. You can learn more here:
32
+ Since this callback is subclassed from `tf.keras.callbacks.ModelCheckpoint`, the
33
+ checkpointing logic is taken care of by the parent callback. You can learn more
34
+ here: https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ModelCheckpoint
35
+
36
+ This callback is to be used in conjunction with training using `model.fit()` to save
37
+ a model or weights (in a checkpoint file) at some interval. The model checkpoints
38
+ will be logged as W&B Artifacts. You can learn more here:
39
39
  https://docs.wandb.ai/guides/artifacts
40
40
 
41
41
  This callback provides the following features:
@@ -14,8 +14,8 @@ class WandbEvalCallback(Callback, abc.ABC):
14
14
  that can be passed to `model.fit()` for classification, object detection,
15
15
  segmentation, etc. tasks.
16
16
 
17
- To use this, inherit from this base callback class and implement the `add_ground_truth`
18
- and `add_model_prediction` methods.
17
+ To use this, inherit from this base callback class and implement the
18
+ `add_ground_truth` and `add_model_prediction` methods.
19
19
 
20
20
  The base class will take care of the following:
21
21
  - Initialize `data_table` for logging the ground truth and
@@ -126,8 +126,11 @@ class WandbEvalCallback(Callback, abc.ABC):
126
126
 
127
127
  @abc.abstractmethod
128
128
  def add_ground_truth(self, logs: Optional[Dict[str, float]] = None) -> None:
129
- """Use this method to write the logic for adding validation/training
130
- data to `data_table` initialized using `init_data_table` method.
129
+ """Add ground truth data to `data_table`.
130
+
131
+ Use this method to write the logic for adding validation/training data to
132
+ `data_table` initialized using `init_data_table` method.
133
+
131
134
  Example:
132
135
  ```
133
136
  for idx, data in enumerate(dataloader):
@@ -144,9 +147,11 @@ class WandbEvalCallback(Callback, abc.ABC):
144
147
  def add_model_predictions(
145
148
  self, epoch: int, logs: Optional[Dict[str, float]] = None
146
149
  ) -> None:
147
- """Use this method to write the logic for adding model prediction for
148
- validation/training data to `pred_table` initialized using
149
- `init_pred_table` method.
150
+ """Add a prediction from a model to `pred_table`.
151
+
152
+ Use this method to write the logic for adding model prediction for validation/
153
+ training data to `pred_table` initialized using `init_pred_table` method.
154
+
150
155
  Example:
151
156
  ```
152
157
  # Assuming the dataloader is not shuffling the samples.
@@ -164,8 +169,10 @@ class WandbEvalCallback(Callback, abc.ABC):
164
169
 
165
170
  def init_data_table(self, column_names: List[str]) -> None:
166
171
  """Initialize the W&B Tables for validation data.
167
- Call this method `on_train_begin` or equivalent hook. This is followed by
168
- adding data to the table row or column wise.
172
+
173
+ Call this method `on_train_begin` or equivalent hook. This is followed by adding
174
+ data to the table row or column wise.
175
+
169
176
  Args:
170
177
  column_names (list): Column names for W&B Tables.
171
178
  """
@@ -173,8 +180,10 @@ class WandbEvalCallback(Callback, abc.ABC):
173
180
 
174
181
  def init_pred_table(self, column_names: List[str]) -> None:
175
182
  """Initialize the W&B Tables for model evaluation.
176
- Call this method `on_epoch_end` or equivalent hook. This is followed by
177
- adding data to the table row or column wise.
183
+
184
+ Call this method `on_epoch_end` or equivalent hook. This is followed by adding
185
+ data to the table row or column wise.
186
+
178
187
  Args:
179
188
  column_names (list): Column names for W&B Tables.
180
189
  """
@@ -183,18 +192,19 @@ class WandbEvalCallback(Callback, abc.ABC):
183
192
  def log_data_table(
184
193
  self, name: str = "val", type: str = "dataset", table_name: str = "val_data"
185
194
  ) -> None:
186
- """Log the `data_table` as W&B artifact and call
187
- `use_artifact` on it so that the evaluation table can use the reference
188
- of already uploaded data (images, text, scalar, etc.).
189
- This allows the data to be uploaded just once.
195
+ """Log the `data_table` as W&B artifact and call `use_artifact` on it.
196
+
197
+ This lets the evaluation table use the reference of already uploaded data
198
+ (images, text, scalar, etc.) without re-uploading.
199
+
190
200
  Args:
191
- name (str): A human-readable name for this artifact, which is how
192
- you can identify this artifact in the UI or reference
193
- it in use_artifact calls. (default is 'val')
201
+ name (str): A human-readable name for this artifact, which is how you can
202
+ identify this artifact in the UI or reference it in use_artifact calls.
203
+ (default is 'val')
194
204
  type (str): The type of the artifact, which is used to organize and
195
205
  differentiate artifacts. (default is 'dataset')
196
206
  table_name (str): The name of the table as will be displayed in the UI.
197
- (default is 'val_data')
207
+ (default is 'val_data').
198
208
  """
199
209
  data_artifact = wandb.Artifact(name, type=type)
200
210
  data_artifact.add(self.data_table, table_name)
@@ -214,8 +224,10 @@ class WandbEvalCallback(Callback, abc.ABC):
214
224
  aliases: Optional[List[str]] = None,
215
225
  ) -> None:
216
226
  """Log the W&B Tables for model evaluation.
227
+
217
228
  The table will be logged multiple times creating new version. Use this
218
229
  to compare models at different intervals interactively.
230
+
219
231
  Args:
220
232
  type (str): The type of the artifact, which is used to organize and
221
233
  differentiate artifacts. (default is 'evaluation')
@@ -123,12 +123,13 @@ def patch_kfp():
123
123
  patch("wandb.integration.kfp", wandb_log)
124
124
 
125
125
 
126
- def wandb_log( # noqa: C901
126
+ def wandb_log(
127
127
  func=None,
128
128
  # /, # py38 only
129
129
  log_component_file=True,
130
130
  ):
131
- """Wrap a standard python function and log to W&B
131
+ """Wrap a standard python function and log to W&B.
132
+
132
133
  NOTE: Because patching failed, this decorator is a no-op.
133
134
  """
134
135
  from functools import wraps
@@ -147,10 +148,11 @@ def wandb_log( # noqa: C901
147
148
 
148
149
 
149
150
  def _get_function_source_definition(func: Callable) -> str:
150
- """This function is modified from KFP. The original source is below:
151
- https://github.com/kubeflow/pipelines/blob/b6406b02f45cdb195c7b99e2f6d22bf85b12268b/sdk/python/kfp/components/_python_op.py#L300-L319
152
- """
151
+ """Get the source code of a function.
153
152
 
153
+ This function is modified from KFP. The original source is below:
154
+ https://github.com/kubeflow/pipelines/blob/b6406b02f45cdb195c7b99e2f6d22bf85b12268b/sdk/python/kfp/components/_python_op.py#L300-L319.
155
+ """
154
156
  func_code = inspect.getsource(func)
155
157
 
156
158
  # Function might be defined in some indented scope (e.g. in another
@@ -181,11 +183,12 @@ def create_component_from_func(
181
183
  packages_to_install: Optional[List[str]] = None,
182
184
  annotations: Optional[Mapping[str, str]] = None,
183
185
  ):
184
- '''This function is modified from KFP. The original source is below:
185
- https://github.com/kubeflow/pipelines/blob/b6406b02f45cdb195c7b99e2f6d22bf85b12268b/sdk/python/kfp/components/_python_op.py#L998-L1110
186
+ '''Convert a Python function to a component and returns a task factory.
187
+
188
+ The returned task factory accepts arguments and returns a task object.
186
189
 
187
- Converts a Python function to a component and returns a task factory
188
- (a function that accepts arguments and returns a task object).
190
+ This function is modified from KFP. The original source is below:
191
+ https://github.com/kubeflow/pipelines/blob/b6406b02f45cdb195c7b99e2f6d22bf85b12268b/sdk/python/kfp/components/_python_op.py#L998-L1110.
189
192
 
190
193
  Args:
191
194
  func: The python function to convert
@@ -202,7 +205,7 @@ def create_component_from_func(
202
205
  The function name and docstring are used as component name and description. Argument and return annotations are used as component input/output types::
203
206
 
204
207
  def add(a: float, b: float) -> float:
205
- """Returns sum of two arguments"""
208
+ """Return sum of two arguments"""
206
209
  return a + b
207
210
 
208
211
  # add_op is a task factory function that creates a task object when given arguments
@@ -231,7 +234,7 @@ def create_component_from_func(
231
234
 
232
235
  @create_component_from_func
233
236
  def add_op(a: float, b: float) -> float:
234
- """Returns sum of two arguments"""
237
+ """Return sum of two arguments"""
235
238
  return a + b
236
239
 
237
240
  To declare a function with multiple return values, use the :code:`NamedTuple` return annotation syntax::
@@ -239,7 +242,7 @@ def create_component_from_func(
239
242
  from typing import NamedTuple
240
243
 
241
244
  def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('Outputs', [('sum', float), ('product', float)]):
242
- """Returns sum and product of two arguments"""
245
+ """Return sum and product of two arguments"""
243
246
  return (a + b, a * b)
244
247
 
245
248
  add_multiply_op = create_component_from_func(add_multiply_two_numbers)
@@ -271,12 +274,11 @@ def create_component_from_func(
271
274
  ('Precision', float), # Small output of type "Float"
272
275
  ('JobUri', 'URI'), # Small output of type "URI"
273
276
  ]):
274
- """Trains CatBoost classification model"""
277
+ """Train CatBoost classification model"""
275
278
  ...
276
279
 
277
280
  return (accuracy, precision, recall)
278
281
  '''
279
-
280
282
  core_packages = ["wandb", "kfp"]
281
283
 
282
284
  if not packages_to_install:
@@ -302,10 +304,11 @@ def create_component_from_func(
302
304
 
303
305
 
304
306
  def strip_type_hints(source_code: str) -> str:
305
- """This function is modified from KFP. The original source is below:
306
- https://github.com/kubeflow/pipelines/blob/b6406b02f45cdb195c7b99e2f6d22bf85b12268b/sdk/python/kfp/components/_python_op.py#L237-L248
307
- """
307
+ """Strip type hints from source code.
308
308
 
309
+ This function is modified from KFP. The original source is below:
310
+ https://github.com/kubeflow/pipelines/blob/b6406b02f45cdb195c7b99e2f6d22bf85b12268b/sdk/python/kfp/components/_python_op.py#L237-L248.
311
+ """
309
312
  # For wandb, do not strip type hints
310
313
 
311
314
  # try:
@@ -3,7 +3,7 @@ def wandb_log( # noqa: C901
3
3
  # /, # py38 only
4
4
  log_component_file=True,
5
5
  ):
6
- """Wrap a standard python function and log to W&B"""
6
+ """Wrap a standard python function and log to W&B."""
7
7
  import json
8
8
  import os
9
9
  from functools import wraps
@@ -131,7 +131,6 @@ def wandb_log( # noqa: C901
131
131
  job_type=func.__name__,
132
132
  group="{{workflow.annotations.pipelines.kubeflow.org/run_name}}",
133
133
  ) as run:
134
-
135
134
  # Link back to the kfp UI
136
135
  kubeflow_url = get_link_back_to_kubeflow()
137
136
  run.notes = kubeflow_url
@@ -111,16 +111,17 @@ def wandb_callback(log_params: bool = True, define_metric: bool = True) -> Calla
111
111
  Example:
112
112
  ```python
113
113
  params = {
114
- 'boosting_type': 'gbdt',
115
- 'objective': 'regression',
116
- .
114
+ "boosting_type": "gbdt",
115
+ "objective": "regression",
117
116
  }
118
- gbm = lgb.train(params,
119
- lgb_train,
120
- num_boost_round=10,
121
- valid_sets=lgb_eval,
122
- valid_names=('validation'),
123
- callbacks=[wandb_callback()])
117
+ gbm = lgb.train(
118
+ params,
119
+ lgb_train,
120
+ num_boost_round=10,
121
+ valid_sets=lgb_eval,
122
+ valid_names=("validation"),
123
+ callbacks=[wandb_callback()],
124
+ )
124
125
  ```
125
126
  """
126
127
  log_params_list: "List[bool]" = [log_params]
@@ -163,7 +164,7 @@ def wandb_callback(log_params: bool = True, define_metric: bool = True) -> Calla
163
164
  def log_summary(
164
165
  model: Booster, feature_importance: bool = True, save_model_checkpoint: bool = False
165
166
  ) -> None:
166
- """Logs useful metrics about lightgbm model after training is done.
167
+ """Log useful metrics about lightgbm model after training is done.
167
168
 
168
169
  Arguments:
169
170
  model: (Booster) is an instance of lightgbm.basic.Booster.
@@ -179,16 +180,17 @@ def log_summary(
179
180
  Example:
180
181
  ```python
181
182
  params = {
182
- 'boosting_type': 'gbdt',
183
- 'objective': 'regression',
184
- .
183
+ "boosting_type": "gbdt",
184
+ "objective": "regression",
185
185
  }
186
- gbm = lgb.train(params,
187
- lgb_train,
188
- num_boost_round=10,
189
- valid_sets=lgb_eval,
190
- valid_names=('validation'),
191
- callbacks=[wandb_callback()])
186
+ gbm = lgb.train(
187
+ params,
188
+ lgb_train,
189
+ num_boost_round=10,
190
+ valid_sets=lgb_eval,
191
+ valid_names=("validation"),
192
+ callbacks=[wandb_callback()],
193
+ )
192
194
 
193
195
  log_summary(gbm)
194
196
  ```
@@ -1,4 +1,4 @@
1
- """Prodigy integration for W&B
1
+ """Prodigy integration for W&B.
2
2
 
3
3
  User can upload Prodigy annotated datasets directly
4
4
  from the local database to W&B in Tables format.
@@ -31,10 +31,10 @@ from wandb.sdk.lib import telemetry as wb_telemetry
31
31
 
32
32
 
33
33
  def named_entity(docs):
34
- """Creates a named entity visualization.
35
- Taken from https://github.com/wandb/wandb/blob/main/wandb/plots/named_entity.py
36
- """
34
+ """Create a named entity visualization.
37
35
 
36
+ Taken from https://github.com/wandb/wandb/blob/main/wandb/plots/named_entity.py.
37
+ """
38
38
  spacy = util.get_module(
39
39
  "spacy",
40
40
  required="part_of_speech requires the spacy library, install with `pip install spacy`",
@@ -68,7 +68,7 @@ def merge(dict1, dict2):
68
68
 
69
69
 
70
70
  def get_schema(list_data_dict, struct, array_dict_types):
71
- """Get a schema of the dataset's structure and data types"""
71
+ """Get a schema of the dataset's structure and data types."""
72
72
  # Get the structure of the JSON objects in the database
73
73
  # This is similar to getting a JSON schema but with slightly different format
74
74
  for _i, item in enumerate(list_data_dict):
@@ -203,7 +203,6 @@ def create_table(data):
203
203
 
204
204
  # Go through each individual row
205
205
  for _i, document in enumerate(matrix):
206
-
207
206
  # Text NER span visualizations
208
207
  if ("spans_visual" in columns) and ("text" in columns):
209
208
  # Add visuals for spans
@@ -270,7 +269,7 @@ def create_table(data):
270
269
 
271
270
 
272
271
  def upload_dataset(dataset_name):
273
- """Uploads dataset from local database to Weights & Biases.
272
+ """Upload dataset from local database to Weights & Biases.
274
273
 
275
274
  Args:
276
275
  dataset_name: The name of the dataset in the Prodigy database.
@@ -8,9 +8,10 @@ import wandb
8
8
 
9
9
 
10
10
  class WandbObserver(RunObserver):
11
- """Logs sacred experiment data to W&B.
11
+ """Log sacred experiment data to W&B.
12
+
12
13
  Arguments:
13
- Accepts all the arguments accepted by wandb.init()
14
+ Accepts all the arguments accepted by wandb.init().
14
15
 
15
16
  name — A display name for this run, which shows up in the UI and is editable, doesn't have to be unique
16
17
  notes — A multiline string description associated with the run
@@ -61,11 +62,8 @@ class WandbObserver(RunObserver):
61
62
  def started_event(
62
63
  self, ex_info, command, host_info, start_time, config, meta_info, _id
63
64
  ):
64
- """
65
- TODO:
66
- * add the source code file
67
- * add dependencies and metadata
68
- """
65
+ # TODO: add the source code file
66
+ # TODO: add dependencies and metadata.
69
67
  self.__update_config(config)
70
68
 
71
69
  def completed_event(self, stop_time, result):
@@ -89,8 +87,9 @@ class WandbObserver(RunObserver):
89
87
  else:
90
88
  warnings.warn(
91
89
  "logging results does not support type '{}' results. Ignoring this result".format(
92
- type(r)
93
- )
90
+ type(r),
91
+ ),
92
+ stacklevel=2,
94
93
  )
95
94
 
96
95
  def artifact_event(self, name, filename, metadata=None, content_type=None):
@@ -101,9 +100,7 @@ class WandbObserver(RunObserver):
101
100
  self.run.log_artifact(artifact)
102
101
 
103
102
  def resource_event(self, filename):
104
- """
105
- TODO: Maintain resources list
106
- """
103
+ """TODO: Maintain resources list."""
107
104
  if filename not in self.resources:
108
105
  md5 = get_digest(filename)
109
106
  self.resources[filename] = md5
@@ -1,6 +1,4 @@
1
- """
2
- wandb integration sagemaker module.
3
- """
1
+ """wandb integration sagemaker module."""
4
2
 
5
3
  from .auth import sagemaker_auth
6
4
  from .config import parse_sm_config
@@ -12,7 +12,6 @@ def sagemaker_auth(overrides=None, path=".", api_key=None):
12
12
  to secrets.env
13
13
  path (str, optional): The path to write the secrets file.
14
14
  """
15
-
16
15
  settings = wandb.setup().settings
17
16
  current_api_key = wandb.wandb_lib.apikey.api_key(settings=settings)
18
17
 
@@ -7,7 +7,7 @@ from . import files as sm_files
7
7
 
8
8
 
9
9
  def parse_sm_config() -> Dict[str, Any]:
10
- """Attempts to parse SageMaker configuration.
10
+ """Attempt to parse SageMaker configuration.
11
11
 
12
12
  Returns:
13
13
  A dictionary of SageMaker config keys/values or empty dict if not found.
@@ -8,7 +8,7 @@ from . import files as sm_files
8
8
 
9
9
 
10
10
  def parse_sm_secrets() -> Dict[str, str]:
11
- """We read our api_key from secrets.env in SageMaker"""
11
+ """We read our api_key from secrets.env in SageMaker."""
12
12
  env_dict = dict()
13
13
  # Set secret variables
14
14
  if os.path.exists(sm_files.SM_SECRETS):
@@ -1,4 +1,4 @@
1
- """W&B callback for sb3
1
+ """W&B callback for sb3.
2
2
 
3
3
  Really simple callback to get logging for each tree
4
4
 
@@ -34,7 +34,9 @@ def make_env():
34
34
 
35
35
 
36
36
  env = DummyVecEnv([make_env])
37
- env = VecVideoRecorder(env, "videos", record_video_trigger=lambda x: x % 2000 == 0, video_length=200)
37
+ env = VecVideoRecorder(
38
+ env, "videos", record_video_trigger=lambda x: x % 2000 == 0, video_length=200
39
+ )
38
40
  model = PPO(config["policy_type"], env, verbose=1, tensorboard_log=f"runs")
39
41
  model.learn(
40
42
  total_timesteps=config["total_timesteps"],
@@ -66,11 +68,13 @@ logger = logging.getLogger(__name__)
66
68
 
67
69
 
68
70
  class WandbCallback(BaseCallback):
69
- """Log SB3 experiments to Weights and Biases
71
+ """Callback for logging experiments to Weights and Biases.
72
+
73
+ Log SB3 experiments to Weights and Biases
70
74
  - Added model tracking and uploading
71
75
  - Added complete hyperparameters recording
72
76
  - Added gradient logging
73
- - Note that `wandb.init(...)` must be called before the WandbCallback can be used
77
+ - Note that `wandb.init(...)` must be called before the WandbCallback can be used.
74
78
 
75
79
  Args:
76
80
  verbose: The verbosity of sb3 output
@@ -1,6 +1,4 @@
1
- """
2
- wandb integration tensorboard module.
3
- """
1
+ """wandb integration tensorboard module."""
4
2
 
5
3
  from .log import _log, log, reset_state, tf_summary_to_dict # noqa: F401
6
4
  from .monkeypatch import patch, unpatch
@@ -62,7 +62,7 @@ def namespaced_tag(tag: str, namespace: str = "") -> str:
62
62
 
63
63
 
64
64
  def history_image_key(key: str, namespace: str = "") -> str:
65
- """Converts invalid filesystem characters to _ for use in History keys.
65
+ """Convert invalid filesystem characters to _ for use in History keys.
66
66
 
67
67
  Unfortunately this means currently certain image keys will collide silently. We
68
68
  implement this mapping up here in the TensorFlow stuff rather than in the History
@@ -75,7 +75,7 @@ def history_image_key(key: str, namespace: str = "") -> str:
75
75
  def tf_summary_to_dict( # noqa: C901
76
76
  tf_summary_str_or_pb: Any, namespace: str = ""
77
77
  ) -> Optional[Dict[str, Any]]:
78
- """Convert a Tensorboard Summary to a dictionary
78
+ """Convert a Tensorboard Summary to a dictionary.
79
79
 
80
80
  Accepts a tensorflow.summary.Summary, one encoded as a string,
81
81
  or a list of such encoded as strings.
@@ -224,13 +224,13 @@ def tf_summary_to_dict( # noqa: C901
224
224
  if len(value.histo.bucket_limit) >= 3:
225
225
  first = (
226
226
  value.histo.bucket_limit[0]
227
- + value.histo.bucket_limit[0] # noqa: W503
228
- - value.histo.bucket_limit[1] # noqa: W503
227
+ + value.histo.bucket_limit[0]
228
+ - value.histo.bucket_limit[1]
229
229
  )
230
230
  last = (
231
231
  value.histo.bucket_limit[-2]
232
- + value.histo.bucket_limit[-2] # noqa: W503
233
- - value.histo.bucket_limit[-3] # noqa: W503
232
+ + value.histo.bucket_limit[-2]
233
+ - value.histo.bucket_limit[-3]
234
234
  )
235
235
  np_histogram = (
236
236
  list(value.histo.bucket),
@@ -272,7 +272,7 @@ def tf_summary_to_dict( # noqa: C901
272
272
 
273
273
 
274
274
  def reset_state() -> None:
275
- """Internal method for resetting state, called by wandb.finish()"""
275
+ """Internal method for resetting state, called by wandb.finish()."""
276
276
  global STEPS
277
277
  STEPS = {"": {"step": 0}, "global": {"step": 0, "last_log": None}}
278
278
 
@@ -284,7 +284,7 @@ def _log(
284
284
  namespace: str = "",
285
285
  **kwargs: Any,
286
286
  ) -> None:
287
- """Logs a tfsummary to wandb
287
+ """Logs a tfsummary to wandb.
288
288
 
289
289
  Can accept a tf summary string or parsed event. Will use wandb.run.history unless a
290
290
  history object is passed. Can optionally namespace events. Results are committed
@@ -1,4 +1,4 @@
1
- """monkeypatch: patch code to add tensorboard hooks"""
1
+ """monkeypatch: patch code to add tensorboard hooks."""
2
2
 
3
3
  import os
4
4
  import re
@@ -17,7 +17,7 @@ TENSORBOARD_PYTORCH_MODULE = "torch.utils.tensorboard.writer"
17
17
 
18
18
  def unpatch() -> None:
19
19
  for module, method in wandb.patched["tensorboard"]:
20
- writer = wandb.util.get_module(module)
20
+ writer = wandb.util.get_module(module, lazy=False)
21
21
  setattr(writer, method, getattr(writer, f"orig_{method}"))
22
22
  wandb.patched["tensorboard"] = []
23
23
 
@@ -36,12 +36,14 @@ def patch(
36
36
 
37
37
  # TODO: Some older versions of tensorflow don't require tensorboard to be present.
38
38
  # we may want to lift this requirement, but it's safer to have it for now
39
- wandb.util.get_module("tensorboard", required="Please install tensorboard package")
40
- c_writer = wandb.util.get_module(TENSORBOARD_C_MODULE)
41
- py_writer = wandb.util.get_module(TENSORFLOW_PY_MODULE)
42
- tb_writer = wandb.util.get_module(TENSORBOARD_WRITER_MODULE)
43
- pt_writer = wandb.util.get_module(TENSORBOARD_PYTORCH_MODULE)
44
- tbx_writer = wandb.util.get_module(TENSORBOARD_X_MODULE)
39
+ wandb.util.get_module(
40
+ "tensorboard", required="Please install tensorboard package", lazy=False
41
+ )
42
+ c_writer = wandb.util.get_module(TENSORBOARD_C_MODULE, lazy=False)
43
+ py_writer = wandb.util.get_module(TENSORFLOW_PY_MODULE, lazy=False)
44
+ tb_writer = wandb.util.get_module(TENSORBOARD_WRITER_MODULE, lazy=False)
45
+ pt_writer = wandb.util.get_module(TENSORBOARD_PYTORCH_MODULE, lazy=False)
46
+ tbx_writer = wandb.util.get_module(TENSORBOARD_X_MODULE, lazy=False)
45
47
 
46
48
  if not pytorch and not tensorboard_x and c_writer:
47
49
  _patch_tensorflow2(
@@ -140,7 +142,7 @@ def _patch_file_writer(
140
142
  # This configures non-TensorFlow Tensorboard logging, or tensorflow <= 1.15
141
143
  logdir_hist = []
142
144
 
143
- class TBXEventFileWriter(writer.EventFileWriter): # type: ignore
145
+ class TBXEventFileWriter(writer.EventFileWriter):
144
146
  def __init__(self, logdir: str, *args: Any, **kwargs: Any) -> None:
145
147
  logdir_hist.append(logdir)
146
148
  root_logdir_arg = root_logdir
@@ -1,6 +1,4 @@
1
- """
2
- api.
3
- """
1
+ """api."""
4
2
 
5
3
  from wandb.integration.tensorboard import log # noqa: F401
6
4
 
@@ -1,11 +1,9 @@
1
- # """W&B callback for xgboost
1
+ """W&B callback for xgboost.
2
2
 
3
- # Really simple callback to get logging for each tree
3
+ Simple callback to get logging for each tree
4
4
 
5
- # Example usage:
6
- """
7
- Use the `wandb_callback` to add `wandb` logging to any `XGboost` model. However, it will be deprecated in favor of
8
- WandbCallback. Use it instead for more features.
5
+ Use the `wandb_callback` to add `wandb` logging to any `XGboost` model. However, it will
6
+ be deprecated in favor of WandbCallback. Use it instead for more features.
9
7
  """
10
8
 
11
9
  from .xgboost import WandbCallback, wandb_callback