wandb 0.13.10__py3-none-any.whl → 0.14.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- wandb/__init__.py +2 -3
- wandb/apis/__init__.py +1 -3
- wandb/apis/importers/__init__.py +4 -0
- wandb/apis/importers/base.py +312 -0
- wandb/apis/importers/mlflow.py +113 -0
- wandb/apis/internal.py +29 -2
- wandb/apis/normalize.py +6 -5
- wandb/apis/public.py +163 -180
- wandb/apis/reports/_templates.py +6 -12
- wandb/apis/reports/report.py +1 -1
- wandb/apis/reports/runset.py +1 -3
- wandb/apis/reports/util.py +12 -10
- wandb/beta/workflows.py +57 -34
- wandb/catboost/__init__.py +1 -2
- wandb/cli/cli.py +215 -133
- wandb/data_types.py +63 -56
- wandb/docker/__init__.py +78 -16
- wandb/docker/auth.py +21 -22
- wandb/env.py +0 -1
- wandb/errors/__init__.py +8 -116
- wandb/errors/term.py +1 -1
- wandb/fastai/__init__.py +1 -2
- wandb/filesync/dir_watcher.py +8 -5
- wandb/filesync/step_prepare.py +76 -75
- wandb/filesync/step_upload.py +1 -2
- wandb/integration/catboost/__init__.py +1 -3
- wandb/integration/catboost/catboost.py +8 -14
- wandb/integration/fastai/__init__.py +7 -13
- wandb/integration/gym/__init__.py +35 -4
- wandb/integration/keras/__init__.py +3 -3
- wandb/integration/keras/callbacks/metrics_logger.py +9 -8
- wandb/integration/keras/callbacks/model_checkpoint.py +9 -9
- wandb/integration/keras/callbacks/tables_builder.py +31 -19
- wandb/integration/kfp/kfp_patch.py +20 -17
- wandb/integration/kfp/wandb_logging.py +1 -2
- wandb/integration/lightgbm/__init__.py +21 -19
- wandb/integration/prodigy/prodigy.py +6 -7
- wandb/integration/sacred/__init__.py +9 -12
- wandb/integration/sagemaker/__init__.py +1 -3
- wandb/integration/sagemaker/auth.py +0 -1
- wandb/integration/sagemaker/config.py +1 -1
- wandb/integration/sagemaker/resources.py +1 -1
- wandb/integration/sb3/sb3.py +8 -4
- wandb/integration/tensorboard/__init__.py +1 -3
- wandb/integration/tensorboard/log.py +8 -8
- wandb/integration/tensorboard/monkeypatch.py +11 -9
- wandb/integration/tensorflow/__init__.py +1 -3
- wandb/integration/xgboost/__init__.py +4 -6
- wandb/integration/yolov8/__init__.py +7 -0
- wandb/integration/yolov8/yolov8.py +250 -0
- wandb/jupyter.py +31 -35
- wandb/lightgbm/__init__.py +1 -2
- wandb/old/settings.py +2 -2
- wandb/plot/bar.py +1 -2
- wandb/plot/confusion_matrix.py +1 -3
- wandb/plot/histogram.py +1 -2
- wandb/plot/line.py +1 -2
- wandb/plot/line_series.py +4 -4
- wandb/plot/pr_curve.py +17 -20
- wandb/plot/roc_curve.py +1 -3
- wandb/plot/scatter.py +1 -2
- wandb/proto/v3/wandb_server_pb2.py +85 -39
- wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
- wandb/proto/v4/wandb_server_pb2.py +51 -39
- wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
- wandb/sdk/__init__.py +1 -3
- wandb/sdk/backend/backend.py +1 -1
- wandb/sdk/data_types/_dtypes.py +38 -30
- wandb/sdk/data_types/base_types/json_metadata.py +1 -3
- wandb/sdk/data_types/base_types/media.py +17 -17
- wandb/sdk/data_types/base_types/wb_value.py +33 -26
- wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +91 -125
- wandb/sdk/data_types/helper_types/classes.py +1 -1
- wandb/sdk/data_types/helper_types/image_mask.py +12 -12
- wandb/sdk/data_types/histogram.py +5 -4
- wandb/sdk/data_types/html.py +1 -2
- wandb/sdk/data_types/image.py +11 -11
- wandb/sdk/data_types/molecule.py +3 -6
- wandb/sdk/data_types/object_3d.py +1 -2
- wandb/sdk/data_types/plotly.py +1 -2
- wandb/sdk/data_types/saved_model.py +10 -8
- wandb/sdk/data_types/video.py +1 -1
- wandb/sdk/integration_utils/data_logging.py +5 -5
- wandb/sdk/interface/artifacts.py +288 -266
- wandb/sdk/interface/interface.py +2 -3
- wandb/sdk/interface/interface_grpc.py +1 -1
- wandb/sdk/interface/interface_queue.py +1 -1
- wandb/sdk/interface/interface_relay.py +1 -1
- wandb/sdk/interface/interface_shared.py +1 -2
- wandb/sdk/interface/interface_sock.py +1 -1
- wandb/sdk/interface/message_future.py +1 -1
- wandb/sdk/interface/message_future_poll.py +1 -1
- wandb/sdk/interface/router.py +1 -1
- wandb/sdk/interface/router_queue.py +1 -1
- wandb/sdk/interface/router_relay.py +1 -1
- wandb/sdk/interface/router_sock.py +1 -1
- wandb/sdk/interface/summary_record.py +1 -1
- wandb/sdk/internal/artifacts.py +1 -1
- wandb/sdk/internal/datastore.py +2 -3
- wandb/sdk/internal/file_pusher.py +5 -3
- wandb/sdk/internal/file_stream.py +22 -19
- wandb/sdk/internal/handler.py +5 -4
- wandb/sdk/internal/internal.py +1 -1
- wandb/sdk/internal/internal_api.py +115 -55
- wandb/sdk/internal/job_builder.py +1 -3
- wandb/sdk/internal/profiler.py +1 -1
- wandb/sdk/internal/progress.py +4 -6
- wandb/sdk/internal/sample.py +1 -3
- wandb/sdk/internal/sender.py +28 -16
- wandb/sdk/internal/settings_static.py +5 -5
- wandb/sdk/internal/system/assets/__init__.py +1 -0
- wandb/sdk/internal/system/assets/cpu.py +3 -9
- wandb/sdk/internal/system/assets/disk.py +2 -4
- wandb/sdk/internal/system/assets/gpu.py +6 -18
- wandb/sdk/internal/system/assets/gpu_apple.py +2 -4
- wandb/sdk/internal/system/assets/interfaces.py +50 -22
- wandb/sdk/internal/system/assets/ipu.py +1 -3
- wandb/sdk/internal/system/assets/memory.py +7 -13
- wandb/sdk/internal/system/assets/network.py +4 -8
- wandb/sdk/internal/system/assets/open_metrics.py +283 -0
- wandb/sdk/internal/system/assets/tpu.py +1 -4
- wandb/sdk/internal/system/assets/trainium.py +26 -14
- wandb/sdk/internal/system/system_info.py +2 -3
- wandb/sdk/internal/system/system_monitor.py +52 -20
- wandb/sdk/internal/tb_watcher.py +12 -13
- wandb/sdk/launch/_project_spec.py +54 -65
- wandb/sdk/launch/agent/agent.py +374 -90
- wandb/sdk/launch/builder/abstract.py +61 -7
- wandb/sdk/launch/builder/build.py +81 -110
- wandb/sdk/launch/builder/docker_builder.py +181 -0
- wandb/sdk/launch/builder/kaniko_builder.py +419 -0
- wandb/sdk/launch/builder/noop.py +31 -12
- wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +70 -20
- wandb/sdk/launch/environment/abstract.py +28 -0
- wandb/sdk/launch/environment/aws_environment.py +276 -0
- wandb/sdk/launch/environment/gcp_environment.py +271 -0
- wandb/sdk/launch/environment/local_environment.py +65 -0
- wandb/sdk/launch/github_reference.py +3 -8
- wandb/sdk/launch/launch.py +38 -29
- wandb/sdk/launch/launch_add.py +6 -8
- wandb/sdk/launch/loader.py +230 -0
- wandb/sdk/launch/registry/abstract.py +54 -0
- wandb/sdk/launch/registry/elastic_container_registry.py +163 -0
- wandb/sdk/launch/registry/google_artifact_registry.py +203 -0
- wandb/sdk/launch/registry/local_registry.py +62 -0
- wandb/sdk/launch/runner/abstract.py +1 -16
- wandb/sdk/launch/runner/{kubernetes.py → kubernetes_runner.py} +83 -95
- wandb/sdk/launch/runner/local_container.py +46 -22
- wandb/sdk/launch/runner/local_process.py +1 -4
- wandb/sdk/launch/runner/{aws.py → sagemaker_runner.py} +53 -212
- wandb/sdk/launch/runner/{gcp_vertex.py → vertex_runner.py} +38 -55
- wandb/sdk/launch/sweeps/__init__.py +3 -2
- wandb/sdk/launch/sweeps/scheduler.py +132 -39
- wandb/sdk/launch/sweeps/scheduler_sweep.py +80 -89
- wandb/sdk/launch/utils.py +101 -30
- wandb/sdk/launch/wandb_reference.py +2 -7
- wandb/sdk/lib/_settings_toposort_generate.py +166 -0
- wandb/sdk/lib/_settings_toposort_generated.py +201 -0
- wandb/sdk/lib/apikey.py +2 -4
- wandb/sdk/lib/config_util.py +4 -1
- wandb/sdk/lib/console.py +1 -3
- wandb/sdk/lib/deprecate.py +3 -3
- wandb/sdk/lib/file_stream_utils.py +7 -5
- wandb/sdk/lib/filenames.py +1 -1
- wandb/sdk/lib/filesystem.py +61 -5
- wandb/sdk/lib/git.py +1 -3
- wandb/sdk/lib/import_hooks.py +4 -7
- wandb/sdk/lib/ipython.py +8 -5
- wandb/sdk/lib/lazyloader.py +1 -3
- wandb/sdk/lib/mailbox.py +14 -4
- wandb/sdk/lib/proto_util.py +10 -5
- wandb/sdk/lib/redirect.py +15 -22
- wandb/sdk/lib/reporting.py +1 -3
- wandb/sdk/lib/retry.py +4 -5
- wandb/sdk/lib/runid.py +1 -3
- wandb/sdk/lib/server.py +15 -9
- wandb/sdk/lib/sock_client.py +1 -1
- wandb/sdk/lib/sparkline.py +1 -1
- wandb/sdk/lib/wburls.py +1 -1
- wandb/sdk/service/port_file.py +1 -2
- wandb/sdk/service/service.py +36 -13
- wandb/sdk/service/service_base.py +12 -1
- wandb/sdk/verify/verify.py +5 -7
- wandb/sdk/wandb_artifacts.py +142 -177
- wandb/sdk/wandb_config.py +5 -8
- wandb/sdk/wandb_helper.py +1 -1
- wandb/sdk/wandb_init.py +24 -13
- wandb/sdk/wandb_login.py +9 -9
- wandb/sdk/wandb_manager.py +39 -4
- wandb/sdk/wandb_metric.py +2 -6
- wandb/sdk/wandb_require.py +4 -15
- wandb/sdk/wandb_require_helpers.py +1 -9
- wandb/sdk/wandb_run.py +95 -141
- wandb/sdk/wandb_save.py +1 -3
- wandb/sdk/wandb_settings.py +149 -54
- wandb/sdk/wandb_setup.py +66 -46
- wandb/sdk/wandb_summary.py +13 -10
- wandb/sdk/wandb_sweep.py +6 -7
- wandb/sdk/wandb_watch.py +1 -1
- wandb/sklearn/calculate/confusion_matrix.py +1 -1
- wandb/sklearn/calculate/learning_curve.py +1 -1
- wandb/sklearn/calculate/summary_metrics.py +1 -3
- wandb/sklearn/plot/__init__.py +1 -1
- wandb/sklearn/plot/classifier.py +27 -18
- wandb/sklearn/plot/clusterer.py +4 -5
- wandb/sklearn/plot/regressor.py +4 -4
- wandb/sklearn/plot/shared.py +2 -2
- wandb/sync/__init__.py +1 -3
- wandb/sync/sync.py +4 -5
- wandb/testing/relay.py +11 -10
- wandb/trigger.py +1 -1
- wandb/util.py +106 -81
- wandb/viz.py +4 -4
- wandb/wandb_agent.py +50 -50
- wandb/wandb_controller.py +2 -3
- wandb/wandb_run.py +1 -2
- wandb/wandb_torch.py +1 -1
- wandb/xgboost/__init__.py +1 -2
- {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/METADATA +6 -2
- {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/RECORD +224 -209
- {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/WHEEL +1 -1
- wandb/sdk/launch/builder/docker.py +0 -80
- wandb/sdk/launch/builder/kaniko.py +0 -393
- wandb/sdk/launch/builder/loader.py +0 -32
- wandb/sdk/launch/runner/loader.py +0 -50
- {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/LICENSE +0 -0
- {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/entry_points.txt +0 -0
- {wandb-0.13.10.dist-info → wandb-0.14.0.dist-info}/top_level.txt +0 -0
wandb/sklearn/plot/shared.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
"""
|
1
|
+
"""Define plots used by multiple sklearn model classes."""
|
2
2
|
from warnings import simplefilter
|
3
3
|
|
4
4
|
import numpy as np
|
@@ -73,7 +73,7 @@ def learning_curve(
|
|
73
73
|
|
74
74
|
Example:
|
75
75
|
```python
|
76
|
-
|
76
|
+
wandb.sklearn.plot_learning_curve(model, X, y)
|
77
77
|
```
|
78
78
|
"""
|
79
79
|
not_missing = utils.test_missing(model=model, X=X, y=y)
|
wandb/sync/__init__.py
CHANGED
wandb/sync/sync.py
CHANGED
@@ -1,6 +1,4 @@
|
|
1
|
-
"""
|
2
|
-
sync.
|
3
|
-
"""
|
1
|
+
"""sync."""
|
4
2
|
|
5
3
|
import datetime
|
6
4
|
import fnmatch
|
@@ -121,7 +119,7 @@ class SyncThread(threading.Thread):
|
|
121
119
|
return tb_event_files, tb_logdirs, tb_root
|
122
120
|
|
123
121
|
def _setup_tensorboard(self, tb_root, tb_logdirs, tb_event_files, sync_item):
|
124
|
-
"""
|
122
|
+
"""Return true if this sync item can be synced as tensorboard."""
|
125
123
|
if tb_root is not None:
|
126
124
|
if tb_event_files > 0 and sync_item.endswith(WANDB_SUFFIX):
|
127
125
|
wandb.termwarn("Found .wandb file, not streaming tensorboard metrics.")
|
@@ -144,6 +142,7 @@ class SyncThread(threading.Thread):
|
|
144
142
|
proto_run.run_id = self._run_id or wandb.util.generate_id()
|
145
143
|
proto_run.project = self._project or wandb.util.auto_project_name(None)
|
146
144
|
proto_run.entity = self._entity
|
145
|
+
proto_run.telemetry.feature.sync_tfevents = True
|
147
146
|
|
148
147
|
url = "{}/{}/{}/runs/{}".format(
|
149
148
|
self._app_url,
|
@@ -218,7 +217,7 @@ class SyncThread(threading.Thread):
|
|
218
217
|
send_manager.finish()
|
219
218
|
|
220
219
|
def _robust_scan(self, ds):
|
221
|
-
"""Attempt to scan data, handling incomplete files"""
|
220
|
+
"""Attempt to scan data, handling incomplete files."""
|
222
221
|
try:
|
223
222
|
return ds.scan_data()
|
224
223
|
except AssertionError as e:
|
wandb/testing/relay.py
CHANGED
@@ -90,10 +90,10 @@ class Timer:
|
|
90
90
|
|
91
91
|
|
92
92
|
class Context:
|
93
|
-
"""
|
94
|
-
|
95
|
-
|
96
|
-
|
93
|
+
"""A container used to store the snooped state/data of a test.
|
94
|
+
|
95
|
+
Includes raw requests and responses, parsed and processed data, and a number of
|
96
|
+
convenience methods and properties for accessing the data.
|
97
97
|
"""
|
98
98
|
|
99
99
|
def __init__(self) -> None:
|
@@ -242,9 +242,9 @@ class Context:
|
|
242
242
|
|
243
243
|
|
244
244
|
class QueryResolver:
|
245
|
-
"""
|
246
|
-
|
247
|
-
|
245
|
+
"""Resolve request/response pairs against a set of known patterns.
|
246
|
+
|
247
|
+
This extracts and processes useful data to be later stored in a Context object.
|
248
248
|
"""
|
249
249
|
|
250
250
|
def __init__(self):
|
@@ -468,9 +468,10 @@ class InjectedResponse:
|
|
468
468
|
self,
|
469
469
|
other: Union["InjectedResponse", requests.Request, requests.PreparedRequest],
|
470
470
|
):
|
471
|
-
"""
|
472
|
-
|
473
|
-
We use this to check if this response should be injected as a replacement of
|
471
|
+
"""Check InjectedResponse object equality.
|
472
|
+
|
473
|
+
We use this to check if this response should be injected as a replacement of
|
474
|
+
`other`.
|
474
475
|
|
475
476
|
:param other:
|
476
477
|
:return:
|
wandb/trigger.py
CHANGED
wandb/util.py
CHANGED
@@ -202,7 +202,7 @@ def sentry_exc(
|
|
202
202
|
|
203
203
|
|
204
204
|
def sentry_reraise(exc: Any, delay: bool = False) -> None:
|
205
|
-
"""Re-raise an exception after logging it to Sentry
|
205
|
+
"""Re-raise an exception after logging it to Sentry.
|
206
206
|
|
207
207
|
Use this for top-level exceptions when you want the user to see the traceback.
|
208
208
|
|
@@ -238,7 +238,8 @@ def sentry_set_scope(
|
|
238
238
|
"sweep_url",
|
239
239
|
"sweep_id",
|
240
240
|
"deployment",
|
241
|
-
"
|
241
|
+
"_disable_service",
|
242
|
+
"launch",
|
242
243
|
]
|
243
244
|
|
244
245
|
s = settings_dict
|
@@ -293,12 +294,19 @@ def sentry_set_scope(
|
|
293
294
|
|
294
295
|
|
295
296
|
def vendor_setup() -> Callable:
|
296
|
-
"""
|
297
|
-
|
298
|
-
|
297
|
+
"""Create a function that restores user paths after vendor imports.
|
298
|
+
|
299
|
+
This enables us to use the vendor directory for packages we don't depend on. Call
|
300
|
+
the returned function after imports are complete. If you don't you may modify the
|
301
|
+
user's path which is never good.
|
302
|
+
|
303
|
+
Usage:
|
304
|
+
|
305
|
+
```python
|
299
306
|
reset_path = vendor_setup()
|
300
307
|
# do any vendor imports...
|
301
308
|
reset_path()
|
309
|
+
```
|
302
310
|
"""
|
303
311
|
original_path = [directory for directory in sys.path]
|
304
312
|
|
@@ -329,8 +337,7 @@ def vendor_import(name: str) -> Any:
|
|
329
337
|
|
330
338
|
|
331
339
|
def import_module_lazy(name: str) -> Any:
|
332
|
-
"""
|
333
|
-
Import a module lazily, only when it is used.
|
340
|
+
"""Import a module lazily, only when it is used.
|
334
341
|
|
335
342
|
:param (str) name: Dot-separated module path. E.g., 'scipy.stats'.
|
336
343
|
"""
|
@@ -356,8 +363,7 @@ def get_module(
|
|
356
363
|
required: Optional[Union[str, bool]] = None,
|
357
364
|
lazy: bool = True,
|
358
365
|
) -> Any:
|
359
|
-
"""
|
360
|
-
Return module or None. Absolute import is required.
|
366
|
+
"""Return module or None. Absolute import is required.
|
361
367
|
|
362
368
|
:param (str) name: Dot-separated module path. E.g., 'scipy.stats'.
|
363
369
|
:param (str) required: A string to raise a ValueError if missing
|
@@ -390,7 +396,7 @@ VALUE_BYTES_LIMIT = 100000
|
|
390
396
|
|
391
397
|
|
392
398
|
def app_url(api_url: str) -> str:
|
393
|
-
"""
|
399
|
+
"""Return the frontend app url without a trailing slash."""
|
394
400
|
# TODO: move me to settings
|
395
401
|
app_url = get_app_url()
|
396
402
|
if app_url is not None:
|
@@ -409,8 +415,9 @@ def app_url(api_url: str) -> str:
|
|
409
415
|
|
410
416
|
|
411
417
|
def get_full_typename(o: Any) -> Any:
|
412
|
-
"""
|
413
|
-
|
418
|
+
"""Determine types based on type names.
|
419
|
+
|
420
|
+
Avoids needing to to import (and therefore depend on) PyTorch, TensorFlow, etc.
|
414
421
|
"""
|
415
422
|
instance_name = o.__class__.__module__ + "." + o.__class__.__name__
|
416
423
|
if instance_name in ["builtins.module", "__builtin__.module"]:
|
@@ -435,8 +442,8 @@ def is_uri(string: str) -> bool:
|
|
435
442
|
|
436
443
|
|
437
444
|
def local_file_uri_to_path(uri: str) -> str:
|
438
|
-
"""
|
439
|
-
|
445
|
+
"""Convert URI to local filesystem path.
|
446
|
+
|
440
447
|
No-op if the uri does not have the expected scheme.
|
441
448
|
"""
|
442
449
|
path = urllib.parse.urlparse(uri).path if uri.startswith("file:") else uri
|
@@ -444,8 +451,10 @@ def local_file_uri_to_path(uri: str) -> str:
|
|
444
451
|
|
445
452
|
|
446
453
|
def get_local_path_or_none(path_or_uri: str) -> Optional[str]:
|
447
|
-
"""
|
448
|
-
|
454
|
+
"""Return path if local, None otherwise.
|
455
|
+
|
456
|
+
Return None if the argument is a local path (not a scheme or file:///). Otherwise
|
457
|
+
return `path_or_uri`.
|
449
458
|
"""
|
450
459
|
parsed_uri = urllib.parse.urlparse(path_or_uri)
|
451
460
|
if (
|
@@ -581,8 +590,10 @@ def is_pandas_data_frame(obj: Any) -> bool:
|
|
581
590
|
|
582
591
|
|
583
592
|
def ensure_matplotlib_figure(obj: Any) -> Any:
|
584
|
-
"""Extract the current figure from a matplotlib object
|
585
|
-
|
593
|
+
"""Extract the current figure from a matplotlib object.
|
594
|
+
|
595
|
+
Return the object itself if it's a figure.
|
596
|
+
Raises ValueError if the object can't be converted.
|
586
597
|
"""
|
587
598
|
import matplotlib # type: ignore
|
588
599
|
from matplotlib.figure import Figure # type: ignore
|
@@ -593,6 +604,7 @@ def ensure_matplotlib_figure(obj: Any) -> Any:
|
|
593
604
|
|
594
605
|
def is_frame_like(self: Any) -> bool:
|
595
606
|
"""Return True if directly on axes frame.
|
607
|
+
|
596
608
|
This is useful for determining if a spine is the edge of an
|
597
609
|
old style MPL plot. If so, this function will return True.
|
598
610
|
"""
|
@@ -671,10 +683,10 @@ def _find_all_matching_keys(
|
|
671
683
|
match_fn: The function to determine if the key is a match.
|
672
684
|
visited: Keep track of visited nodes so we dont recurse forever.
|
673
685
|
key_path: Keep track of all the keys to get to the current node.
|
686
|
+
|
674
687
|
Yields:
|
675
688
|
(key_path, key): The location where the key was found, and the key
|
676
689
|
"""
|
677
|
-
|
678
690
|
if visited is None:
|
679
691
|
visited = set()
|
680
692
|
me = id(d)
|
@@ -706,8 +718,8 @@ def _sanitize_numpy_keys(d: Dict) -> Tuple[Dict, bool]:
|
|
706
718
|
|
707
719
|
def json_friendly( # noqa: C901
|
708
720
|
obj: Any,
|
709
|
-
) -> Union[Tuple[Any, bool], Tuple[Union[None, str, float], bool]]:
|
710
|
-
"""Convert an object into something that's more becoming of JSON"""
|
721
|
+
) -> Union[Tuple[Any, bool], Tuple[Union[None, str, float], bool]]:
|
722
|
+
"""Convert an object into something that's more becoming of JSON."""
|
711
723
|
converted = True
|
712
724
|
typename = get_full_typename(obj)
|
713
725
|
|
@@ -770,7 +782,7 @@ def json_friendly( # noqa: C901
|
|
770
782
|
|
771
783
|
|
772
784
|
def json_friendly_val(val: Any) -> Any:
|
773
|
-
"""Make any value (including dict, slice, sequence, etc) JSON friendly"""
|
785
|
+
"""Make any value (including dict, slice, sequence, etc) JSON friendly."""
|
774
786
|
converted: Union[dict, list]
|
775
787
|
if isinstance(val, dict):
|
776
788
|
converted = {}
|
@@ -844,7 +856,7 @@ def maybe_compress_summary(obj: Any, h5_typename: str) -> Tuple[Any, bool]:
|
|
844
856
|
|
845
857
|
|
846
858
|
def launch_browser(attempt_launch_browser: bool = True) -> bool:
|
847
|
-
"""Decide if we should launch a browser"""
|
859
|
+
"""Decide if we should launch a browser."""
|
848
860
|
_display_variables = ["DISPLAY", "WAYLAND_DISPLAY", "MIR_SOCKET"]
|
849
861
|
_webbrowser_names_blocklist = ["www-browser", "lynx", "links", "elinks", "w3m"]
|
850
862
|
|
@@ -873,7 +885,7 @@ def generate_id(length: int = 8) -> str:
|
|
873
885
|
|
874
886
|
|
875
887
|
def parse_tfjob_config() -> Any:
|
876
|
-
"""
|
888
|
+
"""Attempt to parse TFJob config, returning False if it can't find it."""
|
877
889
|
if os.getenv("TF_CONFIG"):
|
878
890
|
try:
|
879
891
|
return json.loads(os.environ["TF_CONFIG"])
|
@@ -910,7 +922,9 @@ class WandBJSONEncoderOld(json.JSONEncoder):
|
|
910
922
|
|
911
923
|
class WandBHistoryJSONEncoder(json.JSONEncoder):
|
912
924
|
"""A JSON Encoder that handles some extra types.
|
913
|
-
|
925
|
+
|
926
|
+
This encoder turns numpy like objects with a size > 32 into histograms.
|
927
|
+
"""
|
914
928
|
|
915
929
|
def default(self, obj: Any) -> Any:
|
916
930
|
obj, converted = json_friendly(obj)
|
@@ -922,7 +936,9 @@ class WandBHistoryJSONEncoder(json.JSONEncoder):
|
|
922
936
|
|
923
937
|
class JSONEncoderUncompressed(json.JSONEncoder):
|
924
938
|
"""A JSON Encoder that handles some extra types.
|
925
|
-
|
939
|
+
|
940
|
+
This encoder turns numpy like objects with a size > 32 into histograms.
|
941
|
+
"""
|
926
942
|
|
927
943
|
def default(self, obj: Any) -> Any:
|
928
944
|
if is_numpy_array(obj):
|
@@ -949,7 +965,7 @@ def json_dump_uncompressed(obj: Any, fp: IO[str], **kwargs: Any) -> None:
|
|
949
965
|
|
950
966
|
|
951
967
|
def json_dumps_safer_history(obj: Any, **kwargs: Any) -> str:
|
952
|
-
"""Convert obj to json, with some extra encodable types, including histograms"""
|
968
|
+
"""Convert obj to json, with some extra encodable types, including histograms."""
|
953
969
|
return json.dumps(obj, cls=WandBHistoryJSONEncoder, **kwargs)
|
954
970
|
|
955
971
|
|
@@ -1006,11 +1022,21 @@ def no_retry_auth(e: Any) -> bool:
|
|
1006
1022
|
return True
|
1007
1023
|
# Crash w/message on forbidden/unauthorized errors.
|
1008
1024
|
if e.response.status_code == 401:
|
1009
|
-
raise CommError(
|
1025
|
+
raise CommError(
|
1026
|
+
"The API key is either invalid or missing, or the host is incorrect. "
|
1027
|
+
"To resolve this issue, you may try running the 'wandb login --host [hostname]' command. "
|
1028
|
+
"The host defaults to 'https://api.wandb.ai' if not specified. "
|
1029
|
+
f"(Error {e.response.status_code}: {e.response.reason})"
|
1030
|
+
)
|
1010
1031
|
elif wandb.run:
|
1011
1032
|
raise CommError(f"Permission denied to access {wandb.run.path}")
|
1012
1033
|
else:
|
1013
|
-
raise CommError(
|
1034
|
+
raise CommError(
|
1035
|
+
"It appears that you do not have permission to access the requested resource. "
|
1036
|
+
"Please reach out to the project owner to grant you access. "
|
1037
|
+
"If you have the correct permissions, verify that there are no issues with your networking setup."
|
1038
|
+
f"(Error {e.response.status_code}: {e.response.reason})"
|
1039
|
+
)
|
1014
1040
|
|
1015
1041
|
|
1016
1042
|
def check_retry_conflict(e: Any) -> Optional[bool]:
|
@@ -1078,6 +1104,7 @@ def find_runner(program: str) -> Union[None, list, List[str]]:
|
|
1078
1104
|
|
1079
1105
|
Arguments:
|
1080
1106
|
program: The string name of the program to try to run.
|
1107
|
+
|
1081
1108
|
Returns:
|
1082
1109
|
commandline list of strings to run the program (eg. with subprocess.call()) or None
|
1083
1110
|
"""
|
@@ -1096,7 +1123,7 @@ def find_runner(program: str) -> Union[None, list, List[str]]:
|
|
1096
1123
|
|
1097
1124
|
|
1098
1125
|
def downsample(values: Sequence, target_length: int) -> list:
|
1099
|
-
"""
|
1126
|
+
"""Downsample 1d values to target_length, including start and end.
|
1100
1127
|
|
1101
1128
|
Algorithm just rounds index down.
|
1102
1129
|
|
@@ -1131,7 +1158,7 @@ def get_log_file_path() -> str:
|
|
1131
1158
|
|
1132
1159
|
|
1133
1160
|
def docker_image_regex(image: str) -> Any:
|
1134
|
-
"""
|
1161
|
+
"""Regex match for valid docker image names."""
|
1135
1162
|
if image:
|
1136
1163
|
return re.match(
|
1137
1164
|
r"^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(?<!-)(?:\.(?!-)[a-zA-Z0-9-]{1,63}(?<!-))*(?::[0-9]{1,5})?/)?((?![._-])(?:[a-z0-9._-]*)(?<![._-])(?:/(?![._-])[a-z0-9._-]*(?<![._-]))*)(?::(?![.-])[a-zA-Z0-9_.-]{1,128})?$",
|
@@ -1141,9 +1168,11 @@ def docker_image_regex(image: str) -> Any:
|
|
1141
1168
|
|
1142
1169
|
|
1143
1170
|
def image_from_docker_args(args: List[str]) -> Optional[str]:
|
1144
|
-
"""
|
1145
|
-
|
1146
|
-
|
1171
|
+
"""Scan docker run args and attempt to find the most likely docker image argument.
|
1172
|
+
|
1173
|
+
It excludes any arguments that start with a dash, and the argument after it if it
|
1174
|
+
isn't a boolean switch. This can be improved, we currently fallback gracefully when
|
1175
|
+
this fails.
|
1147
1176
|
"""
|
1148
1177
|
bool_args = [
|
1149
1178
|
"-t",
|
@@ -1196,12 +1225,12 @@ def load_yaml(file: Any) -> Any:
|
|
1196
1225
|
|
1197
1226
|
|
1198
1227
|
def image_id_from_k8s() -> Optional[str]:
|
1199
|
-
"""
|
1200
|
-
|
1201
|
-
the
|
1228
|
+
"""Ping the k8s metadata service for the image id.
|
1229
|
+
|
1230
|
+
Specify the KUBERNETES_NAMESPACE environment variable if your pods are not in the
|
1231
|
+
default namespace:
|
1202
1232
|
|
1203
|
-
- name: KUBERNETES_NAMESPACE
|
1204
|
-
valueFrom:
|
1233
|
+
- name: KUBERNETES_NAMESPACE valueFrom:
|
1205
1234
|
fieldRef:
|
1206
1235
|
fieldPath: metadata.namespace
|
1207
1236
|
"""
|
@@ -1233,13 +1262,16 @@ def image_id_from_k8s() -> Optional[str]:
|
|
1233
1262
|
return None
|
1234
1263
|
|
1235
1264
|
|
1236
|
-
def async_call(
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1240
|
-
|
1241
|
-
|
1242
|
-
|
1265
|
+
def async_call(
|
1266
|
+
target: Callable, timeout: Optional[Union[int, float]] = None
|
1267
|
+
) -> Callable:
|
1268
|
+
"""Wrap a method to run in the background with an optional timeout.
|
1269
|
+
|
1270
|
+
Returns a new method that will call the original with any args, waiting for upto
|
1271
|
+
timeout seconds. This new method blocks on the original and returns the result or
|
1272
|
+
None if timeout was reached, along with the thread. You can check thread.is_alive()
|
1273
|
+
to determine if a timeout was reached. If an exception is thrown in the thread, we
|
1274
|
+
reraise it.
|
1243
1275
|
"""
|
1244
1276
|
q: "queue.Queue" = queue.Queue()
|
1245
1277
|
|
@@ -1286,7 +1318,7 @@ def read_many_from_queue(
|
|
1286
1318
|
|
1287
1319
|
|
1288
1320
|
def stopwatch_now() -> float:
|
1289
|
-
"""Get a time value for interval comparisons
|
1321
|
+
"""Get a time value for interval comparisons.
|
1290
1322
|
|
1291
1323
|
When possible it is a monotonic clock to prevent backwards time issues.
|
1292
1324
|
"""
|
@@ -1329,7 +1361,7 @@ def prompt_choices(
|
|
1329
1361
|
input_timeout: Optional[int] = None,
|
1330
1362
|
jupyter: bool = False,
|
1331
1363
|
) -> str:
|
1332
|
-
"""Allow a user to choose from a list of options"""
|
1364
|
+
"""Allow a user to choose from a list of options."""
|
1333
1365
|
for i, choice in enumerate(choices):
|
1334
1366
|
wandb.termlog(f"({i+1}) {choice}")
|
1335
1367
|
|
@@ -1351,7 +1383,7 @@ def prompt_choices(
|
|
1351
1383
|
|
1352
1384
|
|
1353
1385
|
def guess_data_type(shape: Sequence[int], risky: bool = False) -> Optional[str]:
|
1354
|
-
"""Infer the type of data based on the shape of the tensors
|
1386
|
+
"""Infer the type of data based on the shape of the tensors.
|
1355
1387
|
|
1356
1388
|
Arguments:
|
1357
1389
|
shape (Sequence[int]): The shape of the data
|
@@ -1450,7 +1482,6 @@ def parse_sweep_id(parts_dict: dict) -> Optional[str]:
|
|
1450
1482
|
Returns:
|
1451
1483
|
None or str if there is an error
|
1452
1484
|
"""
|
1453
|
-
|
1454
1485
|
entity = None
|
1455
1486
|
project = None
|
1456
1487
|
sweep_id = parts_dict.get("name")
|
@@ -1546,26 +1577,26 @@ def add_import_hook(fullname: str, on_import: Callable) -> None:
|
|
1546
1577
|
|
1547
1578
|
|
1548
1579
|
def host_from_path(path: Optional[str]) -> str:
|
1549
|
-
"""
|
1580
|
+
"""Return the host of the path."""
|
1550
1581
|
url = urllib.parse.urlparse(path)
|
1551
1582
|
return str(url.netloc)
|
1552
1583
|
|
1553
1584
|
|
1554
1585
|
def uri_from_path(path: Optional[str]) -> str:
|
1555
|
-
"""
|
1586
|
+
"""Return the URI of the path."""
|
1556
1587
|
url = urllib.parse.urlparse(path)
|
1557
1588
|
uri = url.path if url.path[0] != "/" else url.path[1:]
|
1558
1589
|
return str(uri)
|
1559
1590
|
|
1560
1591
|
|
1561
1592
|
def is_unicode_safe(stream: TextIO) -> bool:
|
1562
|
-
"""
|
1593
|
+
"""Return True if the stream supports UTF-8."""
|
1563
1594
|
encoding = getattr(stream, "encoding", None)
|
1564
1595
|
return encoding.lower() in {"utf-8", "utf_8"} if encoding else False
|
1565
1596
|
|
1566
1597
|
|
1567
1598
|
def _has_internet() -> bool:
|
1568
|
-
"""
|
1599
|
+
"""Attempt to open a DNS connection to Googles root servers."""
|
1569
1600
|
try:
|
1570
1601
|
s = socket.create_connection(("8.8.8.8", 53), 0.5)
|
1571
1602
|
s.close()
|
@@ -1574,8 +1605,8 @@ def _has_internet() -> bool:
|
|
1574
1605
|
return False
|
1575
1606
|
|
1576
1607
|
|
1577
|
-
def rand_alphanumeric(length: int = 8, rand: Optional[
|
1578
|
-
rand = rand or random
|
1608
|
+
def rand_alphanumeric(length: int = 8, rand: Optional[random.Random] = None) -> str:
|
1609
|
+
rand = rand or random.Random()
|
1579
1610
|
return "".join(rand.choice("0123456789ABCDEF") for _ in range(length))
|
1580
1611
|
|
1581
1612
|
|
@@ -1583,10 +1614,7 @@ def rand_alphanumeric(length: int = 8, rand: Optional[ModuleType] = None) -> str
|
|
1583
1614
|
def fsync_open(
|
1584
1615
|
path: Union[pathlib.Path, str], mode: str = "w", encoding: Optional[str] = None
|
1585
1616
|
) -> Generator[IO[Any], None, None]:
|
1586
|
-
"""
|
1587
|
-
Opens a path for I/O, guaranteeing that the file is flushed and
|
1588
|
-
fsynced when the file's context expires.
|
1589
|
-
"""
|
1617
|
+
"""Open a path for I/O and guarante that the file is flushed and synced."""
|
1590
1618
|
with open(path, mode, encoding=encoding) as f:
|
1591
1619
|
yield f
|
1592
1620
|
|
@@ -1597,7 +1625,7 @@ def fsync_open(
|
|
1597
1625
|
def _is_kaggle() -> bool:
|
1598
1626
|
return (
|
1599
1627
|
os.getenv("KAGGLE_KERNEL_RUN_TYPE") is not None
|
1600
|
-
or "kaggle_environments" in sys.modules
|
1628
|
+
or "kaggle_environments" in sys.modules
|
1601
1629
|
)
|
1602
1630
|
|
1603
1631
|
|
@@ -1633,21 +1661,19 @@ def _is_py_path(path: str) -> bool:
|
|
1633
1661
|
|
1634
1662
|
|
1635
1663
|
def sweep_config_err_text_from_jsonschema_violations(violations: List[str]) -> str:
|
1636
|
-
"""Consolidate violation strings from wandb/sweeps
|
1637
|
-
sweep config violates the allowed schema as a single string.
|
1664
|
+
"""Consolidate schema violation strings from wandb/sweeps into a single string.
|
1638
1665
|
|
1639
1666
|
Parameters
|
1640
1667
|
----------
|
1641
1668
|
violations: list of str
|
1642
1669
|
The warnings to render.
|
1643
1670
|
|
1644
|
-
Returns
|
1671
|
+
Returns:
|
1645
1672
|
-------
|
1646
1673
|
violation: str
|
1647
1674
|
The consolidated violation text.
|
1648
1675
|
|
1649
1676
|
"""
|
1650
|
-
|
1651
1677
|
violation_base = (
|
1652
1678
|
"Malformed sweep config detected! This may cause your sweep to behave in unexpected ways.\n"
|
1653
1679
|
"To avoid this, please fix the sweep config schema violations below:"
|
@@ -1661,15 +1687,13 @@ def sweep_config_err_text_from_jsonschema_violations(violations: List[str]) -> s
|
|
1661
1687
|
|
1662
1688
|
|
1663
1689
|
def handle_sweep_config_violations(warnings: List[str]) -> None:
|
1664
|
-
"""
|
1665
|
-
sweep config violates the allowed schema as terminal warnings.
|
1690
|
+
"""Echo sweep config schema violation warnings from Gorilla to the terminal.
|
1666
1691
|
|
1667
1692
|
Parameters
|
1668
1693
|
----------
|
1669
1694
|
warnings: list of str
|
1670
1695
|
The warnings to render.
|
1671
1696
|
"""
|
1672
|
-
|
1673
1697
|
warning = sweep_config_err_text_from_jsonschema_violations(warnings)
|
1674
1698
|
if len(warnings) > 0:
|
1675
1699
|
term.termwarn(warning)
|
@@ -1677,7 +1701,6 @@ def handle_sweep_config_violations(warnings: List[str]) -> None:
|
|
1677
1701
|
|
1678
1702
|
def _log_thread_stacks() -> None:
|
1679
1703
|
"""Log all threads, useful for debugging."""
|
1680
|
-
|
1681
1704
|
thread_map = {t.ident: t.name for t in threading.enumerate()}
|
1682
1705
|
|
1683
1706
|
for thread_id, frame in sys._current_frames().items():
|
@@ -1745,7 +1768,7 @@ def load_json_yaml_dict(config: str) -> Any:
|
|
1745
1768
|
|
1746
1769
|
|
1747
1770
|
def _parse_entity_project_item(path: str) -> tuple:
|
1748
|
-
"""
|
1771
|
+
"""Parse paths with the following formats: {item}, {project}/{item}, & {entity}/{project}/{item}.
|
1749
1772
|
|
1750
1773
|
Args:
|
1751
1774
|
path: `str`, input path; must be between 0 and 3 in length.
|
@@ -1771,7 +1794,9 @@ def _parse_entity_project_item(path: str) -> tuple:
|
|
1771
1794
|
|
1772
1795
|
|
1773
1796
|
def _resolve_aliases(aliases: Optional[Union[str, Iterable[str]]]) -> List[str]:
|
1774
|
-
"""
|
1797
|
+
"""Add the 'latest' alias and ensure that all aliases are unique.
|
1798
|
+
|
1799
|
+
Takes in `aliases` which can be None, str, or List[str] and returns List[str].
|
1775
1800
|
Ensures that "latest" is always present in the returned list.
|
1776
1801
|
|
1777
1802
|
Args:
|
@@ -1780,13 +1805,15 @@ def _resolve_aliases(aliases: Optional[Union[str, Iterable[str]]]) -> List[str]:
|
|
1780
1805
|
Returns:
|
1781
1806
|
List[str], with "latest" always present.
|
1782
1807
|
|
1783
|
-
|
1784
|
-
aliases = _resolve_aliases(["best", "dev"])
|
1785
|
-
assert aliases == ["best", "dev", "latest"]
|
1808
|
+
Usage:
|
1786
1809
|
|
1787
|
-
|
1788
|
-
|
1810
|
+
```python
|
1811
|
+
aliases = _resolve_aliases(["best", "dev"])
|
1812
|
+
assert aliases == ["best", "dev", "latest"]
|
1789
1813
|
|
1814
|
+
aliases = _resolve_aliases("boom")
|
1815
|
+
assert aliases == ["boom", "latest"]
|
1816
|
+
```
|
1790
1817
|
"""
|
1791
1818
|
aliases = aliases or ["latest"]
|
1792
1819
|
|
@@ -1870,7 +1897,7 @@ def ensure_text(
|
|
1870
1897
|
|
1871
1898
|
|
1872
1899
|
def make_artifact_name_safe(name: str) -> str:
|
1873
|
-
"""Make an artifact name safe for use in artifacts"""
|
1900
|
+
"""Make an artifact name safe for use in artifacts."""
|
1874
1901
|
# artifact names may only contain alphanumeric characters, dashes, underscores, and dots.
|
1875
1902
|
cleaned = re.sub(r"[^a-zA-Z0-9_\-.]", "_", name)
|
1876
1903
|
if len(cleaned) <= 128:
|
@@ -1880,7 +1907,7 @@ def make_artifact_name_safe(name: str) -> str:
|
|
1880
1907
|
|
1881
1908
|
|
1882
1909
|
def make_docker_image_name_safe(name: str) -> str:
|
1883
|
-
"""Make a docker image name safe for use in artifacts"""
|
1910
|
+
"""Make a docker image name safe for use in artifacts."""
|
1884
1911
|
safe_chars = RE_DOCKER_IMAGE_NAME_CHARS.sub("__", name.lower())
|
1885
1912
|
deduped = RE_DOCKER_IMAGE_NAME_SEPARATOR_REPEAT.sub("__", safe_chars)
|
1886
1913
|
trimmed_start = RE_DOCKER_IMAGE_NAME_SEPARATOR_START.sub("", deduped)
|
@@ -1889,9 +1916,7 @@ def make_docker_image_name_safe(name: str) -> str:
|
|
1889
1916
|
|
1890
1917
|
|
1891
1918
|
def merge_dicts(source: Dict[str, Any], destination: Dict[str, Any]) -> Dict[str, Any]:
|
1892
|
-
"""
|
1893
|
-
Recursively merge two dictionaries.
|
1894
|
-
"""
|
1919
|
+
"""Recursively merge two dictionaries."""
|
1895
1920
|
for key, value in source.items():
|
1896
1921
|
if isinstance(value, dict):
|
1897
1922
|
# get node or create one
|
wandb/viz.py
CHANGED
@@ -16,8 +16,8 @@ class Visualize:
|
|
16
16
|
}
|
17
17
|
|
18
18
|
@staticmethod
|
19
|
-
def get_config_key(key: str) -> Tuple[str]:
|
20
|
-
return
|
19
|
+
def get_config_key(key: str) -> Tuple[str, str, str]:
|
20
|
+
return "_wandb", "viz", key
|
21
21
|
|
22
22
|
@property
|
23
23
|
def value(self) -> Table:
|
@@ -54,8 +54,8 @@ class CustomChart:
|
|
54
54
|
}
|
55
55
|
|
56
56
|
@staticmethod
|
57
|
-
def get_config_key(key: str) -> Tuple[str]:
|
58
|
-
return
|
57
|
+
def get_config_key(key: str) -> Tuple[str, str, str]:
|
58
|
+
return "_wandb", "visualize", key
|
59
59
|
|
60
60
|
@staticmethod
|
61
61
|
def user_query(table_key: str) -> Dict[str, Any]:
|