wandb 0.16.5__py3-none-any.whl → 0.17.0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (194) hide show
  1. package_readme.md +95 -0
  2. wandb/__init__.py +2 -3
  3. wandb/agents/pyagent.py +0 -1
  4. wandb/analytics/sentry.py +2 -1
  5. wandb/apis/importers/internals/internal.py +0 -1
  6. wandb/apis/importers/internals/protocols.py +30 -56
  7. wandb/apis/importers/mlflow.py +13 -26
  8. wandb/apis/importers/wandb.py +8 -14
  9. wandb/apis/internal.py +0 -3
  10. wandb/apis/public/api.py +55 -3
  11. wandb/apis/public/artifacts.py +1 -0
  12. wandb/apis/public/files.py +1 -0
  13. wandb/apis/public/history.py +1 -0
  14. wandb/apis/public/jobs.py +17 -4
  15. wandb/apis/public/projects.py +1 -0
  16. wandb/apis/public/reports.py +1 -0
  17. wandb/apis/public/runs.py +15 -17
  18. wandb/apis/public/sweeps.py +1 -0
  19. wandb/apis/public/teams.py +1 -0
  20. wandb/apis/public/users.py +1 -0
  21. wandb/apis/reports/v1/_blocks.py +3 -7
  22. wandb/apis/reports/v2/gql.py +1 -0
  23. wandb/apis/reports/v2/interface.py +3 -4
  24. wandb/apis/reports/v2/internal.py +5 -8
  25. wandb/cli/cli.py +95 -22
  26. wandb/data_types.py +9 -6
  27. wandb/docker/__init__.py +1 -1
  28. wandb/env.py +38 -8
  29. wandb/errors/__init__.py +5 -0
  30. wandb/errors/term.py +10 -2
  31. wandb/filesync/step_checksum.py +1 -4
  32. wandb/filesync/step_prepare.py +4 -24
  33. wandb/filesync/step_upload.py +4 -106
  34. wandb/filesync/upload_job.py +0 -76
  35. wandb/integration/catboost/catboost.py +1 -1
  36. wandb/integration/fastai/__init__.py +1 -0
  37. wandb/integration/huggingface/resolver.py +2 -2
  38. wandb/integration/keras/__init__.py +1 -0
  39. wandb/integration/keras/callbacks/metrics_logger.py +1 -1
  40. wandb/integration/keras/keras.py +7 -7
  41. wandb/integration/langchain/wandb_tracer.py +1 -0
  42. wandb/integration/lightning/fabric/logger.py +1 -3
  43. wandb/integration/metaflow/metaflow.py +41 -6
  44. wandb/integration/openai/fine_tuning.py +77 -40
  45. wandb/integration/prodigy/prodigy.py +1 -1
  46. wandb/old/summary.py +1 -1
  47. wandb/plot/confusion_matrix.py +1 -1
  48. wandb/plot/pr_curve.py +2 -1
  49. wandb/plot/roc_curve.py +2 -1
  50. wandb/{plots → plot}/utils.py +13 -25
  51. wandb/proto/v3/wandb_internal_pb2.py +364 -332
  52. wandb/proto/v3/wandb_settings_pb2.py +2 -2
  53. wandb/proto/v3/wandb_telemetry_pb2.py +10 -10
  54. wandb/proto/v4/wandb_internal_pb2.py +322 -316
  55. wandb/proto/v4/wandb_settings_pb2.py +2 -2
  56. wandb/proto/v4/wandb_telemetry_pb2.py +10 -10
  57. wandb/proto/wandb_deprecated.py +7 -1
  58. wandb/proto/wandb_internal_codegen.py +3 -29
  59. wandb/sdk/artifacts/artifact.py +51 -20
  60. wandb/sdk/artifacts/artifact_download_logger.py +1 -0
  61. wandb/sdk/artifacts/artifact_file_cache.py +18 -4
  62. wandb/sdk/artifacts/artifact_instance_cache.py +1 -0
  63. wandb/sdk/artifacts/artifact_manifest.py +1 -0
  64. wandb/sdk/artifacts/artifact_manifest_entry.py +7 -3
  65. wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +1 -0
  66. wandb/sdk/artifacts/artifact_saver.py +18 -27
  67. wandb/sdk/artifacts/artifact_state.py +1 -0
  68. wandb/sdk/artifacts/artifact_ttl.py +1 -0
  69. wandb/sdk/artifacts/exceptions.py +1 -0
  70. wandb/sdk/artifacts/storage_handlers/azure_handler.py +1 -0
  71. wandb/sdk/artifacts/storage_handlers/gcs_handler.py +13 -18
  72. wandb/sdk/artifacts/storage_handlers/http_handler.py +1 -0
  73. wandb/sdk/artifacts/storage_handlers/local_file_handler.py +1 -0
  74. wandb/sdk/artifacts/storage_handlers/multi_handler.py +1 -0
  75. wandb/sdk/artifacts/storage_handlers/s3_handler.py +5 -3
  76. wandb/sdk/artifacts/storage_handlers/tracking_handler.py +1 -0
  77. wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +1 -0
  78. wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +1 -0
  79. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +3 -42
  80. wandb/sdk/artifacts/storage_policy.py +2 -12
  81. wandb/sdk/data_types/_dtypes.py +8 -8
  82. wandb/sdk/data_types/base_types/media.py +3 -6
  83. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +3 -1
  84. wandb/sdk/data_types/image.py +1 -1
  85. wandb/sdk/data_types/video.py +1 -1
  86. wandb/sdk/integration_utils/auto_logging.py +5 -6
  87. wandb/sdk/integration_utils/data_logging.py +10 -6
  88. wandb/sdk/interface/interface.py +86 -38
  89. wandb/sdk/interface/interface_shared.py +7 -13
  90. wandb/sdk/internal/datastore.py +1 -1
  91. wandb/sdk/internal/file_pusher.py +2 -5
  92. wandb/sdk/internal/file_stream.py +5 -18
  93. wandb/sdk/internal/handler.py +18 -2
  94. wandb/sdk/internal/internal.py +0 -1
  95. wandb/sdk/internal/internal_api.py +1 -129
  96. wandb/sdk/internal/internal_util.py +0 -1
  97. wandb/sdk/internal/job_builder.py +159 -45
  98. wandb/sdk/internal/profiler.py +1 -0
  99. wandb/sdk/internal/progress.py +0 -28
  100. wandb/sdk/internal/run.py +1 -0
  101. wandb/sdk/internal/sender.py +1 -2
  102. wandb/sdk/internal/system/assets/gpu_amd.py +44 -44
  103. wandb/sdk/internal/system/assets/gpu_apple.py +56 -11
  104. wandb/sdk/internal/system/assets/interfaces.py +6 -8
  105. wandb/sdk/internal/system/assets/open_metrics.py +2 -2
  106. wandb/sdk/internal/system/assets/trainium.py +1 -3
  107. wandb/sdk/launch/__init__.py +9 -1
  108. wandb/sdk/launch/_launch.py +9 -24
  109. wandb/sdk/launch/_launch_add.py +1 -3
  110. wandb/sdk/launch/_project_spec.py +188 -241
  111. wandb/sdk/launch/agent/agent.py +115 -48
  112. wandb/sdk/launch/agent/config.py +80 -14
  113. wandb/sdk/launch/builder/abstract.py +69 -1
  114. wandb/sdk/launch/builder/build.py +156 -555
  115. wandb/sdk/launch/builder/context_manager.py +235 -0
  116. wandb/sdk/launch/builder/docker_builder.py +8 -23
  117. wandb/sdk/launch/builder/kaniko_builder.py +161 -159
  118. wandb/sdk/launch/builder/noop.py +1 -0
  119. wandb/sdk/launch/builder/templates/dockerfile.py +92 -0
  120. wandb/sdk/launch/create_job.py +68 -63
  121. wandb/sdk/launch/environment/abstract.py +1 -0
  122. wandb/sdk/launch/environment/gcp_environment.py +1 -0
  123. wandb/sdk/launch/environment/local_environment.py +1 -0
  124. wandb/sdk/launch/inputs/files.py +148 -0
  125. wandb/sdk/launch/inputs/internal.py +217 -0
  126. wandb/sdk/launch/inputs/manage.py +95 -0
  127. wandb/sdk/launch/loader.py +1 -0
  128. wandb/sdk/launch/registry/abstract.py +1 -0
  129. wandb/sdk/launch/registry/azure_container_registry.py +1 -0
  130. wandb/sdk/launch/registry/elastic_container_registry.py +1 -0
  131. wandb/sdk/launch/registry/google_artifact_registry.py +2 -1
  132. wandb/sdk/launch/registry/local_registry.py +1 -0
  133. wandb/sdk/launch/runner/abstract.py +1 -0
  134. wandb/sdk/launch/runner/kubernetes_monitor.py +4 -1
  135. wandb/sdk/launch/runner/kubernetes_runner.py +9 -10
  136. wandb/sdk/launch/runner/local_container.py +2 -3
  137. wandb/sdk/launch/runner/local_process.py +8 -29
  138. wandb/sdk/launch/runner/sagemaker_runner.py +21 -20
  139. wandb/sdk/launch/runner/vertex_runner.py +8 -7
  140. wandb/sdk/launch/sweeps/scheduler.py +7 -4
  141. wandb/sdk/launch/sweeps/scheduler_sweep.py +2 -1
  142. wandb/sdk/launch/sweeps/utils.py +3 -3
  143. wandb/sdk/launch/utils.py +33 -140
  144. wandb/sdk/lib/_settings_toposort_generated.py +1 -5
  145. wandb/sdk/lib/fsm.py +8 -12
  146. wandb/sdk/lib/gitlib.py +4 -4
  147. wandb/sdk/lib/import_hooks.py +1 -1
  148. wandb/sdk/lib/lazyloader.py +0 -1
  149. wandb/sdk/lib/proto_util.py +23 -2
  150. wandb/sdk/lib/redirect.py +19 -14
  151. wandb/sdk/lib/retry.py +3 -2
  152. wandb/sdk/lib/run_moment.py +7 -1
  153. wandb/sdk/lib/tracelog.py +1 -1
  154. wandb/sdk/service/service.py +19 -16
  155. wandb/sdk/verify/verify.py +2 -1
  156. wandb/sdk/wandb_init.py +16 -63
  157. wandb/sdk/wandb_manager.py +2 -2
  158. wandb/sdk/wandb_require.py +5 -0
  159. wandb/sdk/wandb_run.py +164 -90
  160. wandb/sdk/wandb_settings.py +2 -48
  161. wandb/sdk/wandb_setup.py +1 -1
  162. wandb/sklearn/__init__.py +1 -0
  163. wandb/sklearn/plot/__init__.py +1 -0
  164. wandb/sklearn/plot/classifier.py +11 -12
  165. wandb/sklearn/plot/clusterer.py +2 -1
  166. wandb/sklearn/plot/regressor.py +1 -0
  167. wandb/sklearn/plot/shared.py +1 -0
  168. wandb/sklearn/utils.py +1 -0
  169. wandb/testing/relay.py +4 -4
  170. wandb/trigger.py +1 -0
  171. wandb/util.py +67 -54
  172. wandb/wandb_controller.py +2 -3
  173. wandb/wandb_torch.py +1 -2
  174. {wandb-0.16.5.dist-info → wandb-0.17.0.dist-info}/METADATA +67 -70
  175. {wandb-0.16.5.dist-info → wandb-0.17.0.dist-info}/RECORD +178 -188
  176. {wandb-0.16.5.dist-info → wandb-0.17.0.dist-info}/WHEEL +1 -2
  177. wandb/bin/apple_gpu_stats +0 -0
  178. wandb/catboost/__init__.py +0 -9
  179. wandb/fastai/__init__.py +0 -9
  180. wandb/keras/__init__.py +0 -18
  181. wandb/lightgbm/__init__.py +0 -9
  182. wandb/plots/__init__.py +0 -6
  183. wandb/plots/explain_text.py +0 -36
  184. wandb/plots/heatmap.py +0 -81
  185. wandb/plots/named_entity.py +0 -43
  186. wandb/plots/part_of_speech.py +0 -50
  187. wandb/plots/plot_definitions.py +0 -768
  188. wandb/plots/precision_recall.py +0 -121
  189. wandb/plots/roc.py +0 -103
  190. wandb/sacred/__init__.py +0 -3
  191. wandb/xgboost/__init__.py +0 -9
  192. wandb-0.16.5.dist-info/top_level.txt +0 -1
  193. {wandb-0.16.5.dist-info → wandb-0.17.0.dist-info}/entry_points.txt +0 -0
  194. {wandb-0.16.5.dist-info → wandb-0.17.0.dist-info/licenses}/LICENSE +0 -0
@@ -1,7 +1,8 @@
1
1
  """GCS storage handler."""
2
+
2
3
  import time
3
4
  from pathlib import PurePosixPath
4
- from typing import TYPE_CHECKING, Dict, Optional, Sequence, Tuple, Union
5
+ from typing import TYPE_CHECKING, Optional, Sequence, Tuple, Union
5
6
  from urllib.parse import ParseResult, urlparse
6
7
 
7
8
  from wandb import util
@@ -9,7 +10,7 @@ from wandb.errors.term import termlog
9
10
  from wandb.sdk.artifacts.artifact_file_cache import get_artifact_file_cache
10
11
  from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
11
12
  from wandb.sdk.artifacts.storage_handler import DEFAULT_MAX_OBJECTS, StorageHandler
12
- from wandb.sdk.lib.hashutil import B64MD5
13
+ from wandb.sdk.lib.hashutil import ETag
13
14
  from wandb.sdk.lib.paths import FilePathStr, StrPath, URIStr
14
15
 
15
16
  if TYPE_CHECKING:
@@ -51,13 +52,14 @@ class GCSHandler(StorageHandler):
51
52
  manifest_entry: ArtifactManifestEntry,
52
53
  local: bool = False,
53
54
  ) -> Union[URIStr, FilePathStr]:
55
+ assert manifest_entry.ref is not None
54
56
  if not local:
55
- assert manifest_entry.ref is not None
56
57
  return manifest_entry.ref
57
58
 
58
- path, hit, cache_open = self._cache.check_md5_obj_path(
59
- B64MD5(manifest_entry.digest), # TODO(spencerpearson): unsafe cast
60
- manifest_entry.size if manifest_entry.size is not None else 0,
59
+ path, hit, cache_open = self._cache.check_etag_obj_path(
60
+ url=URIStr(manifest_entry.ref),
61
+ etag=ETag(manifest_entry.digest),
62
+ size=manifest_entry.size if manifest_entry.size is not None else 0,
61
63
  )
62
64
  if hit:
63
65
  return path
@@ -81,10 +83,10 @@ class GCSHandler(StorageHandler):
81
83
  raise ValueError(
82
84
  f"Unable to download object {manifest_entry.ref} with generation {version}"
83
85
  )
84
- md5 = obj.md5_hash
85
- if md5 != manifest_entry.digest:
86
+ if obj.etag != manifest_entry.digest:
86
87
  raise ValueError(
87
- f"Digest mismatch for object {manifest_entry.ref}: expected {manifest_entry.digest} but found {md5}"
88
+ f"Digest mismatch for object {manifest_entry.ref}: "
89
+ f"expected {manifest_entry.digest} but found {obj.etag}"
88
90
  )
89
91
 
90
92
  with cache_open(mode="wb") as f:
@@ -187,14 +189,7 @@ class GCSHandler(StorageHandler):
187
189
  return ArtifactManifestEntry(
188
190
  path=posix_name,
189
191
  ref=URIStr(f"{self._scheme}://{str(posix_ref)}"),
190
- digest=obj.md5_hash,
192
+ digest=obj.etag,
191
193
  size=obj.size,
192
- extra=self._extra_from_obj(obj),
194
+ extra={"versionID": obj.generation},
193
195
  )
194
-
195
- @staticmethod
196
- def _extra_from_obj(obj: "gcs_module.blob.Blob") -> Dict[str, str]:
197
- return {
198
- "etag": obj.etag,
199
- "versionID": obj.generation,
200
- }
@@ -1,4 +1,5 @@
1
1
  """HTTP storage handler."""
2
+
2
3
  import os
3
4
  from typing import TYPE_CHECKING, Dict, Optional, Sequence, Tuple, Union
4
5
  from urllib.parse import ParseResult
@@ -1,4 +1,5 @@
1
1
  """Local file storage handler."""
2
+
2
3
  import os
3
4
  import shutil
4
5
  import time
@@ -1,4 +1,5 @@
1
1
  """Multi storage handler."""
2
+
2
3
  from typing import TYPE_CHECKING, List, Optional, Sequence, Union
3
4
  from urllib.parse import urlparse
4
5
 
@@ -1,4 +1,5 @@
1
1
  """S3 storage handler."""
2
+
2
3
  import os
3
4
  import time
4
5
  from pathlib import PurePosixPath
@@ -182,9 +183,10 @@ class S3Handler(StorageHandler):
182
183
  multi = True
183
184
  else:
184
185
  raise CommError(
185
- "Unable to connect to S3 ({}): {}".format(
186
- e.response["Error"]["Code"], e.response["Error"]["Message"]
187
- )
186
+ f"Unable to connect to S3 ({e.response['Error']['Code']}): "
187
+ f"{e.response['Error']['Message']}. Check that your "
188
+ "authentication credentials are valid and that your region is "
189
+ "set correctly."
188
190
  )
189
191
  else:
190
192
  multi = True
@@ -1,4 +1,5 @@
1
1
  """Tracking storage handler."""
2
+
2
3
  from typing import TYPE_CHECKING, Optional, Sequence, Union
3
4
  from urllib.parse import urlparse
4
5
 
@@ -1,4 +1,5 @@
1
1
  """WB artifact storage handler."""
2
+
2
3
  import os
3
4
  from typing import TYPE_CHECKING, Optional, Sequence, Union
4
5
  from urllib.parse import urlparse
@@ -1,4 +1,5 @@
1
1
  """WB local artifact storage handler."""
2
+
2
3
  import os
3
4
  from typing import TYPE_CHECKING, Optional, Sequence, Union
4
5
 
@@ -1,4 +1,5 @@
1
1
  """WandB storage policy."""
2
+
2
3
  import hashlib
3
4
  import math
4
5
  import os
@@ -262,7 +263,7 @@ class WandbStoragePolicy(StoragePolicy):
262
263
  return math.ceil(file_size / S3_MAX_PART_NUMBERS)
263
264
  return default_chunk_size
264
265
 
265
- def store_file_sync(
266
+ def store_file(
266
267
  self,
267
268
  artifact_id: str,
268
269
  artifact_manifest_id: str,
@@ -300,7 +301,7 @@ class WandbStoragePolicy(StoragePolicy):
300
301
  hex_digests[part_number] = hex_digest
301
302
  part_number += 1
302
303
 
303
- resp = preparer.prepare_sync(
304
+ resp = preparer.prepare(
304
305
  {
305
306
  "artifactID": artifact_id,
306
307
  "artifactManifestID": artifact_manifest_id,
@@ -346,46 +347,6 @@ class WandbStoragePolicy(StoragePolicy):
346
347
 
347
348
  return False
348
349
 
349
- async def store_file_async(
350
- self,
351
- artifact_id: str,
352
- artifact_manifest_id: str,
353
- entry: "ArtifactManifestEntry",
354
- preparer: "StepPrepare",
355
- progress_callback: Optional["progress.ProgressFn"] = None,
356
- ) -> bool:
357
- """Async equivalent to `store_file_sync`."""
358
- resp = await preparer.prepare_async(
359
- {
360
- "artifactID": artifact_id,
361
- "artifactManifestID": artifact_manifest_id,
362
- "name": entry.path,
363
- "md5": entry.digest,
364
- }
365
- )
366
-
367
- entry.birth_artifact_id = resp.birth_artifact_id
368
- if resp.upload_url is None:
369
- return True
370
- if entry.local_path is None:
371
- return False
372
-
373
- with open(entry.local_path, "rb") as file:
374
- # This fails if we don't send the first byte before the signed URL expires.
375
- await self._api.upload_file_retry_async(
376
- resp.upload_url,
377
- file,
378
- progress_callback,
379
- extra_headers={
380
- header.split(":", 1)[0]: header.split(":", 1)[1]
381
- for header in (resp.upload_headers or {})
382
- },
383
- )
384
-
385
- self._write_cache(entry)
386
-
387
- return False
388
-
389
350
  def _write_cache(self, entry: "ArtifactManifestEntry") -> None:
390
351
  if entry.local_path is None:
391
352
  return
@@ -1,4 +1,5 @@
1
1
  """Storage policy."""
2
+
2
3
  from typing import TYPE_CHECKING, Dict, Optional, Sequence, Type, Union
3
4
 
4
5
  from wandb.sdk.internal.internal_api import Api as InternalApi
@@ -42,17 +43,7 @@ class StoragePolicy:
42
43
  ) -> FilePathStr:
43
44
  raise NotImplementedError
44
45
 
45
- def store_file_sync(
46
- self,
47
- artifact_id: str,
48
- artifact_manifest_id: str,
49
- entry: "ArtifactManifestEntry",
50
- preparer: "StepPrepare",
51
- progress_callback: Optional["ProgressFn"] = None,
52
- ) -> bool:
53
- raise NotImplementedError
54
-
55
- async def store_file_async(
46
+ def store_file(
56
47
  self,
57
48
  artifact_id: str,
58
49
  artifact_manifest_id: str,
@@ -60,7 +51,6 @@ class StoragePolicy:
60
51
  preparer: "StepPrepare",
61
52
  progress_callback: Optional["ProgressFn"] = None,
62
53
  ) -> bool:
63
- """Async equivalent to `store_file_sync`."""
64
54
  raise NotImplementedError
65
55
 
66
56
  def store_reference(
@@ -14,7 +14,7 @@ np = get_module("numpy") # intentionally not required
14
14
  if t.TYPE_CHECKING:
15
15
  from wandb.sdk.artifacts.artifact import Artifact
16
16
 
17
- ConvertableToType = t.Union["Type", t.Type["Type"], type, t.Any]
17
+ ConvertibleToType = t.Union["Type", t.Type["Type"], type, t.Any]
18
18
 
19
19
 
20
20
  class TypeRegistry:
@@ -84,7 +84,7 @@ class TypeRegistry:
84
84
  return _type.from_json(json_dict, artifact)
85
85
 
86
86
  @staticmethod
87
- def type_from_dtype(dtype: ConvertableToType) -> "Type":
87
+ def type_from_dtype(dtype: ConvertibleToType) -> "Type":
88
88
  # The dtype is already an instance of Type
89
89
  if isinstance(dtype, Type):
90
90
  wbtype: Type = dtype
@@ -528,7 +528,7 @@ class UnionType(Type):
528
528
 
529
529
  def __init__(
530
530
  self,
531
- allowed_types: t.Optional[t.Sequence[ConvertableToType]] = None,
531
+ allowed_types: t.Optional[t.Sequence[ConvertibleToType]] = None,
532
532
  ):
533
533
  assert allowed_types is None or (allowed_types.__class__ == list)
534
534
  if allowed_types is None:
@@ -576,7 +576,7 @@ class UnionType(Type):
576
576
  return "{}".format(" or ".join([str(t) for t in self.params["allowed_types"]]))
577
577
 
578
578
 
579
- def OptionalType(dtype: ConvertableToType) -> UnionType: # noqa: N802
579
+ def OptionalType(dtype: ConvertibleToType) -> UnionType: # noqa: N802
580
580
  """Function that mimics the Type class API for constructing an "Optional Type".
581
581
 
582
582
  This is just a Union[wb_type, NoneType].
@@ -591,14 +591,14 @@ def OptionalType(dtype: ConvertableToType) -> UnionType: # noqa: N802
591
591
 
592
592
 
593
593
  class ListType(Type):
594
- """A list of homogenous types."""
594
+ """A list of homogeneous types."""
595
595
 
596
596
  name = "list"
597
597
  types: t.ClassVar[t.List[type]] = [list, tuple, set, frozenset]
598
598
 
599
599
  def __init__(
600
600
  self,
601
- element_type: t.Optional[ConvertableToType] = None,
601
+ element_type: t.Optional[ConvertibleToType] = None,
602
602
  length: t.Optional[int] = None,
603
603
  ):
604
604
  if element_type is None:
@@ -691,7 +691,7 @@ class ListType(Type):
691
691
 
692
692
 
693
693
  class NDArrayType(Type):
694
- """Represents a list of homogenous types."""
694
+ """Represents a list of homogeneous types."""
695
695
 
696
696
  name = "ndarray"
697
697
  types: t.ClassVar[t.List[type]] = [] # will manually add type if np is available
@@ -786,7 +786,7 @@ class TypedDictType(Type):
786
786
 
787
787
  def __init__(
788
788
  self,
789
- type_map: t.Optional[t.Dict[str, ConvertableToType]] = None,
789
+ type_map: t.Optional[t.Dict[str, ConvertibleToType]] = None,
790
790
  ):
791
791
  if type_map is None:
792
792
  type_map = {}
@@ -177,9 +177,7 @@ class Media(WBValue):
177
177
  json_obj["_latest_artifact_path"] = artifact_entry_latest_url
178
178
 
179
179
  if artifact_entry_url is None or self.is_bound():
180
- assert (
181
- self.is_bound()
182
- ), "Value of type {} must be bound to a run with bind_to_run() before being serialized to JSON.".format(
180
+ assert self.is_bound(), "Value of type {} must be bound to a run with bind_to_run() before being serialized to JSON.".format(
183
181
  type(self).__name__
184
182
  )
185
183
 
@@ -221,8 +219,7 @@ class Media(WBValue):
221
219
 
222
220
  # if not, check to see if there is a source artifact for this object
223
221
  if (
224
- self._artifact_source
225
- is not None
222
+ self._artifact_source is not None
226
223
  # and self._artifact_source.artifact != artifact
227
224
  ):
228
225
  default_root = self._artifact_source.artifact._default_root()
@@ -292,7 +289,7 @@ class BatchableMedia(Media):
292
289
 
293
290
 
294
291
  def _numpy_arrays_to_lists(
295
- payload: Union[dict, Sequence, "np.ndarray"]
292
+ payload: Union[dict, Sequence, "np.ndarray"],
296
293
  ) -> Union[Sequence, dict, str, int, float, bool]:
297
294
  # Casts all numpy arrays to lists so we don't convert them to histograms, primarily for Plotly
298
295
 
@@ -231,8 +231,10 @@ class BoundingBoxes2D(JSONMetadata):
231
231
 
232
232
  for box in boxes:
233
233
  # Required arguments
234
- error_str = "Each box must contain a position with: middle, width, and height or \
234
+ error_str = (
235
+ "Each box must contain a position with: middle, width, and height or \
235
236
  \nminX, maxX, minY, maxY."
237
+ )
236
238
  if "position" not in box:
237
239
  raise TypeError(error_str)
238
240
  else:
@@ -167,7 +167,7 @@ class Image(BatchableMedia):
167
167
  self._file_type = None
168
168
 
169
169
  # Allows the user to pass an Image object as the first parameter and have a perfect copy,
170
- # only overriding additional metdata passed in. If this pattern is compelling, we can generalize.
170
+ # only overriding additional metadata passed in. If this pattern is compelling, we can generalize.
171
171
  if isinstance(data_or_path, Image):
172
172
  self._initialize_from_wbimage(data_or_path)
173
173
  elif isinstance(data_or_path, str):
@@ -24,7 +24,7 @@ if TYPE_CHECKING: # pragma: no cover
24
24
  # https://github.com/wandb/wandb/issues/3472
25
25
  #
26
26
  # Essentially, the issue is that moviepy's write_gif function fails to close
27
- # the open write / file descripter returned from `imageio.save`. The following
27
+ # the open write / file descriptor returned from `imageio.save`. The following
28
28
  # function is a simplified copy of the function in the moviepy source code.
29
29
  # See https://github.com/Zulko/moviepy/blob/7e3e8bb1b739eb6d1c0784b0cb2594b587b93b39/moviepy/video/io/gif_writers.py#L428
30
30
  #
@@ -27,11 +27,11 @@ V = TypeVar("V")
27
27
 
28
28
 
29
29
  class Response(Protocol[K, V]):
30
- def __getitem__(self, key: K) -> V:
31
- ... # pragma: no cover
30
+ def __getitem__(self, key: K) -> V: ... # pragma: no cover
32
31
 
33
- def get(self, key: K, default: Optional[V] = None) -> Optional[V]:
34
- ... # pragma: no cover
32
+ def get(
33
+ self, key: K, default: Optional[V] = None
34
+ ) -> Optional[V]: ... # pragma: no cover
35
35
 
36
36
 
37
37
  class ArgumentResponseResolver(Protocol):
@@ -42,8 +42,7 @@ class ArgumentResponseResolver(Protocol):
42
42
  response: Response,
43
43
  start_time: float,
44
44
  time_elapsed: float,
45
- ) -> Optional[Dict[str, Any]]:
46
- ... # pragma: no cover
45
+ ) -> Optional[Dict[str, Any]]: ... # pragma: no cover
47
46
 
48
47
 
49
48
  class PatchAPI:
@@ -78,7 +78,7 @@ class ValidationDataLogger:
78
78
  Defaults to `"wb_validation_data"`.
79
79
  artifact_type: The artifact type to use for the validation data.
80
80
  Defaults to `"validation_dataset"`.
81
- class_labels: Optional list of lables to use in the inferred
81
+ class_labels: Optional list of labels to use in the inferred
82
82
  processors. If the model's `target` or `output` is inferred to be a class,
83
83
  we will attempt to map the class to these labels. Defaults to `None`.
84
84
  infer_missing_processors: Determines if processors are inferred if
@@ -262,7 +262,7 @@ def _infer_single_example_keyed_processor(
262
262
  ):
263
263
  np = wandb.util.get_module(
264
264
  "numpy",
265
- required="Infering processors require numpy",
265
+ required="Inferring processors require numpy",
266
266
  )
267
267
  # Assume these are logits
268
268
  class_names = class_labels_table.get_column("label")
@@ -291,13 +291,17 @@ def _infer_single_example_keyed_processor(
291
291
  ):
292
292
  # assume this is a class
293
293
  if class_labels_table is not None:
294
- processors["class"] = lambda n, d, p: class_labels_table.index_ref(d[0]) if d[0] < len(class_labels_table.data) else d[0] # type: ignore
294
+ processors["class"] = (
295
+ lambda n, d, p: class_labels_table.index_ref(d[0])
296
+ if d[0] < len(class_labels_table.data)
297
+ else d[0]
298
+ ) # type: ignore
295
299
  else:
296
300
  processors["val"] = lambda n, d, p: d[0]
297
301
  elif len(shape) == 1:
298
302
  np = wandb.util.get_module(
299
303
  "numpy",
300
- required="Infering processors require numpy",
304
+ required="Inferring processors require numpy",
301
305
  )
302
306
  # This could be anything
303
307
  if shape[0] <= 10:
@@ -350,7 +354,7 @@ def _infer_validation_row_processor(
350
354
  input_col_name: str = "input",
351
355
  target_col_name: str = "target",
352
356
  ) -> Callable:
353
- """Infers the composit processor for the validation data."""
357
+ """Infers the composite processor for the validation data."""
354
358
  single_processors = {}
355
359
  if isinstance(example_input, dict):
356
360
  for key in example_input:
@@ -427,7 +431,7 @@ def _infer_prediction_row_processor(
427
431
  input_col_name: str = "input",
428
432
  output_col_name: str = "output",
429
433
  ) -> Callable:
430
- """Infers the composit processor for the prediction output data."""
434
+ """Infers the composite processor for the prediction output data."""
431
435
  single_processors = {}
432
436
 
433
437
  if isinstance(example_prediction, dict):
@@ -25,6 +25,7 @@ from typing import (
25
25
  Union,
26
26
  )
27
27
 
28
+ from wandb import termwarn
28
29
  from wandb.proto import wandb_internal_pb2 as pb
29
30
  from wandb.proto import wandb_telemetry_pb2 as tpb
30
31
  from wandb.sdk.artifacts.artifact import Artifact
@@ -386,18 +387,30 @@ class InterfaceBase:
386
387
  def _make_partial_source_str(
387
388
  source: Any, job_info: Dict[str, Any], metadata: Dict[str, Any]
388
389
  ) -> str:
389
- """Construct use_artifact.partial.source_info.sourc as str."""
390
+ """Construct use_artifact.partial.source_info.source as str."""
390
391
  source_type = job_info.get("source_type", "").strip()
391
392
  if source_type == "artifact":
392
393
  info_source = job_info.get("source", {})
393
394
  source.artifact.artifact = info_source.get("artifact", "")
394
395
  source.artifact.entrypoint.extend(info_source.get("entrypoint", []))
395
396
  source.artifact.notebook = info_source.get("notebook", False)
397
+ build_context = info_source.get("build_context")
398
+ if build_context:
399
+ source.artifact.build_context = build_context
400
+ dockerfile = info_source.get("dockerfile")
401
+ if dockerfile:
402
+ source.artifact.dockerfile = dockerfile
396
403
  elif source_type == "repo":
397
404
  source.git.git_info.remote = metadata.get("git", {}).get("remote", "")
398
405
  source.git.git_info.commit = metadata.get("git", {}).get("commit", "")
399
406
  source.git.entrypoint.extend(metadata.get("entrypoint", []))
400
407
  source.git.notebook = metadata.get("notebook", False)
408
+ build_context = metadata.get("build_context")
409
+ if build_context:
410
+ source.git.build_context = build_context
411
+ dockerfile = metadata.get("dockerfile")
412
+ if dockerfile:
413
+ source.git.dockerfile = dockerfile
401
414
  elif source_type == "image":
402
415
  source.image.image = metadata.get("docker", "")
403
416
  else:
@@ -423,7 +436,7 @@ class InterfaceBase:
423
436
  job_info=job_info,
424
437
  metadata=metadata,
425
438
  )
426
- use_artifact.partial.source_info.source.ParseFromString(src_str)
439
+ use_artifact.partial.source_info.source.ParseFromString(src_str) # type: ignore[arg-type]
427
440
 
428
441
  return use_artifact
429
442
 
@@ -447,16 +460,27 @@ class InterfaceBase:
447
460
  path = artifact.get_entry("wandb-job.json").download()
448
461
  with open(path) as f:
449
462
  job_info = json.load(f)
463
+
450
464
  except Exception as e:
451
465
  logger.warning(
452
466
  f"Failed to download partial job info from artifact {artifact}, : {e}"
453
467
  )
454
- use_artifact = self._make_proto_use_artifact(
455
- use_artifact=use_artifact,
456
- job_name=artifact.name,
457
- job_info=job_info,
458
- metadata=artifact.metadata,
459
- )
468
+ termwarn(
469
+ f"Failed to download partial job info from artifact {artifact}, : {e}"
470
+ )
471
+ return
472
+
473
+ try:
474
+ use_artifact = self._make_proto_use_artifact(
475
+ use_artifact=use_artifact,
476
+ job_name=artifact.name,
477
+ job_info=job_info,
478
+ metadata=artifact.metadata,
479
+ )
480
+ except Exception as e:
481
+ logger.warning(f"Failed to construct use artifact proto: {e}")
482
+ termwarn(f"Failed to construct use artifact proto: {e}")
483
+ return
460
484
 
461
485
  self._publish_use_artifact(use_artifact)
462
486
 
@@ -504,11 +528,15 @@ class InterfaceBase:
504
528
  artifact_id: str,
505
529
  download_root: str,
506
530
  allow_missing_references: bool,
531
+ skip_cache: bool,
532
+ path_prefix: Optional[str],
507
533
  ) -> MailboxHandle:
508
534
  download_artifact = pb.DownloadArtifactRequest()
509
535
  download_artifact.artifact_id = artifact_id
510
536
  download_artifact.download_root = download_root
511
537
  download_artifact.allow_missing_references = allow_missing_references
538
+ download_artifact.skip_cache = skip_cache
539
+ download_artifact.path_prefix = path_prefix or ""
512
540
  resp = self._deliver_download_artifact(download_artifact)
513
541
  return resp
514
542
 
@@ -717,6 +745,56 @@ class InterfaceBase:
717
745
  def _publish_keepalive(self, keepalive: pb.KeepaliveRequest) -> None:
718
746
  raise NotImplementedError
719
747
 
748
+ def publish_job_input(
749
+ self,
750
+ include_paths: List[List[str]],
751
+ exclude_paths: List[List[str]],
752
+ run_config: bool = False,
753
+ file_path: str = "",
754
+ ):
755
+ """Publishes a request to add inputs to the job.
756
+
757
+ If run_config is True, the wandb.config will be added as a job input.
758
+ If file_path is provided, the file at file_path will be added as a job
759
+ input.
760
+
761
+ The paths provided as arguments are sequences of dictionary keys that
762
+ specify a path within the wandb.config. If a path is included, the
763
+ corresponding field will be treated as a job input. If a path is
764
+ excluded, the corresponding field will not be treated as a job input.
765
+
766
+ Args:
767
+ include_paths: paths within config to include as job inputs.
768
+ exclude_paths: paths within config to exclude as job inputs.
769
+ run_config: bool indicating whether wandb.config is the input source.
770
+ file_path: path to file to include as a job input.
771
+ """
772
+ if run_config and file_path:
773
+ raise ValueError(
774
+ "run_config and file_path are mutually exclusive arguments."
775
+ )
776
+ request = pb.JobInputRequest()
777
+ include_records = [pb.JobInputPath(path=path) for path in include_paths]
778
+ exclude_records = [pb.JobInputPath(path=path) for path in exclude_paths]
779
+ request.include_paths.extend(include_records)
780
+ request.exclude_paths.extend(exclude_records)
781
+ source = pb.JobInputSource(
782
+ run_config=pb.JobInputSource.RunConfigSource(),
783
+ )
784
+ if run_config:
785
+ source.run_config.CopyFrom(pb.JobInputSource.RunConfigSource())
786
+ else:
787
+ source.file.CopyFrom(
788
+ pb.JobInputSource.ConfigFileSource(path=file_path),
789
+ )
790
+ request.input_source.CopyFrom(source)
791
+
792
+ return self._publish_job_input(request)
793
+
794
+ @abstractmethod
795
+ def _publish_job_input(self, request: pb.JobInputRequest) -> MailboxHandle:
796
+ raise NotImplementedError
797
+
720
798
  def join(self) -> None:
721
799
  # Drop indicates that the internal process has already been shutdown
722
800
  if self._drop:
@@ -767,36 +845,6 @@ class InterfaceBase:
767
845
  run_start.run.CopyFrom(run_pb)
768
846
  return self._deliver_run_start(run_start)
769
847
 
770
- def publish_launch_wandb_config_parameters(
771
- self, include_paths: List[List[str]], exclude_paths: List[List[str]]
772
- ):
773
- """Tells the internal process to treat wandb.config fields as job inputs.
774
-
775
- The paths provided as arguments are sequences of dictionary keys that
776
- specify a path within the wandb.config. If a path is included, the
777
- corresponding field will be treated as a job input. If a path is
778
- excluded, the corresponding field will not be treated as a job input.
779
-
780
- Args:
781
- include_paths: paths within config to include as job inputs.
782
- exclude_paths: paths within config to exclude as job inputs.
783
-
784
- Returns:
785
- None
786
- """
787
- config_parameters = pb.LaunchWandbConfigParametersRecord()
788
- include_records = [pb.ConfigFilterPath(path=path) for path in include_paths]
789
- exclude_records = [pb.ConfigFilterPath(path=path) for path in exclude_paths]
790
- config_parameters.include_paths.extend(include_records)
791
- config_parameters.exclude_paths.extend(exclude_records)
792
- return self._publish_launch_wandb_config_parameters(config_parameters)
793
-
794
- @abstractmethod
795
- def _publish_launch_wandb_config_parameters(
796
- self, config_parameters: pb.LaunchWandbConfigParametersRecord
797
- ) -> None:
798
- raise NotImplementedError
799
-
800
848
  @abstractmethod
801
849
  def _deliver_run_start(self, run_start: pb.RunStartRequest) -> MailboxHandle:
802
850
  raise NotImplementedError