wandb 0.15.3__py3-none-any.whl → 0.15.5__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (156) hide show
  1. wandb/__init__.py +1 -1
  2. wandb/analytics/sentry.py +1 -0
  3. wandb/apis/importers/base.py +20 -5
  4. wandb/apis/importers/mlflow.py +7 -1
  5. wandb/apis/internal.py +12 -0
  6. wandb/apis/public.py +247 -1387
  7. wandb/apis/reports/_panels.py +58 -35
  8. wandb/beta/workflows.py +6 -7
  9. wandb/cli/cli.py +130 -60
  10. wandb/data_types.py +3 -1
  11. wandb/filesync/dir_watcher.py +21 -27
  12. wandb/filesync/step_checksum.py +8 -8
  13. wandb/filesync/step_prepare.py +23 -10
  14. wandb/filesync/step_upload.py +13 -13
  15. wandb/filesync/upload_job.py +4 -8
  16. wandb/integration/cohere/__init__.py +3 -0
  17. wandb/integration/cohere/cohere.py +21 -0
  18. wandb/integration/cohere/resolver.py +347 -0
  19. wandb/integration/gym/__init__.py +4 -6
  20. wandb/integration/huggingface/__init__.py +3 -0
  21. wandb/integration/huggingface/huggingface.py +18 -0
  22. wandb/integration/huggingface/resolver.py +213 -0
  23. wandb/integration/langchain/wandb_tracer.py +16 -179
  24. wandb/integration/openai/__init__.py +1 -3
  25. wandb/integration/openai/openai.py +11 -143
  26. wandb/integration/openai/resolver.py +111 -38
  27. wandb/integration/sagemaker/config.py +2 -2
  28. wandb/integration/tensorboard/log.py +4 -4
  29. wandb/old/settings.py +24 -7
  30. wandb/proto/v3/wandb_telemetry_pb2.py +12 -12
  31. wandb/proto/v4/wandb_telemetry_pb2.py +12 -12
  32. wandb/proto/wandb_deprecated.py +3 -1
  33. wandb/sdk/__init__.py +1 -1
  34. wandb/sdk/artifacts/__init__.py +0 -0
  35. wandb/sdk/artifacts/artifact.py +2101 -0
  36. wandb/sdk/artifacts/artifact_download_logger.py +42 -0
  37. wandb/sdk/artifacts/artifact_manifest.py +67 -0
  38. wandb/sdk/artifacts/artifact_manifest_entry.py +159 -0
  39. wandb/sdk/artifacts/artifact_manifests/__init__.py +0 -0
  40. wandb/sdk/artifacts/artifact_manifests/artifact_manifest_v1.py +91 -0
  41. wandb/sdk/{internal → artifacts}/artifact_saver.py +6 -5
  42. wandb/sdk/artifacts/artifact_state.py +10 -0
  43. wandb/sdk/{interface/artifacts/artifact_cache.py → artifacts/artifacts_cache.py} +22 -12
  44. wandb/sdk/artifacts/exceptions.py +55 -0
  45. wandb/sdk/artifacts/storage_handler.py +59 -0
  46. wandb/sdk/artifacts/storage_handlers/__init__.py +0 -0
  47. wandb/sdk/artifacts/storage_handlers/azure_handler.py +192 -0
  48. wandb/sdk/artifacts/storage_handlers/gcs_handler.py +224 -0
  49. wandb/sdk/artifacts/storage_handlers/http_handler.py +112 -0
  50. wandb/sdk/artifacts/storage_handlers/local_file_handler.py +134 -0
  51. wandb/sdk/artifacts/storage_handlers/multi_handler.py +53 -0
  52. wandb/sdk/artifacts/storage_handlers/s3_handler.py +301 -0
  53. wandb/sdk/artifacts/storage_handlers/tracking_handler.py +67 -0
  54. wandb/sdk/artifacts/storage_handlers/wb_artifact_handler.py +132 -0
  55. wandb/sdk/artifacts/storage_handlers/wb_local_artifact_handler.py +72 -0
  56. wandb/sdk/artifacts/storage_layout.py +6 -0
  57. wandb/sdk/artifacts/storage_policies/__init__.py +0 -0
  58. wandb/sdk/artifacts/storage_policies/s3_bucket_policy.py +61 -0
  59. wandb/sdk/artifacts/storage_policies/wandb_storage_policy.py +386 -0
  60. wandb/sdk/{interface/artifacts/artifact_storage.py → artifacts/storage_policy.py} +5 -57
  61. wandb/sdk/data_types/_dtypes.py +7 -12
  62. wandb/sdk/data_types/base_types/json_metadata.py +3 -2
  63. wandb/sdk/data_types/base_types/media.py +8 -8
  64. wandb/sdk/data_types/base_types/wb_value.py +12 -13
  65. wandb/sdk/data_types/helper_types/bounding_boxes_2d.py +5 -6
  66. wandb/sdk/data_types/helper_types/classes.py +6 -8
  67. wandb/sdk/data_types/helper_types/image_mask.py +5 -6
  68. wandb/sdk/data_types/histogram.py +4 -3
  69. wandb/sdk/data_types/html.py +3 -4
  70. wandb/sdk/data_types/image.py +11 -9
  71. wandb/sdk/data_types/molecule.py +5 -3
  72. wandb/sdk/data_types/object_3d.py +7 -5
  73. wandb/sdk/data_types/plotly.py +3 -2
  74. wandb/sdk/data_types/saved_model.py +11 -11
  75. wandb/sdk/data_types/trace_tree.py +5 -4
  76. wandb/sdk/data_types/utils.py +3 -5
  77. wandb/sdk/data_types/video.py +5 -4
  78. wandb/sdk/integration_utils/auto_logging.py +215 -0
  79. wandb/sdk/interface/interface.py +15 -15
  80. wandb/sdk/internal/file_pusher.py +8 -16
  81. wandb/sdk/internal/file_stream.py +5 -11
  82. wandb/sdk/internal/handler.py +13 -1
  83. wandb/sdk/internal/internal_api.py +287 -13
  84. wandb/sdk/internal/job_builder.py +119 -30
  85. wandb/sdk/internal/sender.py +6 -26
  86. wandb/sdk/internal/settings_static.py +2 -0
  87. wandb/sdk/internal/system/assets/__init__.py +2 -0
  88. wandb/sdk/internal/system/assets/gpu.py +42 -0
  89. wandb/sdk/internal/system/assets/gpu_amd.py +216 -0
  90. wandb/sdk/internal/system/env_probe_helpers.py +13 -0
  91. wandb/sdk/internal/system/system_info.py +3 -3
  92. wandb/sdk/internal/tb_watcher.py +32 -22
  93. wandb/sdk/internal/thread_local_settings.py +18 -0
  94. wandb/sdk/launch/_project_spec.py +57 -11
  95. wandb/sdk/launch/agent/agent.py +147 -65
  96. wandb/sdk/launch/agent/job_status_tracker.py +34 -0
  97. wandb/sdk/launch/agent/run_queue_item_file_saver.py +45 -0
  98. wandb/sdk/launch/builder/abstract.py +5 -1
  99. wandb/sdk/launch/builder/build.py +21 -18
  100. wandb/sdk/launch/builder/docker_builder.py +10 -4
  101. wandb/sdk/launch/builder/kaniko_builder.py +113 -23
  102. wandb/sdk/launch/builder/noop.py +6 -3
  103. wandb/sdk/launch/builder/templates/_wandb_bootstrap.py +46 -14
  104. wandb/sdk/launch/environment/aws_environment.py +3 -2
  105. wandb/sdk/launch/environment/azure_environment.py +124 -0
  106. wandb/sdk/launch/environment/gcp_environment.py +2 -4
  107. wandb/sdk/launch/environment/local_environment.py +1 -1
  108. wandb/sdk/launch/errors.py +19 -0
  109. wandb/sdk/launch/github_reference.py +32 -19
  110. wandb/sdk/launch/launch.py +3 -8
  111. wandb/sdk/launch/launch_add.py +6 -2
  112. wandb/sdk/launch/loader.py +21 -2
  113. wandb/sdk/launch/registry/azure_container_registry.py +132 -0
  114. wandb/sdk/launch/registry/elastic_container_registry.py +39 -5
  115. wandb/sdk/launch/registry/google_artifact_registry.py +68 -26
  116. wandb/sdk/launch/registry/local_registry.py +2 -1
  117. wandb/sdk/launch/runner/abstract.py +24 -3
  118. wandb/sdk/launch/runner/kubernetes_runner.py +479 -26
  119. wandb/sdk/launch/runner/local_container.py +103 -51
  120. wandb/sdk/launch/runner/local_process.py +1 -1
  121. wandb/sdk/launch/runner/sagemaker_runner.py +60 -10
  122. wandb/sdk/launch/runner/vertex_runner.py +10 -5
  123. wandb/sdk/launch/sweeps/__init__.py +7 -9
  124. wandb/sdk/launch/sweeps/scheduler.py +307 -77
  125. wandb/sdk/launch/sweeps/scheduler_sweep.py +2 -1
  126. wandb/sdk/launch/sweeps/utils.py +82 -35
  127. wandb/sdk/launch/utils.py +89 -75
  128. wandb/sdk/lib/_settings_toposort_generated.py +7 -0
  129. wandb/sdk/lib/capped_dict.py +26 -0
  130. wandb/sdk/lib/{git.py → gitlib.py} +76 -59
  131. wandb/sdk/lib/hashutil.py +12 -4
  132. wandb/sdk/lib/paths.py +96 -8
  133. wandb/sdk/lib/sock_client.py +2 -2
  134. wandb/sdk/lib/timer.py +1 -0
  135. wandb/sdk/service/server.py +22 -9
  136. wandb/sdk/service/server_sock.py +1 -1
  137. wandb/sdk/service/service.py +27 -8
  138. wandb/sdk/verify/verify.py +4 -7
  139. wandb/sdk/wandb_config.py +2 -6
  140. wandb/sdk/wandb_init.py +57 -53
  141. wandb/sdk/wandb_require.py +7 -0
  142. wandb/sdk/wandb_run.py +61 -223
  143. wandb/sdk/wandb_settings.py +28 -4
  144. wandb/testing/relay.py +15 -2
  145. wandb/util.py +74 -36
  146. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/METADATA +15 -9
  147. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/RECORD +151 -116
  148. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/entry_points.txt +1 -0
  149. wandb/integration/langchain/util.py +0 -191
  150. wandb/sdk/interface/artifacts/__init__.py +0 -33
  151. wandb/sdk/interface/artifacts/artifact.py +0 -615
  152. wandb/sdk/interface/artifacts/artifact_manifest.py +0 -131
  153. wandb/sdk/wandb_artifacts.py +0 -2226
  154. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/LICENSE +0 -0
  155. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/WHEEL +0 -0
  156. {wandb-0.15.3.dist-info → wandb-0.15.5.dist-info}/top_level.txt +0 -0
@@ -2,7 +2,7 @@ from typing import Optional, Union
2
2
 
3
3
  from .helpers import LineKey, PCColumn
4
4
  from .util import Attr, Panel, coalesce, nested_get, nested_set
5
- from .validators import ( # MARKS,; Between,
5
+ from .validators import (
6
6
  AGGFUNCS,
7
7
  CODE_COMPARE_DIFF,
8
8
  FONT_SIZES,
@@ -570,9 +570,9 @@ class ParallelCoordinatesPlot(Panel):
570
570
  validators=[TypeValidator(Union[PCColumn, str], how="keys")],
571
571
  )
572
572
  title: Optional[str] = Attr(json_path="spec.config.chartTitle")
573
+ gradient: Optional[list] = Attr(json_path="spec.config.customGradient")
573
574
 
574
575
  # Attr(json_path="spec.config.dimensions")
575
- # Attr(json_path="spec.config.customGradient")
576
576
  # Attr(json_path="spec.config.gradientColor")
577
577
  # Attr(json_path="spec.config.legendFields")
578
578
  font_size: Optional[str] = Attr(
@@ -720,14 +720,32 @@ class Vega(Panel):
720
720
  class CustomChart(Panel):
721
721
  query: dict = Attr(json_path="spec.config.userQuery.queryFields")
722
722
  chart_name: str = Attr(json_path="spec.config.panelDefId")
723
- user_fields: dict = Attr(json_path="spec.config.fieldSettings")
723
+ chart_fields: dict = Attr(json_path="spec.config.fieldSettings")
724
+ chart_strings: dict = Attr(json_path="spec.config.stringSettings")
724
725
 
725
- def __init__(self, query=None, chart_name="", user_fields=None, *args, **kwargs):
726
+ def __init__(
727
+ self,
728
+ query=None,
729
+ chart_name="",
730
+ chart_fields=None,
731
+ chart_strings=None,
732
+ *args,
733
+ **kwargs,
734
+ ):
726
735
  super().__init__(*args, **kwargs)
727
736
  self.spec["config"] = {"transform": {"name": "tableWithLeafColNames"}}
728
737
  self.query = coalesce(query, {})
729
738
  self.chart_name = chart_name
730
- self.user_fields = coalesce(user_fields, {})
739
+ self.chart_fields = coalesce(chart_fields, {})
740
+ self.chart_strings = coalesce(chart_strings, {})
741
+
742
+ @classmethod
743
+ def from_table(cls, table_name, chart_fields=None, chart_strings=None):
744
+ return CustomChart(
745
+ query={"summaryTable": {"tableKey": table_name}},
746
+ chart_fields=chart_fields,
747
+ chart_strings=chart_strings,
748
+ )
731
749
 
732
750
  @property
733
751
  def view_type(self) -> str:
@@ -735,46 +753,51 @@ class CustomChart(Panel):
735
753
 
736
754
  @query.getter
737
755
  def query(self):
738
- d = nested_get(self, self._get_path("query"))
739
- fields = d[0]["fields"]
740
- query = {o["name"]: o.get("args") for o in fields}
741
- query = {
742
- k: v[0]["value"] if v is not None else None
743
- for k, v in query.items()
744
- if k not in ("id", "name")
745
- }
746
- return query
756
+ def fields_to_dict(fields):
757
+ d = {}
758
+ for field in fields:
759
+ keys = set(field.keys())
760
+ name = field["name"]
761
+
762
+ if keys == {"name", "fields"}:
763
+ d[name] = {}
764
+ elif keys == {"name", "value"}:
765
+ d[name] = field["value"]
766
+ elif keys == {"name", "args", "fields"}:
767
+ d[name] = fields_to_dict(field["args"])
768
+ return d
769
+
770
+ fields = nested_get(self, self._get_path("query"))
771
+ return fields_to_dict(fields)
747
772
 
748
773
  @query.setter
749
774
  def query(self, d):
750
- def make_fields(d):
775
+ def dict_to_fields(d):
751
776
  fields = []
752
777
  for k, v in d.items():
753
- if v is None:
778
+ if isinstance(v, dict) and len(v) > 0:
779
+ field = {"name": k, "args": dict_to_fields(v), "fields": []}
780
+ elif isinstance(v, dict) and len(v) == 0 or v is None:
754
781
  field = {"name": k, "fields": []}
755
782
  else:
756
- field = {
757
- "name": k,
758
- "fields": [],
759
- "args": [{"name": "keys", "value": v}],
760
- }
783
+ field = {"name": k, "value": v}
761
784
  fields.append(field)
762
- if "id" not in d:
763
- fields.append({"name": "id", "fields": []})
764
- if "name" not in d:
765
- fields.append({"name": "name", "fields": []})
766
785
  return fields
767
786
 
768
- query = {
769
- "args": [
770
- {"name": "runSets", "value": "${runSets}"},
771
- {"name": "limit", "value": 500},
772
- ],
773
- "name": "runSets",
774
- }
775
- query["fields"] = make_fields(d)
776
- query = [query]
777
- nested_set(self, self._get_path("query"), query)
787
+ d.setdefault("id", [])
788
+ d.setdefault("name", [])
789
+
790
+ _query = [
791
+ {
792
+ "args": [
793
+ {"name": "runSets", "value": r"${runSets}"},
794
+ {"name": "limit", "value": 500},
795
+ ],
796
+ "fields": dict_to_fields(d),
797
+ "name": "runSets",
798
+ }
799
+ ]
800
+ nested_set(self, self._get_path("query"), _query)
778
801
 
779
802
 
780
803
  class Vega3(Panel):
wandb/beta/workflows.py CHANGED
@@ -5,12 +5,12 @@ from typing import Any, Dict, List, Optional, Union
5
5
  import wandb
6
6
  import wandb.data_types as data_types
7
7
  from wandb.data_types import _SavedModel
8
- from wandb.sdk.interface.artifacts import Artifact as ArtifactInterface
9
- from wandb.sdk.interface.artifacts import ArtifactManifestEntry
8
+ from wandb.sdk.artifacts.artifact import Artifact
9
+ from wandb.sdk.artifacts.artifact_manifest_entry import ArtifactManifestEntry
10
10
 
11
11
 
12
12
  def _add_any(
13
- artifact: ArtifactInterface,
13
+ artifact: Artifact,
14
14
  path_or_obj: Union[
15
15
  str, ArtifactManifestEntry, data_types.WBValue
16
16
  ], # todo: add dataframe
@@ -23,8 +23,7 @@ def _add_any(
23
23
  be moved to the Artifact class in the future.
24
24
 
25
25
  Args:
26
- artifact: `ArtifactInterface` - most likely a LocalArtifact created with
27
- `wandb.Artifact(...)`
26
+ artifact: `Artifact` - artifact created with `wandb.Artifact(...)`
28
27
  path_or_obj: `Union[str, ArtifactManifestEntry, data_types.WBValue]` - either a
29
28
  str or valid object which indicates what to add to an artifact.
30
29
 
@@ -62,7 +61,7 @@ def _log_artifact_version(
62
61
  project: Optional[str] = None,
63
62
  scope_project: Optional[bool] = None,
64
63
  job_type: str = "auto",
65
- ) -> ArtifactInterface:
64
+ ) -> Artifact:
66
65
  """Create an artifact, populate it, and log it with a run.
67
66
 
68
67
  If a run is not present, we create one.
@@ -83,7 +82,7 @@ def _log_artifact_version(
83
82
  Used to identify runs of a certain job type, i.e "evaluation".
84
83
 
85
84
  Returns:
86
- ArtifactInterface
85
+ Artifact
87
86
 
88
87
  """
89
88
  if wandb.run is None:
wandb/cli/cli.py CHANGED
@@ -15,7 +15,7 @@ import textwrap
15
15
  import time
16
16
  import traceback
17
17
  from functools import wraps
18
- from typing import Any, Dict
18
+ from typing import Any, Dict, Optional
19
19
 
20
20
  import click
21
21
  import yaml
@@ -32,13 +32,13 @@ import wandb.sdk.verify.verify as wandb_verify
32
32
  from wandb import Config, Error, env, util, wandb_agent, wandb_sdk
33
33
  from wandb.apis import InternalApi, PublicApi
34
34
  from wandb.integration.magic import magic_install
35
+ from wandb.sdk.artifacts.artifacts_cache import get_artifacts_cache
36
+ from wandb.sdk.launch.errors import ExecutionError, LaunchError
35
37
  from wandb.sdk.launch.launch_add import _launch_add
36
- from wandb.sdk.launch.sweeps import SCHEDULER_URI
37
38
  from wandb.sdk.launch.sweeps import utils as sweep_utils
39
+ from wandb.sdk.launch.sweeps.scheduler import Scheduler
38
40
  from wandb.sdk.launch.utils import (
39
41
  LAUNCH_DEFAULT_PROJECT,
40
- ExecutionError,
41
- LaunchError,
42
42
  check_logged_in,
43
43
  construct_launch_spec,
44
44
  )
@@ -921,12 +921,6 @@ def sweep(
921
921
  default=None,
922
922
  help="Resume a launch sweep by passing an 8-char sweep id. Queue required",
923
923
  )
924
- @click.option(
925
- "--num_workers",
926
- "-n",
927
- default=1,
928
- help="Number of concurrent jobs a scheduler can run",
929
- )
930
924
  @click.argument("config", required=False, type=click.Path(exists=True))
931
925
  @click.pass_context
932
926
  @display_error
@@ -937,36 +931,14 @@ def launch_sweep(
937
931
  queue,
938
932
  config,
939
933
  resume_id,
940
- num_workers,
941
934
  ):
942
935
  api = _get_cling_api()
936
+ env = os.environ
943
937
  if api.api_key is None:
944
938
  wandb.termlog("Login to W&B to use the sweep feature")
945
939
  ctx.invoke(login, no_offline=True)
946
940
  api = _get_cling_api(reset=True)
947
941
 
948
- # if not sweep_config XOR resume_id
949
- if not (config or resume_id):
950
- wandb.termerror("'config' and/or 'resume_id' required")
951
- return
952
-
953
- parsed_config = sweep_utils.load_launch_sweep_config(config)
954
- # Rip special keys out of config
955
- scheduler_args: Dict[str, Any] = parsed_config.get("scheduler", {})
956
- launch_args: Dict[str, Any] = parsed_config.get("launch", {})
957
- if scheduler_args:
958
- del parsed_config["scheduler"]
959
- if launch_args:
960
- del parsed_config["launch"]
961
-
962
- queue = queue or launch_args.get("queue")
963
- if not queue:
964
- wandb.termerror(
965
- "Launch-sweeps require setting a 'queue', use --queue option or a 'queue' key in the 'launch' section in the config"
966
- )
967
- return
968
-
969
- env = os.environ
970
942
  entity = entity or env.get("WANDB_ENTITY") or api.settings("entity")
971
943
  if entity is None:
972
944
  wandb.termerror("Must specify entity when using launch")
@@ -977,51 +949,146 @@ def launch_sweep(
977
949
  wandb.termerror("A project must be configured when using launch")
978
950
  return
979
951
 
980
- parsed_sweep_config, sweep_obj_id = None, None
981
- if resume_id: # Resuming an existing sweep
952
+ # get personal username, not team name or service account, default to entity
953
+ author = api.viewer().get("username") or entity
954
+
955
+ # if not sweep_config XOR resume_id
956
+ if not (config or resume_id):
957
+ wandb.termerror("'config' and/or 'resume_id' required")
958
+ return
959
+
960
+ parsed_user_config = sweep_utils.load_launch_sweep_config(config)
961
+ # Rip special keys out of config, store in scheduler run_config
962
+ launch_args: Dict[str, Any] = parsed_user_config.pop("launch", {})
963
+ scheduler_args: Dict[str, Any] = parsed_user_config.pop("scheduler", {})
964
+ settings: Dict[str, Any] = scheduler_args.pop("settings", {})
965
+
966
+ scheduler_job: Optional[str] = scheduler_args.get("job")
967
+ if scheduler_job:
968
+ wandb.termwarn(
969
+ "Using a scheduler job for launch sweeps is *experimental* and may change without warning"
970
+ )
971
+ queue: Optional[str] = queue or launch_args.get("queue")
972
+
973
+ sweep_config, sweep_obj_id = None, None
974
+ if not resume_id:
975
+ sweep_config = parsed_user_config
976
+
977
+ # check method
978
+ method = sweep_config.get("method")
979
+ if scheduler_job and not method:
980
+ sweep_config["method"] = "custom"
981
+ elif scheduler_job and method != "custom":
982
+ # TODO(gst): Check if using Anaconda2
983
+ wandb.termwarn(
984
+ "Use 'method': 'custom' in the sweep config when using scheduler jobs, "
985
+ "or omit it entirely. For jobs using the wandb optimization engine (WandbScheduler), "
986
+ "set the method in the sweep config under scheduler.settings.method "
987
+ )
988
+ settings["method"] = method
989
+
990
+ if settings.get("method"):
991
+ # assume WandbScheduler, and user is using this right
992
+ sweep_config["method"] = settings["method"]
993
+
994
+ else: # Resuming an existing sweep
982
995
  found = api.sweep(resume_id, "{}", entity=entity, project=project)
983
996
  if not found:
984
997
  wandb.termerror(f"Could not find sweep {entity}/{project}/{resume_id}")
985
998
  return
999
+
1000
+ if found.get("state") == "RUNNING":
1001
+ wandb.termerror(
1002
+ f"Cannot resume sweep {entity}/{project}/{resume_id}, it is already running"
1003
+ )
1004
+ return
1005
+
986
1006
  sweep_obj_id = found["id"]
987
- parsed_sweep_config = yaml.safe_load(found["config"])
1007
+ sweep_config = yaml.safe_load(found["config"])
988
1008
  wandb.termlog(f"Resuming from existing sweep {entity}/{project}/{resume_id}")
989
- if len(parsed_config.keys()) > 0:
1009
+ if len(parsed_user_config.keys()) > 0:
990
1010
  wandb.termwarn(
991
- "Sweep params loaded from resumed sweep, ignoring provided keys"
1011
+ "Sweep parameters loaded from resumed sweep, ignoring provided config"
992
1012
  )
993
- else:
994
- parsed_sweep_config = parsed_config
995
1013
 
996
- num_workers = num_workers or scheduler_args.get("num_workers", 8)
997
- scheduler_entrypoint = sweep_utils.construct_scheduler_entrypoint(
998
- sweep_config=parsed_sweep_config,
1014
+ prev_scheduler = json.loads(found.get("scheduler") or "{}")
1015
+ run_spec = json.loads(prev_scheduler.get("run_spec", "{}"))
1016
+ if (
1017
+ scheduler_job
1018
+ and run_spec.get("job")
1019
+ and run_spec.get("job") != scheduler_job
1020
+ ):
1021
+ wandb.termerror(
1022
+ f"Resuming a launch sweep with a different scheduler job is not supported. Job loaded from sweep: {run_spec.get('job')}, job in config: {scheduler_job}"
1023
+ )
1024
+ return
1025
+
1026
+ prev_scheduler_args, prev_settings = sweep_utils.get_previous_args(run_spec)
1027
+ # Passed in scheduler_args and settings override previous
1028
+ scheduler_args.update(prev_scheduler_args)
1029
+ settings.update(prev_settings)
1030
+ if not queue:
1031
+ wandb.termerror(
1032
+ "Launch-sweeps require setting a 'queue', use --queue option or a 'queue' key in the 'launch' section in the config"
1033
+ )
1034
+ return
1035
+
1036
+ entrypoint = Scheduler.ENTRYPOINT if not scheduler_job else None
1037
+ args = sweep_utils.construct_scheduler_args(
1038
+ return_job=scheduler_job is not None,
1039
+ sweep_config=sweep_config,
999
1040
  queue=queue,
1000
1041
  project=project,
1001
- num_workers=num_workers,
1002
- author=entity,
1042
+ author=author,
1003
1043
  )
1004
- if not scheduler_entrypoint:
1005
- # error already logged
1044
+ if not args:
1006
1045
  return
1007
1046
 
1047
+ # validate training job existence
1048
+ if not sweep_utils.check_job_exists(PublicApi(), sweep_config.get("job")):
1049
+ return False
1050
+
1051
+ # validate scheduler job existence, if present
1052
+ if not sweep_utils.check_job_exists(PublicApi(), scheduler_job):
1053
+ return False
1054
+
1055
+ # Set run overrides for the Scheduler
1056
+ overrides = {"run_config": {}}
1057
+ if launch_args:
1058
+ overrides["run_config"]["launch"] = launch_args
1059
+ if scheduler_args:
1060
+ overrides["run_config"]["scheduler"] = scheduler_args
1061
+ if settings:
1062
+ overrides["run_config"]["settings"] = settings
1063
+
1064
+ if scheduler_job:
1065
+ overrides["run_config"]["sweep_args"] = args
1066
+ else:
1067
+ overrides["args"] = args
1068
+
1069
+ # set name of scheduler
1070
+ prefix = "sweep" if scheduler_job else "wandb"
1071
+ name = f"{prefix}-scheduler-WANDB_SWEEP_ID"
1072
+ if scheduler_args.get("name"):
1073
+ name = scheduler_args["name"]
1074
+
1008
1075
  # Launch job spec for the Scheduler
1009
1076
  launch_scheduler_spec = construct_launch_spec(
1010
- uri=SCHEDULER_URI,
1077
+ uri=Scheduler.PLACEHOLDER_URI,
1011
1078
  api=api,
1012
- name="Scheduler.WANDB_SWEEP_ID",
1079
+ name=name,
1013
1080
  project=project,
1014
1081
  entity=entity,
1015
1082
  docker_image=scheduler_args.get("docker_image"),
1016
1083
  resource=scheduler_args.get("resource", "local-process"),
1017
- entry_point=scheduler_entrypoint,
1084
+ entry_point=entrypoint,
1018
1085
  resource_args=scheduler_args.get("resource_args", {}),
1019
1086
  repository=launch_args.get("registry", {}).get("url", None),
1020
- job=None,
1087
+ job=scheduler_job,
1021
1088
  version=None,
1022
- launch_config=None,
1023
- run_id=None,
1024
- author=None, # author gets passed into scheduler command
1089
+ launch_config={"overrides": overrides},
1090
+ run_id="WANDB_SWEEP_ID", # scheduler inits run with sweep_id=run_id
1091
+ author=None, # author gets passed into scheduler override args
1025
1092
  )
1026
1093
  launch_scheduler_with_queue = json.dumps(
1027
1094
  {
@@ -1032,7 +1099,7 @@ def launch_sweep(
1032
1099
  )
1033
1100
 
1034
1101
  sweep_id, warnings = api.upsert_sweep(
1035
- parsed_sweep_config,
1102
+ sweep_config,
1036
1103
  project=project,
1037
1104
  entity=entity,
1038
1105
  obj_id=sweep_obj_id, # if resuming
@@ -1040,7 +1107,6 @@ def launch_sweep(
1040
1107
  state="PENDING",
1041
1108
  )
1042
1109
  sweep_utils.handle_sweep_config_violations(warnings)
1043
-
1044
1110
  # Log nicely formatted sweep information
1045
1111
  styled_id = click.style(sweep_id, fg="yellow")
1046
1112
  wandb.termlog(f"{'Resumed' if resume_id else 'Created'} sweep with ID: {styled_id}")
@@ -1169,6 +1235,7 @@ def launch_sweep(
1169
1235
  "--project-queue",
1170
1236
  "-pq",
1171
1237
  default=None,
1238
+ hidden=True,
1172
1239
  help="Name of the project containing the queue to push to. If none, defaults to entity level queues.",
1173
1240
  )
1174
1241
  @display_error
@@ -1359,6 +1426,7 @@ def launch_agent(
1359
1426
 
1360
1427
  from wandb.sdk.launch import launch as wandb_launch
1361
1428
 
1429
+ api = _get_cling_api()
1362
1430
  wandb._sentry.configure_scope(process_context="launch_agent")
1363
1431
  agent_config, api = wandb_launch.resolve_agent_config(
1364
1432
  entity, project, max_jobs, queues, config
@@ -1439,7 +1507,8 @@ def scheduler(
1439
1507
  _args = int(_args)
1440
1508
  kwargs[_key] = _args
1441
1509
  try:
1442
- _scheduler = load_scheduler("sweep")(
1510
+ sweep_type = kwargs.get("sweep_type", "wandb")
1511
+ _scheduler = load_scheduler(scheduler_type=sweep_type)(
1443
1512
  api,
1444
1513
  sweep_id=sweep_id,
1445
1514
  **kwargs,
@@ -1937,11 +2006,12 @@ def cache():
1937
2006
  help="Clean up less frequently used files from the artifacts cache",
1938
2007
  )
1939
2008
  @click.argument("target_size")
2009
+ @click.option("--remove-temp/--no-remove-temp", default=False, help="Remove temp files")
1940
2010
  @display_error
1941
- def cleanup(target_size):
2011
+ def cleanup(target_size, remove_temp):
1942
2012
  target_size = util.from_human_size(target_size)
1943
- cache = wandb_sdk.wandb_artifacts.get_artifacts_cache()
1944
- reclaimed_bytes = cache.cleanup(target_size)
2013
+ cache = get_artifacts_cache()
2014
+ reclaimed_bytes = cache.cleanup(target_size, remove_temp)
1945
2015
  print(f"Reclaimed {util.to_human_size(reclaimed_bytes)} of space")
1946
2016
 
1947
2017
 
wandb/data_types.py CHANGED
@@ -47,6 +47,7 @@ from .sdk.data_types.molecule import Molecule
47
47
  from .sdk.data_types.object_3d import Object3D
48
48
  from .sdk.data_types.plotly import Plotly
49
49
  from .sdk.data_types.saved_model import _SavedModel
50
+ from .sdk.data_types.trace_tree import WBTraceTree
50
51
  from .sdk.data_types.video import Video
51
52
  from .sdk.lib import runid
52
53
 
@@ -67,6 +68,7 @@ __all__ = [
67
68
  "Object3D",
68
69
  "Plotly",
69
70
  "Video",
71
+ "WBTraceTree",
70
72
  "_SavedModel",
71
73
  # Typed Legacy Exports (I'd like to remove these)
72
74
  "ImageMask",
@@ -633,7 +635,7 @@ class Table(Media):
633
635
  }
634
636
  )
635
637
 
636
- elif isinstance(run_or_artifact, wandb.wandb_sdk.wandb_artifacts.Artifact):
638
+ elif isinstance(run_or_artifact, wandb.Artifact):
637
639
  artifact = run_or_artifact
638
640
  mapped_data = []
639
641
  data = self._to_table_json(Table.MAX_ARTIFACT_ROWS)["data"]
@@ -5,18 +5,11 @@ import logging
5
5
  import os
6
6
  import queue
7
7
  import time
8
- from typing import (
9
- TYPE_CHECKING,
10
- Any,
11
- Mapping,
12
- MutableMapping,
13
- MutableSet,
14
- NewType,
15
- Optional,
16
- )
8
+ from typing import TYPE_CHECKING, Any, Mapping, MutableMapping, MutableSet, Optional
17
9
 
18
10
  from wandb import util
19
11
  from wandb.sdk.interface.interface import GlobStr
12
+ from wandb.sdk.lib.paths import LogicalPath
20
13
 
21
14
  if TYPE_CHECKING:
22
15
  import wandb.vendor.watchdog_0_9_0.observers.api as wd_api
@@ -30,7 +23,7 @@ else:
30
23
  wd_events = util.vendor_import("wandb_watchdog.events")
31
24
 
32
25
  PathStr = str # TODO(spencerpearson): would be nice to use Path here
33
- SaveName = NewType("SaveName", str)
26
+
34
27
 
35
28
  logger = logging.getLogger(__name__)
36
29
 
@@ -39,15 +32,14 @@ class FileEventHandler(abc.ABC):
39
32
  def __init__(
40
33
  self,
41
34
  file_path: PathStr,
42
- save_name: SaveName,
35
+ save_name: LogicalPath,
43
36
  file_pusher: "FilePusher",
44
37
  *args: Any,
45
38
  **kwargs: Any,
46
39
  ) -> None:
47
40
  self.file_path = file_path
48
41
  # Convert windows paths to unix paths
49
- save_name = SaveName(util.to_forward_slash_path(save_name))
50
- self.save_name = save_name
42
+ self.save_name = LogicalPath(save_name)
51
43
  self._file_pusher = file_pusher
52
44
  self._last_sync: Optional[float] = None
53
45
 
@@ -64,7 +56,7 @@ class FileEventHandler(abc.ABC):
64
56
  def finish(self) -> None:
65
57
  raise NotImplementedError
66
58
 
67
- def on_renamed(self, new_path: PathStr, new_name: SaveName) -> None:
59
+ def on_renamed(self, new_path: PathStr, new_name: LogicalPath) -> None:
68
60
  self.file_path = new_path
69
61
  self.save_name = new_name
70
62
  self.on_modified()
@@ -120,7 +112,7 @@ class PolicyLive(FileEventHandler):
120
112
  def __init__(
121
113
  self,
122
114
  file_path: PathStr,
123
- save_name: SaveName,
115
+ save_name: LogicalPath,
124
116
  file_pusher: "FilePusher",
125
117
  settings: Optional["wandb_settings.Settings"] = None,
126
118
  *args: Any,
@@ -203,14 +195,14 @@ class DirWatcher:
203
195
  self._file_count = 0
204
196
  self._dir = file_dir or settings.files_dir
205
197
  self._settings = settings
206
- self._savename_file_policies: MutableMapping[SaveName, "PolicyName"] = {}
198
+ self._savename_file_policies: MutableMapping[LogicalPath, "PolicyName"] = {}
207
199
  self._user_file_policies: Mapping["PolicyName", MutableSet[GlobStr]] = {
208
200
  "end": set(),
209
201
  "live": set(),
210
202
  "now": set(),
211
203
  }
212
204
  self._file_pusher = file_pusher
213
- self._file_event_handlers: MutableMapping[SaveName, FileEventHandler] = {}
205
+ self._file_event_handlers: MutableMapping[LogicalPath, FileEventHandler] = {}
214
206
  self._file_observer = wd_polling.PollingObserver()
215
207
  self._file_observer.schedule(
216
208
  self._per_file_event_handler(), self._dir, recursive=True
@@ -232,7 +224,9 @@ class DirWatcher:
232
224
  # faster if the name happens to include glob escapable characters. In
233
225
  # the future we may add a flag to "files" records that indicates it's
234
226
  # policy is not dynamic and doesn't need to be stored / checked.
235
- save_name = SaveName(os.path.relpath(os.path.join(self._dir, path), self._dir))
227
+ save_name = LogicalPath(
228
+ os.path.relpath(os.path.join(self._dir, path), self._dir)
229
+ )
236
230
  if save_name.startswith("media/"):
237
231
  pass
238
232
  elif path == glob.escape(path):
@@ -240,7 +234,7 @@ class DirWatcher:
240
234
  else:
241
235
  self._user_file_policies[policy].add(path)
242
236
  for src_path in glob.glob(os.path.join(self._dir, path)):
243
- save_name = SaveName(os.path.relpath(src_path, self._dir))
237
+ save_name = LogicalPath(os.path.relpath(src_path, self._dir))
244
238
  feh = self._get_file_event_handler(src_path, save_name)
245
239
  # handle the case where the policy changed
246
240
  if feh.policy != policy:
@@ -284,18 +278,18 @@ class DirWatcher:
284
278
  emitter = self.emitter
285
279
  if emitter:
286
280
  emitter._timeout = int(self._file_count / 100) + 1
287
- save_name = SaveName(os.path.relpath(event.src_path, self._dir))
281
+ save_name = LogicalPath(os.path.relpath(event.src_path, self._dir))
288
282
  self._get_file_event_handler(event.src_path, save_name).on_modified()
289
283
 
290
284
  # TODO(spencerpearson): this pattern repeats so many times we should have a method/function for it
291
- # def _save_name(self, path: PathStr) -> SaveName:
292
- # return SaveName(os.path.relpath(path, self._dir))
285
+ # def _save_name(self, path: PathStr) -> LogicalPath:
286
+ # return LogicalPath(os.path.relpath(path, self._dir))
293
287
 
294
288
  def _on_file_modified(self, event: "wd_events.FileModifiedEvent") -> None:
295
289
  logger.info(f"file/dir modified: { event.src_path}")
296
290
  if os.path.isdir(event.src_path):
297
291
  return None
298
- save_name = SaveName(os.path.relpath(event.src_path, self._dir))
292
+ save_name = LogicalPath(os.path.relpath(event.src_path, self._dir))
299
293
  self._get_file_event_handler(event.src_path, save_name).on_modified()
300
294
 
301
295
  def _on_file_moved(self, event: "wd_events.FileMovedEvent") -> None:
@@ -303,8 +297,8 @@ class DirWatcher:
303
297
  logger.info(f"file/dir moved: {event.src_path} -> {event.dest_path}")
304
298
  if os.path.isdir(event.dest_path):
305
299
  return None
306
- old_save_name = SaveName(os.path.relpath(event.src_path, self._dir))
307
- new_save_name = SaveName(os.path.relpath(event.dest_path, self._dir))
300
+ old_save_name = LogicalPath(os.path.relpath(event.src_path, self._dir))
301
+ new_save_name = LogicalPath(os.path.relpath(event.dest_path, self._dir))
308
302
 
309
303
  # We have to move the existing file handler to the new name
310
304
  handler = self._get_file_event_handler(event.src_path, old_save_name)
@@ -314,7 +308,7 @@ class DirWatcher:
314
308
  handler.on_renamed(event.dest_path, new_save_name)
315
309
 
316
310
  def _get_file_event_handler(
317
- self, file_path: PathStr, save_name: SaveName
311
+ self, file_path: PathStr, save_name: LogicalPath
318
312
  ) -> FileEventHandler:
319
313
  """Get or create an event handler for a particular file.
320
314
 
@@ -397,7 +391,7 @@ class DirWatcher:
397
391
  for dirpath, _, filenames in os.walk(self._dir):
398
392
  for fname in filenames:
399
393
  file_path = os.path.join(dirpath, fname)
400
- save_name = SaveName(os.path.relpath(file_path, self._dir))
394
+ save_name = LogicalPath(os.path.relpath(file_path, self._dir))
401
395
  ignored = False
402
396
  for glb in self._settings.ignore_globs:
403
397
  if len(fnmatch.filter([save_name], glb)) > 0: