wandb 0.16.5__py3-none-any.whl → 0.17.0__py3-none-any.whl

wandb/sdk/internal/profiler.py

@@ -1,4 +1,5 @@
 """Integration with pytorch profiler."""
+
 import os
 
 import wandb

wandb/sdk/internal/progress.py

@@ -81,31 +81,3 @@ class Progress:
         return self.len
 
     next = __next__
-
-
-class AsyncProgress:
-    """Wrapper around Progress, to make it async iterable.
-
-    httpx, for streaming uploads, requires the data source to be an async iterable.
-    If we pass in a sync iterable (like a bare `Progress` instance), httpx will
-    get confused, think we're trying to make a synchronous request, and raise.
-    So we need this wrapper class to be an async iterable but *not* a sync iterable.
-    """
-
-    def __init__(self, progress: Progress) -> None:
-        self._progress = progress
-
-    def __aiter__(self):
-        return self
-
-    async def __anext__(self):
-        try:
-            return next(self._progress)
-        except StopIteration:
-            raise StopAsyncIteration
-
-    def __len__(self):
-        return len(self._progress)
-
-    def rewind(self) -> None:
-        self._progress.rewind()

wandb/sdk/internal/run.py

@@ -4,6 +4,7 @@
 Semi-stubbed run for internal process use.
 
 """
+
 from wandb._globals import _datatypes_set_callback
 
 from .. import wandb_run

wandb/sdk/internal/sender.py

@@ -327,7 +327,6 @@ class SendManager:
             # ignore_globs=(),
             _sync=True,
             disable_job_creation=False,
-            _async_upload_concurrency_limit=None,
             _file_stream_timeout_seconds=0,
         )
         record_q: Queue[Record] = queue.Queue()
@@ -910,7 +909,7 @@ class SendManager:
         is_wandb_init = self._run is None
 
         # save start time of a run
-        self._start_time = run.start_time.ToMicroseconds() // 1e6
+        self._start_time = int(run.start_time.ToMicroseconds() // 1e6)
 
         # update telemetry
         if run.telemetry:

wandb/sdk/internal/system/assets/gpu_amd.py

@@ -28,14 +28,6 @@ logger = logging.getLogger(__name__)
 ROCM_SMI_CMD: Final[str] = shutil.which("rocm-smi") or "/usr/bin/rocm-smi"
 
 
-def get_rocm_smi_stats() -> Dict[str, Any]:
-    command = [str(ROCM_SMI_CMD), "-a", "--json"]
-    output = subprocess.check_output(command, universal_newlines=True).strip()
-    if "No AMD GPUs specified" in output:
-        return {}
-    return json.loads(output.split("\n")[0])  # type: ignore
-
-
 _StatsKeys = Literal[
     "gpu",
     "memoryAllocated",
@@ -49,6 +41,48 @@ _Stats = Dict[_StatsKeys, float]
 _InfoDict = Dict[str, Union[int, List[Dict[str, Any]]]]
 
 
+def get_rocm_smi_stats() -> Dict[str, Any]:
+    command = [str(ROCM_SMI_CMD), "-a", "--json"]
+    output = subprocess.check_output(command, universal_newlines=True).strip()
+    if "No AMD GPUs specified" in output:
+        return {}
+    return json.loads(output.split("\n")[0])  # type: ignore
+
+
+def parse_stats(stats: Dict[str, str]) -> _Stats:
+    """Parse stats from rocm-smi output."""
+    parsed_stats: _Stats = {}
+
+    try:
+        parsed_stats["gpu"] = float(stats.get("GPU use (%)"))  # type: ignore
+    except (TypeError, ValueError):
+        logger.warning("Could not parse GPU usage as float")
+    try:
+        parsed_stats["memoryAllocated"] = float(stats.get("GPU memory use (%)"))  # type: ignore
+    except (TypeError, ValueError):
+        logger.warning("Could not parse GPU memory allocation as float")
+    try:
+        parsed_stats["temp"] = float(stats.get("Temperature (Sensor memory) (C)"))  # type: ignore
+    except (TypeError, ValueError):
+        logger.warning("Could not parse GPU temperature as float")
+    try:
+        parsed_stats["powerWatts"] = float(
+            stats.get("Average Graphics Package Power (W)")  # type: ignore
+        )
+    except (TypeError, ValueError):
+        logger.warning("Could not parse GPU power as float")
+    try:
+        parsed_stats["powerPercent"] = (
+            float(stats.get("Average Graphics Package Power (W)"))  # type: ignore
+            / float(stats.get("Max Graphics Package Power (W)"))  # type: ignore
+            * 100
+        )
+    except (TypeError, ValueError):
+        logger.warning("Could not parse GPU average/max power as float")
+
+    return parsed_stats
+
+
 class GPUAMDStats:
     """Stats for AMD GPU devices."""
 
@@ -58,40 +92,6 @@ class GPUAMDStats:
     def __init__(self) -> None:
         self.samples = deque()
 
-    @staticmethod
-    def parse_stats(stats: Dict[str, str]) -> _Stats:
-        """Parse stats from rocm-smi output."""
-        parsed_stats: _Stats = {}
-
-        try:
-            parsed_stats["gpu"] = float(stats.get("GPU use (%)"))  # type: ignore
-        except (TypeError, ValueError):
-            logger.warning("Could not parse GPU usage as float")
-        try:
-            parsed_stats["memoryAllocated"] = float(stats.get("GPU memory use (%)"))  # type: ignore
-        except (TypeError, ValueError):
-            logger.warning("Could not parse GPU memory allocation as float")
-        try:
-            parsed_stats["temp"] = float(stats.get("Temperature (Sensor memory) (C)"))  # type: ignore
-        except (TypeError, ValueError):
-            logger.warning("Could not parse GPU temperature as float")
-        try:
-            parsed_stats["powerWatts"] = float(
-                stats.get("Average Graphics Package Power (W)")  # type: ignore
-            )
-        except (TypeError, ValueError):
-            logger.warning("Could not parse GPU power as float")
-        try:
-            parsed_stats["powerPercent"] = (
-                float(stats.get("Average Graphics Package Power (W)"))  # type: ignore
-                / float(stats.get("Max Graphics Package Power (W)"))  # type: ignore
-                * 100
-            )
-        except (TypeError, ValueError):
-            logger.warning("Could not parse GPU average/max power as float")
-
-        return parsed_stats
-
     def sample(self) -> None:
         try:
             raw_stats = get_rocm_smi_stats()
@@ -103,7 +103,7 @@ class GPUAMDStats:
 
             for card_key in card_keys:
                 card_stats = raw_stats[card_key]
-                stats = self.parse_stats(card_stats)
+                stats = parse_stats(card_stats)
                 if stats:
                     cards.append(stats)
 
@@ -183,7 +183,7 @@ class GPUAMD:
 
         can_read_rocm_smi = False
         try:
-            if get_rocm_smi_stats():
+            if parse_stats(get_rocm_smi_stats()):
                 can_read_rocm_smi = True
         except Exception:
             pass

wandb/sdk/internal/system/assets/gpu_apple.py

@@ -37,6 +37,12 @@ class _Stats(TypedDict):
     # cpuWaitMs: float
 
 
+def get_apple_gpu_path() -> pathlib.Path:
+    return (
+        pathlib.Path(sys.modules["wandb"].__path__[0]) / "bin" / "apple_gpu_stats"
+    ).resolve()
+
+
 class GPUAppleStats:
     """Apple GPU stats available on Arm Macs."""
 
@@ -49,9 +55,7 @@ class GPUAppleStats:
 
     def __init__(self) -> None:
         self.samples = deque()
-        self.binary_path = (
-            pathlib.Path(sys.modules["wandb"].__path__[0]) / "bin" / "apple_gpu_stats"
-        ).resolve()
+        self.binary_path = get_apple_gpu_path()
 
     def sample(self) -> None:
         try:
@@ -63,22 +67,47 @@ class GPUAppleStats:
             )[0]
             raw_stats = json.loads(output)
 
+            temp_keys = [
+                "m1Gpu1",
+                "m1Gpu2",
+                "m1Gpu3",
+                "m1Gpu4",
+                "m2Gpu1",
+                "m2Gpu2",
+                "m3Gpu1",
+                "m3Gpu2",
+                "m3Gpu3",
+                "m3Gpu4",
+                "m3Gpu5",
+                "m3Gpu6",
+                "m3Gpu7",
+                "m3Gpu8",
+            ]
+            temp, count = 0, 0
+            for k in temp_keys:
+                if raw_stats.get(k, 0) > 0:
+                    temp += raw_stats[k]
+                    count += 1
+
             stats: _Stats = {
                 "gpu": raw_stats["utilization"],
-                "memoryAllocated": raw_stats["mem_used"],
-                "temp": raw_stats["temperature"],
-                "powerWatts": raw_stats["power"],
-                "powerPercent": (raw_stats["power"] / self.MAX_POWER_WATTS) * 100,
+                "memoryAllocated": (
+                    raw_stats["inUseSystemMemory"]
+                    / raw_stats["allocatedSystemMemory"]
+                    * 100
+                ),
+                "powerWatts": raw_stats["systemPower"],
+                "powerPercent": (raw_stats["systemPower"] / self.MAX_POWER_WATTS) * 100,
+                "temp": temp / count if count > 0 else 0,
                 # TODO: this stat could be useful eventually, it was consistently
                 #  0 in my experimentation and requires a frontend change
                 #  so leaving it out for now.
                 # "cpuWaitMs": raw_stats["cpu_wait_ms"],
             }
-
             self.samples.append(stats)
 
         except (OSError, ValueError, TypeError, subprocess.CalledProcessError) as e:
-            logger.exception(f"GPU stats error: {e}")
+            logger.exception("GPU stats error: %s", e)
 
     def clear(self) -> None:
         self.samples.clear()
@@ -116,6 +145,7 @@ class GPUApple:
         telemetry_record = telemetry.TelemetryRecord()
         telemetry_record.env.m1_gpu = True
         interface._publish_telemetry(telemetry_record)
+        self.binary_path = get_apple_gpu_path()
 
     @classmethod
     def is_available(cls) -> bool:
@@ -128,5 +158,20 @@ class GPUApple:
         self.metrics_monitor.finish()
 
     def probe(self) -> dict:
-        # todo: make this actually meaningful
-        return {self.name: {"type": "arm", "vendor": "Apple"}}
+        try:
+            command = [str(self.binary_path), "--json"]
+            output = (
+                subprocess.check_output(command, universal_newlines=True)
+                .strip()
+                .split("\n")
+            )[0]
+            raw_stats = json.loads(output)
+            return {
+                self.name: {
+                    "type": raw_stats["name"],
+                    "vendor": raw_stats["vendor"],
+                }
+            }
+        except (OSError, ValueError, TypeError, subprocess.CalledProcessError) as e:
+            logger.exception("GPU stats error: %s", e)
+            return {self.name: {"type": "arm", "vendor": "Apple"}}

wandb/sdk/internal/system/assets/interfaces.py

@@ -68,8 +68,7 @@ class Asset(Protocol):
     metrics: List[Metric]
     metrics_monitor: "MetricsMonitor"
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        ...  # pragma: no cover
+    def __init__(self, *args: Any, **kwargs: Any) -> None: ...  # pragma: no cover
 
     @classmethod
     def is_available(cls) -> bool:
@@ -90,14 +89,13 @@ class Asset(Protocol):
 
 
 class Interface(Protocol):
-    def publish_stats(self, stats: dict) -> None:
-        ...  # pragma: no cover
+    def publish_stats(self, stats: dict) -> None: ...  # pragma: no cover
 
-    def _publish_telemetry(self, telemetry: "TelemetryRecord") -> None:
-        ...  # pragma: no cover
+    def _publish_telemetry(
+        self, telemetry: "TelemetryRecord"
+    ) -> None: ...  # pragma: no cover
 
-    def publish_files(self, files_dict: "FilesDict") -> None:
-        ...  # pragma: no cover
+    def publish_files(self, files_dict: "FilesDict") -> None: ...  # pragma: no cover
 
 
 class MetricsMonitor:

wandb/sdk/internal/system/assets/open_metrics.py

@@ -65,13 +65,13 @@ def _setup_requests_session() -> requests.Session:
 
 
 def _nested_dict_to_tuple(
-    nested_dict: Mapping[str, Mapping[str, str]]
+    nested_dict: Mapping[str, Mapping[str, str]],
 ) -> Tuple[Tuple[str, Tuple[str, str]], ...]:
     return tuple((k, *v.items()) for k, v in nested_dict.items())  # type: ignore
 
 
 def _tuple_to_nested_dict(
-    nested_tuple: Tuple[Tuple[str, Tuple[str, str]], ...]
+    nested_tuple: Tuple[Tuple[str, Tuple[str, str]], ...],
 ) -> Dict[str, Dict[str, str]]:
     return {k: dict(v) for k, *v in nested_tuple}
 

wandb/sdk/internal/system/assets/trainium.py

@@ -197,9 +197,7 @@ class NeuronCoreStats:
                 entry["report"]
                 for entry in raw_stats["neuron_runtime_data"]
                 if self._is_matching_entry(entry)
-            ][
-                0
-            ]  # there should be only one entry with the pid
+            ][0]  # there should be only one entry with the pid
 
             neuroncores_in_use = neuron_runtime_data["neuroncore_counters"][
                 "neuroncores_in_use"

wandb/sdk/launch/__init__.py

@@ -1,6 +1,14 @@
 from ._launch import launch
 from ._launch_add import launch_add
 from .agent.agent import LaunchAgent
+from .inputs.manage import manage_config_file, manage_wandb_config
 from .utils import load_wandb_config
 
-__all__ = ["LaunchAgent", "launch", "launch_add", "load_wandb_config"]
+__all__ = [
+    "LaunchAgent",
+    "launch",
+    "launch_add",
+    "load_wandb_config",
+    "manage_config_file",
+    "manage_wandb_config",
+]

wandb/sdk/launch/_launch.py

@@ -12,13 +12,12 @@ from wandb.apis.internal import Api
 from . import loader
 from ._project_spec import LaunchProject
 from .agent import LaunchAgent
-from .builder.build import construct_agent_configs
+from .agent.agent import construct_agent_configs
 from .environment.local_environment import LocalEnvironment
 from .errors import ExecutionError, LaunchError
 from .runner.abstract import AbstractRun
 from .utils import (
     LAUNCH_CONFIG_FILE,
-    LAUNCH_DEFAULT_PROJECT,
     PROJECT_SYNCHRONOUS,
     construct_launch_spec,
     validate_launch_spec_source,
@@ -58,33 +57,32 @@ def set_launch_logfile(logfile: str) -> None:
 
 def resolve_agent_config(  # noqa: C901
     entity: Optional[str],
-    project: Optional[str],
     max_jobs: Optional[int],
     queues: Optional[Tuple[str]],
     config: Optional[str],
+    verbosity: Optional[int],
 ) -> Tuple[Dict[str, Any], Api]:
     """Resolve the agent config.
 
     Arguments:
         api (Api): The api.
         entity (str): The entity.
-        project (str): The project.
         max_jobs (int): The max number of jobs.
         queues (Tuple[str]): The queues.
         config (str): The config.
+        verbosity (int): How verbose to print, 0 or None = default, 1 = print status every 20 seconds, 2 = also print debugging information
 
     Returns:
         Tuple[Dict[str, Any], Api]: The resolved config and api.
     """
     defaults = {
-        "project": LAUNCH_DEFAULT_PROJECT,
         "max_jobs": 1,
         "max_schedulers": 1,
         "queues": [],
         "registry": {},
         "builder": {},
+        "verbosity": 0,
     }
-    user_set_project = False
     resolved_config: Dict[str, Any] = defaults
     config_path = config or os.path.expanduser(LAUNCH_CONFIG_FILE)
     if os.path.isfile(config_path):
@@ -97,16 +95,11 @@ def resolve_agent_config(  # noqa: C901
                     launch_config = {}  # type: ignore
             except yaml.YAMLError as e:
                 raise LaunchError(f"Invalid launch agent config: {e}")
-        if launch_config.get("project") is not None:
-            user_set_project = True
         resolved_config.update(launch_config.items())
     elif config is not None:
         raise LaunchError(
             f"Could not find use specified launch config file: {config_path}"
         )
-    if os.environ.get("WANDB_PROJECT") is not None:
-        resolved_config.update({"project": os.environ.get("WANDB_PROJECT")})
-        user_set_project = True
     if os.environ.get("WANDB_ENTITY") is not None:
         resolved_config.update({"entity": os.environ.get("WANDB_ENTITY")})
     if os.environ.get("WANDB_LAUNCH_MAX_JOBS") is not None:
@@ -114,15 +107,14 @@ def resolve_agent_config(  # noqa: C901
             {"max_jobs": int(os.environ.get("WANDB_LAUNCH_MAX_JOBS", 1))}
         )
 
-    if project is not None:
-        resolved_config.update({"project": project})
-        user_set_project = True
     if entity is not None:
         resolved_config.update({"entity": entity})
     if max_jobs is not None:
         resolved_config.update({"max_jobs": int(max_jobs)})
     if queues:
         resolved_config.update({"queues": list(queues)})
+    if verbosity:
+        resolved_config.update({"verbosity": int(verbosity)})
     # queue -> queues
     if resolved_config.get("queue"):
         if isinstance(resolved_config.get("queue"), str):
@@ -133,7 +125,7 @@ def resolve_agent_config(  # noqa: C901
                 + " (expected str). Specify multiple queues with the 'queues' key"
             )
 
-    keys = ["project", "entity"]
+    keys = ["entity"]
     settings = {
         k: resolved_config.get(k) for k in keys if resolved_config.get(k) is not None
     }
@@ -142,10 +134,6 @@ def resolve_agent_config(  # noqa: C901
 
     if resolved_config.get("entity") is None:
         resolved_config.update({"entity": api.default_entity})
-    if user_set_project:
-        wandb.termwarn(
-            "Specifying a project for the launch agent is deprecated. Please use queues found in the Launch application at https://wandb.ai/launch."
-        )
 
     return resolved_config, api
 
@@ -183,7 +171,6 @@ def create_and_run_agent(
 
 async def _launch(
     api: Api,
-    uri: Optional[str] = None,
     job: Optional[str] = None,
     name: Optional[str] = None,
     project: Optional[str] = None,
@@ -204,7 +191,7 @@ async def _launch(
     if resource is None:
         resource = "local-container"
     launch_spec = construct_launch_spec(
-        uri,
+        None,
         job,
         api,
         name,
@@ -223,7 +210,7 @@ async def _launch(
     validate_launch_spec_source(launch_spec)
     launch_project = LaunchProject.from_spec(launch_spec, api)
     launch_project.fetch_and_validate_project()
-    entrypoint = launch_project.get_single_entry_point()
+    entrypoint = launch_project.get_job_entry_point()
     image_uri = launch_project.docker_image  # Either set by user or None.
 
     # construct runner config.
@@ -321,8 +308,6 @@ def launch(
     """
     submitted_run_obj = asyncio.run(
         _launch(
-            # TODO: fully deprecate URI path
-            uri=None,
             job=job,
             name=name,
             project=project,

wandb/sdk/launch/_launch_add.py

@@ -109,7 +109,6 @@ def launch_add(
 
     return _launch_add(
         api,
-        uri,
         job,
         config,
         template_variables,
@@ -134,7 +133,6 @@ def launch_add(
 
 def _launch_add(
     api: Api,
-    uri: Optional[str],
     job: Optional[str],
     config: Optional[Dict[str, Any]],
     template_variables: Optional[dict],
@@ -156,7 +154,7 @@ def _launch_add(
     priority: Optional[int] = None,
 ) -> "public.QueuedRun":
     launch_spec = construct_launch_spec(
-        uri,
+        None,
         job,
         api,
         name,