wandb 0.20.1__py3-none-win32.whl → 0.20.2rc20250616__py3-none-win32.whl

wandb/sdk/internal/internal_util.py

@@ -1,97 +0,0 @@
-"""Internal utility routines.
-
-Collection of classes to support the internal process.
-
-"""
-
-import logging
-import queue
-import sys
-import threading
-import time
-from typing import TYPE_CHECKING, Optional, Tuple, Type, Union
-
-if TYPE_CHECKING:
-    from queue import Queue
-    from threading import Event
-    from types import TracebackType
-
-    from wandb.proto.wandb_internal_pb2 import Record, Result
-
-    ExceptionType = Union[
-        Tuple[Type[BaseException], BaseException, TracebackType],
-        Tuple[None, None, None],
-    ]
-
-
-logger = logging.getLogger(__name__)
-
-
-class ExceptionThread(threading.Thread):
-    """Class to catch exceptions when running a thread."""
-
-    __stopped: "Event"
-    __exception: Optional["ExceptionType"]
-
-    def __init__(self, stopped: "Event") -> None:
-        threading.Thread.__init__(self)
-        self.__stopped = stopped
-        self.__exception = None
-
-    def _run(self) -> None:
-        raise NotImplementedError
-
-    def run(self) -> None:
-        try:
-            self._run()
-        except Exception:
-            self.__exception = sys.exc_info()
-        finally:
-            if self.__exception and self.__stopped:
-                self.__stopped.set()
-
-    def get_exception(self) -> Optional["ExceptionType"]:
-        return self.__exception
-
-
-class RecordLoopThread(ExceptionThread):
-    """Class to manage reading from queues safely."""
-
-    def __init__(
-        self,
-        input_record_q: "Queue[Record]",
-        result_q: "Queue[Result]",
-        stopped: "Event",
-        debounce_interval_ms: "float" = 1000,
-    ) -> None:
-        ExceptionThread.__init__(self, stopped=stopped)
-        self._input_record_q = input_record_q
-        self._result_q = result_q
-        self._stopped = stopped
-        self._debounce_interval_ms = debounce_interval_ms
-
-    def _setup(self) -> None:
-        raise NotImplementedError
-
-    def _process(self, record: "Record") -> None:
-        raise NotImplementedError
-
-    def _finish(self) -> None:
-        raise NotImplementedError
-
-    def _debounce(self) -> None:
-        raise NotImplementedError
-
-    def _run(self) -> None:
-        self._setup()
-        start = time.time()
-        while not self._stopped.is_set():
-            if time.time() - start >= self._debounce_interval_ms / 1000.0:
-                self._debounce()
-                start = time.time()
-            try:
-                record = self._input_record_q.get(timeout=1)
-            except queue.Empty:
-                continue
-            self._process(record)
-        self._finish()

wandb/sdk/internal/system/assets/__init__.py

@@ -1,25 +0,0 @@
-__all__ = (
-    "asset_registry",
-    "CPU",
-    "Disk",
-    "GPU",
-    "GPUAMD",
-    "IPU",
-    "Memory",
-    "Network",
-    "OpenMetrics",
-    "TPU",
-    "Trainium",
-)
-
-from .asset_registry import asset_registry
-from .cpu import CPU
-from .disk import Disk
-from .gpu import GPU
-from .gpu_amd import GPUAMD
-from .ipu import IPU
-from .memory import Memory
-from .network import Network
-from .open_metrics import OpenMetrics
-from .tpu import TPU
-from .trainium import Trainium

wandb/sdk/internal/system/assets/aggregators.py

@@ -1,31 +0,0 @@
-from typing import Sequence, Union
-
-Number = Union[int, float]
-
-
-def aggregate_mean(samples: Sequence[Number], precision: int = 2) -> float:
-    return round(sum(samples) / len(samples), precision)
-
-
-def aggregate_last(samples: Sequence[Number], precision: int = 2) -> Union[float, int]:
-    if isinstance(samples[-1], int):
-        return samples[-1]
-    return round(samples[-1], precision)
-
-
-def aggregate_max(samples: Sequence[Number], precision: int = 2) -> Union[float, int]:
-    if isinstance(samples[-1], int):
-        return max(samples)
-    return round(max(samples), precision)
-
-
-def aggregate_min(samples: Sequence[Number], precision: int = 2) -> Union[float, int]:
-    if isinstance(samples[-1], int):
-        return min(samples)
-    return round(min(samples), precision)
-
-
-def aggregate_sum(samples: Sequence[Number], precision: int = 2) -> Union[float, int]:
-    if isinstance(samples[-1], int):
-        return sum(samples)
-    return round(sum(samples), precision)

wandb/sdk/internal/system/assets/asset_registry.py

@@ -1,20 +0,0 @@
-from typing import Iterator, List, Type
-
-from .interfaces import Asset
-
-
-class AssetRegistry:
-    def __init__(self) -> None:
-        self._registry: List[Type[Asset]] = []
-
-    def register(self, asset: Type[Asset]) -> Type[Asset]:
-        self._registry.append(asset)
-        return asset
-
-    def __iter__(self) -> Iterator[Type[Asset]]:
-        for asset in self._registry:
-            if asset.is_available():
-                yield asset
-
-
-asset_registry = AssetRegistry()

wandb/sdk/internal/system/assets/cpu.py

@@ -1,163 +0,0 @@
-import threading
-from collections import deque
-from typing import TYPE_CHECKING, List, Optional
-
-try:
-    import psutil
-except ImportError:
-    psutil = None
-from .aggregators import aggregate_last, aggregate_mean
-from .asset_registry import asset_registry
-from .interfaces import Interface, Metric, MetricsMonitor
-
-if TYPE_CHECKING:
-    from typing import Deque
-
-    from wandb.sdk.internal.settings_static import SettingsStatic
-
-
-# CPU Metrics
-
-
-class ProcessCpuPercent:
-    """CPU usage of the process in percent normalized by the number of CPUs."""
-
-    # name = "process_cpu_percent"
-    name = "cpu"
-
-    def __init__(self, pid: int) -> None:
-        self.pid = pid
-        self.samples: Deque[float] = deque([])
-        self.process: Optional[psutil.Process] = None
-
-    def sample(self) -> None:
-        # todo: this is what we'd eventually want to do
-        # self.samples.append(
-        #     (
-        #         datetime.datetime.utcnow(),
-        #         psutil.Process(self.pid).cpu_percent(),
-        #     )
-        # )
-        if self.process is None:
-            self.process = psutil.Process(self.pid)
-
-        self.samples.append(self.process.cpu_percent() / psutil.cpu_count())
-
-    def clear(self) -> None:
-        self.samples.clear()
-
-    def aggregate(self) -> dict:
-        # todo: create a statistics class with helper methods to compute
-        #      mean, median, min, max, etc.
-        if not self.samples:
-            return {}
-        aggregate = aggregate_mean(self.samples)
-        return {self.name: aggregate}
-
-
-class CpuPercent:
-    """CPU usage of the system in percent per core."""
-
-    name = "cpu.{i}.cpu_percent"
-
-    def __init__(self, interval: Optional[float] = None) -> None:
-        self.samples: Deque[List[float]] = deque([])
-        self.interval = interval
-
-    def sample(self) -> None:
-        self.samples.append(psutil.cpu_percent(interval=self.interval, percpu=True))
-
-    def clear(self) -> None:
-        self.samples.clear()
-
-    def aggregate(self) -> dict:
-        if not self.samples:
-            return {}
-        num_cpu = len(self.samples[0])
-        cpu_metrics = {}
-        for i in range(num_cpu):
-            aggregate_i = aggregate_mean([sample[i] for sample in self.samples])
-            cpu_metrics[self.name.format(i=i)] = aggregate_i
-
-        return cpu_metrics
-
-
-class ProcessCpuThreads:
-    """Number of threads used by the process."""
-
-    name = "proc.cpu.threads"
-
-    def __init__(self, pid: int) -> None:
-        self.samples: Deque[int] = deque([])
-        self.pid = pid
-        self.process: Optional[psutil.Process] = None
-
-    def sample(self) -> None:
-        if self.process is None:
-            self.process = psutil.Process(self.pid)
-
-        self.samples.append(self.process.num_threads())
-
-    def clear(self) -> None:
-        self.samples.clear()
-
-    def aggregate(self) -> dict:
-        if not self.samples:
-            return {}
-        return {self.name: aggregate_last(self.samples)}
-
-
-@asset_registry.register
-class CPU:
-    def __init__(
-        self,
-        interface: "Interface",
-        settings: "SettingsStatic",
-        shutdown_event: threading.Event,
-    ) -> None:
-        self.name: str = self.__class__.__name__.lower()
-        self.metrics: List[Metric] = [
-            ProcessCpuPercent(settings.x_stats_pid),
-            # CpuPercent(),
-            ProcessCpuThreads(settings.x_stats_pid),
-        ]
-        self.metrics_monitor: MetricsMonitor = MetricsMonitor(
-            self.name,
-            self.metrics,
-            interface,
-            settings,
-            shutdown_event,
-        )
-
-    @classmethod
-    def is_available(cls) -> bool:
-        return psutil is not None
-
-    def probe(self) -> dict:
-        asset_info = {
-            "cpu_count": psutil.cpu_count(logical=False),
-            "cpu_count_logical": psutil.cpu_count(logical=True),
-        }
-        try:
-            asset_info["cpu_freq"] = {
-                "current": psutil.cpu_freq().current,
-                "min": psutil.cpu_freq().min,
-                "max": psutil.cpu_freq().max,
-            }
-            asset_info["cpu_freq_per_core"] = [
-                {
-                    "current": freq.current,
-                    "min": freq.min,
-                    "max": freq.max,
-                }
-                for freq in psutil.cpu_freq(percpu=True)
-            ]
-        except Exception:
-            pass
-        return asset_info
-
-    def start(self) -> None:
-        self.metrics_monitor.start()
-
-    def finish(self) -> None:
-        self.metrics_monitor.finish()

wandb/sdk/internal/system/assets/disk.py

@@ -1,210 +0,0 @@
-import threading
-from collections import deque
-from typing import TYPE_CHECKING, List, Optional
-
-try:
-    import psutil
-except ImportError:
-    psutil = None
-
-from wandb.errors.term import termwarn
-
-from .aggregators import aggregate_mean
-from .asset_registry import asset_registry
-from .interfaces import Interface, Metric, MetricsMonitor
-
-if TYPE_CHECKING:
-    from typing import Deque
-
-    from wandb.sdk.internal.settings_static import SettingsStatic
-
-
-class DiskUsagePercent:
-    """Total system disk usage in percent."""
-
-    name = "disk.{path}.usagePercent"
-    samples: "Deque[List[float]]"
-
-    def __init__(self, paths: List[str]) -> None:
-        self.samples = deque([])
-        # check if we have access to the disk paths:
-        self.paths: List[str] = []
-        for path in paths:
-            try:
-                psutil.disk_usage(path)
-                self.paths.append(path)
-            except Exception as e:
-                termwarn(f"Could not access disk path {path}: {e}", repeat=False)
-
-    def sample(self) -> None:
-        # self.samples.append(psutil.disk_usage("/").percent)
-        disk_usage: List[float] = []
-        for path in self.paths:
-            disk_usage.append(psutil.disk_usage(path).percent)
-        if disk_usage:
-            self.samples.append(disk_usage)
-
-    def clear(self) -> None:
-        self.samples.clear()
-
-    def aggregate(self) -> dict:
-        if not self.samples:
-            return {}
-        disk_metrics = {}
-        for i, _path in enumerate(self.paths):
-            aggregate_i = aggregate_mean([sample[i] for sample in self.samples])
-            # ugly hack to please the frontend:
-            _path = _path.replace("/", "\\")
-            disk_metrics[self.name.format(path=_path)] = aggregate_i
-
-        return disk_metrics
-
-
-class DiskUsage:
-    """Total system disk usage in GB."""
-
-    name = "disk.{path}.usageGB"
-    samples: "Deque[List[float]]"
-
-    def __init__(self, paths: List[str]) -> None:
-        self.samples = deque([])
-        # check if we have access to the disk paths:
-        self.paths: List[str] = []
-        for path in paths:
-            try:
-                psutil.disk_usage(path)
-                self.paths.append(path)
-            except Exception as e:
-                termwarn(f"Could not access disk path {path}: {e}", repeat=False)
-
-    def sample(self) -> None:
-        disk_usage: List[float] = []
-        for path in self.paths:
-            disk_usage.append(psutil.disk_usage(path).used / 1024 / 1024 / 1024)
-        if disk_usage:
-            self.samples.append(disk_usage)
-
-    def clear(self) -> None:
-        self.samples.clear()
-
-    def aggregate(self) -> dict:
-        if not self.samples:
-            return {}
-        disk_metrics = {}
-        for i, _path in enumerate(self.paths):
-            aggregate_i = aggregate_mean([sample[i] for sample in self.samples])
-            # ugly hack to please the frontend:
-            _path = _path.replace("/", "\\")
-            disk_metrics[self.name.format(path=_path)] = aggregate_i
-
-        return disk_metrics
-
-
-class DiskIn:
-    """Total system disk read in MB."""
-
-    name = "disk.in"
-    samples: "Deque[float]"
-
-    def __init__(self) -> None:
-        self.samples = deque([])
-        self.read_init: Optional[int] = None
-
-    def sample(self) -> None:
-        if self.read_init is None:
-            # initialize the read_init value on first sample
-            self.read_init = psutil.disk_io_counters().read_bytes
-        self.samples.append(
-            (psutil.disk_io_counters().read_bytes - self.read_init) / 1024 / 1024
-        )
-
-    def clear(self) -> None:
-        self.samples.clear()
-
-    def aggregate(self) -> dict:
-        if not self.samples:
-            return {}
-        aggregate = aggregate_mean(self.samples)
-        return {self.name: aggregate}
-
-
-class DiskOut:
-    """Total system disk write in MB."""
-
-    name = "disk.out"
-    samples: "Deque[float]"
-
-    def __init__(self) -> None:
-        self.samples = deque([])
-        self.write_init: Optional[int] = None
-
-    def sample(self) -> None:
-        if self.write_init is None:
-            # init on first sample
-            self.write_init = psutil.disk_io_counters().write_bytes
-        self.samples.append(
-            (psutil.disk_io_counters().write_bytes - self.write_init) / 1024 / 1024
-        )
-
-    def clear(self) -> None:
-        self.samples.clear()
-
-    def aggregate(self) -> dict:
-        if not self.samples:
-            return {}
-        aggregate = aggregate_mean(self.samples)
-        return {self.name: aggregate}
-
-
-@asset_registry.register
-class Disk:
-    def __init__(
-        self,
-        interface: "Interface",
-        settings: "SettingsStatic",
-        shutdown_event: threading.Event,
-    ) -> None:
-        self.name = self.__class__.__name__.lower()
-        self.settings = settings
-        self.metrics: List[Metric] = [
-            DiskUsagePercent(list(settings.x_stats_disk_paths or ["/"])),
-            DiskUsage(list(settings.x_stats_disk_paths or ["/"])),
-            DiskIn(),
-            DiskOut(),
-        ]
-        self.metrics_monitor = MetricsMonitor(
-            self.name,
-            self.metrics,
-            interface,
-            settings,
-            shutdown_event,
-        )
-
-    @classmethod
-    def is_available(cls) -> bool:
-        """Return a new instance of the CPU metrics."""
-        return psutil is not None
-
-    def probe(self) -> dict:
-        disk_paths = list(self.settings.x_stats_disk_paths or ["/"])
-        disk_metrics = {}
-        for disk_path in disk_paths:
-            try:
-                # total disk space in Bytes:
-                total = psutil.disk_usage(disk_path).total
-                # total disk space used in Bytes:
-                used = psutil.disk_usage(disk_path).used
-                disk_metrics[disk_path] = {
-                    "total": total,
-                    "used": used,
-                }
-            except Exception as e:
-                termwarn(f"Could not access disk path {disk_path}: {e}", repeat=False)
-
-        return {self.name: disk_metrics}
-
-    def start(self) -> None:
-        self.metrics_monitor.start()
-
-    def finish(self) -> None:
-        self.metrics_monitor.finish()