wandb 0.19.12rc1__py3-none-win32.whl → 0.20.1__py3-none-win32.whl

wandb/sdk/data_types/table_decorators.py

@@ -0,0 +1,108 @@
+"""Decorators for W&B Table operations."""
+
+from __future__ import annotations
+
+from functools import wraps
+from typing import Any, Callable, TypeVar
+
+from typing_extensions import Concatenate, ParamSpec
+
+import wandb
+
+_P = ParamSpec("_P")
+_T = TypeVar("_T")
+
+
+def allow_relogging_after_mutation(
+    method: Callable[Concatenate[wandb.Table, _P], _T],
+) -> Callable[Concatenate[wandb.Table, _P], _T]:
+    """Decorator that handles table state after mutations based on log_mode.
+
+    For MUTABLE tables, resets the run and artifact target to allow re-logging.
+    For IMMUTABLE tables, warns if attempting to mutate after logging.
+    """
+
+    @wraps(method)
+    def wrapper(self: wandb.Table, *args: Any, **kwargs: Any) -> _T:
+        has_been_logged = self._run is not None or self._artifact_target is not None
+
+        if self.log_mode == "MUTABLE":
+            self._run = None
+            self._artifact_target = None
+            self._path = None
+            self._sha256 = None
+        elif self.log_mode == "IMMUTABLE" and has_been_logged:
+            wandb.termwarn(
+                "You are mutating a Table with log_mode='IMMUTABLE' that has been "
+                "logged already. Subsequent log() calls will have no effect. "
+                "Set log_mode='MUTABLE' to enable re-logging after mutations",
+                repeat=False,
+            )
+
+        return method(self, *args, **kwargs)
+
+    return wrapper
+
+
+def allow_incremental_logging_after_append(
+    method: Callable[Concatenate[wandb.Table, _P], _T],
+) -> Callable[Concatenate[wandb.Table, _P], _T]:
+    """Decorator that handles incremental logging state after append operations.
+
+    For INCREMENTAL tables, manages artifact references and increments counters
+    to support partial data logging.
+    """
+
+    @wraps(method)
+    def wrapper(self: wandb.Table, *args: Any, **kwargs: Any) -> _T:
+        res = method(self, *args, **kwargs)
+
+        if self.log_mode != "INCREMENTAL" or self._artifact_target is None:
+            return res
+
+        art_entry_url = self._get_artifact_entry_ref_url()
+        if art_entry_url is not None:
+            if self._previous_increments_paths is None:
+                raise ValueError(
+                    "_previous_increments_paths must be set if incremental"
+                    " table has been logged already"
+                )
+            self._previous_increments_paths.append(art_entry_url)
+        self._run = None
+        self._artifact_target = None
+        self._path = None
+        self._sha256 = None
+
+        if self._increment_num is None:
+            raise ValueError(
+                "_increment_num must be set if incremental table has been"
+                " logged already"
+            )
+
+        self._increment_num += 1
+        if self._increment_num > 99:
+            wandb.termwarn(
+                "You have exceeded 100 increments for this table. "
+                "Only the latest 100 increments will be visualized in the run workspace.",
+                repeat=False,
+            )
+        return res
+
+    return wrapper
+
+
+def ensure_not_incremental(
+    method: Callable[Concatenate[wandb.Table, _P], _T],
+) -> Callable[Concatenate[wandb.Table, _P], _T]:
+    """Decorator that checks if log mode is incremental to disallow methods from being called."""
+
+    @wraps(method)
+    def wrapper(self: wandb.Table, *args: Any, **kwargs: Any) -> _T:
+        if self.log_mode == "INCREMENTAL":
+            raise wandb.Error(
+                f"Operation '{method.__name__}' is not supported for tables with "
+                "log_mode='INCREMENTAL'. Use a different log mode like 'MUTABLE' or 'IMMUTABLE'."
+            )
+        return method(self, *args, **kwargs)
+
+    return wrapper

wandb/sdk/data_types/utils.py

@@ -1,13 +1,13 @@
 import datetime
 import logging
 import os
-import re
 from decimal import Decimal
 from typing import TYPE_CHECKING, Optional, Sequence, Union, cast
 
 import wandb
 from wandb import util
 
+from ..internal import incremental_table_util
 from .base_types.media import BatchableMedia, Media
 from .base_types.wb_value import WBValue
 from .image import _server_accepts_image_filenames
@@ -148,11 +148,7 @@ def val_to_json(
                 "partitioned-table",
                 "joined-table",
             ]:
-                # Sanitize the key to meet the constraints of artifact names.
-                sanitized_key = re.sub(r"[^a-zA-Z0-9_\-.]+", "", key)
-                art = wandb.Artifact(f"run-{run.id}-{sanitized_key}", "run_table")
-                art.add(val, key)
-                run.log_artifact(art)
+                _log_table_artifact(val, key, run)
 
             # Partitioned tables and joined tables do not support being bound to runs.
             if not (
@@ -161,11 +157,51 @@ def val_to_json(
             ):
                 val.bind_to_run(run, key, namespace)
 
-        return val.to_json(run)
+        res = val.to_json(run)
+
+        if isinstance(val, wandb.Table) and val.log_mode == "INCREMENTAL":
+            # Set the _last_logged_idx AFTER the Table has been logged and
+            # bound to the run.
+            val._last_logged_idx = len(val.data) - 1
+        return res
 
     return converted  # type: ignore
 
 
+def _log_table_artifact(val: "Media", key: str, run: "LocalRun") -> None:
+    """Log a table to the run based on the table type and logging mode.
+
+    Creates and logs a `run_table` type for Table, PartitionedTable, and
+    JoinedTable values. For tables with log_mode="INCREMENTAL", creates and
+    logs an incremental artifact of type `wandb-run-incremental-table.`
+
+    Args:
+        val: A wbvalue with log_type "table", "partitioned-table",
+            or "joined-table."
+        key: The key used to log val.
+        run: The LocalRun used to log val.
+    """
+    from wandb.sdk.artifacts._internal_artifact import InternalArtifact
+
+    if isinstance(val, wandb.Table) and val.log_mode == "INCREMENTAL":
+        if (
+            run.resumed
+            and val._previous_increments_paths is None
+            and val._increment_num is None
+        ):
+            val._load_incremental_table_state_from_resumed_run(run, key)
+        else:
+            val._set_incremental_table_run_target(run)
+        art = incremental_table_util.init_artifact(run, key)
+        entry_name = incremental_table_util.get_entry_name(val, key)
+    else:
+        art = InternalArtifact(f"run-{run.id}-{key}", "run_table")
+        entry_name = key
+
+    art.add(val, entry_name)
+    run.log_artifact(art)
+
+
 def _prune_max_seq(seq: Sequence["BatchableMedia"]) -> Sequence["BatchableMedia"]:
     # If media type has a max respect it
     items = seq

wandb/sdk/data_types/video.py

@@ -1,6 +1,7 @@
 import functools
 import logging
 import os
+import pathlib
 from io import BytesIO
 from typing import TYPE_CHECKING, Any, Literal, Optional, Sequence, Type, Union
 
@@ -34,12 +35,26 @@ if TYPE_CHECKING:  # pragma: no cover
 def write_gif_with_image_io(
     clip: Any, filename: str, fps: Optional[int] = None
 ) -> None:
+    from packaging.version import parse
+
     imageio = util.get_module(
         "imageio",
         required='wandb.Video requires imageio when passing raw data. Install with "pip install wandb[media]"',
     )
 
-    writer = imageio.save(filename, fps=clip.fps, quantizer=0, palettesize=256, loop=0)
+    if parse(imageio.__version__) < parse("2.28.1"):
+        raise ValueError(
+            "imageio version 2.28.1 or higher is required to encode gifs. "
+            "Please upgrade imageio with `pip install imageio>=2.28.1`"
+        )
+
+    writer = imageio.save(
+        filename,
+        quantizer=0,
+        palettesize=256,
+        loop=0,
+        duration=1000 / clip.fps,
+    )
 
     for frame in clip.iter_frames(fps=fps, dtype="uint8"):
         writer.append_data(frame)
@@ -57,7 +72,7 @@ class Video(BatchableMedia):
 
     def __init__(
         self,
-        data_or_path: Union["np.ndarray", str, "TextIO", "BytesIO"],
+        data_or_path: Union[str, pathlib.Path, "np.ndarray", "TextIO", "BytesIO"],
         caption: Optional[str] = None,
         fps: Optional[int] = None,
         format: Optional[Literal["gif", "mp4", "webm", "ogg"]] = None,
@@ -84,6 +99,7 @@ class Video(BatchableMedia):
                 or io object. This parameter will be used to determine the format
                 to use when encoding the video data. Accepted values are "gif",
                 "mp4", "webm", or "ogg".
+                If no value is provided, the default format will be "gif".
 
         Examples:
             ### Log a numpy array as a video
@@ -130,7 +146,9 @@ class Video(BatchableMedia):
             with open(filename, "wb") as f:
                 f.write(data_or_path.read())
             self._set_file(filename, is_tmp=True)
-        elif isinstance(data_or_path, str):
+        elif isinstance(data_or_path, (str, pathlib.Path)):
+            data_or_path = str(data_or_path)
+
             _, ext = os.path.splitext(data_or_path)
             ext = ext[1:].lower()
             if ext not in Video.EXTS:

wandb/sdk/interface/interface.py

@@ -166,6 +166,16 @@ class InterfaceBase:
             proto_run.host = run._settings.host
         if run._settings.resumed:
             proto_run.resumed = run._settings.resumed
+        if run._settings.fork_from:
+            run_moment = run._settings.fork_from
+            proto_run.branch_point.run = run_moment.run
+            proto_run.branch_point.metric = run_moment.metric
+            proto_run.branch_point.value = run_moment.value
+        if run._settings.resume_from:
+            run_moment = run._settings.resume_from
+            proto_run.branch_point.run = run_moment.run
+            proto_run.branch_point.metric = run_moment.metric
+            proto_run.branch_point.value = run_moment.value
         if run._forked:
             proto_run.forked = run._forked
         if run._config is not None:

wandb/sdk/internal/datastore.py

@@ -126,9 +126,7 @@ class DataStore:
             return None
         assert (
             len(header) == LEVELDBLOG_HEADER_LEN
-        ), "record header is {} bytes instead of the expected {}".format(
-            len(header), LEVELDBLOG_HEADER_LEN
-        )
+        ), f"record header is {len(header)} bytes instead of the expected {LEVELDBLOG_HEADER_LEN}"
         fields = struct.unpack("<IHB", header)
         checksum, dlength, dtype = fields
         # check len, better fit in the block
@@ -195,9 +193,7 @@ class DataStore:
         header = self._fp.read(LEVELDBLOG_HEADER_LEN)
         assert (
             len(header) == LEVELDBLOG_HEADER_LEN
-        ), "header is {} bytes instead of the expected {}".format(
-            len(header), LEVELDBLOG_HEADER_LEN
-        )
+        ), f"header is {len(header)} bytes instead of the expected {LEVELDBLOG_HEADER_LEN}"
         ident, magic, version = struct.unpack("<4sHB", header)
         if ident != strtobytes(LEVELDBLOG_HEADER_IDENT):
             raise Exception("Invalid header")

wandb/sdk/internal/file_pusher.py

@@ -98,11 +98,7 @@ class FilePusher:
             if not self.is_alive():
                 stop = True
             summary = self._stats.summary()
-            line = " {:.2f}MB of {:.2f}MB uploaded ({:.2f}MB deduped)\r".format(
-                summary.uploaded_bytes / 1048576.0,
-                summary.total_bytes / 1048576.0,
-                summary.deduped_bytes / 1048576.0,
-            )
+            line = f" {summary.uploaded_bytes / 1048576.0:.2f}MB of {summary.total_bytes / 1048576.0:.2f}MB uploaded ({summary.deduped_bytes / 1048576.0:.2f}MB deduped)\r"
             line = spinner_states[step % 4] + line
             step += 1
             wandb.termlog(line, newline=False, prefix=prefix)

wandb/sdk/internal/file_stream.py

@@ -75,7 +75,7 @@ class DefaultFilePolicy:
 
         # get key size and convert to MB
         key_sizes = [(k, len(json.dumps(v))) for k, v in loaded.items()]
-        key_msg = [f"{k}: {v/1048576:.5f} MB" for k, v in key_sizes]
+        key_msg = [f"{k}: {v / 1048576:.5f} MB" for k, v in key_sizes]
         wandb.termerror(f"Step: {loaded['_step']} | {key_msg}", repeat=False)
         self.has_debug_log = True
 
@@ -88,10 +88,7 @@ class JsonlFilePolicy(DefaultFilePolicy):
         chunk_data = []
         for chunk in chunks:
             if len(chunk.data) > util.MAX_LINE_BYTES:
-                msg = "Metric data exceeds maximum size of {} ({})".format(
-                    util.to_human_size(util.MAX_LINE_BYTES),
-                    util.to_human_size(len(chunk.data)),
-                )
+                msg = f"Metric data exceeds maximum size of {util.to_human_size(util.MAX_LINE_BYTES)} ({util.to_human_size(len(chunk.data))})"
                 wandb.termerror(msg, repeat=False)
                 wandb._sentry.message(msg, repeat=False)
                 self._debug_log(chunk.data)
@@ -108,9 +105,7 @@ class SummaryFilePolicy(DefaultFilePolicy):
     def process_chunks(self, chunks: List[Chunk]) -> Union[bool, "ProcessedChunk"]:
         data = chunks[-1].data
         if len(data) > util.MAX_LINE_BYTES:
-            msg = "Summary data exceeds maximum size of {}. Dropping it.".format(
-                util.to_human_size(util.MAX_LINE_BYTES)
-            )
+            msg = f"Summary data exceeds maximum size of {util.to_human_size(util.MAX_LINE_BYTES)}. Dropping it."
             wandb.termerror(msg, repeat=False)
             wandb._sentry.message(msg, repeat=False)
             self._debug_log(data)
@@ -494,12 +489,12 @@ class FileStreamApi:
         # TODO: Consolidate with internal_util.ExceptionThread
         try:
             self._thread_body()
-        except Exception as e:
+        except Exception:
             exc_info = sys.exc_info()
             self._exc_info = exc_info
             logger.exception("generic exception in filestream thread")
             wandb._sentry.exception(exc_info)
-            raise e
+            raise
 
     def _handle_response(self, response: Union[Exception, "requests.Response"]) -> None:
         """Log dropped chunks and updates dynamic settings."""
@@ -507,7 +502,7 @@ class FileStreamApi:
             wandb.termerror(
                 "Dropped streaming file chunk (see wandb/debug-internal.log)"
             )
-            logger.exception("dropped chunk {}".format(response))
+            logger.exception(f"dropped chunk {response}")
             self._dropped_chunks += 1
         else:
             parsed: Optional[dict] = None
@@ -664,8 +659,7 @@ def request_with_retry(
                 e.response is not None and e.response.status_code == 429
             ):
                 err_str = (
-                    "Filestream rate limit exceeded, "
-                    f"retrying in {delay:.1f} seconds. "
+                    f"Filestream rate limit exceeded, retrying in {delay:.1f} seconds. "
                 )
                 if retry_callback:
                     retry_callback(e.response.status_code, err_str)
@@ -688,8 +682,5 @@ def request_with_retry(
                 error_message = response.json()["error"]  # todo: clean this up
             except Exception:
                 pass
-            logger.error(f"requests_with_retry error: {error_message}")
-            logger.exception(
-                "requests_with_retry encountered unretryable exception: %s", e
-            )
+            logger.exception(f"requests_with_retry error: {error_message}")
             return e

wandb/sdk/internal/handler.py

@@ -704,8 +704,8 @@ class HandleManager:
         ):
             try:
                 self._metadata = Metadata(**self._system_monitor.probe(publish=True))
-            except Exception as e:
-                logger.error("Error probing system metadata: %s", e)
+            except Exception:
+                logger.exception("Error probing system metadata.")
 
         self._tb_watcher = tb_watcher.TBWatcher(
             self._settings, interface=self._interface, run_proto=run_start.run

wandb/sdk/internal/incremental_table_util.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from wandb import Table
+    from wandb.sdk.artifacts.artifact import Artifact
+
+    from ..wandb_run import Run as LocalRun
+
+ART_TYPE = "wandb-run-incremental-table"
+
+
+def _get_artifact_name(run: LocalRun, key: str) -> str:
+    from wandb.sdk.artifacts._internal_artifact import sanitize_artifact_name
+
+    return sanitize_artifact_name(f"run-{run.id}-incr-{key}")
+
+
+def init_artifact(run: LocalRun, sanitized_key: str) -> Artifact:
+    """Initialize a new artifact for an incremental table.
+
+    Args:
+        run: The wandb run associated with this artifact
+        sanitized_key: Sanitized string key to identify the table
+
+    Returns:
+        A wandb Artifact configured for incremental table storage
+    """
+    from wandb.sdk.artifacts._internal_artifact import InternalArtifact
+
+    artifact = InternalArtifact(
+        _get_artifact_name(run, sanitized_key),
+        ART_TYPE,
+        incremental=True,
+    )
+    return artifact
+
+
+def get_entry_name(incr_table: Table, key: str) -> str:
+    """Generate a unique entry name for a table increment.
+
+    Args:
+        run: The wandb run associated with this table
+        incr_table: The incremental table being updated
+        key: String key for the table entry
+
+    Returns:
+        A unique string name for the table entry
+    """
+    epoch = time.time_ns() // 1_000_000
+    return f"{incr_table._increment_num}-{epoch}.{key}"

wandb/sdk/internal/internal.py

@@ -169,11 +169,9 @@ def wandb_internal(
             traceback.print_exception(*exc_info)
             wandb._sentry.exception(exc_info)
             wandb.termerror("Internal wandb error: file data was not synced")
-            if not settings.x_disable_service:
-                # TODO: We can make this more graceful by returning an error to streams.py
-                # and potentially just fail the one stream.
-                os._exit(-1)
-            sys.exit(-1)
+            # TODO: We can make this more graceful by returning an error to streams.py
+            # and potentially just fail the one stream.
+            os._exit(-1)
 
     close_internal_log()