wandb 0.16.5__py3-none-any.whl → 0.17.0__py3-none-any.whl

wandb/integration/openai/fine_tuning.py

@@ -1,9 +1,11 @@
 import datetime
 import io
 import json
+import os
 import re
+import tempfile
 import time
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import wandb
 from wandb import util
@@ -26,7 +28,10 @@ if parse_version(openai.__version__) < parse_version("1.0.1"):
 
 from openai import OpenAI  # noqa: E402
 from openai.types.fine_tuning import FineTuningJob  # noqa: E402
-from openai.types.fine_tuning.fine_tuning_job import Hyperparameters  # noqa: E402
+from openai.types.fine_tuning.fine_tuning_job import (  # noqa: E402
+    Error,
+    Hyperparameters,
+)
 
 np = util.get_module(
     name="numpy",
@@ -59,6 +64,7 @@ class WandbLogger:
         entity: Optional[str] = None,
         overwrite: bool = False,
         wait_for_job_success: bool = True,
+        log_datasets: bool = True,
         **kwargs_wandb_init: Dict[str, Any],
     ) -> str:
         """Sync fine-tunes to Weights & Biases.
@@ -150,6 +156,7 @@ class WandbLogger:
                 entity,
                 overwrite,
                 show_individual_warnings,
+                log_datasets,
                 **kwargs_wandb_init,
             )
 
@@ -160,11 +167,14 @@ class WandbLogger:
 
     @classmethod
     def _wait_for_job_success(cls, fine_tune: FineTuningJob) -> FineTuningJob:
-        wandb.termlog("Waiting for the OpenAI fine-tuning job to be finished...")
+        wandb.termlog("Waiting for the OpenAI fine-tuning job to finish training...")
+        wandb.termlog(
+            "To avoid blocking, you can call `WandbLogger.sync` with `wait_for_job_success=False` after OpenAI training completes."
+        )
         while True:
             if fine_tune.status == "succeeded":
                 wandb.termlog(
-                    "Fine-tuning finished, logging metrics, model metadata, and more to W&B"
+                    "Fine-tuning finished, logging metrics, model metadata, and run metadata to Weights & Biases"
                 )
                 return fine_tune
             if fine_tune.status == "failed":
@@ -190,6 +200,7 @@ class WandbLogger:
         entity: Optional[str],
         overwrite: bool,
         show_individual_warnings: bool,
+        log_datasets: bool,
         **kwargs_wandb_init: Dict[str, Any],
     ):
         fine_tune_id = fine_tune.id
@@ -209,7 +220,7 @@ class WandbLogger:
         # check results are present
         try:
             results_id = fine_tune.result_files[0]
-            results = cls.openai_client.files.retrieve_content(file_id=results_id)
+            results = cls.openai_client.files.content(file_id=results_id).text
         except openai.NotFoundError:
             if show_individual_warnings:
                 wandb.termwarn(
@@ -233,7 +244,7 @@ class WandbLogger:
             cls._run.summary["fine_tuned_model"] = fine_tuned_model
 
         # training/validation files and fine-tune details
-        cls._log_artifacts(fine_tune, project, entity)
+        cls._log_artifacts(fine_tune, project, entity, log_datasets, overwrite)
 
         # mark run as complete
         cls._run.summary["status"] = "succeeded"
@@ -249,7 +260,7 @@ class WandbLogger:
             else:
                 raise Exception(
                     "It appears you are not currently logged in to Weights & Biases. "
-                    "Please run `wandb login` in your terminal. "
+                    "Please run `wandb login` in your terminal or `wandb.login()` in a notebook."
                     "When prompted, you can obtain your API key by visiting wandb.ai/authorize."
                 )
 
@@ -286,15 +297,9 @@ class WandbLogger:
                 config["finished_at"]
             ).strftime("%Y-%m-%d %H:%M:%S")
         if config.get("hyperparameters"):
-            hyperparameters = config.pop("hyperparameters")
-            hyperparams = cls._unpack_hyperparameters(hyperparameters)
-            if hyperparams is None:
-                # If unpacking fails, log the object which will render as string
-                config["hyperparameters"] = hyperparameters
-            else:
-                # nested rendering on hyperparameters
-                config["hyperparameters"] = hyperparams
-
+            config["hyperparameters"] = cls.sanitize(config["hyperparameters"])
+        if config.get("error"):
+            config["error"] = cls.sanitize(config["error"])
         return config
 
     @classmethod
@@ -305,30 +310,53 @@ class WandbLogger:
         try:
             hyperparams["n_epochs"] = hyperparameters.n_epochs
             hyperparams["batch_size"] = hyperparameters.batch_size
-            hyperparams[
-                "learning_rate_multiplier"
-            ] = hyperparameters.learning_rate_multiplier
+            hyperparams["learning_rate_multiplier"] = (
+                hyperparameters.learning_rate_multiplier
+            )
         except Exception:
             # If unpacking fails, return the object to be logged as config
             return None
 
         return hyperparams
 
+    @staticmethod
+    def sanitize(input: Any) -> Union[Dict, List, str]:
+        valid_types = [bool, int, float, str]
+        if isinstance(input, (Hyperparameters, Error)):
+            return dict(input)
+        if isinstance(input, dict):
+            return {
+                k: v if type(v) in valid_types else str(v) for k, v in input.items()
+            }
+        elif isinstance(input, list):
+            return [v if type(v) in valid_types else str(v) for v in input]
+        else:
+            return str(input)
+
     @classmethod
     def _log_artifacts(
-        cls, fine_tune: FineTuningJob, project: str, entity: Optional[str]
+        cls,
+        fine_tune: FineTuningJob,
+        project: str,
+        entity: Optional[str],
+        log_datasets: bool,
+        overwrite: bool,
     ) -> None:
-        # training/validation files
-        training_file = fine_tune.training_file if fine_tune.training_file else None
-        validation_file = (
-            fine_tune.validation_file if fine_tune.validation_file else None
-        )
-        for file, prefix, artifact_type in (
-            (training_file, "train", "training_files"),
-            (validation_file, "valid", "validation_files"),
-        ):
-            if file is not None:
-                cls._log_artifact_inputs(file, prefix, artifact_type, project, entity)
+        if log_datasets:
+            wandb.termlog("Logging training/validation files...")
+            # training/validation files
+            training_file = fine_tune.training_file if fine_tune.training_file else None
+            validation_file = (
+                fine_tune.validation_file if fine_tune.validation_file else None
+            )
+            for file, prefix, artifact_type in (
+                (training_file, "train", "training_files"),
+                (validation_file, "valid", "validation_files"),
+            ):
+                if file is not None:
+                    cls._log_artifact_inputs(
+                        file, prefix, artifact_type, project, entity, overwrite
+                    )
 
         # fine-tune details
         fine_tune_id = fine_tune.id
@@ -337,9 +365,14 @@ class WandbLogger:
             type="model",
             metadata=dict(fine_tune),
         )
+
         with artifact.new_file("model_metadata.json", mode="w", encoding="utf-8") as f:
             dict_fine_tune = dict(fine_tune)
-            dict_fine_tune["hyperparameters"] = dict(dict_fine_tune["hyperparameters"])
+            dict_fine_tune["hyperparameters"] = cls.sanitize(
+                dict_fine_tune["hyperparameters"]
+            )
+            dict_fine_tune["error"] = cls.sanitize(dict_fine_tune["error"])
+            dict_fine_tune = cls.sanitize(dict_fine_tune)
             json.dump(dict_fine_tune, f, indent=2)
         cls._run.log_artifact(
             artifact,
@@ -354,6 +387,7 @@ class WandbLogger:
         artifact_type: str,
         project: str,
         entity: Optional[str],
+        overwrite: bool,
     ) -> None:
         # get input artifact
         artifact_name = f"{prefix}-{file_id}"
@@ -366,23 +400,26 @@ class WandbLogger:
         artifact = cls._get_wandb_artifact(artifact_path)
 
         # create artifact if file not already logged previously
-        if artifact is None:
+        if artifact is None or overwrite:
             # get file content
             try:
-                file_content = cls.openai_client.files.retrieve_content(file_id=file_id)
+                file_content = cls.openai_client.files.content(file_id=file_id)
             except openai.NotFoundError:
                 wandb.termerror(
-                    f"File {file_id} could not be retrieved. Make sure you are allowed to download training/validation files"
+                    f"File {file_id} could not be retrieved. Make sure you have OpenAI permissions to download training/validation files"
                 )
                 return
 
             artifact = wandb.Artifact(artifact_name, type=artifact_type)
-            with artifact.new_file(file_id, mode="w", encoding="utf-8") as f:
-                f.write(file_content)
+            with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+                tmp_file.write(file_content.content)
+                tmp_file_path = tmp_file.name
+            artifact.add_file(tmp_file_path, file_id)
+            os.unlink(tmp_file_path)
 
             # create a Table
             try:
-                table, n_items = cls._make_table(file_content)
+                table, n_items = cls._make_table(file_content.text)
                 # Add table to the artifact.
                 artifact.add(table, file_id)
                 # Add the same table to the workspace.
@@ -390,9 +427,9 @@ class WandbLogger:
                 # Update the run config and artifact metadata
                 cls._run.config.update({f"n_{prefix}": n_items})
                 artifact.metadata["items"] = n_items
-            except Exception:
+            except Exception as e:
                 wandb.termerror(
-                    f"File {file_id} could not be read as a valid JSON file"
+                    f"Issue saving {file_id} as a Table to Artifacts, exception:\n  '{e}'"
                 )
         else:
             # log number of items

wandb/integration/prodigy/prodigy.py

@@ -26,7 +26,7 @@ from PIL import Image
 
 import wandb
 from wandb import util
-from wandb.plots.utils import test_missing
+from wandb.plot.utils import test_missing
 from wandb.sdk.lib import telemetry as wb_telemetry
 
 

wandb/old/summary.py

@@ -61,7 +61,7 @@ class SummarySubDict:
         This should only be implemented by the "_root" child class.
 
         We pass the child_dict so the item can be set on it or not as
-        appropriate. Returning None for a nonexistant path wouldn't be
+        appropriate. Returning None for a nonexistent path wouldn't be
         distinguishable from that path being set to the value None.
         """
         raise NotImplementedError

wandb/plot/confusion_matrix.py

@@ -48,7 +48,7 @@ def confusion_matrix(
 
     assert (probs is None or preds is None) and not (
         probs is None and preds is None
-    ), "Must provide probabilties or predictions but not both to confusion matrix"
+    ), "Must provide probabilities or predictions but not both to confusion matrix"
 
     if probs is not None:
         preds = np.argmax(probs, axis=1).tolist()

wandb/plot/pr_curve.py

@@ -2,7 +2,8 @@ from typing import Optional
 
 import wandb
 from wandb import util
-from wandb.plots.utils import test_missing, test_types
+
+from .utils import test_missing, test_types
 
 
 def pr_curve(

wandb/plot/roc_curve.py

@@ -2,7 +2,8 @@ from typing import Optional
 
 import wandb
 from wandb import util
-from wandb.plots.utils import test_missing, test_types
+
+from .utils import test_missing, test_types
 
 
 def roc_curve(

wandb/{plots → plot}/utils.py

@@ -1,21 +1,9 @@
-from collections.abc import Iterable, Sequence
+from typing import Iterable, Sequence
 
 import wandb
 from wandb import util
-from wandb.sdk.lib import deprecate
 
 
-def deprecation_notice() -> None:
-    deprecate.deprecate(
-        field_name=deprecate.Deprecated.plots,
-        warning_message=(
-            "wandb.plots.* functions are deprecated and will be removed in a future release. "
-            "Please use wandb.plot.* instead."
-        ),
-    )
-
-
-# Test assumptions for plotting parameters and datasets
 def test_missing(**kwargs):
     np = util.get_module("numpy", required="Logging plots requires numpy")
     pd = util.get_module("pandas", required="Logging dataframes requires pandas")
@@ -25,7 +13,7 @@ def test_missing(**kwargs):
     for k, v in kwargs.items():
         # Missing/empty params/datapoint arrays
         if v is None:
-            wandb.termerror("%s is None. Please try again." % (k))
+            wandb.termerror("{} is None. Please try again.".format(k))
             test_passed = False
         if (k == "X") or (k == "X_test"):
             if isinstance(v, scipy.sparse.csr.csr_matrix):
@@ -64,8 +52,8 @@ def test_missing(**kwargs):
                 )
             if non_nums > 0:
                 wandb.termerror(
-                    "%s contains values that are not numbers. Please vectorize, label encode or one hot encode %s and call the plotting function again."
-                    % (k, k)
+                    f"{k} contains values that are not numbers. Please vectorize, "
+                    f"label encode or one hot encode {k} and call the plotting function again."
                 )
                 test_passed = False
     return test_passed
@@ -73,8 +61,8 @@ def test_missing(**kwargs):
 
 def test_fitted(model):
     np = util.get_module("numpy", required="Logging plots requires numpy")
-    pd = util.get_module("pandas", required="Logging dataframes requires pandas")
-    scipy = util.get_module("scipy", required="Logging scipy matrices requires scipy")
+    _ = util.get_module("pandas", required="Logging dataframes requires pandas")
+    _ = util.get_module("scipy", required="Logging scipy matrices requires scipy")
     scikit_utils = util.get_module(
         "sklearn.utils",
         required="roc requires the scikit utils submodule, install with `pip install scikit-learn`",
@@ -120,7 +108,7 @@ def test_fitted(model):
 
 
 def encode_labels(df):
-    pd = util.get_module("pandas", required="Logging dataframes requires pandas")
+    _ = util.get_module("pandas", required="Logging dataframes requires pandas")
     preprocessing = util.get_module(
         "sklearn.preprocessing",
         "roc requires the scikit preprocessing submodule, install with `pip install scikit-learn`",
@@ -137,7 +125,7 @@ def encode_labels(df):
 def test_types(**kwargs):
     np = util.get_module("numpy", required="Logging plots requires numpy")
     pd = util.get_module("pandas", required="Logging dataframes requires pandas")
-    scipy = util.get_module("scipy", required="Logging scipy matrices requires scipy")
+    _ = util.get_module("scipy", required="Logging scipy matrices requires scipy")
 
     base = util.get_module(
         "sklearn.base",
@@ -171,25 +159,25 @@ def test_types(**kwargs):
                     list,
                 ),
             ):
-                wandb.termerror("%s is not an array. Please try again." % (k))
+                wandb.termerror("{} is not an array. Please try again.".format(k))
                 test_passed = False
         # check for classifier types
         if k == "model":
             if (not base.is_classifier(v)) and (not base.is_regressor(v)):
                 wandb.termerror(
-                    "%s is not a classifier or regressor. Please try again." % (k)
+                    "{} is not a classifier or regressor. Please try again.".format(k)
                 )
                 test_passed = False
         elif k == "clf" or k == "binary_clf":
             if not (base.is_classifier(v)):
-                wandb.termerror("%s is not a classifier. Please try again." % (k))
+                wandb.termerror("{} is not a classifier. Please try again.".format(k))
                 test_passed = False
         elif k == "regressor":
             if not base.is_regressor(v):
-                wandb.termerror("%s is not a regressor. Please try again." % (k))
+                wandb.termerror("{} is not a regressor. Please try again.".format(k))
                 test_passed = False
         elif k == "clusterer":
             if not (getattr(v, "_estimator_type", None) == "clusterer"):
-                wandb.termerror("%s is not a clusterer. Please try again." % (k))
+                wandb.termerror("{} is not a clusterer. Please try again.".format(k))
                 test_passed = False
     return test_passed