wandb 0.17.0rc1__py3-none-any.whl → 0.17.1__py3-none-any.whl

wandb/filesync/step_upload.py

@@ -1,6 +1,5 @@
 """Batching file prepare requests to our API."""
 
-import asyncio
 import concurrent.futures
 import logging
 import queue
@@ -8,7 +7,6 @@ import sys
 import threading
 from typing import (
     TYPE_CHECKING,
-    Awaitable,
     Callable,
     MutableMapping,
     MutableSequence,
@@ -43,7 +41,6 @@ if TYPE_CHECKING:
 PreCommitFn = Callable[[], None]
 OnRequestFinishFn = Callable[[], None]
 SaveFn = Callable[["progress.ProgressFn"], bool]
-SaveFnAsync = Callable[["progress.ProgressFn"], Awaitable[bool]]
 
 logger = logging.getLogger(__name__)
 
@@ -55,7 +52,6 @@ class RequestUpload(NamedTuple):
     md5: Optional[str]
     copied: bool
     save_fn: Optional[SaveFn]
-    save_fn_async: Optional[SaveFnAsync]
     digest: Optional[str]
 
 
@@ -78,47 +74,6 @@ class EventJobDone(NamedTuple):
 Event = Union[RequestUpload, RequestCommitArtifact, RequestFinish, EventJobDone]
 
 
-class AsyncExecutor:
-    """Runs async file uploads in a background thread."""
-
-    def __init__(
-        self,
-        pool: concurrent.futures.ThreadPoolExecutor,
-        concurrency_limit: Optional[int],
-    ) -> None:
-        self.loop = asyncio.new_event_loop()
-        self.loop.set_default_executor(pool)
-        self.loop_thread = threading.Thread(
-            target=self.loop.run_forever,
-            daemon=True,
-            name="wandb-upload-async",
-        )
-
-        self.concurrency_limiter = asyncio.Semaphore(
-            value=concurrency_limit or 128,
-            # Before Python 3.10: if we don't set `loop=loop`,
-            #   then the Semaphore will bind to the wrong event loop,
-            #   causing errors when a coroutine tries to wait for it;
-            #   see https://pastebin.com/XcrS9suX .
-            # After 3.10: the `loop` argument doesn't exist.
-            # So we need to only conditionally pass in `loop`.
-            **({} if sys.version_info >= (3, 10) else {"loop": self.loop}),
-        )
-
-    def start(self) -> None:
-        self.loop_thread.start()
-
-    def stop(self) -> None:
-        self.loop.call_soon_threadsafe(self.loop.stop)
-
-    def submit(self, coro: Awaitable[None]) -> None:
-        async def run_with_limiter() -> None:
-            async with self.concurrency_limiter:
-                await coro
-
-        asyncio.run_coroutine_threadsafe(run_with_limiter(), self.loop)
-
-
 class StepUpload:
     def __init__(
         self,
@@ -142,15 +97,6 @@ class StepUpload:
             max_workers=max_threads,
         )
 
-        self._async_executor = (
-            AsyncExecutor(
-                pool=self._pool,
-                concurrency_limit=settings._async_upload_concurrency_limit,
-            )
-            if settings is not None and settings._async_upload_concurrency_limit
-            else None
-        )
-
         # Indexed by files' `save_name`'s, which are their ID's in the Run.
         self._running_jobs: MutableMapping[LogicalPath, RequestUpload] = {}
         self._pending_jobs: MutableSequence[RequestUpload] = []
@@ -186,8 +132,6 @@ class StepUpload:
             elif not self._running_jobs:
                 # Queue was empty and no jobs left.
                 self._pool.shutdown(wait=False)
-                if self._async_executor:
-                    self._async_executor.stop()
                 if finish_callback:
                     finish_callback()
                 break
@@ -235,7 +179,7 @@ class StepUpload:
                 self._artifacts[event.artifact_id]["pending_count"] += 1
             self._start_upload_job(event)
         else:
-            raise Exception("Programming error: unhandled event: %s" % str(event))
+            raise Exception("Programming error: unhandled event: {}".format(str(event)))
 
     def _start_upload_job(self, event: RequestUpload) -> None:
         # Operations on a single backend file must be serialized. if
@@ -245,18 +189,9 @@ class StepUpload:
             self._pending_jobs.append(event)
             return
 
-        if self._async_executor and event.save_fn_async is not None:
-            # (The `and save_fn_async is not None` is because the async code path
-            # doesn't support all uploads yet: even if the user has requested async,
-            # we sometimes need to use the sync method instead.)
-            self._spawn_upload_async(
-                event,
-                async_executor=self._async_executor,
-            )
-        else:
-            self._spawn_upload_sync(event)
+        self._spawn_upload(event)
 
-    def _spawn_upload_sync(self, event: RequestUpload) -> None:
+    def _spawn_upload(self, event: RequestUpload) -> None:
         """Spawn an upload job, and handles the bookkeeping of `self._running_jobs`.
 
         Context: it's important that, whenever we add an entry to `self._running_jobs`,
@@ -285,35 +220,13 @@ class StepUpload:
 
         def run_and_notify() -> None:
             try:
-                self._do_upload_sync(event)
+                self._do_upload(event)
             finally:
                 self._event_queue.put(EventJobDone(event, exc=sys.exc_info()[1]))
 
         self._pool.submit(run_and_notify)
 
-    def _spawn_upload_async(
-        self,
-        event: RequestUpload,
-        async_executor: AsyncExecutor,
-    ) -> None:
-        """Equivalent to _spawn_upload_sync, but uses the async event loop instead of a thread, and requires `event.save_fn_async`.
-
-        Raises:
-            AssertionError: if `event.save_fn_async` is None.
-        """
-        assert event.save_fn_async is not None
-
-        self._running_jobs[event.save_name] = event
-
-        async def run_and_notify() -> None:
-            try:
-                await self._do_upload_async(event)
-            finally:
-                self._event_queue.put(EventJobDone(event, exc=sys.exc_info()[1]))
-
-        async_executor.submit(run_and_notify())
-
-    def _do_upload_sync(self, event: RequestUpload) -> None:
+    def _do_upload(self, event: RequestUpload) -> None:
         job = upload_job.UploadJob(
             self._stats,
             self._api,
@@ -329,19 +242,6 @@ class StepUpload:
         )
         job.run()
 
-    async def _do_upload_async(self, event: RequestUpload) -> None:
-        """Upload a file and returns when it's done. Requires `event.save_fn_async`."""
-        assert event.save_fn_async is not None
-        job = upload_job.UploadJobAsync(
-            stats=self._stats,
-            api=self._api,
-            file_stream=self._file_stream,
-            silent=self.silent,
-            request=event,
-            save_fn_async=event.save_fn_async,
-        )
-        await job.run()
-
     def _init_artifact(self, artifact_id: str) -> None:
         self._artifacts[artifact_id] = {
             "finalize": False,
@@ -385,8 +285,6 @@ class StepUpload:
 
     def start(self) -> None:
         self._thread.start()
-        if self._async_executor:
-            self._async_executor.start()
 
     def is_alive(self) -> bool:
         return self._thread.is_alive()

wandb/filesync/upload_job.py

@@ -1,4 +1,3 @@
-import asyncio
 import logging
 import os
 from typing import TYPE_CHECKING, Optional
@@ -141,78 +140,3 @@ class UploadJob:
 
     def progress(self, total_bytes: int) -> None:
         self._stats.update_uploaded_file(self.save_name, total_bytes)
-
-
-class UploadJobAsync:
-    """Roughly an async equivalent of UploadJob.
-
-    Important differences:
-    - `run` is a coroutine
-    - If `run()` fails, it falls back to the synchronous UploadJob
-    """
-
-    def __init__(
-        self,
-        stats: "stats.Stats",
-        api: "internal_api.Api",
-        file_stream: "file_stream.FileStreamApi",
-        silent: bool,
-        request: "step_upload.RequestUpload",
-        save_fn_async: "step_upload.SaveFnAsync",
-    ) -> None:
-        self._stats = stats
-        self._api = api
-        self._file_stream = file_stream
-        self.silent = silent
-        self._request = request
-        self._save_fn_async = save_fn_async
-
-    async def run(self) -> None:
-        try:
-            deduped = await self._save_fn_async(
-                lambda _, t: self._stats.update_uploaded_file(self._request.path, t)
-            )
-        except Exception as e:
-            # Async uploads aren't yet (2023-01) battle-tested.
-            # Fall back to the "normal" synchronous upload.
-            loop = asyncio.get_event_loop()
-            logger.exception("async upload failed", exc_info=e)
-            loop.run_in_executor(None, wandb._sentry.exception, e)
-            wandb.termwarn(
-                "Async file upload failed; falling back to sync", repeat=False
-            )
-            sync_job = UploadJob(
-                self._stats,
-                self._api,
-                self._file_stream,
-                self.silent,
-                self._request.save_name,
-                self._request.path,
-                self._request.artifact_id,
-                self._request.md5,
-                self._request.copied,
-                self._request.save_fn,
-                self._request.digest,
-            )
-
-            await loop.run_in_executor(None, sync_job.run)
-        else:
-            self._file_stream.push_success(
-                self._request.artifact_id,  # type: ignore
-                self._request.save_name,
-            )
-
-            if deduped:
-                logger.info("Skipped uploading %s", self._request.path)
-                self._stats.set_file_deduped(self._request.path)
-            else:
-                logger.info("Uploaded file %s", self._request.path)
-        finally:
-            # If we fell back to the sync impl, the file will have already been deleted.
-            # Doesn't matter, we only try to delete it if it exists.
-            if self._request.copied:
-                try:
-                    os.remove(self._request.path)
-                except OSError:
-                    # The file has already been deleted, we don't have permissions, or something else we can't fix.
-                    pass

wandb/integration/gym/__init__.py

@@ -12,6 +12,8 @@ else:
 
 
 _gym_version_lt_0_26: Optional[bool] = None
+_gymnasium_version_lt_1_0_0: Optional[bool] = None
+
 _required_error_msg = (
     "Couldn't import the gymnasium python package, "
     "install with `pip install gymnasium`"
@@ -35,14 +37,10 @@ def monitor():
     if gym_lib is None:
         raise wandb.Error(_required_error_msg)
 
-    vcr = wandb.util.get_module(
-        f"{gym_lib}.wrappers.monitoring.video_recorder",
-        required=_required_error_msg,
-    )
-
     global _gym_version_lt_0_26
+    global _gymnasium_version_lt_1_0_0
 
-    if _gym_version_lt_0_26 is None:
+    if _gym_version_lt_0_26 is None or _gymnasium_version_lt_1_0_0 is None:
         if gym_lib == "gym":
             import gym
         else:
@@ -50,15 +48,31 @@ def monitor():
 
         from wandb.util import parse_version
 
-        if parse_version(gym.__version__) < parse_version("0.26.0"):
-            _gym_version_lt_0_26 = True
+        gym_lib_version = parse_version(gym.__version__)
+        _gym_version_lt_0_26 = gym_lib_version < parse_version("0.26.0")
+        _gymnasium_version_lt_1_0_0 = gym_lib_version < parse_version("1.0.0a1")
+
+    path = "path"  # Default path
+    if gym_lib == "gymnasium" and not _gymnasium_version_lt_1_0_0:
+        vcr_recorder_attribute = "RecordVideo"
+        wrappers = wandb.util.get_module(
+            f"{gym_lib}.wrappers",
+            required=_required_error_msg,
+        )
+        recorder = getattr(wrappers, vcr_recorder_attribute)
+    else:
+        vcr = wandb.util.get_module(
+            f"{gym_lib}.wrappers.monitoring.video_recorder",
+            required=_required_error_msg,
+        )
+        # Breaking change in gym 0.26.0
+        if _gym_version_lt_0_26:
+            vcr_recorder_attribute = "ImageEncoder"
+            recorder = getattr(vcr, vcr_recorder_attribute)
+            path = "output_path"  # Override path for older gym versions
         else:
-            _gym_version_lt_0_26 = False
-
-    # breaking change in gym 0.26.0
-    vcr_recorder_attribute = "ImageEncoder" if _gym_version_lt_0_26 else "VideoRecorder"
-    recorder = getattr(vcr, vcr_recorder_attribute)
-    path = "output_path" if _gym_version_lt_0_26 else "path"
+            vcr_recorder_attribute = "VideoRecorder"
+            recorder = getattr(vcr, vcr_recorder_attribute)
 
     recorder.orig_close = recorder.close
 
@@ -77,9 +91,15 @@ def monitor():
     if not _gym_version_lt_0_26:
         recorder.__del__ = del_
     recorder.close = close
+
+    if gym_lib == "gymnasium" and not _gymnasium_version_lt_1_0_0:
+        wrapper_name = vcr_recorder_attribute
+    else:
+        wrapper_name = f"monitoring.video_recorder.{vcr_recorder_attribute}"
+
     wandb.patched["gym"].append(
         [
-            f"{gym_lib}.wrappers.monitoring.video_recorder.{vcr_recorder_attribute}",
+            f"{gym_lib}.wrappers.{wrapper_name}",
             "close",
         ]
     )

wandb/integration/huggingface/resolver.py

@@ -91,8 +91,8 @@ class HuggingFacePipelineRequestResponseResolver:
 
     # TODO: This should have a dependency on PreTrainedModel. i.e. isinstance(PreTrainedModel)
     # from transformers.modeling_utils import PreTrainedModel
-    # We do not want this dependency explicity in our codebase so we make a very general assumption about
-    # the structure of the pipeline which may have unintended consequences
+    # We do not want this dependency explicitly in our codebase so we make a very general
+    # assumption about the structure of the pipeline which may have unintended consequences
     def _get_model(self, pipe) -> Optional[Any]:
         """Extracts model from the pipeline.
 

wandb/integration/keras/callbacks/metrics_logger.py

@@ -43,7 +43,7 @@ class WandbMetricsLogger(callbacks.Callback):
             at the end of each epoch. If "batch", logs metrics at the end
             of each batch. If an integer, logs metrics at the end of that
             many batches. Defaults to "epoch".
-        initial_global_step: (int) Use this argument to correcly log the
+        initial_global_step: (int) Use this argument to correctly log the
             learning rate when you resume training from some `initial_epoch`,
             and a learning rate scheduler is used. This can be computed as
             `step_size * initial_step`. Defaults to 0.

wandb/integration/keras/keras.py

@@ -576,7 +576,7 @@ class WandbCallback(tf.keras.callbacks.Callback):
                     )
                     self._model_trained_since_last_eval = False
             except Exception as e:
-                wandb.termwarn("Error durring prediction logging for epoch: " + str(e))
+                wandb.termwarn("Error during prediction logging for epoch: " + str(e))
 
     def on_epoch_end(self, epoch, logs=None):
         if logs is None:

wandb/integration/openai/fine_tuning.py

@@ -65,6 +65,8 @@ class WandbLogger:
         overwrite: bool = False,
         wait_for_job_success: bool = True,
         log_datasets: bool = True,
+        model_artifact_name: str = "model-metadata",
+        model_artifact_type: str = "model",
         **kwargs_wandb_init: Dict[str, Any],
     ) -> str:
         """Sync fine-tunes to Weights & Biases.
@@ -76,6 +78,8 @@ class WandbLogger:
         :param entity: Username or team name where you're sending runs. By default, your default entity is used, which is usually your username.
         :param overwrite: Forces logging and overwrite existing wandb run of the same fine-tune.
         :param wait_for_job_success: Waits for the fine-tune to be complete and then log metrics to W&B. By default, it is True.
+        :param model_artifact_name: Name of the model artifact that is logged
+        :param model_artifact_type: Type of the model artifact that is logged
         """
         if openai_client is None:
             openai_client = OpenAI()
@@ -157,6 +161,8 @@ class WandbLogger:
                 overwrite,
                 show_individual_warnings,
                 log_datasets,
+                model_artifact_name,
+                model_artifact_type,
                 **kwargs_wandb_init,
             )
 
@@ -201,6 +207,8 @@ class WandbLogger:
         overwrite: bool,
         show_individual_warnings: bool,
         log_datasets: bool,
+        model_artifact_name: str,
+        model_artifact_type: str,
         **kwargs_wandb_init: Dict[str, Any],
     ):
         fine_tune_id = fine_tune.id
@@ -244,7 +252,15 @@ class WandbLogger:
             cls._run.summary["fine_tuned_model"] = fine_tuned_model
 
         # training/validation files and fine-tune details
-        cls._log_artifacts(fine_tune, project, entity, log_datasets, overwrite)
+        cls._log_artifacts(
+            fine_tune,
+            project,
+            entity,
+            log_datasets,
+            overwrite,
+            model_artifact_name,
+            model_artifact_type,
+        )
 
         # mark run as complete
         cls._run.summary["status"] = "succeeded"
@@ -341,6 +357,8 @@ class WandbLogger:
         entity: Optional[str],
         log_datasets: bool,
         overwrite: bool,
+        model_artifact_name: str,
+        model_artifact_type: str,
     ) -> None:
         if log_datasets:
             wandb.termlog("Logging training/validation files...")
@@ -361,8 +379,8 @@ class WandbLogger:
         # fine-tune details
         fine_tune_id = fine_tune.id
         artifact = wandb.Artifact(
-            "model_metadata",
-            type="model",
+            model_artifact_name,
+            type=model_artifact_type,
             metadata=dict(fine_tune),
         )
 

wandb/integration/prodigy/prodigy.py

@@ -26,7 +26,7 @@ from PIL import Image
 
 import wandb
 from wandb import util
-from wandb.plots.utils import test_missing
+from wandb.plot.utils import test_missing
 from wandb.sdk.lib import telemetry as wb_telemetry
 
 

wandb/jupyter.py

@@ -288,35 +288,34 @@ def attempt_colab_login(app_url):
     display.display(
         display.Javascript(
             """
-        window._wandbApiKey = new Promise((resolve, reject) => {
-            function loadScript(url) {
-            return new Promise(function(resolve, reject) {
+        window._wandbApiKey = new Promise((resolve, reject) => {{
+            function loadScript(url) {{
+            return new Promise(function(resolve, reject) {{
                 let newScript = document.createElement("script");
                 newScript.onerror = reject;
                 newScript.onload = resolve;
                 document.body.appendChild(newScript);
                 newScript.src = url;
-            });
-            }
-            loadScript("https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js").then(() => {
+            }});
+            }}
+            loadScript("https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js").then(() => {{
             const iframe = document.createElement('iframe')
             iframe.style.cssText = "width:0;height:0;border:none"
             document.body.appendChild(iframe)
-            const handshake = new Postmate({
+            const handshake = new Postmate({{
                 container: iframe,
-                url: '%s/authorize'
-            });
+                url: '{}/authorize'
+            }});
             const timeout = setTimeout(() => reject("Couldn't auto authenticate"), 5000)
-            handshake.then(function(child) {
-                child.on('authorize', data => {
+            handshake.then(function(child) {{
+                child.on('authorize', data => {{
                     clearTimeout(timeout)
                     resolve(data)
-                });
-            });
-            })
-        });
-    """
-            % app_url.replace("http:", "https:")
+                }});
+            }});
+            }})
+        }});
+    """.format(app_url.replace("http:", "https:"))
         )
     )
     try:

wandb/old/summary.py

@@ -61,7 +61,7 @@ class SummarySubDict:
         This should only be implemented by the "_root" child class.
 
         We pass the child_dict so the item can be set on it or not as
-        appropriate. Returning None for a nonexistant path wouldn't be
+        appropriate. Returning None for a nonexistent path wouldn't be
         distinguishable from that path being set to the value None.
         """
         raise NotImplementedError

wandb/plot/confusion_matrix.py

@@ -48,7 +48,7 @@ def confusion_matrix(
 
     assert (probs is None or preds is None) and not (
         probs is None and preds is None
-    ), "Must provide probabilties or predictions but not both to confusion matrix"
+    ), "Must provide probabilities or predictions but not both to confusion matrix"
 
     if probs is not None:
         preds = np.argmax(probs, axis=1).tolist()

wandb/plot/pr_curve.py

@@ -2,7 +2,8 @@ from typing import Optional
 
 import wandb
 from wandb import util
-from wandb.plots.utils import test_missing, test_types
+
+from .utils import test_missing, test_types
 
 
 def pr_curve(

wandb/plot/roc_curve.py

@@ -2,7 +2,8 @@ from typing import Optional
 
 import wandb
 from wandb import util
-from wandb.plots.utils import test_missing, test_types
+
+from .utils import test_missing, test_types
 
 
 def roc_curve(

wandb/{plots → plot}/utils.py

@@ -1,21 +1,9 @@
-from collections.abc import Iterable, Sequence
+from typing import Iterable, Sequence
 
 import wandb
 from wandb import util
-from wandb.sdk.lib import deprecate
 
 
-def deprecation_notice() -> None:
-    deprecate.deprecate(
-        field_name=deprecate.Deprecated.plots,
-        warning_message=(
-            "wandb.plots.* functions are deprecated and will be removed in a future release. "
-            "Please use wandb.plot.* instead."
-        ),
-    )
-
-
-# Test assumptions for plotting parameters and datasets
 def test_missing(**kwargs):
     np = util.get_module("numpy", required="Logging plots requires numpy")
     pd = util.get_module("pandas", required="Logging dataframes requires pandas")
@@ -25,7 +13,7 @@ def test_missing(**kwargs):
     for k, v in kwargs.items():
         # Missing/empty params/datapoint arrays
         if v is None:
-            wandb.termerror("%s is None. Please try again." % (k))
+            wandb.termerror("{} is None. Please try again.".format(k))
             test_passed = False
         if (k == "X") or (k == "X_test"):
             if isinstance(v, scipy.sparse.csr.csr_matrix):
@@ -64,8 +52,8 @@ def test_missing(**kwargs):
                 )
             if non_nums > 0:
                 wandb.termerror(
-                    "%s contains values that are not numbers. Please vectorize, label encode or one hot encode %s and call the plotting function again."
-                    % (k, k)
+                    f"{k} contains values that are not numbers. Please vectorize, "
+                    f"label encode or one hot encode {k} and call the plotting function again."
                 )
                 test_passed = False
     return test_passed
@@ -73,8 +61,8 @@ def test_missing(**kwargs):
 
 def test_fitted(model):
     np = util.get_module("numpy", required="Logging plots requires numpy")
-    pd = util.get_module("pandas", required="Logging dataframes requires pandas")
-    scipy = util.get_module("scipy", required="Logging scipy matrices requires scipy")
+    _ = util.get_module("pandas", required="Logging dataframes requires pandas")
+    _ = util.get_module("scipy", required="Logging scipy matrices requires scipy")
     scikit_utils = util.get_module(
         "sklearn.utils",
         required="roc requires the scikit utils submodule, install with `pip install scikit-learn`",
@@ -120,7 +108,7 @@ def test_fitted(model):
 
 
 def encode_labels(df):
-    pd = util.get_module("pandas", required="Logging dataframes requires pandas")
+    _ = util.get_module("pandas", required="Logging dataframes requires pandas")
     preprocessing = util.get_module(
         "sklearn.preprocessing",
         "roc requires the scikit preprocessing submodule, install with `pip install scikit-learn`",
@@ -137,7 +125,7 @@ def encode_labels(df):
 def test_types(**kwargs):
     np = util.get_module("numpy", required="Logging plots requires numpy")
     pd = util.get_module("pandas", required="Logging dataframes requires pandas")
-    scipy = util.get_module("scipy", required="Logging scipy matrices requires scipy")
+    _ = util.get_module("scipy", required="Logging scipy matrices requires scipy")
 
     base = util.get_module(
         "sklearn.base",
@@ -171,25 +159,25 @@ def test_types(**kwargs):
                     list,
                 ),
             ):
-                wandb.termerror("%s is not an array. Please try again." % (k))
+                wandb.termerror("{} is not an array. Please try again.".format(k))
                 test_passed = False
         # check for classifier types
         if k == "model":
             if (not base.is_classifier(v)) and (not base.is_regressor(v)):
                 wandb.termerror(
-                    "%s is not a classifier or regressor. Please try again." % (k)
+                    "{} is not a classifier or regressor. Please try again.".format(k)
                 )
                 test_passed = False
         elif k == "clf" or k == "binary_clf":
             if not (base.is_classifier(v)):
-                wandb.termerror("%s is not a classifier. Please try again." % (k))
+                wandb.termerror("{} is not a classifier. Please try again.".format(k))
                 test_passed = False
         elif k == "regressor":
             if not base.is_regressor(v):
-                wandb.termerror("%s is not a regressor. Please try again." % (k))
+                wandb.termerror("{} is not a regressor. Please try again.".format(k))
                 test_passed = False
         elif k == "clusterer":
             if not (getattr(v, "_estimator_type", None) == "clusterer"):
-                wandb.termerror("%s is not a clusterer. Please try again." % (k))
+                wandb.termerror("{} is not a clusterer. Please try again.".format(k))
                 test_passed = False
     return test_passed