valor-lite 0.33.15__tar.gz → 0.33.17__tar.gz

{valor_lite-0.33.15/valor_lite.egg-info → valor_lite-0.33.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.15
+Version: 0.33.17
 Summary: Compute valor metrics locally.
 License: MIT License
         
@@ -28,15 +28,23 @@ Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: Pillow>=9.1.0
+Requires-Dist: evaluate
 Requires-Dist: importlib_metadata; python_version < "3.8"
-Requires-Dist: tqdm
-Requires-Dist: requests
+Requires-Dist: nltk
 Requires-Dist: numpy
+Requires-Dist: Pillow>=9.1.0
+Requires-Dist: requests
+Requires-Dist: rouge_score
 Requires-Dist: shapely
+Requires-Dist: tqdm
+Provides-Extra: mistral
+Requires-Dist: mistralai>=1.0; extra == "mistral"
+Provides-Extra: openai
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: coverage; extra == "test"
+Requires-Dist: pre-commit; extra == "test"
 
 # valor-lite: Fast, local machine learning evaluation.
 

valor_lite-0.33.17/examples/text_generation.ipynb

@@ -0,0 +1,500 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Text Generation Example\n",
+    "\n",
+    "## Introduction\n",
+    "\n",
+    "In this notebook, we'll walk-through a detailed example of how you can use Valor to evaluate LLM's.\n",
+    "\n",
+    "For a conceptual introduction to Valor, [check out our project overview](https://striveworks.github.io/valor/). For a higher-level example notebook, [check out our \"Getting Started\" notebook](https://github.com/Striveworks/valor/blob/main/examples/getting_started.ipynb)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/czaloom/valor/.env-valor/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import json\n",
+    "import torch\n",
+    "from transformers import pipeline\n",
+    "from valor_lite.text_generation import Evaluator, QueryResponse, Context, MetricType\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Set up an LLM using Huggingface."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class LlamaWrapper:\n",
+    "\n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        model_name: str = \"meta-llama/Llama-3.2-1B-Instruct\",\n",
+    "    ) -> None:\n",
+    "        self.model_name = model_name\n",
+    "        self.pipe = pipeline(\n",
+    "            \"text-generation\", \n",
+    "            model=model_name, \n",
+    "            torch_dtype=torch.bfloat16, \n",
+    "            device_map=\"auto\"\n",
+    "        )\n",
+    "\n",
+    "    def __call__(\n",
+    "        self,\n",
+    "        messages: list[dict[str, str]],\n",
+    "    ) -> str:\n",
+    "        output = self.pipe(messages, max_new_tokens=256)\n",
+    "        return output[0]['generated_text'][-1][\"content\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client = LlamaWrapper()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Setting `pad_token_id` to `eos_token_id`:None for open-end generation.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'I\\'m an artificial intelligence model known as Llama. Llama stands for \"Large Language Model Meta AI.\"'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "client([{\"role\": \"user\", \"content\": \"Who are you?\"}])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Now, lets evaluate a query!"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "First, lets choose a model to perform the evaluation requests."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "evaluator = Evaluator.openai()\n",
+    "# evaluator = Evaluator.mistral()\n",
+    "# evaluator = Evaluator(client=LlamaWrapper())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = QueryResponse(\n",
+    "    query=\"Did John Adams get along with Alexander Hamilton?\",\n",
+    "    response=\"Based on the provided context, John Adams and Alexander Hamilton did not get along. John Adams, during his presidency, had grown independent of his cabinet, often making decisions despite opposition from it. Hamilton, who was accustomed to being regularly consulted by Washington, sent Adams a detailed letter with policy suggestions after his inauguration, which Adams dismissively ignored.\\n\",\n",
+    "    context=Context(\n",
+    "        groundtruth=[\n",
+    "            \"John Adams and Alexander Hamilton did not get along. John Adams had grown independent of his cabinet, often making decisions despite opposition from it.\\n\",\n",
+    "        ],\n",
+    "        prediction=[\n",
+    "            \"\"\"Although aware of Hamilton\\'s influence, Adams was convinced that their retention ensured a smoother succession. Adams maintained the economic programs of Hamilton, who regularly consulted with key cabinet members, especially the powerful Treasury Secretary, Oliver Wolcott Jr. Adams was in other respects quite independent of his cabinet, often making decisions despite opposition from it. Hamilton had grown accustomed to being regularly consulted by Washington. Shortly after Adams was inaugurated, Hamilton sent him a detailed letter with policy suggestions. Adams dismissively ignored it.\\n\\nFailed peace commission and XYZ affair\\nHistorian Joseph Ellis writes that \"[t]he Adams presidency was destined to be dominated by a single question of American policy to an extent seldom if ever encountered by any succeeding occupant of the office.\" That question was whether to make war with France or find peace. Britain and France were at war as a result of the French Revolution. Hamilton and the Federalists strongly favored the British monarchy against what they denounced as the political radicalism and anti-religious frenzy of the French Revolution. Jefferson and the Republicans, with their firm opposition to monarchy, strongly supported the French overthrowing their king. The French had supported Jefferson for president in 1796 and became belligerent at his loss.\"\"\",\n",
+    "            \"\"\"Led by Revolutionary War veteran John Fries, rural German-speaking farmers protested what they saw as a threat to their liberties. They intimidated tax collectors, who often found themselves unable to go about their business. The disturbance was quickly ended with Hamilton leading the army to restore peace.Fries and two other leaders were arrested, found guilty of treason, and sentenced to hang. They appealed to Adams requesting a pardon. The cabinet unanimously advised Adams to refuse, but he instead granted the pardon, arguing the men had instigated a mere riot as opposed to a rebellion. In his pamphlet attacking Adams before the election, Hamilton wrote that \\\"it was impossible to commit a greater error.\\\"\\n\\nFederalist divisions and peace\\nOn May 5, 1800, Adams's frustrations with the Hamilton wing of the party exploded during a meeting with McHenry, a Hamilton loyalist who was universally regarded, even by Hamilton, as an inept Secretary of War. Adams accused him of subservience to Hamilton and declared that he would rather serve as Jefferson's vice president or minister at The Hague than be beholden to Hamilton for the presidency. McHenry offered to resign at once, and Adams accepted. On May 10, he asked Pickering to resign.\"\"\",\n",
+    "            \"\"\"Indeed, Adams did not consider himself a strong member of the Federalist Party. He had remarked that Hamilton\\'s economic program, centered around banks, would \"swindle\" the poor and unleash the \"gangrene of avarice.\" Desiring \"a more pliant president than Adams,\" Hamilton maneuvered to tip the election to Pinckney. He coerced South Carolina Federalist electors, pledged to vote for \"favorite son\" Pinckney, to scatter their second votes among candidates other than Adams. Hamilton\\'s scheme was undone when several New England state electors heard of it and agreed not to vote for Pinckney. Adams wrote shortly after the election that Hamilton was a \"proud Spirited, conceited, aspiring Mortal always pretending to Morality, with as debauched Morals as old Franklin who is more his Model than any one I know.\" Throughout his life, Adams made highly critical statements about Hamilton. He made derogatory references to his womanizing, real or alleged, and slurred him as the \"Creole bastard.\"\"\",\n",
+    "            \"\"\"The pair\\'s exchange was respectful; Adams promised to do all that he could to restore friendship and cordiality \"between People who, tho Seperated [sic] by an Ocean and under different Governments have the Same Language, a Similar Religion and kindred Blood,\" and the King agreed to \"receive with Pleasure, the Assurances of the friendly Dispositions of the United States.\" The King added that although \"he had been the last to consent\" to American independence, he had always done what he thought was right. He startled Adams by commenting that \"There is an Opinion, among Some People, that you are not the most attached of all Your Countrymen, to the manners of France.\" Adams replied, \"That Opinion sir, is not mistaken... I have no Attachments but to my own Country.\" King George responded, \"An honest Man will never have any other.\"\\nAdams was joined by Abigail in London. Suffering the hostility of the King\\'s courtiers, they escaped when they could by seeking out Richard Price, minister of Newington Green Unitarian Church and instigator of the debate over the Revolution within Britain.\"\"\",\n",
+    "        ],\n",
+    "    )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"AnswerCorrectness\",\n",
+      "    \"value\": 0.6666666666666666,\n",
+      "    \"parameters\": {\n",
+      "        \"evaluator\": \"gpt-3.5-turbo\",\n",
+      "        \"retries\": 0\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_answer_correctness(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"AnswerRelevance\",\n",
+      "    \"value\": 0.16666666666666666,\n",
+      "    \"parameters\": {\n",
+      "        \"evaluator\": \"gpt-3.5-turbo\",\n",
+      "        \"retries\": 0\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_answer_relevance(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"Bias\",\n",
+      "    \"value\": 0.0,\n",
+      "    \"parameters\": {\n",
+      "        \"evaluator\": \"gpt-3.5-turbo\",\n",
+      "        \"retries\": 0\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_bias(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"BLEU\",\n",
+      "    \"value\": 0.3502270395690205,\n",
+      "    \"parameters\": {\n",
+      "        \"weights\": [\n",
+      "            0.25,\n",
+      "            0.25,\n",
+      "            0.25,\n",
+      "            0.25\n",
+      "        ]\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_sentence_bleu(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"ContextPrecision\",\n",
+      "    \"value\": 0.8333333333333333,\n",
+      "    \"parameters\": {\n",
+      "        \"evaluator\": \"gpt-3.5-turbo\",\n",
+      "        \"retries\": 0\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_context_precision(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"ContextRecall\",\n",
+      "    \"value\": 0.6666666666666666,\n",
+      "    \"parameters\": {\n",
+      "        \"evaluator\": \"gpt-3.5-turbo\",\n",
+      "        \"retries\": 0\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_context_recall(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"Faithfulness\",\n",
+      "    \"value\": 0.8333333333333334,\n",
+      "    \"parameters\": {\n",
+      "        \"evaluator\": \"gpt-3.5-turbo\",\n",
+      "        \"retries\": 0\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_faithfulness(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"Hallucination\",\n",
+      "    \"value\": 0.5,\n",
+      "    \"parameters\": {\n",
+      "        \"evaluator\": \"gpt-3.5-turbo\",\n",
+      "        \"retries\": 0\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_hallucination(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"ROUGE\",\n",
+      "    \"value\": 0.5925925925925926,\n",
+      "    \"parameters\": {\n",
+      "        \"rouge_type\": \"rouge1\",\n",
+      "        \"use_stemmer\": false\n",
+      "    }\n",
+      "}\n",
+      "{\n",
+      "    \"type\": \"ROUGE\",\n",
+      "    \"value\": 0.5569620253164557,\n",
+      "    \"parameters\": {\n",
+      "        \"rouge_type\": \"rouge2\",\n",
+      "        \"use_stemmer\": false\n",
+      "    }\n",
+      "}\n",
+      "{\n",
+      "    \"type\": \"ROUGE\",\n",
+      "    \"value\": 0.5925925925925926,\n",
+      "    \"parameters\": {\n",
+      "        \"rouge_type\": \"rougeL\",\n",
+      "        \"use_stemmer\": false\n",
+      "    }\n",
+      "}\n",
+      "{\n",
+      "    \"type\": \"ROUGE\",\n",
+      "    \"value\": 0.5925925925925926,\n",
+      "    \"parameters\": {\n",
+      "        \"rouge_type\": \"rougeLsum\",\n",
+      "        \"use_stemmer\": false\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metrics = evaluator.compute_rouge(query)\n",
+    "for m in metrics:\n",
+    "    print(json.dumps(m.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"SummaryCoherence\",\n",
+      "    \"value\": 4,\n",
+      "    \"parameters\": {\n",
+      "        \"evaluator\": \"gpt-3.5-turbo\",\n",
+      "        \"retries\": 0\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_summary_coherence(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\n",
+      "    \"type\": \"Toxicity\",\n",
+      "    \"value\": 0.3333333333333333,\n",
+      "    \"parameters\": {\n",
+      "        \"evaluator\": \"gpt-3.5-turbo\",\n",
+      "        \"retries\": 0\n",
+      "    }\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "metric = evaluator.compute_toxicity(query)\n",
+    "print(json.dumps(metric.to_dict(), indent=4))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".env-valor",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

{valor_lite-0.33.15 → valor_lite-0.33.17}/pyproject.toml

@@ -6,12 +6,15 @@ readme = "README.md"
 requires-python = ">=3.10"
 license = { file = "LICENSE" }
 dependencies = [
-    "Pillow >= 9.1.0",
+    "evaluate",
     "importlib_metadata; python_version < '3.8'",
-    "tqdm",
-    "requests",
+    "nltk",
     "numpy",
+    "Pillow >= 9.1.0",
+    "requests",
+    "rouge_score",
     "shapely",
+    "tqdm",
 ]
 
 [project.urls]
@@ -22,7 +25,9 @@ requires = ["setuptools>=61.0", "setuptools_scm[toml]>=6.2"]
 build-backend = "setuptools.build_meta"
 
 [project.optional-dependencies]
-test = ["pytest", "coverage"]
+mistral = ["mistralai >= 1.0"]
+openai = ["openai"]
+test = ["pytest", "coverage", "pre-commit"]
 
 [tool.black]
 line-length = 79

valor_lite-0.33.17/tests/classification/test_metric.py

@@ -0,0 +1,39 @@
+import pytest
+from valor_lite.classification import Metric
+
+
+def test_metric_type_validation():
+
+    # test type attribute
+    with pytest.raises(TypeError):
+        Metric(
+            type=1234,  # type: ignore - testing
+            value=1234,
+            parameters={},
+        )
+
+    # test value attribute
+    with pytest.raises(TypeError):
+        Metric(
+            type="SomeMetric",
+            value=[1, 2, 3],  # type: ignore - testing
+            parameters={},
+        )
+
+    # test parameters attribute
+    with pytest.raises(TypeError):
+        Metric(
+            type="SomeMetric",
+            value=1,
+            parameters=123,  # type: ignore - testing
+        )
+
+    # test parameter keys
+    with pytest.raises(TypeError):
+        Metric(
+            type="SomeMetric",
+            value=1,
+            parameters={
+                1: "hello",
+            },  # type: ignore - testing
+        )

{valor_lite-0.33.15 → valor_lite-0.33.17}/tests/object_detection/test_dataloader.py

@@ -16,6 +16,49 @@ def test_no_data():
         loader.finalize()
 
 
+def test_iou_computation():
+
+    detection = Detection(
+        uid="uid",
+        groundtruths=[
+            BoundingBox(xmin=0, xmax=10, ymin=0, ymax=10, labels=["0"]),
+            BoundingBox(xmin=100, xmax=110, ymin=100, ymax=110, labels=["0"]),
+            BoundingBox(
+                xmin=1000, xmax=1100, ymin=1000, ymax=1100, labels=["0"]
+            ),
+        ],
+        predictions=[
+            BoundingBox(
+                xmin=1,
+                xmax=11,
+                ymin=1,
+                ymax=11,
+                labels=["0", "1", "2"],
+                scores=[0.5, 0.25, 0.25],
+            ),
+            BoundingBox(
+                xmin=105,
+                xmax=116,
+                ymin=105,
+                ymax=116,
+                labels=["0", "1", "2"],
+                scores=[0.5, 0.25, 0.25],
+            ),
+        ],
+    )
+
+    loader = DataLoader()
+    loader.add_bounding_boxes([detection])
+
+    assert len(loader.pairs) == 1
+
+    # show that three unique IOUs exist
+    unique_ious = np.unique(loader.pairs[0][:, 3])
+    assert np.isclose(
+        unique_ious, np.array([0.0, 0.12755102, 0.68067227])
+    ).all()
+
+
 def test_mixed_annotations(
     rect1: tuple[float, float, float, float],
     rect1_rotated_5_degrees_around_origin: tuple[float, float, float, float],
@@ -87,7 +130,15 @@ def test_mixed_annotations(
 
     loader = DataLoader()
 
-    for input_ in mixed_detections:
-        with pytest.raises(ValueError) as e:
-            loader.add_bounding_boxes([input_])
-        assert "but annotation is of type" in str(e)
+    for detection in mixed_detections:
+
+        # anything can be converted to a bbox
+        loader.add_bounding_boxes([detection])
+
+        with pytest.raises(AttributeError) as e:
+            loader.add_polygons([detection])
+        assert "no attribute 'shape'" in str(e)
+
+        with pytest.raises(AttributeError) as e:
+            loader.add_bitmasks([detection])
+        assert "no attribute 'mask'" in str(e)