valor-lite 0.33.12__tar.gz → 0.33.14__tar.gz

{valor_lite-0.33.12/valor_lite.egg-info → valor_lite-0.33.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.12
+Version: 0.33.14
 Summary: Compute valor metrics locally.
 License: MIT License
         

{valor_lite-0.33.12 → valor_lite-0.33.14}/benchmarks/benchmark_classification.py

@@ -211,7 +211,7 @@ def run_benchmarking_analysis(
             )
 
         # evaluate
-        eval_time, _ = time_it(evaluator.compute_precision_recall)()
+        eval_time, _ = time_it(evaluator.compute_precision_recall_rocauc)()
         if eval_time > evaluation_timeout and evaluation_timeout != -1:
             raise TimeoutError(
                 f"Base evaluation timed out with {evaluator.n_datums} datums."

{valor_lite-0.33.12 → valor_lite-0.33.14}/examples/object-detection.ipynb

@@ -234,7 +234,7 @@
           "name": "stderr",
           "output_type": "stream",
           "text": [
-            "100%|██████████| 5000/5000 [00:01<00:00, 3682.79it/s]\n"
+            "100%|██████████| 5000/5000 [00:00<00:00, 8122.07it/s]\n"
           ]
         }
       ],
@@ -321,86 +321,86 @@
         {
           "data": {
             "text/plain": [
-              "[AP(value=0.03931711480949181, iou_threshold=0.25, label='person'),\n",
-              " AP(value=0.00574667500702136, iou_threshold=0.25, label='chair'),\n",
-              " AP(value=0.012869301715994604, iou_threshold=0.25, label='potted plant'),\n",
-              " AP(value=0.12468056653228336, iou_threshold=0.25, label='dining table'),\n",
-              " AP(value=0.025555256805287804, iou_threshold=0.25, label='tv'),\n",
-              " AP(value=0.0873892652423137, iou_threshold=0.25, label='microwave'),\n",
-              " AP(value=0.07156154587928347, iou_threshold=0.25, label='refrigerator'),\n",
-              " AP(value=0.0, iou_threshold=0.25, label='book'),\n",
-              " AP(value=0.017410438586802798, iou_threshold=0.25, label='clock'),\n",
-              " AP(value=0.01582175783225678, iou_threshold=0.25, label='vase'),\n",
-              " AP(value=0.231753782860388, iou_threshold=0.25, label='bear'),\n",
-              " AP(value=0.23569425610798753, iou_threshold=0.25, label='bed'),\n",
-              " AP(value=0.000695619913279384, iou_threshold=0.25, label='bottle'),\n",
-              " AP(value=0.002421632122336015, iou_threshold=0.25, label='car'),\n",
-              " AP(value=0.059802019119639865, iou_threshold=0.25, label='truck'),\n",
-              " AP(value=0.12696681676449215, iou_threshold=0.25, label='stop sign'),\n",
-              " AP(value=0.14102282165570407, iou_threshold=0.25, label='teddy bear'),\n",
-              " AP(value=0.0007519739315703722, iou_threshold=0.25, label='skis'),\n",
-              " AP(value=0.027301028578575685, iou_threshold=0.25, label='oven'),\n",
-              " AP(value=0.0007072135785007071, iou_threshold=0.25, label='sports ball'),\n",
-              " AP(value=0.014991066775850518, iou_threshold=0.25, label='baseball glove'),\n",
-              " AP(value=0.012910114540865852, iou_threshold=0.25, label='tennis racket'),\n",
-              " AP(value=0.002605523710265763, iou_threshold=0.25, label='backpack'),\n",
-              " AP(value=0.0012280297797221582, iou_threshold=0.25, label='handbag'),\n",
-              " AP(value=0.014437478085599116, iou_threshold=0.25, label='boat'),\n",
-              " AP(value=0.02472893969494828, iou_threshold=0.25, label='bird'),\n",
-              " AP(value=0.0173023682386164, iou_threshold=0.25, label='cell phone'),\n",
-              " AP(value=0.21514217763688503, iou_threshold=0.25, label='train'),\n",
-              " AP(value=0.039401614478727934, iou_threshold=0.25, label='bowl'),\n",
-              " AP(value=0.09553809158302397, iou_threshold=0.25, label='sandwich'),\n",
-              " AP(value=0.008772195312241687, iou_threshold=0.25, label='cup'),\n",
-              " AP(value=0.003529433618784017, iou_threshold=0.25, label='surfboard'),\n",
-              " AP(value=0.057312433796814985, iou_threshold=0.25, label='laptop'),\n",
-              " AP(value=0.003592328379843429, iou_threshold=0.25, label='mouse'),\n",
-              " AP(value=0.007705426769738388, iou_threshold=0.25, label='keyboard'),\n",
-              " AP(value=0.0015763816845228704, iou_threshold=0.25, label='traffic light'),\n",
-              " AP(value=0.14803132929413712, iou_threshold=0.25, label='bus'),\n",
-              " AP(value=0.2706385426934155, iou_threshold=0.25, label='cat'),\n",
-              " AP(value=0.06197253140976744, iou_threshold=0.25, label='airplane'),\n",
-              " AP(value=0.11098807002863581, iou_threshold=0.25, label='zebra'),\n",
-              " AP(value=0.000456968773800457, iou_threshold=0.25, label='tie'),\n",
-              " AP(value=0.01789546873590162, iou_threshold=0.25, label='apple'),\n",
-              " AP(value=0.00497941318169436, iou_threshold=0.25, label='baseball bat'),\n",
-              " AP(value=0.008054041938151662, iou_threshold=0.25, label='wine glass'),\n",
-              " AP(value=0.0013656538067599864, iou_threshold=0.25, label='knife'),\n",
-              " AP(value=0.034283643667397244, iou_threshold=0.25, label='cake'),\n",
-              " AP(value=0.0010068803490518544, iou_threshold=0.25, label='spoon'),\n",
-              " AP(value=0.005547968589962444, iou_threshold=0.25, label='snowboard'),\n",
-              " AP(value=0.03775782935785413, iou_threshold=0.25, label='banana'),\n",
-              " AP(value=0.06007451642097835, iou_threshold=0.25, label='donut'),\n",
-              " AP(value=0.04456774994898758, iou_threshold=0.25, label='toilet'),\n",
-              " AP(value=0.006179001878088361, iou_threshold=0.25, label='sink'),\n",
-              " AP(value=0.03680123352285498, iou_threshold=0.25, label='broccoli'),\n",
-              " AP(value=0.014936047541778902, iou_threshold=0.25, label='skateboard'),\n",
-              " AP(value=0.022694552738821507, iou_threshold=0.25, label='bench'),\n",
-              " AP(value=0.002867675716650755, iou_threshold=0.25, label='fork'),\n",
-              " AP(value=0.0080174142984774, iou_threshold=0.25, label='carrot'),\n",
-              " AP(value=0.04041483896880335, iou_threshold=0.25, label='couch'),\n",
-              " AP(value=0.0007135848720007136, iou_threshold=0.25, label='remote'),\n",
-              " AP(value=0.02584120622988414, iou_threshold=0.25, label='bicycle'),\n",
-              " AP(value=0.06534653465346534, iou_threshold=0.25, label='scissors'),\n",
-              " AP(value=0.046674509472276055, iou_threshold=0.25, label='orange'),\n",
-              " AP(value=0.023613502704789624, iou_threshold=0.25, label='sheep'),\n",
-              " AP(value=0.14258072068496863, iou_threshold=0.25, label='elephant'),\n",
-              " AP(value=0.013407926158469507, iou_threshold=0.25, label='frisbee'),\n",
-              " AP(value=0.04803841606991949, iou_threshold=0.25, label='umbrella'),\n",
-              " AP(value=0.10212714134235075, iou_threshold=0.25, label='horse'),\n",
-              " AP(value=0.09399604539879332, iou_threshold=0.25, label='motorcycle'),\n",
-              " AP(value=0.1316711323706997, iou_threshold=0.25, label='dog'),\n",
-              " AP(value=0.016052361400942596, iou_threshold=0.25, label='kite'),\n",
-              " AP(value=0.13853737273699804, iou_threshold=0.25, label='pizza'),\n",
-              " AP(value=0.03162000973589944, iou_threshold=0.25, label='cow'),\n",
-              " AP(value=0.0702284791057795, iou_threshold=0.25, label='fire hydrant'),\n",
-              " AP(value=0.04231869341571282, iou_threshold=0.25, label='suitcase'),\n",
-              " AP(value=0.16491634254279497, iou_threshold=0.25, label='giraffe'),\n",
-              " AP(value=0.048692537906881606, iou_threshold=0.25, label='hot dog'),\n",
-              " AP(value=0.02254763291455196, iou_threshold=0.25, label='parking meter'),\n",
-              " AP(value=0.0, iou_threshold=0.25, label='toothbrush'),\n",
-              " AP(value=0.0, iou_threshold=0.25, label='toaster'),\n",
-              " AP(value=0.0, iou_threshold=0.25, label='hair drier')]"
+              "[Metric(type='AP', value=0.03931711480949181, parameters={'iou_threshold': 0.25, 'label': 'person'}),\n",
+              " Metric(type='AP', value=0.00574667500702136, parameters={'iou_threshold': 0.25, 'label': 'chair'}),\n",
+              " Metric(type='AP', value=0.012869301715994604, parameters={'iou_threshold': 0.25, 'label': 'potted plant'}),\n",
+              " Metric(type='AP', value=0.12468056653228336, parameters={'iou_threshold': 0.25, 'label': 'dining table'}),\n",
+              " Metric(type='AP', value=0.025555256805287804, parameters={'iou_threshold': 0.25, 'label': 'tv'}),\n",
+              " Metric(type='AP', value=0.0873892652423137, parameters={'iou_threshold': 0.25, 'label': 'microwave'}),\n",
+              " Metric(type='AP', value=0.07156154587928347, parameters={'iou_threshold': 0.25, 'label': 'refrigerator'}),\n",
+              " Metric(type='AP', value=0.0, parameters={'iou_threshold': 0.25, 'label': 'book'}),\n",
+              " Metric(type='AP', value=0.017410438586802798, parameters={'iou_threshold': 0.25, 'label': 'clock'}),\n",
+              " Metric(type='AP', value=0.01582175783225678, parameters={'iou_threshold': 0.25, 'label': 'vase'}),\n",
+              " Metric(type='AP', value=0.231753782860388, parameters={'iou_threshold': 0.25, 'label': 'bear'}),\n",
+              " Metric(type='AP', value=0.23569425610798753, parameters={'iou_threshold': 0.25, 'label': 'bed'}),\n",
+              " Metric(type='AP', value=0.000695619913279384, parameters={'iou_threshold': 0.25, 'label': 'bottle'}),\n",
+              " Metric(type='AP', value=0.002421632122336015, parameters={'iou_threshold': 0.25, 'label': 'car'}),\n",
+              " Metric(type='AP', value=0.059802019119639865, parameters={'iou_threshold': 0.25, 'label': 'truck'}),\n",
+              " Metric(type='AP', value=0.12696681676449215, parameters={'iou_threshold': 0.25, 'label': 'stop sign'}),\n",
+              " Metric(type='AP', value=0.14102282165570407, parameters={'iou_threshold': 0.25, 'label': 'teddy bear'}),\n",
+              " Metric(type='AP', value=0.0007519739315703722, parameters={'iou_threshold': 0.25, 'label': 'skis'}),\n",
+              " Metric(type='AP', value=0.027301028578575685, parameters={'iou_threshold': 0.25, 'label': 'oven'}),\n",
+              " Metric(type='AP', value=0.0007072135785007071, parameters={'iou_threshold': 0.25, 'label': 'sports ball'}),\n",
+              " Metric(type='AP', value=0.014991066775850518, parameters={'iou_threshold': 0.25, 'label': 'baseball glove'}),\n",
+              " Metric(type='AP', value=0.012910114540865852, parameters={'iou_threshold': 0.25, 'label': 'tennis racket'}),\n",
+              " Metric(type='AP', value=0.002605523710265763, parameters={'iou_threshold': 0.25, 'label': 'backpack'}),\n",
+              " Metric(type='AP', value=0.0012280297797221582, parameters={'iou_threshold': 0.25, 'label': 'handbag'}),\n",
+              " Metric(type='AP', value=0.014437478085599116, parameters={'iou_threshold': 0.25, 'label': 'boat'}),\n",
+              " Metric(type='AP', value=0.02472893969494828, parameters={'iou_threshold': 0.25, 'label': 'bird'}),\n",
+              " Metric(type='AP', value=0.0173023682386164, parameters={'iou_threshold': 0.25, 'label': 'cell phone'}),\n",
+              " Metric(type='AP', value=0.21514217763688503, parameters={'iou_threshold': 0.25, 'label': 'train'}),\n",
+              " Metric(type='AP', value=0.039401614478727934, parameters={'iou_threshold': 0.25, 'label': 'bowl'}),\n",
+              " Metric(type='AP', value=0.09553809158302397, parameters={'iou_threshold': 0.25, 'label': 'sandwich'}),\n",
+              " Metric(type='AP', value=0.008772195312241687, parameters={'iou_threshold': 0.25, 'label': 'cup'}),\n",
+              " Metric(type='AP', value=0.003529433618784017, parameters={'iou_threshold': 0.25, 'label': 'surfboard'}),\n",
+              " Metric(type='AP', value=0.057312433796814985, parameters={'iou_threshold': 0.25, 'label': 'laptop'}),\n",
+              " Metric(type='AP', value=0.003592328379843429, parameters={'iou_threshold': 0.25, 'label': 'mouse'}),\n",
+              " Metric(type='AP', value=0.007705426769738388, parameters={'iou_threshold': 0.25, 'label': 'keyboard'}),\n",
+              " Metric(type='AP', value=0.0015763816845228704, parameters={'iou_threshold': 0.25, 'label': 'traffic light'}),\n",
+              " Metric(type='AP', value=0.14803132929413712, parameters={'iou_threshold': 0.25, 'label': 'bus'}),\n",
+              " Metric(type='AP', value=0.2706385426934155, parameters={'iou_threshold': 0.25, 'label': 'cat'}),\n",
+              " Metric(type='AP', value=0.06197253140976744, parameters={'iou_threshold': 0.25, 'label': 'airplane'}),\n",
+              " Metric(type='AP', value=0.11098807002863581, parameters={'iou_threshold': 0.25, 'label': 'zebra'}),\n",
+              " Metric(type='AP', value=0.000456968773800457, parameters={'iou_threshold': 0.25, 'label': 'tie'}),\n",
+              " Metric(type='AP', value=0.01789546873590162, parameters={'iou_threshold': 0.25, 'label': 'apple'}),\n",
+              " Metric(type='AP', value=0.00497941318169436, parameters={'iou_threshold': 0.25, 'label': 'baseball bat'}),\n",
+              " Metric(type='AP', value=0.008054041938151662, parameters={'iou_threshold': 0.25, 'label': 'wine glass'}),\n",
+              " Metric(type='AP', value=0.0013656538067599864, parameters={'iou_threshold': 0.25, 'label': 'knife'}),\n",
+              " Metric(type='AP', value=0.034283643667397244, parameters={'iou_threshold': 0.25, 'label': 'cake'}),\n",
+              " Metric(type='AP', value=0.0010068803490518544, parameters={'iou_threshold': 0.25, 'label': 'spoon'}),\n",
+              " Metric(type='AP', value=0.005547968589962444, parameters={'iou_threshold': 0.25, 'label': 'snowboard'}),\n",
+              " Metric(type='AP', value=0.03775782935785413, parameters={'iou_threshold': 0.25, 'label': 'banana'}),\n",
+              " Metric(type='AP', value=0.06007451642097835, parameters={'iou_threshold': 0.25, 'label': 'donut'}),\n",
+              " Metric(type='AP', value=0.04456774994898758, parameters={'iou_threshold': 0.25, 'label': 'toilet'}),\n",
+              " Metric(type='AP', value=0.006179001878088361, parameters={'iou_threshold': 0.25, 'label': 'sink'}),\n",
+              " Metric(type='AP', value=0.03680123352285498, parameters={'iou_threshold': 0.25, 'label': 'broccoli'}),\n",
+              " Metric(type='AP', value=0.014936047541778902, parameters={'iou_threshold': 0.25, 'label': 'skateboard'}),\n",
+              " Metric(type='AP', value=0.022694552738821507, parameters={'iou_threshold': 0.25, 'label': 'bench'}),\n",
+              " Metric(type='AP', value=0.002867675716650755, parameters={'iou_threshold': 0.25, 'label': 'fork'}),\n",
+              " Metric(type='AP', value=0.0080174142984774, parameters={'iou_threshold': 0.25, 'label': 'carrot'}),\n",
+              " Metric(type='AP', value=0.04041483896880335, parameters={'iou_threshold': 0.25, 'label': 'couch'}),\n",
+              " Metric(type='AP', value=0.0007135848720007136, parameters={'iou_threshold': 0.25, 'label': 'remote'}),\n",
+              " Metric(type='AP', value=0.02584120622988414, parameters={'iou_threshold': 0.25, 'label': 'bicycle'}),\n",
+              " Metric(type='AP', value=0.06534653465346534, parameters={'iou_threshold': 0.25, 'label': 'scissors'}),\n",
+              " Metric(type='AP', value=0.046674509472276055, parameters={'iou_threshold': 0.25, 'label': 'orange'}),\n",
+              " Metric(type='AP', value=0.023613502704789624, parameters={'iou_threshold': 0.25, 'label': 'sheep'}),\n",
+              " Metric(type='AP', value=0.14258072068496863, parameters={'iou_threshold': 0.25, 'label': 'elephant'}),\n",
+              " Metric(type='AP', value=0.013407926158469507, parameters={'iou_threshold': 0.25, 'label': 'frisbee'}),\n",
+              " Metric(type='AP', value=0.04803841606991949, parameters={'iou_threshold': 0.25, 'label': 'umbrella'}),\n",
+              " Metric(type='AP', value=0.10212714134235075, parameters={'iou_threshold': 0.25, 'label': 'horse'}),\n",
+              " Metric(type='AP', value=0.09399604539879332, parameters={'iou_threshold': 0.25, 'label': 'motorcycle'}),\n",
+              " Metric(type='AP', value=0.1316711323706997, parameters={'iou_threshold': 0.25, 'label': 'dog'}),\n",
+              " Metric(type='AP', value=0.016052361400942596, parameters={'iou_threshold': 0.25, 'label': 'kite'}),\n",
+              " Metric(type='AP', value=0.13853737273699804, parameters={'iou_threshold': 0.25, 'label': 'pizza'}),\n",
+              " Metric(type='AP', value=0.03162000973589944, parameters={'iou_threshold': 0.25, 'label': 'cow'}),\n",
+              " Metric(type='AP', value=0.0702284791057795, parameters={'iou_threshold': 0.25, 'label': 'fire hydrant'}),\n",
+              " Metric(type='AP', value=0.04231869341571282, parameters={'iou_threshold': 0.25, 'label': 'suitcase'}),\n",
+              " Metric(type='AP', value=0.16491634254279497, parameters={'iou_threshold': 0.25, 'label': 'giraffe'}),\n",
+              " Metric(type='AP', value=0.048692537906881606, parameters={'iou_threshold': 0.25, 'label': 'hot dog'}),\n",
+              " Metric(type='AP', value=0.02254763291455196, parameters={'iou_threshold': 0.25, 'label': 'parking meter'}),\n",
+              " Metric(type='AP', value=0.0, parameters={'iou_threshold': 0.25, 'label': 'toothbrush'}),\n",
+              " Metric(type='AP', value=0.0, parameters={'iou_threshold': 0.25, 'label': 'toaster'}),\n",
+              " Metric(type='AP', value=0.0, parameters={'iou_threshold': 0.25, 'label': 'hair drier'})]"
             ]
           },
           "execution_count": 7,
@@ -430,7 +430,7 @@
         {
           "data": {
             "text/plain": [
-              "[mAP(value=0.049578458401377326, iou_threshold=0.25)]"
+              "[Metric(type='mAP', value=0.049578458401377326, parameters={'iou_threshold': 0.25})]"
             ]
           },
           "execution_count": 8,
@@ -473,7 +473,11 @@
         "plt.figure()\n",
         "recall = [x / 100 for x in range(0,101)]\n",
         "for curve in metrics[MetricType.PrecisionRecallCurve]:\n",
-        "    plt.plot(recall, curve.precisions, label=curve.label)\n",
+        "    plt.plot(\n",
+        "        recall, \n",
+        "        curve.value[\"precisions\"],\n",
+        "        label=curve.parameters[\"label\"],\n",
+        "    )\n",
         "plt.title(\"Precision-Recall Curve\")\n",
         "plt.legend(prop={'size': 6})\n",
         "plt.show()"
@@ -481,7 +485,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 15,
+      "execution_count": 10,
       "id": "c4877536",
       "metadata": {},
       "outputs": [
@@ -503,7 +507,7 @@
         "plt.figure()\n",
         "recall = [x / 100 for x in range(0,101)]\n",
         "for curve in metrics[MetricType.PrecisionRecallCurve]:\n",
-        "    plt.plot(recall, curve.scores, label=curve.label)\n",
+        "    plt.plot(recall, curve.value[\"scores\"], label=curve.parameters[\"label\"])\n",
         "plt.title(\"Score-Recall Curve\")\n",
         "plt.legend(prop={'size': 6})\n",
         "plt.show()"
@@ -554,7 +558,7 @@
         ")[MetricType.ConfusionMatrix]\n",
         "metric = matrices[0]\n",
         "\n",
-        "metric.confusion_matrix[\"person\"][\"person\"][\"examples\"]"
+        "metric.value[\"confusion_matrix\"][\"person\"][\"person\"][\"examples\"]"
       ]
     },
     {
@@ -941,10 +945,10 @@
       "source": [
         "cm = {\n",
         "    gt: {\n",
-        "        pd: metric.confusion_matrix[gt][pd][\"count\"]\n",
-        "        for pd in metric.confusion_matrix[gt]\n",
+        "        pd: metric.value[\"confusion_matrix\"][gt][pd][\"count\"]\n",
+        "        for pd in metric.value[\"confusion_matrix\"][gt]\n",
         "    }\n",
-        "    for gt in metric.confusion_matrix\n",
+        "    for gt in metric.value[\"confusion_matrix\"]\n",
         "}\n",
         "pd.DataFrame(cm)"
       ]
@@ -1760,7 +1764,7 @@
         }
       ],
       "source": [
-        "metric.hallucinations"
+        "metric.value[\"hallucinations\"]"
       ]
     },
     {
@@ -2991,7 +2995,7 @@
         }
       ],
       "source": [
-        "metric.missing_predictions"
+        "metric.value[\"missing_predictions\"]"
       ]
     }
   ],