valor-lite 0.33.18__py3-none-any.whl → 0.33.19__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of valor-lite might be problematic. Click here for more details.

@@ -282,7 +282,7 @@ def compute_confusion_matrix(
282
282
  NDArray[np.float64]
283
283
  Confusion matrix.
284
284
  NDArray[np.int32]
285
- Ground truths with missing predictions.
285
+ Unmatched Ground Truths.
286
286
  """
287
287
 
288
288
  n_labels = label_metadata.shape[0]
@@ -292,7 +292,7 @@ def compute_confusion_matrix(
292
292
  (n_scores, n_labels, n_labels, 2 * n_examples + 1),
293
293
  dtype=np.float32,
294
294
  )
295
- missing_predictions = -1 * np.ones(
295
+ unmatched_ground_truths = -1 * np.ones(
296
296
  (n_scores, n_labels, n_examples + 1),
297
297
  dtype=np.int32,
298
298
  )
@@ -339,7 +339,7 @@ def compute_confusion_matrix(
339
339
  score_idx, misclf_labels[:, 0], misclf_labels[:, 1], 0
340
340
  ] = misclf_counts
341
341
 
342
- missing_predictions[score_idx, misprd_labels, 0] = misprd_counts
342
+ unmatched_ground_truths[score_idx, misprd_labels, 0] = misprd_counts
343
343
 
344
344
  if n_examples > 0:
345
345
  for label_idx in range(n_labels):
@@ -375,16 +375,16 @@ def compute_confusion_matrix(
375
375
  1 : 2 * misclf_label_examples.shape[0] + 1,
376
376
  ] = misclf_label_examples[:, [0, 3]].flatten()
377
377
 
378
- # missing prediction examples
378
+ # unmatched ground truth examples
379
379
  mask_misprd_label = misprd_examples[:, 1] == label_idx
380
380
  if misprd_examples.size > 0:
381
381
  misprd_label_examples = misprd_examples[mask_misprd_label][
382
382
  :n_examples
383
383
  ]
384
- missing_predictions[
384
+ unmatched_ground_truths[
385
385
  score_idx,
386
386
  label_idx,
387
387
  1 : misprd_label_examples.shape[0] + 1,
388
388
  ] = misprd_label_examples[:, 0].flatten()
389
389
 
390
- return confusion_matrix, missing_predictions
390
+ return confusion_matrix, unmatched_ground_truths
@@ -321,7 +321,7 @@ class Metric(BaseMetric):
321
321
  ],
322
322
  ],
323
323
  ],
324
- missing_predictions: dict[
324
+ unmatched_ground_truths: dict[
325
325
  str, # ground truth label value
326
326
  dict[
327
327
  str, # either `count` or `examples`
@@ -335,8 +335,8 @@ class Metric(BaseMetric):
335
335
  The confusion matrix and related metrics for the classification task.
336
336
 
337
337
  This class encapsulates detailed information about the model's performance, including correct
338
- predictions, misclassifications, hallucinations (false positives), and missing predictions
339
- (false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
338
+ predictions, misclassifications, unmatched predictions (subset of false positives), and unmatched ground truths
339
+ (subset of false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
340
340
 
341
341
  Confusion Matrix Structure:
342
342
  {
@@ -358,7 +358,7 @@ class Metric(BaseMetric):
358
358
  ...
359
359
  }
360
360
 
361
- Missing Prediction Structure:
361
+ Unmatched Ground Truths Structure:
362
362
  {
363
363
  ground_truth_label: {
364
364
  'count': int,
@@ -379,7 +379,7 @@ class Metric(BaseMetric):
379
379
  A nested dictionary where the first key is the ground truth label value, the second key
380
380
  is the prediction label value, and the innermost dictionary contains either a `count`
381
381
  or a list of `examples`. Each example includes the datum UID and prediction score.
382
- missing_predictions : dict
382
+ unmatched_ground_truths : dict
383
383
  A dictionary where each key is a ground truth label value for which the model failed to predict
384
384
  (false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
385
385
  Each example includes the datum UID.
@@ -396,7 +396,7 @@ class Metric(BaseMetric):
396
396
  type=MetricType.ConfusionMatrix.value,
397
397
  value={
398
398
  "confusion_matrix": confusion_matrix,
399
- "missing_predictions": missing_predictions,
399
+ "unmatched_ground_truths": unmatched_ground_truths,
400
400
  },
401
401
  parameters={
402
402
  "score_threshold": score_threshold,
@@ -153,20 +153,20 @@ def _unpack_confusion_matrix_value(
153
153
  }
154
154
 
155
155
 
156
- def _unpack_missing_predictions_value(
157
- missing_predictions: NDArray[np.int32],
156
+ def _unpack_unmatched_ground_truths_value(
157
+ unmatched_ground_truths: NDArray[np.int32],
158
158
  number_of_labels: int,
159
159
  number_of_examples: int,
160
160
  index_to_uid: dict[int, str],
161
161
  index_to_label: dict[int, str],
162
162
  ) -> dict[str, dict[str, int | list[dict[str, str]]]]:
163
163
  """
164
- Unpacks a numpy array of missing prediction counts and examples.
164
+ Unpacks a numpy array of unmatched ground truth counts and examples.
165
165
  """
166
166
 
167
167
  datum_idx = (
168
168
  lambda gt_label_idx, example_idx: int( # noqa: E731 - lambda fn
169
- missing_predictions[
169
+ unmatched_ground_truths[
170
170
  gt_label_idx,
171
171
  example_idx + 1,
172
172
  ]
@@ -176,7 +176,7 @@ def _unpack_missing_predictions_value(
176
176
  return {
177
177
  index_to_label[gt_label_idx]: {
178
178
  "count": max(
179
- int(missing_predictions[gt_label_idx, 0]),
179
+ int(unmatched_ground_truths[gt_label_idx, 0]),
180
180
  0,
181
181
  ),
182
182
  "examples": [
@@ -197,7 +197,7 @@ def unpack_confusion_matrix_into_metric_list(
197
197
  index_to_label: dict[int, str],
198
198
  ) -> list[Metric]:
199
199
 
200
- (confusion_matrix, missing_predictions) = results
200
+ (confusion_matrix, unmatched_ground_truths) = results
201
201
  n_scores, n_labels, _, _ = confusion_matrix.shape
202
202
  return [
203
203
  Metric.confusion_matrix(
@@ -210,8 +210,10 @@ def unpack_confusion_matrix_into_metric_list(
210
210
  index_to_label=index_to_label,
211
211
  index_to_uid=index_to_uid,
212
212
  ),
213
- missing_predictions=_unpack_missing_predictions_value(
214
- missing_predictions=missing_predictions[score_idx, :, :],
213
+ unmatched_ground_truths=_unpack_unmatched_ground_truths_value(
214
+ unmatched_ground_truths=unmatched_ground_truths[
215
+ score_idx, :, :
216
+ ],
215
217
  number_of_labels=n_labels,
216
218
  number_of_examples=number_of_examples,
217
219
  index_to_label=index_to_label,
@@ -669,9 +669,9 @@ def compute_confusion_matrix(
669
669
  NDArray[np.float64]
670
670
  Confusion matrix.
671
671
  NDArray[np.float64]
672
- Hallucinations.
672
+ Unmatched Predictions.
673
673
  NDArray[np.int32]
674
- Missing Predictions.
674
+ Unmatched Ground Truths.
675
675
  """
676
676
 
677
677
  n_labels = label_metadata.shape[0]
@@ -683,12 +683,12 @@ def compute_confusion_matrix(
683
683
  (n_ious, n_scores, n_labels, n_labels, 4 * n_examples + 1),
684
684
  dtype=np.float32,
685
685
  )
686
- hallucinations = -1 * np.ones(
686
+ unmatched_predictions = -1 * np.ones(
687
687
  # (datum idx, pd idx, pd score) * n_examples + count
688
688
  (n_ious, n_scores, n_labels, 3 * n_examples + 1),
689
689
  dtype=np.float32,
690
690
  )
691
- missing_predictions = -1 * np.ones(
691
+ unmatched_ground_truths = -1 * np.ones(
692
692
  # (datum idx, gt idx) * n_examples + count
693
693
  (n_ious, n_scores, n_labels, 2 * n_examples + 1),
694
694
  dtype=np.int32,
@@ -793,7 +793,7 @@ def compute_confusion_matrix(
793
793
  data[mask_misclf], unique_idx=[0, 1, 2, 4, 5], label_idx=[3, 4]
794
794
  )
795
795
 
796
- # count hallucinations
796
+ # count unmatched predictions
797
797
  (
798
798
  halluc_examples,
799
799
  halluc_labels,
@@ -802,7 +802,7 @@ def compute_confusion_matrix(
802
802
  data[mask_halluc], unique_idx=[0, 2, 5], label_idx=2
803
803
  )
804
804
 
805
- # count missing predictions
805
+ # count unmatched ground truths
806
806
  (
807
807
  misprd_examples,
808
808
  misprd_labels,
@@ -822,13 +822,13 @@ def compute_confusion_matrix(
822
822
  misclf_labels[:, 1],
823
823
  0,
824
824
  ] = misclf_counts
825
- hallucinations[
825
+ unmatched_predictions[
826
826
  iou_idx,
827
827
  score_idx,
828
828
  halluc_labels,
829
829
  0,
830
830
  ] = halluc_counts
831
- missing_predictions[
831
+ unmatched_ground_truths[
832
832
  iou_idx,
833
833
  score_idx,
834
834
  misprd_labels,
@@ -877,26 +877,26 @@ def compute_confusion_matrix(
877
877
  :, [0, 1, 2, 6]
878
878
  ].flatten()
879
879
 
880
- # hallucination examples
880
+ # unmatched prediction examples
881
881
  mask_halluc_label = halluc_examples[:, 5] == label_idx
882
882
  if mask_halluc_label.sum() > 0:
883
883
  halluc_label_examples = halluc_examples[
884
884
  mask_halluc_label
885
885
  ][:n_examples]
886
- hallucinations[
886
+ unmatched_predictions[
887
887
  iou_idx,
888
888
  score_idx,
889
889
  label_idx,
890
890
  1 : 3 * halluc_label_examples.shape[0] + 1,
891
891
  ] = halluc_label_examples[:, [0, 2, 6]].flatten()
892
892
 
893
- # missing prediction examples
893
+ # unmatched ground truth examples
894
894
  mask_misprd_label = misprd_examples[:, 4] == label_idx
895
895
  if misprd_examples.size > 0:
896
896
  misprd_label_examples = misprd_examples[
897
897
  mask_misprd_label
898
898
  ][:n_examples]
899
- missing_predictions[
899
+ unmatched_ground_truths[
900
900
  iou_idx,
901
901
  score_idx,
902
902
  label_idx,
@@ -905,6 +905,6 @@ def compute_confusion_matrix(
905
905
 
906
906
  return (
907
907
  confusion_matrix,
908
- hallucinations,
909
- missing_predictions,
908
+ unmatched_predictions,
909
+ unmatched_ground_truths,
910
910
  )
@@ -619,7 +619,7 @@ class Metric(BaseMetric):
619
619
  ],
620
620
  ],
621
621
  ],
622
- hallucinations: dict[
622
+ unmatched_predictions: dict[
623
623
  str, # prediction label value
624
624
  dict[
625
625
  str, # either `count` or `examples`
@@ -636,7 +636,7 @@ class Metric(BaseMetric):
636
636
  ],
637
637
  ],
638
638
  ],
639
- missing_predictions: dict[
639
+ unmatched_ground_truths: dict[
640
640
  str, # ground truth label value
641
641
  dict[
642
642
  str, # either `count` or `examples`
@@ -660,8 +660,8 @@ class Metric(BaseMetric):
660
660
  Confusion matrix for object detection tasks.
661
661
 
662
662
  This class encapsulates detailed information about the model's performance, including correct
663
- predictions, misclassifications, hallucinations (false positives), and missing predictions
664
- (false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
663
+ predictions, misclassifications, unmatched_predictions (subset of false positives), and unmatched ground truths
664
+ (subset of false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
665
665
 
666
666
  Confusion Matrix Format:
667
667
  {
@@ -683,7 +683,7 @@ class Metric(BaseMetric):
683
683
  ...
684
684
  }
685
685
 
686
- Hallucinations Format:
686
+ Unmatched Predictions Format:
687
687
  {
688
688
  <prediction label>: {
689
689
  'count': int,
@@ -699,7 +699,7 @@ class Metric(BaseMetric):
699
699
  ...
700
700
  }
701
701
 
702
- Missing Prediction Format:
702
+ Unmatched Ground Truths Format:
703
703
  {
704
704
  <ground truth label>: {
705
705
  'count': int,
@@ -721,13 +721,13 @@ class Metric(BaseMetric):
721
721
  is the prediction label value, and the innermost dictionary contains either a `count`
722
722
  or a list of `examples`. Each example includes the datum UID, ground truth bounding box,
723
723
  predicted bounding box, and prediction scores.
724
- hallucinations : dict
724
+ unmatched_predictions : dict
725
725
  A dictionary where each key is a prediction label value with no corresponding ground truth
726
- (false positives). The value is a dictionary containing either a `count` or a list of
726
+ (subset of false positives). The value is a dictionary containing either a `count` or a list of
727
727
  `examples`. Each example includes the datum UID, predicted bounding box, and prediction score.
728
- missing_predictions : dict
728
+ unmatched_ground_truths : dict
729
729
  A dictionary where each key is a ground truth label value for which the model failed to predict
730
- (false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
730
+ (subset of false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
731
731
  Each example includes the datum UID and ground truth bounding box.
732
732
  score_threshold : float
733
733
  The confidence score threshold used to filter predictions.
@@ -744,8 +744,8 @@ class Metric(BaseMetric):
744
744
  type=MetricType.ConfusionMatrix.value,
745
745
  value={
746
746
  "confusion_matrix": confusion_matrix,
747
- "hallucinations": hallucinations,
748
- "missing_predictions": missing_predictions,
747
+ "unmatched_predictions": unmatched_predictions,
748
+ "unmatched_ground_truths": unmatched_ground_truths,
749
749
  },
750
750
  parameters={
751
751
  "score_threshold": score_threshold,
@@ -321,8 +321,8 @@ def _unpack_confusion_matrix_value(
321
321
  }
322
322
 
323
323
 
324
- def _unpack_hallucinations_value(
325
- hallucinations: NDArray[np.float64],
324
+ def _unpack_unmatched_predictions_value(
325
+ unmatched_predictions: NDArray[np.float64],
326
326
  number_of_labels: int,
327
327
  number_of_examples: int,
328
328
  index_to_uid: dict[int, str],
@@ -336,12 +336,12 @@ def _unpack_hallucinations_value(
336
336
  ],
337
337
  ]:
338
338
  """
339
- Unpacks a numpy array of hallucination counts and examples.
339
+ Unpacks a numpy array of unmatched_prediction counts and examples.
340
340
  """
341
341
 
342
342
  datum_idx = (
343
343
  lambda pd_label_idx, example_idx: int( # noqa: E731 - lambda fn
344
- hallucinations[
344
+ unmatched_predictions[
345
345
  pd_label_idx,
346
346
  example_idx * 3 + 1,
347
347
  ]
@@ -350,7 +350,7 @@ def _unpack_hallucinations_value(
350
350
 
351
351
  prediction_idx = (
352
352
  lambda pd_label_idx, example_idx: int( # noqa: E731 - lambda fn
353
- hallucinations[
353
+ unmatched_predictions[
354
354
  pd_label_idx,
355
355
  example_idx * 3 + 2,
356
356
  ]
@@ -359,7 +359,7 @@ def _unpack_hallucinations_value(
359
359
 
360
360
  score_idx = (
361
361
  lambda pd_label_idx, example_idx: float( # noqa: E731 - lambda fn
362
- hallucinations[
362
+ unmatched_predictions[
363
363
  pd_label_idx,
364
364
  example_idx * 3 + 3,
365
365
  ]
@@ -369,7 +369,7 @@ def _unpack_hallucinations_value(
369
369
  return {
370
370
  index_to_label[pd_label_idx]: {
371
371
  "count": max(
372
- int(hallucinations[pd_label_idx, 0]),
372
+ int(unmatched_predictions[pd_label_idx, 0]),
373
373
  0,
374
374
  ),
375
375
  "examples": [
@@ -392,8 +392,8 @@ def _unpack_hallucinations_value(
392
392
  }
393
393
 
394
394
 
395
- def _unpack_missing_predictions_value(
396
- missing_predictions: NDArray[np.int32],
395
+ def _unpack_unmatched_ground_truths_value(
396
+ unmatched_ground_truths: NDArray[np.int32],
397
397
  number_of_labels: int,
398
398
  number_of_examples: int,
399
399
  index_to_uid: dict[int, str],
@@ -401,12 +401,12 @@ def _unpack_missing_predictions_value(
401
401
  groundtruth_examples: dict[int, NDArray[np.float16]],
402
402
  ) -> dict[str, dict[str, int | list[dict[str, str | dict[str, float]]]]]:
403
403
  """
404
- Unpacks a numpy array of missing prediction counts and examples.
404
+ Unpacks a numpy array of unmatched ground truth counts and examples.
405
405
  """
406
406
 
407
407
  datum_idx = (
408
408
  lambda gt_label_idx, example_idx: int( # noqa: E731 - lambda fn
409
- missing_predictions[
409
+ unmatched_ground_truths[
410
410
  gt_label_idx,
411
411
  example_idx * 2 + 1,
412
412
  ]
@@ -415,7 +415,7 @@ def _unpack_missing_predictions_value(
415
415
 
416
416
  groundtruth_idx = (
417
417
  lambda gt_label_idx, example_idx: int( # noqa: E731 - lambda fn
418
- missing_predictions[
418
+ unmatched_ground_truths[
419
419
  gt_label_idx,
420
420
  example_idx * 2 + 2,
421
421
  ]
@@ -425,7 +425,7 @@ def _unpack_missing_predictions_value(
425
425
  return {
426
426
  index_to_label[gt_label_idx]: {
427
427
  "count": max(
428
- int(missing_predictions[gt_label_idx, 0]),
428
+ int(unmatched_ground_truths[gt_label_idx, 0]),
429
429
  0,
430
430
  ),
431
431
  "examples": [
@@ -463,8 +463,8 @@ def unpack_confusion_matrix_into_metric_list(
463
463
  ) -> list[Metric]:
464
464
  (
465
465
  confusion_matrix,
466
- hallucinations,
467
- missing_predictions,
466
+ unmatched_predictions,
467
+ unmatched_ground_truths,
468
468
  ) = results
469
469
  n_labels = len(index_to_label)
470
470
  return [
@@ -481,16 +481,18 @@ def unpack_confusion_matrix_into_metric_list(
481
481
  groundtruth_examples=groundtruth_examples,
482
482
  prediction_examples=prediction_examples,
483
483
  ),
484
- hallucinations=_unpack_hallucinations_value(
485
- hallucinations=hallucinations[iou_idx, score_idx, :, :],
484
+ unmatched_predictions=_unpack_unmatched_predictions_value(
485
+ unmatched_predictions=unmatched_predictions[
486
+ iou_idx, score_idx, :, :
487
+ ],
486
488
  number_of_labels=n_labels,
487
489
  number_of_examples=number_of_examples,
488
490
  index_to_label=index_to_label,
489
491
  index_to_uid=index_to_uid,
490
492
  prediction_examples=prediction_examples,
491
493
  ),
492
- missing_predictions=_unpack_missing_predictions_value(
493
- missing_predictions=missing_predictions[
494
+ unmatched_ground_truths=_unpack_unmatched_ground_truths_value(
495
+ unmatched_ground_truths=unmatched_ground_truths[
494
496
  iou_idx, score_idx, :, :
495
497
  ],
496
498
  number_of_labels=n_labels,
@@ -98,9 +98,9 @@ def compute_metrics(
98
98
  NDArray[np.float64]
99
99
  Confusion matrix containing IOU values.
100
100
  NDArray[np.float64]
101
- Hallucination ratios.
101
+ Unmatched prediction ratios.
102
102
  NDArray[np.float64]
103
- Missing prediction ratios.
103
+ Unmatched ground truth ratios.
104
104
  """
105
105
  n_labels = label_metadata.shape[0]
106
106
  gt_counts = label_metadata[:, 0]
@@ -108,7 +108,7 @@ def compute_metrics(
108
108
 
109
109
  counts = data.sum(axis=0)
110
110
 
111
- # compute iou, missing_predictions and hallucinations
111
+ # compute iou, unmatched_ground_truth and unmatched predictions
112
112
  intersection_ = counts[1:, 1:]
113
113
  union_ = (
114
114
  gt_counts[:, np.newaxis] + pd_counts[np.newaxis, :] - intersection_
@@ -122,20 +122,20 @@ def compute_metrics(
122
122
  out=ious,
123
123
  )
124
124
 
125
- hallucination_ratio = np.zeros((n_labels), dtype=np.float64)
125
+ unmatched_prediction_ratio = np.zeros((n_labels), dtype=np.float64)
126
126
  np.divide(
127
127
  counts[0, 1:],
128
128
  pd_counts,
129
129
  where=pd_counts > 1e-9,
130
- out=hallucination_ratio,
130
+ out=unmatched_prediction_ratio,
131
131
  )
132
132
 
133
- missing_prediction_ratio = np.zeros((n_labels), dtype=np.float64)
133
+ unmatched_ground_truth_ratio = np.zeros((n_labels), dtype=np.float64)
134
134
  np.divide(
135
135
  counts[1:, 0],
136
136
  gt_counts,
137
137
  where=gt_counts > 1e-9,
138
- out=missing_prediction_ratio,
138
+ out=unmatched_ground_truth_ratio,
139
139
  )
140
140
 
141
141
  # compute precision, recall, f1
@@ -168,6 +168,6 @@ def compute_metrics(
168
168
  f1_score,
169
169
  accuracy,
170
170
  ious,
171
- hallucination_ratio,
172
- missing_prediction_ratio,
171
+ unmatched_prediction_ratio,
172
+ unmatched_ground_truth_ratio,
173
173
  )
@@ -209,11 +209,11 @@ class Metric(BaseMetric):
209
209
  dict[str, float], # iou
210
210
  ],
211
211
  ],
212
- hallucinations: dict[
212
+ unmatched_predictions: dict[
213
213
  str, # prediction label value
214
214
  dict[str, float], # pixel ratio
215
215
  ],
216
- missing_predictions: dict[
216
+ unmatched_ground_truths: dict[
217
217
  str, # ground truth label value
218
218
  dict[str, float], # pixel ratio
219
219
  ],
@@ -222,8 +222,8 @@ class Metric(BaseMetric):
222
222
  The confusion matrix and related metrics for semantic segmentation tasks.
223
223
 
224
224
  This class encapsulates detailed information about the model's performance, including correct
225
- predictions, misclassifications, hallucinations (false positives), and missing predictions
226
- (false negatives). It provides counts for each category to facilitate in-depth analysis.
225
+ predictions, misclassifications, unmatched_predictions (subset of false positives), and unmatched ground truths
226
+ (subset of false negatives). It provides counts for each category to facilitate in-depth analysis.
227
227
 
228
228
  Confusion Matrix Format:
229
229
  {
@@ -234,14 +234,14 @@ class Metric(BaseMetric):
234
234
  },
235
235
  }
236
236
 
237
- Hallucinations Format:
237
+ Unmatched Predictions Format:
238
238
  {
239
239
  <prediction label>: {
240
240
  'iou': <float>,
241
241
  },
242
242
  }
243
243
 
244
- Missing Predictions Format:
244
+ Unmatched Ground Truths Format:
245
245
  {
246
246
  <ground truth label>: {
247
247
  'iou': <float>,
@@ -253,10 +253,10 @@ class Metric(BaseMetric):
253
253
  confusion_matrix : dict
254
254
  Nested dictionaries representing the Intersection over Union (IOU) scores for each
255
255
  ground truth label and prediction label pair.
256
- hallucinations : dict
256
+ unmatched_predictions : dict
257
257
  Dictionary representing the pixel ratios for predicted labels that do not correspond
258
258
  to any ground truth labels (false positives).
259
- missing_predictions : dict
259
+ unmatched_ground_truths : dict
260
260
  Dictionary representing the pixel ratios for ground truth labels that were not predicted
261
261
  (false negatives).
262
262
 
@@ -268,8 +268,8 @@ class Metric(BaseMetric):
268
268
  type=MetricType.ConfusionMatrix.value,
269
269
  value={
270
270
  "confusion_matrix": confusion_matrix,
271
- "hallucinations": hallucinations,
272
- "missing_predictions": missing_predictions,
271
+ "unmatched_predictions": unmatched_predictions,
272
+ "unmatched_ground_truths": unmatched_ground_truths,
273
273
  },
274
274
  parameters={},
275
275
  )
@@ -18,8 +18,8 @@ def unpack_precision_recall_iou_into_metric_lists(
18
18
  f1_score,
19
19
  accuracy,
20
20
  ious,
21
- hallucination_ratios,
22
- missing_prediction_ratios,
21
+ unmatched_prediction_ratios,
22
+ unmatched_ground_truth_ratios,
23
23
  ) = results
24
24
 
25
25
  metrics = defaultdict(list)
@@ -43,16 +43,16 @@ def unpack_precision_recall_iou_into_metric_lists(
43
43
  for gt_label_idx in range(n_labels)
44
44
  if label_metadata[gt_label_idx, 0] > 0
45
45
  },
46
- hallucinations={
46
+ unmatched_predictions={
47
47
  index_to_label[pd_label_idx]: {
48
- "ratio": float(hallucination_ratios[pd_label_idx])
48
+ "ratio": float(unmatched_prediction_ratios[pd_label_idx])
49
49
  }
50
50
  for pd_label_idx in range(n_labels)
51
51
  if label_metadata[pd_label_idx, 0] > 0
52
52
  },
53
- missing_predictions={
53
+ unmatched_ground_truths={
54
54
  index_to_label[gt_label_idx]: {
55
- "ratio": float(missing_prediction_ratios[gt_label_idx])
55
+ "ratio": float(unmatched_ground_truth_ratios[gt_label_idx])
56
56
  }
57
57
  for gt_label_idx in range(n_labels)
58
58
  if label_metadata[gt_label_idx, 0] > 0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: valor-lite
3
- Version: 0.33.18
3
+ Version: 0.33.19
4
4
  Summary: Compute valor metrics locally.
5
5
  License: MIT License
6
6
 
@@ -29,22 +29,22 @@ Requires-Python: >=3.10
29
29
  Description-Content-Type: text/markdown
30
30
  License-File: LICENSE
31
31
  Requires-Dist: evaluate
32
+ Requires-Dist: importlib_metadata; python_version < "3.8"
32
33
  Requires-Dist: nltk
33
34
  Requires-Dist: numpy
34
- Requires-Dist: Pillow >=9.1.0
35
+ Requires-Dist: Pillow>=9.1.0
35
36
  Requires-Dist: requests
36
- Requires-Dist: rouge-score
37
+ Requires-Dist: rouge_score
37
38
  Requires-Dist: shapely
38
39
  Requires-Dist: tqdm
39
- Requires-Dist: importlib-metadata ; python_version < "3.8"
40
40
  Provides-Extra: mistral
41
- Requires-Dist: mistralai >=1.0 ; extra == 'mistral'
41
+ Requires-Dist: mistralai>=1.0; extra == "mistral"
42
42
  Provides-Extra: openai
43
- Requires-Dist: openai ; extra == 'openai'
43
+ Requires-Dist: openai; extra == "openai"
44
44
  Provides-Extra: test
45
- Requires-Dist: pytest ; extra == 'test'
46
- Requires-Dist: coverage ; extra == 'test'
47
- Requires-Dist: pre-commit ; extra == 'test'
45
+ Requires-Dist: pytest; extra == "test"
46
+ Requires-Dist: coverage; extra == "test"
47
+ Requires-Dist: pre-commit; extra == "test"
48
48
 
49
49
  # valor-lite: Fast, local machine learning evaluation.
50
50
 
@@ -4,23 +4,23 @@ valor_lite/profiling.py,sha256=TLIROA1qccFw9NoEkMeQcrvvGGO75c4K5yTIWoCUix8,11746
4
4
  valor_lite/schemas.py,sha256=pB0MrPx5qFLbwBWDiOUUm-vmXdWvbJLFCBmKgbcbI5g,198
5
5
  valor_lite/classification/__init__.py,sha256=8MI8bGwCxYGqRP7KxG7ezhYv4qQ5947XGvvlF8WPM5g,392
6
6
  valor_lite/classification/annotation.py,sha256=0aUOvcwBAZgiNOJuyh-pXyNTG7vP7r8CUfnU3OmpUwQ,1113
7
- valor_lite/classification/computation.py,sha256=qfBhhuDYCiY8h2RdBG3shzgJbHLXDVNujkYFg9xZa6U,12116
7
+ valor_lite/classification/computation.py,sha256=XU-55bzR2JPKDRr8CvCbHHavM5vjBw1TrUX1mOIm_TY,12121
8
8
  valor_lite/classification/manager.py,sha256=8GXZECSx4CBbG5NfPrA19BPENqmrjo-wZBmaulWHY20,16676
9
- valor_lite/classification/metric.py,sha256=fkAo-_3s4EIRSkyn3owBSf4_Gp6lBK9xdToDYMWmT8A,12236
10
- valor_lite/classification/utilities.py,sha256=PmQar06Vt-ew4Jvnn0IM63mq730QVTsdRtFdVu1HMFU,6885
9
+ valor_lite/classification/metric.py,sha256=_mW3zynmpW8jUIhK2OeX4usdftHgHM9_l7EAbEe2N3w,12288
10
+ valor_lite/classification/utilities.py,sha256=7P3H2wsUR_qmK4WJBFPkViOvhhKVMkCHvDKJbeentMM,6963
11
11
  valor_lite/object_detection/__init__.py,sha256=Ql8rju2q7y0Zd9zFvtBJDRhgQFDm1RSYkTsyH3ZE6pA,648
12
12
  valor_lite/object_detection/annotation.py,sha256=x9bsl8b75yvkMByXXiIYI9d9T03olDqtykSvKJc3aFw,7729
13
- valor_lite/object_detection/computation.py,sha256=P5ijxEBuZ3mxYjBQy24TiQpGxRmPuS40Gwn44uv0J7M,28064
13
+ valor_lite/object_detection/computation.py,sha256=70XkgyxAbvZC8FcTKqdAATD5Muebw_jGjoTKPaecwjE,28141
14
14
  valor_lite/object_detection/manager.py,sha256=utdILUUCx04EWC0_bHGpEPaxcCOhmsOx5lxT9qU1a9s,23033
15
- valor_lite/object_detection/metric.py,sha256=8QhdauuaRrzE39idetkFYTPxA12wrBalQDIR4IUzEbg,24794
16
- valor_lite/object_detection/utilities.py,sha256=98VSW-g8EYI8Cdd9KHLHdm6F4fI89jaX5I4z99zny4s,16271
15
+ valor_lite/object_detection/metric.py,sha256=npK2sxiwCUTKlRlFym1AlZTvP9herf9lakbsBDwljGM,24901
16
+ valor_lite/object_detection/utilities.py,sha256=F0sfhkledX3h3jXLDpLro7zHbUF9ig7r35J7QbogQVY,16437
17
17
  valor_lite/semantic_segmentation/__init__.py,sha256=BhTUbwbdJa1FdS4ZA3QSIZ8TuJmdGGLGCd5hX6SzKa4,297
18
18
  valor_lite/semantic_segmentation/annotation.py,sha256=xd2qJyIeTW8CT_Goyu3Kvl_51b9b6D3WvUfqwShR0Sk,4990
19
19
  valor_lite/semantic_segmentation/benchmark.py,sha256=iVdxUo9LgDbbXUa6eRhZ49LOYw-yyr2W4p9FP3KHg0k,3848
20
- valor_lite/semantic_segmentation/computation.py,sha256=myHjJZ70f2Xc-PGHx3DcLWvXXRu_H8w9z20n7qV-Abo,4687
20
+ valor_lite/semantic_segmentation/computation.py,sha256=l98h8s9RTWQOB_eg2rconqGL1ZbTS4GMtz69vbyEdQ0,4741
21
21
  valor_lite/semantic_segmentation/manager.py,sha256=TtwJI7Bsn3zHL2ECOqCmymG-JqREo7I6qxYtycbz54Y,14322
22
- valor_lite/semantic_segmentation/metric.py,sha256=aJv3wPEl6USLhZ3c4yz6prnBU-EaG4Kz16f0BXcodd4,7046
23
- valor_lite/semantic_segmentation/utilities.py,sha256=vZM66YNMz9VJclhuKvcWp74nF65s6bscnnD5U9iDW7Q,2925
22
+ valor_lite/semantic_segmentation/metric.py,sha256=T9RfPJf4WgqGQTXYvSy08vJG5bjXXJnyYZeW0mlxMa8,7132
23
+ valor_lite/semantic_segmentation/utilities.py,sha256=y9SgArhrk-u_r54SgQxGEKrDjow5KvPG8Zj1GeZefqY,2958
24
24
  valor_lite/text_generation/__init__.py,sha256=pGhpWCSZjLM0pPHCtPykAfos55B8ie3mi9EzbNxfj-U,356
25
25
  valor_lite/text_generation/annotation.py,sha256=O5aXiwCS4WjA-fqn4ly-O0MsTHoIOmqxqCaAp9IeI3M,1270
26
26
  valor_lite/text_generation/computation.py,sha256=cG35qMpxNPEYHXN2fz8wcanESriSHoWMl1idpm9-ous,18638
@@ -33,8 +33,8 @@ valor_lite/text_generation/llm/instructions.py,sha256=fz2onBZZWcl5W8iy7zEWkPGU9N
33
33
  valor_lite/text_generation/llm/integrations.py,sha256=-rTfdAjq1zH-4ixwYuMQEOQ80pIFzMTe0BYfroVx3Pg,6974
34
34
  valor_lite/text_generation/llm/utilities.py,sha256=bjqatGgtVTcl1PrMwiDKTYPGJXKrBrx7PDtzIblGSys,1178
35
35
  valor_lite/text_generation/llm/validators.py,sha256=Wzr5RlfF58_2wOU-uTw7C8skan_fYdhy4Gfn0jSJ8HM,2700
36
- valor_lite-0.33.18.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
37
- valor_lite-0.33.18.dist-info/METADATA,sha256=oo3sEQQvJJvAIelgFRB1Me2Jmkk-nb_dkphL2k4wo7Y,5888
38
- valor_lite-0.33.18.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
39
- valor_lite-0.33.18.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
40
- valor_lite-0.33.18.dist-info/RECORD,,
36
+ valor_lite-0.33.19.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
37
+ valor_lite-0.33.19.dist-info/METADATA,sha256=lwmxYzLPlKkCZM0mVkepQYYy4mjbg34UEOeeqM7ZE2k,5880
38
+ valor_lite-0.33.19.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
39
+ valor_lite-0.33.19.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
40
+ valor_lite-0.33.19.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.5.0)
2
+ Generator: setuptools (75.6.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5