valor-lite 0.34.3__py3-none-any.whl → 0.36.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of valor-lite might be problematic. Click here for more details.
- valor_lite/classification/computation.py +147 -38
- valor_lite/classification/manager.py +229 -236
- valor_lite/classification/metric.py +5 -8
- valor_lite/classification/utilities.py +18 -14
- valor_lite/object_detection/__init__.py +2 -15
- valor_lite/object_detection/annotation.py +24 -48
- valor_lite/object_detection/computation.py +324 -384
- valor_lite/object_detection/manager.py +549 -456
- valor_lite/object_detection/metric.py +16 -34
- valor_lite/object_detection/utilities.py +134 -305
- valor_lite/semantic_segmentation/__init__.py +3 -3
- valor_lite/semantic_segmentation/annotation.py +32 -103
- valor_lite/semantic_segmentation/benchmark.py +87 -1
- valor_lite/semantic_segmentation/computation.py +96 -14
- valor_lite/semantic_segmentation/manager.py +199 -222
- valor_lite/semantic_segmentation/utilities.py +3 -3
- {valor_lite-0.34.3.dist-info → valor_lite-0.36.0.dist-info}/METADATA +2 -2
- {valor_lite-0.34.3.dist-info → valor_lite-0.36.0.dist-info}/RECORD +20 -20
- {valor_lite-0.34.3.dist-info → valor_lite-0.36.0.dist-info}/WHEEL +1 -1
- {valor_lite-0.34.3.dist-info → valor_lite-0.36.0.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
from dataclasses import dataclass
|
|
1
|
+
import warnings
|
|
2
|
+
from dataclasses import asdict, dataclass
|
|
3
3
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
from numpy.typing import NDArray
|
|
@@ -8,7 +8,9 @@ from tqdm import tqdm
|
|
|
8
8
|
from valor_lite.semantic_segmentation.annotation import Segmentation
|
|
9
9
|
from valor_lite.semantic_segmentation.computation import (
|
|
10
10
|
compute_intermediate_confusion_matrices,
|
|
11
|
+
compute_label_metadata,
|
|
11
12
|
compute_metrics,
|
|
13
|
+
filter_cache,
|
|
12
14
|
)
|
|
13
15
|
from valor_lite.semantic_segmentation.metric import Metric, MetricType
|
|
14
16
|
from valor_lite.semantic_segmentation.utilities import (
|
|
@@ -31,16 +33,43 @@ metrics = evaluator.evaluate()
|
|
|
31
33
|
f1_metrics = metrics[MetricType.F1]
|
|
32
34
|
accuracy_metrics = metrics[MetricType.Accuracy]
|
|
33
35
|
|
|
34
|
-
filter_mask = evaluator.create_filter(
|
|
36
|
+
filter_mask = evaluator.create_filter(datum_ids=["uid1", "uid2"])
|
|
35
37
|
filtered_metrics = evaluator.evaluate(filter_mask=filter_mask)
|
|
36
38
|
"""
|
|
37
39
|
|
|
38
40
|
|
|
41
|
+
@dataclass
|
|
42
|
+
class Metadata:
|
|
43
|
+
number_of_labels: int = 0
|
|
44
|
+
number_of_pixels: int = 0
|
|
45
|
+
number_of_datums: int = 0
|
|
46
|
+
number_of_ground_truths: int = 0
|
|
47
|
+
number_of_predictions: int = 0
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def create(
|
|
51
|
+
cls,
|
|
52
|
+
confusion_matrices: NDArray[np.int64],
|
|
53
|
+
):
|
|
54
|
+
if confusion_matrices.size == 0:
|
|
55
|
+
return cls()
|
|
56
|
+
return cls(
|
|
57
|
+
number_of_labels=confusion_matrices.shape[1] - 1,
|
|
58
|
+
number_of_pixels=confusion_matrices.sum(),
|
|
59
|
+
number_of_datums=confusion_matrices.shape[0],
|
|
60
|
+
number_of_ground_truths=confusion_matrices[:, 1:, :].sum(),
|
|
61
|
+
number_of_predictions=confusion_matrices[:, :, 1:].sum(),
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def to_dict(self) -> dict[str, int | bool]:
|
|
65
|
+
return asdict(self)
|
|
66
|
+
|
|
67
|
+
|
|
39
68
|
@dataclass
|
|
40
69
|
class Filter:
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
70
|
+
datum_mask: NDArray[np.bool_]
|
|
71
|
+
label_mask: NDArray[np.bool_]
|
|
72
|
+
metadata: Metadata
|
|
44
73
|
|
|
45
74
|
|
|
46
75
|
class Evaluator:
|
|
@@ -49,29 +78,21 @@ class Evaluator:
|
|
|
49
78
|
"""
|
|
50
79
|
|
|
51
80
|
def __init__(self):
|
|
81
|
+
"""Initializes evaluator caches."""
|
|
82
|
+
# external references
|
|
83
|
+
self.datum_id_to_index: dict[str, int] = {}
|
|
84
|
+
self.index_to_datum_id: list[str] = []
|
|
85
|
+
self.label_to_index: dict[str, int] = {}
|
|
86
|
+
self.index_to_label: list[str] = []
|
|
87
|
+
|
|
88
|
+
# internal caches
|
|
89
|
+
self._confusion_matrices = np.array([], dtype=np.int64)
|
|
90
|
+
self._label_metadata = np.array([], dtype=np.int64)
|
|
91
|
+
self._metadata = Metadata()
|
|
52
92
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
self.
|
|
56
|
-
self.n_predictions = 0
|
|
57
|
-
self.n_pixels = 0
|
|
58
|
-
self.n_groundtruth_pixels = 0
|
|
59
|
-
self.n_prediction_pixels = 0
|
|
60
|
-
self.n_labels = 0
|
|
61
|
-
|
|
62
|
-
# datum reference
|
|
63
|
-
self.uid_to_index: dict[str, int] = dict()
|
|
64
|
-
self.index_to_uid: dict[int, str] = dict()
|
|
65
|
-
|
|
66
|
-
# label reference
|
|
67
|
-
self.label_to_index: dict[str, int] = dict()
|
|
68
|
-
self.index_to_label: dict[int, str] = dict()
|
|
69
|
-
|
|
70
|
-
# computation caches
|
|
71
|
-
self._confusion_matrices = np.array([])
|
|
72
|
-
self._label_metadata = np.array([], dtype=np.int32)
|
|
73
|
-
self._label_metadata_per_datum = np.array([], dtype=np.int32)
|
|
74
|
-
self._n_pixels_per_datum = np.array([], dtype=np.int32)
|
|
93
|
+
@property
|
|
94
|
+
def metadata(self) -> Metadata:
|
|
95
|
+
return self._metadata
|
|
75
96
|
|
|
76
97
|
@property
|
|
77
98
|
def ignored_prediction_labels(self) -> list[str]:
|
|
@@ -95,129 +116,144 @@ class Evaluator:
|
|
|
95
116
|
self.index_to_label[label_id] for label_id in (glabels - plabels)
|
|
96
117
|
]
|
|
97
118
|
|
|
98
|
-
@property
|
|
99
|
-
def metadata(self) -> dict:
|
|
100
|
-
"""
|
|
101
|
-
Evaluation metadata.
|
|
102
|
-
"""
|
|
103
|
-
return {
|
|
104
|
-
"number_of_datums": self.n_datums,
|
|
105
|
-
"number_of_groundtruths": self.n_groundtruths,
|
|
106
|
-
"number_of_predictions": self.n_predictions,
|
|
107
|
-
"number_of_groundtruth_pixels": self.n_groundtruth_pixels,
|
|
108
|
-
"number_of_prediction_pixels": self.n_prediction_pixels,
|
|
109
|
-
"number_of_labels": self.n_labels,
|
|
110
|
-
"ignored_prediction_labels": self.ignored_prediction_labels,
|
|
111
|
-
"missing_prediction_labels": self.missing_prediction_labels,
|
|
112
|
-
}
|
|
113
|
-
|
|
114
119
|
def create_filter(
|
|
115
120
|
self,
|
|
116
|
-
|
|
117
|
-
labels: list[str] |
|
|
121
|
+
datum_ids: list[str] | None = None,
|
|
122
|
+
labels: list[str] | None = None,
|
|
118
123
|
) -> Filter:
|
|
119
124
|
"""
|
|
120
|
-
Creates a
|
|
125
|
+
Creates a filter for use with the evaluator.
|
|
121
126
|
|
|
122
127
|
Parameters
|
|
123
128
|
----------
|
|
124
|
-
|
|
125
|
-
An optional list of string uids
|
|
126
|
-
labels : list[
|
|
127
|
-
An optional list of labels
|
|
129
|
+
datum_ids : list[str], optional
|
|
130
|
+
An optional list of string uids representing datums.
|
|
131
|
+
labels : list[str], optional
|
|
132
|
+
An optional list of labels.
|
|
128
133
|
|
|
129
134
|
Returns
|
|
130
135
|
-------
|
|
131
136
|
Filter
|
|
132
|
-
|
|
137
|
+
The filter object containing a mask and metadata.
|
|
133
138
|
"""
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
139
|
+
datum_mask = np.ones(self._confusion_matrices.shape[0], dtype=np.bool_)
|
|
140
|
+
label_mask = np.zeros(
|
|
141
|
+
self.metadata.number_of_labels + 1, dtype=np.bool_
|
|
142
|
+
)
|
|
143
|
+
if datum_ids is not None:
|
|
144
|
+
if not datum_ids:
|
|
145
|
+
filtered_confusion_matrices = np.array([], dtype=np.int64)
|
|
146
|
+
warnings.warn("datum filter results in empty data array")
|
|
147
|
+
return Filter(
|
|
148
|
+
datum_mask=np.zeros_like(datum_mask),
|
|
149
|
+
label_mask=label_mask,
|
|
150
|
+
metadata=Metadata(),
|
|
145
151
|
)
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
).
|
|
153
|
-
|
|
154
|
-
|
|
152
|
+
datum_id_array = np.array(
|
|
153
|
+
[self.datum_id_to_index[uid] for uid in datum_ids],
|
|
154
|
+
dtype=np.int64,
|
|
155
|
+
)
|
|
156
|
+
datum_id_array.sort()
|
|
157
|
+
mask_valid_datums = (
|
|
158
|
+
np.arange(self._confusion_matrices.shape[0]).reshape(-1, 1)
|
|
159
|
+
== datum_id_array.reshape(1, -1)
|
|
160
|
+
).any(axis=1)
|
|
161
|
+
datum_mask[~mask_valid_datums] = False
|
|
155
162
|
if labels is not None:
|
|
156
|
-
if
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
163
|
+
if not labels:
|
|
164
|
+
filtered_confusion_matrices = np.array([], dtype=np.int64)
|
|
165
|
+
warnings.warn("label filter results in empty data array")
|
|
166
|
+
return Filter(
|
|
167
|
+
datum_mask=datum_mask,
|
|
168
|
+
label_mask=np.ones_like(label_mask),
|
|
169
|
+
metadata=Metadata(),
|
|
160
170
|
)
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
mask = (
|
|
165
|
-
np.arange(n_labels).reshape(-1, 1) == labels.reshape(1, -1)
|
|
166
|
-
).any(axis=1)
|
|
167
|
-
mask_labels[~mask] = False
|
|
168
|
-
|
|
169
|
-
mask = mask_datums[:, np.newaxis] & mask_labels[np.newaxis, :]
|
|
170
|
-
label_metadata_per_datum = self._label_metadata_per_datum.copy()
|
|
171
|
-
label_metadata_per_datum[:, ~mask] = 0
|
|
172
|
-
|
|
173
|
-
label_metadata = np.zeros_like(self._label_metadata, dtype=np.int32)
|
|
174
|
-
label_metadata = np.transpose(
|
|
175
|
-
np.sum(
|
|
176
|
-
label_metadata_per_datum,
|
|
177
|
-
axis=1,
|
|
171
|
+
labels_id_array = np.array(
|
|
172
|
+
[self.label_to_index[label] for label in labels] + [-1],
|
|
173
|
+
dtype=np.int64,
|
|
178
174
|
)
|
|
175
|
+
label_range = np.arange(self.metadata.number_of_labels + 1) - 1
|
|
176
|
+
mask_valid_labels = (
|
|
177
|
+
label_range.reshape(-1, 1) == labels_id_array.reshape(1, -1)
|
|
178
|
+
).any(axis=1)
|
|
179
|
+
label_mask[~mask_valid_labels] = True
|
|
180
|
+
|
|
181
|
+
filtered_confusion_matrices, _ = filter_cache(
|
|
182
|
+
confusion_matrices=self._confusion_matrices.copy(),
|
|
183
|
+
datum_mask=datum_mask,
|
|
184
|
+
label_mask=label_mask,
|
|
185
|
+
number_of_labels=self.metadata.number_of_labels,
|
|
179
186
|
)
|
|
180
|
-
n_datums = int(np.sum(label_metadata[:, 0]))
|
|
181
187
|
|
|
182
188
|
return Filter(
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
189
|
+
datum_mask=datum_mask,
|
|
190
|
+
label_mask=label_mask,
|
|
191
|
+
metadata=Metadata.create(
|
|
192
|
+
confusion_matrices=filtered_confusion_matrices,
|
|
193
|
+
),
|
|
186
194
|
)
|
|
187
195
|
|
|
188
|
-
def
|
|
189
|
-
self,
|
|
190
|
-
|
|
191
|
-
) -> dict[MetricType, list]:
|
|
196
|
+
def filter(
|
|
197
|
+
self, filter_: Filter
|
|
198
|
+
) -> tuple[NDArray[np.int64], NDArray[np.int64]]:
|
|
192
199
|
"""
|
|
193
|
-
Performs
|
|
200
|
+
Performs the filter operation over the internal cache.
|
|
194
201
|
|
|
195
202
|
Parameters
|
|
196
203
|
----------
|
|
197
|
-
filter_ : Filter
|
|
198
|
-
An
|
|
204
|
+
filter_ : Filter
|
|
205
|
+
An object describing the filter operation.
|
|
206
|
+
|
|
207
|
+
Returns
|
|
208
|
+
-------
|
|
209
|
+
NDArray[int64]
|
|
210
|
+
Filtered confusion matrices.
|
|
211
|
+
NDArray[int64]
|
|
212
|
+
Filtered label metadata
|
|
213
|
+
"""
|
|
214
|
+
empty_datum_mask = not filter_.datum_mask.any()
|
|
215
|
+
empty_label_mask = filter_.label_mask.all()
|
|
216
|
+
if empty_datum_mask or empty_label_mask:
|
|
217
|
+
if empty_datum_mask:
|
|
218
|
+
warnings.warn("filter does not allow any datum")
|
|
219
|
+
if empty_label_mask:
|
|
220
|
+
warnings.warn("filter removes all labels")
|
|
221
|
+
return (
|
|
222
|
+
np.array([], dtype=np.int64),
|
|
223
|
+
np.zeros((self.metadata.number_of_labels, 2), dtype=np.int64),
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
return filter_cache(
|
|
227
|
+
confusion_matrices=self._confusion_matrices.copy(),
|
|
228
|
+
datum_mask=filter_.datum_mask,
|
|
229
|
+
label_mask=filter_.label_mask,
|
|
230
|
+
number_of_labels=self.metadata.number_of_labels,
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def compute_precision_recall_iou(
|
|
234
|
+
self, filter_: Filter | None = None
|
|
235
|
+
) -> dict[MetricType, list]:
|
|
236
|
+
"""
|
|
237
|
+
Performs an evaluation and returns metrics.
|
|
199
238
|
|
|
200
239
|
Returns
|
|
201
240
|
-------
|
|
202
241
|
dict[MetricType, list]
|
|
203
242
|
A dictionary mapping MetricType enumerations to lists of computed metrics.
|
|
204
243
|
"""
|
|
205
|
-
|
|
206
|
-
# apply filters
|
|
207
|
-
data = self._confusion_matrices
|
|
208
|
-
label_metadata = self._label_metadata
|
|
209
|
-
n_pixels = self.n_pixels
|
|
210
244
|
if filter_ is not None:
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
245
|
+
confusion_matrices, label_metadata = self.filter(filter_)
|
|
246
|
+
n_pixels = filter_.metadata.number_of_pixels
|
|
247
|
+
else:
|
|
248
|
+
confusion_matrices = self._confusion_matrices
|
|
249
|
+
label_metadata = self._label_metadata
|
|
250
|
+
n_pixels = self.metadata.number_of_pixels
|
|
214
251
|
|
|
215
252
|
results = compute_metrics(
|
|
216
|
-
|
|
253
|
+
confusion_matrices=confusion_matrices,
|
|
217
254
|
label_metadata=label_metadata,
|
|
218
255
|
n_pixels=n_pixels,
|
|
219
256
|
)
|
|
220
|
-
|
|
221
257
|
return unpack_precision_recall_iou_into_metric_lists(
|
|
222
258
|
results=results,
|
|
223
259
|
label_metadata=label_metadata,
|
|
@@ -225,17 +261,11 @@ class Evaluator:
|
|
|
225
261
|
)
|
|
226
262
|
|
|
227
263
|
def evaluate(
|
|
228
|
-
self,
|
|
229
|
-
filter_: Filter | None = None,
|
|
264
|
+
self, filter_: Filter | None = None
|
|
230
265
|
) -> dict[MetricType, list[Metric]]:
|
|
231
266
|
"""
|
|
232
267
|
Computes all available metrics.
|
|
233
268
|
|
|
234
|
-
Parameters
|
|
235
|
-
----------
|
|
236
|
-
filter_ : Filter, optional
|
|
237
|
-
An optional filter object.
|
|
238
|
-
|
|
239
269
|
Returns
|
|
240
270
|
-------
|
|
241
271
|
dict[MetricType, list[Metric]]
|
|
@@ -244,10 +274,6 @@ class Evaluator:
|
|
|
244
274
|
return self.compute_precision_recall_iou(filter_=filter_)
|
|
245
275
|
|
|
246
276
|
|
|
247
|
-
def defaultdict_int():
|
|
248
|
-
return defaultdict(int)
|
|
249
|
-
|
|
250
|
-
|
|
251
277
|
class DataLoader:
|
|
252
278
|
"""
|
|
253
279
|
Segmentation DataLoader.
|
|
@@ -255,10 +281,7 @@ class DataLoader:
|
|
|
255
281
|
|
|
256
282
|
def __init__(self):
|
|
257
283
|
self._evaluator = Evaluator()
|
|
258
|
-
self.groundtruth_count = defaultdict(defaultdict_int)
|
|
259
|
-
self.prediction_count = defaultdict(defaultdict_int)
|
|
260
284
|
self.matrices = list()
|
|
261
|
-
self.pixel_count = list()
|
|
262
285
|
|
|
263
286
|
def _add_datum(self, uid: str) -> int:
|
|
264
287
|
"""
|
|
@@ -274,11 +297,11 @@ class DataLoader:
|
|
|
274
297
|
int
|
|
275
298
|
The datum index.
|
|
276
299
|
"""
|
|
277
|
-
if uid in self._evaluator.
|
|
278
|
-
raise ValueError(f"Datum with uid `{uid}`
|
|
279
|
-
index = len(self._evaluator.
|
|
280
|
-
self._evaluator.
|
|
281
|
-
self._evaluator.
|
|
300
|
+
if uid in self._evaluator.datum_id_to_index:
|
|
301
|
+
raise ValueError(f"Datum with uid `{uid}` already exists.")
|
|
302
|
+
index = len(self._evaluator.datum_id_to_index)
|
|
303
|
+
self._evaluator.datum_id_to_index[uid] = index
|
|
304
|
+
self._evaluator.index_to_datum_id.append(uid)
|
|
282
305
|
return index
|
|
283
306
|
|
|
284
307
|
def _add_label(self, label: str) -> int:
|
|
@@ -298,7 +321,7 @@ class DataLoader:
|
|
|
298
321
|
if label not in self._evaluator.label_to_index:
|
|
299
322
|
label_id = len(self._evaluator.index_to_label)
|
|
300
323
|
self._evaluator.label_to_index[label] = label_id
|
|
301
|
-
self._evaluator.index_to_label
|
|
324
|
+
self._evaluator.index_to_label.append(label)
|
|
302
325
|
return self._evaluator.label_to_index[label]
|
|
303
326
|
|
|
304
327
|
def add_data(
|
|
@@ -319,56 +342,50 @@ class DataLoader:
|
|
|
319
342
|
|
|
320
343
|
disable_tqdm = not show_progress
|
|
321
344
|
for segmentation in tqdm(segmentations, disable=disable_tqdm):
|
|
322
|
-
|
|
323
|
-
# update metadata
|
|
324
|
-
self._evaluator.n_datums += 1
|
|
325
|
-
self._evaluator.n_groundtruths += len(segmentation.groundtruths)
|
|
326
|
-
self._evaluator.n_predictions += len(segmentation.predictions)
|
|
327
|
-
self._evaluator.n_pixels += segmentation.size
|
|
328
|
-
self._evaluator.n_groundtruth_pixels += segmentation.size * len(
|
|
329
|
-
segmentation.groundtruths
|
|
330
|
-
)
|
|
331
|
-
self._evaluator.n_prediction_pixels += segmentation.size * len(
|
|
332
|
-
segmentation.predictions
|
|
333
|
-
)
|
|
334
|
-
|
|
335
345
|
# update datum cache
|
|
336
|
-
|
|
346
|
+
self._add_datum(segmentation.uid)
|
|
337
347
|
|
|
338
|
-
groundtruth_labels = np.
|
|
339
|
-
len(segmentation.groundtruths),
|
|
348
|
+
groundtruth_labels = -1 * np.ones(
|
|
349
|
+
len(segmentation.groundtruths), dtype=np.int64
|
|
340
350
|
)
|
|
341
351
|
for idx, groundtruth in enumerate(segmentation.groundtruths):
|
|
342
352
|
label_idx = self._add_label(groundtruth.label)
|
|
343
353
|
groundtruth_labels[idx] = label_idx
|
|
344
|
-
self.groundtruth_count[label_idx][
|
|
345
|
-
uid_index
|
|
346
|
-
] += groundtruth.mask.sum()
|
|
347
354
|
|
|
348
|
-
prediction_labels = np.
|
|
349
|
-
len(segmentation.predictions),
|
|
355
|
+
prediction_labels = -1 * np.ones(
|
|
356
|
+
len(segmentation.predictions), dtype=np.int64
|
|
350
357
|
)
|
|
351
358
|
for idx, prediction in enumerate(segmentation.predictions):
|
|
352
359
|
label_idx = self._add_label(prediction.label)
|
|
353
360
|
prediction_labels[idx] = label_idx
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
361
|
+
|
|
362
|
+
if segmentation.groundtruths:
|
|
363
|
+
combined_groundtruths = np.stack(
|
|
364
|
+
[
|
|
365
|
+
groundtruth.mask.flatten()
|
|
366
|
+
for groundtruth in segmentation.groundtruths
|
|
367
|
+
],
|
|
368
|
+
axis=0,
|
|
369
|
+
)
|
|
370
|
+
else:
|
|
371
|
+
combined_groundtruths = np.zeros(
|
|
372
|
+
(1, segmentation.shape[0] * segmentation.shape[1]),
|
|
373
|
+
dtype=np.bool_,
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
if segmentation.predictions:
|
|
377
|
+
combined_predictions = np.stack(
|
|
378
|
+
[
|
|
379
|
+
prediction.mask.flatten()
|
|
380
|
+
for prediction in segmentation.predictions
|
|
381
|
+
],
|
|
382
|
+
axis=0,
|
|
383
|
+
)
|
|
384
|
+
else:
|
|
385
|
+
combined_predictions = np.zeros(
|
|
386
|
+
(1, segmentation.shape[0] * segmentation.shape[1]),
|
|
387
|
+
dtype=np.bool_,
|
|
388
|
+
)
|
|
372
389
|
|
|
373
390
|
self.matrices.append(
|
|
374
391
|
compute_intermediate_confusion_matrices(
|
|
@@ -379,7 +396,6 @@ class DataLoader:
|
|
|
379
396
|
n_labels=len(self._evaluator.index_to_label),
|
|
380
397
|
)
|
|
381
398
|
)
|
|
382
|
-
self.pixel_count.append(segmentation.size)
|
|
383
399
|
|
|
384
400
|
def finalize(self) -> Evaluator:
|
|
385
401
|
"""
|
|
@@ -394,58 +410,19 @@ class DataLoader:
|
|
|
394
410
|
if len(self.matrices) == 0:
|
|
395
411
|
raise ValueError("No data available to create evaluator.")
|
|
396
412
|
|
|
397
|
-
n_datums = self._evaluator.n_datums
|
|
398
413
|
n_labels = len(self._evaluator.index_to_label)
|
|
399
|
-
|
|
400
|
-
self._evaluator.n_labels = n_labels
|
|
401
|
-
|
|
402
|
-
self._evaluator._label_metadata_per_datum = np.zeros(
|
|
403
|
-
(2, n_datums, n_labels), dtype=np.int32
|
|
404
|
-
)
|
|
405
|
-
for datum_idx in range(n_datums):
|
|
406
|
-
for label_idx in range(n_labels):
|
|
407
|
-
gt_count = (
|
|
408
|
-
self.groundtruth_count[label_idx].get(datum_idx, 0)
|
|
409
|
-
if label_idx in self.groundtruth_count
|
|
410
|
-
else 0
|
|
411
|
-
)
|
|
412
|
-
pd_count = (
|
|
413
|
-
self.prediction_count[label_idx].get(datum_idx, 0)
|
|
414
|
-
if label_idx in self.prediction_count
|
|
415
|
-
else 0
|
|
416
|
-
)
|
|
417
|
-
self._evaluator._label_metadata_per_datum[
|
|
418
|
-
:, datum_idx, label_idx
|
|
419
|
-
] = np.array([gt_count, pd_count])
|
|
420
|
-
|
|
421
|
-
self._evaluator._label_metadata = np.array(
|
|
422
|
-
[
|
|
423
|
-
[
|
|
424
|
-
np.sum(
|
|
425
|
-
self._evaluator._label_metadata_per_datum[
|
|
426
|
-
0, :, label_idx
|
|
427
|
-
]
|
|
428
|
-
),
|
|
429
|
-
np.sum(
|
|
430
|
-
self._evaluator._label_metadata_per_datum[
|
|
431
|
-
1, :, label_idx
|
|
432
|
-
]
|
|
433
|
-
),
|
|
434
|
-
]
|
|
435
|
-
for label_idx in range(n_labels)
|
|
436
|
-
],
|
|
437
|
-
dtype=np.int32,
|
|
438
|
-
)
|
|
439
|
-
|
|
440
|
-
self._evaluator._n_pixels_per_datum = np.array(
|
|
441
|
-
self.pixel_count, dtype=np.int32
|
|
442
|
-
)
|
|
443
|
-
|
|
414
|
+
n_datums = len(self._evaluator.index_to_datum_id)
|
|
444
415
|
self._evaluator._confusion_matrices = np.zeros(
|
|
445
|
-
(n_datums, n_labels + 1, n_labels + 1), dtype=np.
|
|
416
|
+
(n_datums, n_labels + 1, n_labels + 1), dtype=np.int64
|
|
446
417
|
)
|
|
447
418
|
for idx, matrix in enumerate(self.matrices):
|
|
448
419
|
h, w = matrix.shape
|
|
449
420
|
self._evaluator._confusion_matrices[idx, :h, :w] = matrix
|
|
450
|
-
|
|
421
|
+
self._evaluator._label_metadata = compute_label_metadata(
|
|
422
|
+
confusion_matrices=self._evaluator._confusion_matrices,
|
|
423
|
+
n_labels=n_labels,
|
|
424
|
+
)
|
|
425
|
+
self._evaluator._metadata = Metadata.create(
|
|
426
|
+
confusion_matrices=self._evaluator._confusion_matrices,
|
|
427
|
+
)
|
|
451
428
|
return self._evaluator
|
|
@@ -8,8 +8,8 @@ from valor_lite.semantic_segmentation.metric import Metric, MetricType
|
|
|
8
8
|
|
|
9
9
|
def unpack_precision_recall_iou_into_metric_lists(
|
|
10
10
|
results: tuple,
|
|
11
|
-
label_metadata: NDArray[np.
|
|
12
|
-
index_to_label:
|
|
11
|
+
label_metadata: NDArray[np.int64],
|
|
12
|
+
index_to_label: list[str],
|
|
13
13
|
) -> dict[MetricType, list[Metric]]:
|
|
14
14
|
|
|
15
15
|
n_labels = len(index_to_label)
|
|
@@ -67,7 +67,7 @@ def unpack_precision_recall_iou_into_metric_lists(
|
|
|
67
67
|
)
|
|
68
68
|
]
|
|
69
69
|
|
|
70
|
-
for label_idx, label in index_to_label
|
|
70
|
+
for label_idx, label in enumerate(index_to_label):
|
|
71
71
|
|
|
72
72
|
kwargs = {
|
|
73
73
|
"label": label,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: valor-lite
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.36.0
|
|
4
4
|
Summary: Evaluate machine learning models.
|
|
5
5
|
Project-URL: homepage, https://www.striveworks.com
|
|
6
6
|
Requires-Python: >=3.10
|
|
@@ -13,7 +13,7 @@ Requires-Dist: evaluate; extra == "nlp"
|
|
|
13
13
|
Requires-Dist: nltk; extra == "nlp"
|
|
14
14
|
Requires-Dist: rouge_score; extra == "nlp"
|
|
15
15
|
Provides-Extra: mistral
|
|
16
|
-
Requires-Dist: mistralai
|
|
16
|
+
Requires-Dist: mistralai<1.8.0,>=1.0.0; extra == "mistral"
|
|
17
17
|
Provides-Extra: openai
|
|
18
18
|
Requires-Dist: openai; extra == "openai"
|
|
19
19
|
Provides-Extra: docs
|
|
@@ -4,24 +4,24 @@ valor_lite/profiling.py,sha256=TLIROA1qccFw9NoEkMeQcrvvGGO75c4K5yTIWoCUix8,11746
|
|
|
4
4
|
valor_lite/schemas.py,sha256=pB0MrPx5qFLbwBWDiOUUm-vmXdWvbJLFCBmKgbcbI5g,198
|
|
5
5
|
valor_lite/classification/__init__.py,sha256=8MI8bGwCxYGqRP7KxG7ezhYv4qQ5947XGvvlF8WPM5g,392
|
|
6
6
|
valor_lite/classification/annotation.py,sha256=0aUOvcwBAZgiNOJuyh-pXyNTG7vP7r8CUfnU3OmpUwQ,1113
|
|
7
|
-
valor_lite/classification/computation.py,sha256=
|
|
8
|
-
valor_lite/classification/manager.py,sha256=
|
|
9
|
-
valor_lite/classification/metric.py,sha256=
|
|
7
|
+
valor_lite/classification/computation.py,sha256=B5Y5K_ksbRnCCvkemYb23PKojKmhvSb2sF5JWpIdgD8,16271
|
|
8
|
+
valor_lite/classification/manager.py,sha256=ZE8tf2C85lr0KaA4KuUg1j0eHASrECYiIcKCeAYUYbE,16635
|
|
9
|
+
valor_lite/classification/metric.py,sha256=BJn82GZ7h-350ugXdRKYNPczidtjW_dvdNE194_i7BM,11905
|
|
10
10
|
valor_lite/classification/numpy_compatibility.py,sha256=roqtTetsm1_HxuaejrthQdydjsRIy-FpXpGb86cLh_E,365
|
|
11
|
-
valor_lite/classification/utilities.py,sha256=
|
|
12
|
-
valor_lite/object_detection/__init__.py,sha256=
|
|
13
|
-
valor_lite/object_detection/annotation.py,sha256=
|
|
14
|
-
valor_lite/object_detection/computation.py,sha256=
|
|
15
|
-
valor_lite/object_detection/manager.py,sha256=
|
|
16
|
-
valor_lite/object_detection/metric.py,sha256=
|
|
17
|
-
valor_lite/object_detection/utilities.py,sha256=
|
|
18
|
-
valor_lite/semantic_segmentation/__init__.py,sha256=
|
|
19
|
-
valor_lite/semantic_segmentation/annotation.py,sha256=
|
|
20
|
-
valor_lite/semantic_segmentation/benchmark.py,sha256=
|
|
21
|
-
valor_lite/semantic_segmentation/computation.py,sha256=
|
|
22
|
-
valor_lite/semantic_segmentation/manager.py,sha256=
|
|
11
|
+
valor_lite/classification/utilities.py,sha256=awKz-OdT2y5ydJuwDi5Y9FCFL1gdWyiGBZpqQxWER2A,7063
|
|
12
|
+
valor_lite/object_detection/__init__.py,sha256=y3aQbjP2Y5XXSbzGkf1E1c0c0RuonGUamsQZ-lBvtdM,317
|
|
13
|
+
valor_lite/object_detection/annotation.py,sha256=LVec-rIk408LuFxcOoIkPk0QZMWSSxbmsady4wapC1s,7007
|
|
14
|
+
valor_lite/object_detection/computation.py,sha256=IacBsfaA02v82oi_yl_jbzE93FzqXVF8zuBw3HW269U,24966
|
|
15
|
+
valor_lite/object_detection/manager.py,sha256=iNK7zaczp4ln7_RlOKKipOIYjKsMF6NUMCCNxDPjeJs,28185
|
|
16
|
+
valor_lite/object_detection/metric.py,sha256=sUYSZwXYfIyfmXG6_7Tje1_ZL_QwvecPq85jrGmwOWE,22739
|
|
17
|
+
valor_lite/object_detection/utilities.py,sha256=tNdv5dL7JhzOamGQkZ8x3ocZoTwPI6K8rcRAGMhp2nc,11217
|
|
18
|
+
valor_lite/semantic_segmentation/__init__.py,sha256=_BOClAErxZCpSnnl4C_ofigQfs_9Ak_AFh7EZECBW_I,267
|
|
19
|
+
valor_lite/semantic_segmentation/annotation.py,sha256=XRMV32Sx9A1bAVMFQdBGc3tN5Xz2RfmlyKGXCzdee7A,3705
|
|
20
|
+
valor_lite/semantic_segmentation/benchmark.py,sha256=uxd0SiDY3npsgU5pdeT4HvNP_au9GVRWzoqT6br9DtM,5961
|
|
21
|
+
valor_lite/semantic_segmentation/computation.py,sha256=ZO0qAFmq8lN73UjCyiynSv18qQDtn35FNOmvuXY4rOw,7380
|
|
22
|
+
valor_lite/semantic_segmentation/manager.py,sha256=ljJBFo4tYT8u19PU2cTYsmzGSHHV4qcOpyDIFhfwa1g,13787
|
|
23
23
|
valor_lite/semantic_segmentation/metric.py,sha256=T9RfPJf4WgqGQTXYvSy08vJG5bjXXJnyYZeW0mlxMa8,7132
|
|
24
|
-
valor_lite/semantic_segmentation/utilities.py,sha256=
|
|
24
|
+
valor_lite/semantic_segmentation/utilities.py,sha256=zgVmV8nyKWQK-T4Ov8cZFQzOmTKc5EL7errKFvc2H0g,2957
|
|
25
25
|
valor_lite/text_generation/__init__.py,sha256=pGhpWCSZjLM0pPHCtPykAfos55B8ie3mi9EzbNxfj-U,356
|
|
26
26
|
valor_lite/text_generation/annotation.py,sha256=O5aXiwCS4WjA-fqn4ly-O0MsTHoIOmqxqCaAp9IeI3M,1270
|
|
27
27
|
valor_lite/text_generation/computation.py,sha256=hGDkPfzWY9SDTdozd-nArexJ3ZSNlCIWqHGoD8vO2Cc,18652
|
|
@@ -34,7 +34,7 @@ valor_lite/text_generation/llm/instructions.py,sha256=fz2onBZZWcl5W8iy7zEWkPGU9N
|
|
|
34
34
|
valor_lite/text_generation/llm/integrations.py,sha256=-rTfdAjq1zH-4ixwYuMQEOQ80pIFzMTe0BYfroVx3Pg,6974
|
|
35
35
|
valor_lite/text_generation/llm/utilities.py,sha256=bjqatGgtVTcl1PrMwiDKTYPGJXKrBrx7PDtzIblGSys,1178
|
|
36
36
|
valor_lite/text_generation/llm/validators.py,sha256=Wzr5RlfF58_2wOU-uTw7C8skan_fYdhy4Gfn0jSJ8HM,2700
|
|
37
|
-
valor_lite-0.
|
|
38
|
-
valor_lite-0.
|
|
39
|
-
valor_lite-0.
|
|
40
|
-
valor_lite-0.
|
|
37
|
+
valor_lite-0.36.0.dist-info/METADATA,sha256=DcB4986a5xhaKIUGPFTF7HIDwgpkkhyhUp_BbejbG_g,5071
|
|
38
|
+
valor_lite-0.36.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
39
|
+
valor_lite-0.36.0.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
|
|
40
|
+
valor_lite-0.36.0.dist-info/RECORD,,
|
|
File without changes
|