code-loader 0.2.58.2rc0__tar.gz → 0.2.59.dev2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/PKG-INFO +3 -3
  2. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/helpers/detection/yolo/loss.py +78 -17
  3. code_loader-0.2.59.dev2/code_loader/helpers/detection/yolo/pytorch_utils.py +250 -0
  4. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/leaploader.py +2 -14
  5. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/pyproject.toml +1 -1
  6. code_loader-0.2.59.dev2/setup.py +42 -0
  7. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/LICENSE +0 -0
  8. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/README.md +0 -0
  9. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/__init__.py +0 -0
  10. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/contract/__init__.py +0 -0
  11. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/contract/datasetclasses.py +0 -0
  12. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/contract/enums.py +0 -0
  13. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/contract/exceptions.py +0 -0
  14. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/contract/responsedataclasses.py +0 -0
  15. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/contract/visualizer_classes.py +0 -0
  16. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/helpers/__init__.py +0 -0
  17. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/helpers/detection/__init__.py +0 -0
  18. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/helpers/detection/utils.py +0 -0
  19. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/helpers/detection/yolo/__init__.py +0 -0
  20. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/helpers/detection/yolo/decoder.py +0 -0
  21. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/helpers/detection/yolo/grid.py +0 -0
  22. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/helpers/detection/yolo/utils.py +0 -0
  23. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/leap_binder/__init__.py +0 -0
  24. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/leap_binder/leapbinder.py +0 -0
  25. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/metrics/__init__.py +0 -0
  26. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/metrics/default_metrics.py +0 -0
  27. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/utils.py +0 -0
  28. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/visualizers/__init__.py +0 -0
  29. {code_loader-0.2.58.2rc0 → code_loader-0.2.59.dev2}/code_loader/visualizers/default_visualizers.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: code-loader
3
- Version: 0.2.58.2rc0
3
+ Version: 0.2.59.dev2
4
4
  Summary:
5
5
  Home-page: https://github.com/tensorleap/code-loader
6
6
  License: MIT
@@ -13,8 +13,8 @@ Classifier: Programming Language :: Python :: 3.8
13
13
  Classifier: Programming Language :: Python :: 3.9
14
14
  Classifier: Programming Language :: Python :: 3.10
15
15
  Requires-Dist: numpy (>=1.22.3,<2.0.0)
16
- Requires-Dist: tensorflow (>=2.11.0,<3.0.0) ; platform_machine == "x86_64"
17
- Requires-Dist: tensorflow-macos (>=2.11.0,<3.0.0) ; platform_machine == "arm64"
16
+ Requires-Dist: tensorflow (>=2.11.0,<3.0.0); platform_machine == "x86_64"
17
+ Requires-Dist: tensorflow-macos (>=2.11.0,<3.0.0); platform_machine == "arm64"
18
18
  Requires-Dist: typeguard (>=2.13.3,<3.0.0)
19
19
  Project-URL: Repository, https://github.com/tensorleap/code-loader
20
20
  Description-Content-Type: text/markdown
@@ -1,11 +1,13 @@
1
- from typing import List, Tuple
1
+ from typing import List, Tuple, Union, Optional
2
2
 
3
3
  import numpy as np
4
4
  import tensorflow as tf # type: ignore
5
+ import torch # type: ignore
5
6
  from numpy.typing import NDArray
6
-
7
7
  from code_loader.helpers.detection.utils import true_coords_labels, ciou, xywh_to_xyxy_format
8
8
  from code_loader.helpers.detection.yolo.utils import match
9
+ from code_loader.helpers.detection.yolo.pytorch_utils import build_targets
10
+ import collections.abc
9
11
 
10
12
 
11
13
  class YoloLoss:
@@ -18,10 +20,12 @@ class YoloLoss:
18
20
  """
19
21
 
20
22
  def __init__(self, num_classes: int, default_boxes: List[NDArray[np.int32]],
21
- overlap_thresh: float, background_label: int,
23
+ overlap_thresh: float, background_label: int, features: List[Tuple[int, int]] = [],
24
+ anchors: Optional[NDArray[np.int32]] = None,
22
25
  from_logits: bool = True, weights: List[float] = [4.0, 1.0, 0.4],
23
- max_match_per_gt: int = 10, image_size: int = 640,
24
- cls_w: float = 0.3, obj_w: float = 0.7, box_w: float = 0.05):
26
+ max_match_per_gt: int = 10, image_size: Union[Tuple[int, int], int] = (640, 640),
27
+ cls_w: float = 0.3, obj_w: float = 0.7, box_w: float = 0.05,
28
+ yolo_match: bool = False):
25
29
  self.background_label = background_label
26
30
  self.default_boxes = [tf.convert_to_tensor(box_arr) for box_arr in default_boxes]
27
31
  self.num_classes = num_classes
@@ -31,11 +35,17 @@ class YoloLoss:
31
35
  self.weights = weights # Following yolov7 weights
32
36
  scale_factor = 3. / len(weights)
33
37
  class_factor = self.num_classes / 80
34
- image_factor = image_size / 640.
38
+ if not isinstance(image_size, collections.abc.Sequence):
39
+ image_size = (image_size, image_size)
40
+ image_factor = (image_size[0] + image_size[1]) / 2 / 640.
35
41
  self.obj_w = obj_w * scale_factor
36
42
  self.cls_w = cls_w * class_factor * scale_factor
37
43
  self.box_w = box_w * image_factor ** 2 * scale_factor
38
44
  self.max_match_per_gt = max_match_per_gt
45
+ self.feature_maps = features
46
+ self.anchors = anchors
47
+ self.image_size = image_size
48
+ self.yolo_match = yolo_match
39
49
 
40
50
  def __call__(self, y_true: tf.Tensor, y_pred: Tuple[List[tf.Tensor], List[tf.Tensor]]) -> \
41
51
  Tuple[List[tf.Tensor], List[tf.Tensor], List[tf.Tensor]]:
@@ -52,6 +62,14 @@ class YoloLoss:
52
62
  l_losses = []
53
63
  c_losses = []
54
64
  o_losses = []
65
+ b, a, gj, gi, target = None, None, None, None, None
66
+ anchor_num = len(self.default_boxes)
67
+ if self.yolo_match:
68
+ b, a, gj, gi, target, _ = self.get_yolo_match(batch_size=num,
69
+ y_true=y_true,
70
+ loc_data=loc_data,
71
+ conf_data=conf_data,
72
+ )
55
73
  for i in range(len(self.default_boxes)):
56
74
  default_box_layer = self.default_boxes[i]
57
75
  loc_data_layer = loc_data[i]
@@ -62,17 +80,24 @@ class YoloLoss:
62
80
  conf_t = []
63
81
  priors = tf.cast(priors, dtype=tf.float32) #
64
82
  y_true = tf.cast(y_true, dtype=tf.float32)
65
- for idx in range(num):
66
- truths, labels = true_coords_labels(idx, y_true, self.background_label) #
67
- loc, conf = match(
68
- threshold=self.threshold, truths=truths, priors=priors,
69
- labels=labels,
70
- background_label=self.background_label,
71
- max_match_per_gt=self.max_match_per_gt) # encoded_gt_loc, label_pred, ciou between gt and pred
72
- loc_t.append(loc)
73
- conf_t.append(conf)
74
- loc_t_tensor: tf.Tensor = tf.stack(values=loc_t, axis=0) # this is the location predictions (relative and logged)
75
- conf_t_tensor = tf.stack(values=conf_t, axis=0) # these are the labels
83
+ if not self.yolo_match:
84
+ for idx in range(num):
85
+ truths, labels = true_coords_labels(idx, y_true, self.background_label)
86
+ loc, conf = match(
87
+ threshold=self.threshold, truths=truths, priors=priors,
88
+ labels=labels,
89
+ background_label=self.background_label,
90
+ max_match_per_gt=self.max_match_per_gt) # encoded_gt_loc, label_pred, ciou between gt and pred
91
+ loc_t.append(loc)
92
+ conf_t.append(conf)
93
+ loc_t_tensor: tf.Tensor = tf.stack(values=loc_t,
94
+ axis=0) # this is the location predictions (relative and logged)
95
+ conf_t_tensor = tf.stack(values=conf_t, axis=0)
96
+ else:
97
+ assert (b is not None) and (a is not None) and (gj is not None) and (gi is not None) and\
98
+ (target is not None) # for typing - how do we remove?
99
+ loc_t_tensor, conf_t_tensor = self.get_scale_matched_gt_tf(i, num, b, a, gj, gi, target)
100
+ # these are the labels
76
101
  pos = conf_t_tensor != self.background_label
77
102
  num_pos = tf.reduce_sum(tf.cast(pos, dtype=tf.int32))
78
103
 
@@ -135,3 +160,39 @@ class YoloLoss:
135
160
  c_losses.append(loss_c_tensor)
136
161
  o_losses.append(loss_o_tensor)
137
162
  return l_losses, c_losses, o_losses
163
+
164
+ def get_scale_matched_gt_tf(self, i: int, batch_size: int, b: List[torch.Tensor], a: List[torch.Tensor],
165
+ gj: List[torch.Tensor], gi: List[torch.Tensor],
166
+ target: List[torch.Tensor]) -> Tuple[tf.Tensor, tf.Tensor]:
167
+ # temp
168
+ gt_class = tf.ones((batch_size,
169
+ len(self.feature_maps), *self.feature_maps[i]), dtype=tf.int32) * self.background_label
170
+ if len(b[i]) > 0:
171
+ gt_class = tf.tensor_scatter_nd_update(gt_class, torch.stack([b[i], a[i], gj[i], gi[i]]).T.numpy(),
172
+ target[i][:, 1])
173
+ conf_t_tensor = tf.reshape(gt_class, [gt_class.shape[0], -1])
174
+ assert self.anchors is not None
175
+ gt_loc = tf.zeros((batch_size, len(self.anchors[i]), *self.feature_maps[i], 4), dtype=tf.float32)
176
+ if len(b[i]) > 0:
177
+ gt_loc = tf.tensor_scatter_nd_update(gt_loc, torch.stack([b[i], a[i], gj[i], gi[i]]).T.numpy(),
178
+ target[i][:, -4:])
179
+ loc_t_tensor = tf.reshape(gt_loc, [gt_class.shape[0], -1, 4])
180
+ return loc_t_tensor, conf_t_tensor
181
+
182
+ def get_yolo_match(self, batch_size: int, y_true: tf.Tensor, loc_data: List[tf.Tensor], conf_data: List[tf.Tensor]) \
183
+ -> Tuple[List[torch.Tensor], ...]:
184
+ yolo_targets: List[NDArray[np.float32]] = []
185
+ scales_num = len(loc_data)
186
+ for i in range(batch_size):
187
+ batch_gt = y_true[i][y_true[i, ..., -1] != self.background_label]
188
+ yolo_targets.append(np.concatenate([np.ones((batch_gt.shape[0], 1)) * i, batch_gt[:, 4:], batch_gt[:, :4]], axis=1))
189
+ yolo_targets_cat: NDArray[np.float32] = np.concatenate(yolo_targets, axis=0)
190
+ orig_pred = [torch.from_numpy(tf.concat([loc_data[i], conf_data[i]], axis=-1).numpy()) for i in
191
+ range(scales_num)]
192
+ fin_pred = [pred.reshape([pred.shape[0], scales_num, *self.feature_maps[i], -1]) for i, pred in
193
+ enumerate(orig_pred)]
194
+ yolo_anchors = np.array(self.anchors) * np.swapaxes(np.array([*self.feature_maps])[..., None], 1, 2) / 640
195
+ b, a, gj, gi, target, anch = build_targets(fin_pred, torch.from_numpy(yolo_targets_cat.astype(np.float32)),
196
+ torch.from_numpy(yolo_anchors.astype(np.float32)), self.image_size,
197
+ self.num_classes)
198
+ return b, a, gj, gi, target, anch
@@ -0,0 +1,250 @@
1
+ #Taken from Yolov7 https://github.com/WongKinYiu/yolov7
2
+ import numpy as np
3
+ import torch # type: ignore
4
+ import torch.nn.functional as F # type: ignore
5
+ from code_loader.helpers.detection.utils import xywh_to_xyxy_format
6
+ from typing import List, Tuple
7
+
8
+
9
+ def find_3_positive(p: List[torch.Tensor], targets: torch.Tensor, anchors: torch.Tensor) ->\
10
+ Tuple[List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]], List[torch.Tensor]]:
11
+ # Build targets for compute_loss(), input targets(x,y,w,h)
12
+ # p.shape = [B,3, GX, GW, 5+CLASSES]
13
+ # targers.shape = [B,6=[image, class, x, y, w, h,]]
14
+ # targets=torch.from_numpy(truths.numpy())
15
+ na, nt = anchors.shape[0], targets.shape[0] # number of anchors, targets
16
+ indices, anch = [], []
17
+ gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain
18
+ ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt)
19
+ targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices
20
+
21
+ g = 0.5 # bias
22
+ off = torch.tensor([[0, 0],
23
+ [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m
24
+ # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm
25
+ ], device=targets.device).float() * g # offsets
26
+ for i in range(len(p)):
27
+ gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain
28
+ layer_anchors = anchors[i]
29
+
30
+ # Match targets to anchors
31
+ t = targets * gain
32
+ if nt:
33
+ # Matches
34
+ r = t[:, :, 2:4] / layer_anchors[:, None] # wh ratio
35
+ j = torch.max(r, 1. / r).max(2)[0] < 4 # compare
36
+ # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))
37
+ t = t[j] # filter only relevant anchors
38
+
39
+ # Offsets
40
+ gxy = t[:, 2:4] # grid xy
41
+ gxi = gain[[2, 3]] - gxy # inverse offset in the feature space
42
+ j, k = ((gxy % 1. < g) & (gxy > 1.)).T # x closer to left, y closest to up
43
+ l, m = ((gxi % 1. < g) & (gxi > 1.)).T # x closer to right, y closet to down
44
+ j = torch.stack((torch.ones_like(j), j, k, l, m)) # [True, x-left, y-up, x-right, y-down]
45
+ t = t.repeat((5, 1, 1))[j]
46
+ offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]
47
+ else:
48
+ t = targets[0]
49
+ offsets = 0
50
+
51
+ # Define
52
+ b, c = t[:, :2].long().T # image, class
53
+ gxy = t[:, 2:4] # grid xy
54
+ gwh = t[:, 4:6] # grid wh
55
+ gij = (gxy - offsets).long()
56
+ gi, gj = gij.T # grid xy indices
57
+
58
+ # Append
59
+ a = t[:, 6].long() # anchor indices
60
+ indices.append(
61
+ (b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices [y,x]]
62
+ anch.append(layer_anchors[a]) # anchors
63
+
64
+ return indices, anch
65
+
66
+
67
+ def box_iou(box1: torch.Tensor, box2: torch.Tensor) -> torch.Tensor:
68
+ # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py
69
+ """
70
+ Return intersection-over-union (Jaccard index) of boxes.
71
+ Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
72
+ Arguments:
73
+ box1 (Tensor[N, 4])
74
+ box2 (Tensor[M, 4])
75
+ Returns:
76
+ iou (Tensor[N, M]): the NxM matrix containing the pairwise
77
+ IoU values for every element in boxes1 and boxes2
78
+ """
79
+
80
+ def box_area(box: torch.Tensor) -> torch.Tensor:
81
+ # box = 4xn
82
+ return (box[2] - box[0]) * (box[3] - box[1])
83
+
84
+ area1 = box_area(box1.T)
85
+ area2 = box_area(box2.T)
86
+
87
+ # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)
88
+ inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)
89
+ return inter / (area1[:, None] + area2 - inter) # iou = inter / (area1 + area2 - inter)
90
+
91
+
92
+ def build_targets(p: List[torch.Tensor], targets: torch.Tensor, anchors: torch.Tensor,
93
+ image_size: Tuple[int, int], num_classes: int) -> Tuple[List[torch.Tensor], ...]:
94
+ # targets [Image, class, x, y, w, h]
95
+ # anchors [scale x anchor-per-scale]
96
+ # P == LIST[[B,ANCHORS,H_scale_i,W_scale_i,CLASSES+5]...]
97
+ # len(p) == scales
98
+
99
+ # indices, anch = self.find_positive(p, targets)
100
+ indices, anch = find_3_positive(p, targets, anchors)
101
+ # indices, anch = self.find_4_positive(p, targets)
102
+ # indices, anch = self.find_5_positive(p, targets)
103
+ # indices, anch = self.find_9_positive(p, targets)
104
+
105
+ matching_bs: List[torch.Tensor] = [[] for pp in p]
106
+ matching_as: List[torch.Tensor] = [[] for pp in p]
107
+ matching_gjs: List[torch.Tensor] = [[] for pp in p]
108
+ matching_gis: List[torch.Tensor] = [[] for pp in p]
109
+ matching_targets: List[torch.Tensor] = [[] for pp in p]
110
+ matching_anchs: List[torch.Tensor] = [[] for pp in p]
111
+
112
+ nl = len(p)
113
+
114
+ for batch_idx in range(p[0].shape[0]):
115
+
116
+ b_idx = targets[:, 0] == batch_idx
117
+ this_target = targets[b_idx]
118
+ if this_target.shape[0] == 0:
119
+ continue
120
+
121
+ txywh = this_target[:, 2:6] * np.array([*image_size[::-1], *image_size[::-1]], dtype=np.float32)
122
+ txyxy = torch.from_numpy(xywh_to_xyxy_format(txywh).astype(np.float32))
123
+
124
+ pxyxys = []
125
+ p_cls = []
126
+ p_obj = []
127
+ from_which_layer = []
128
+ all_b = []
129
+ all_a = []
130
+ all_gj = []
131
+ all_gi = []
132
+ all_anch = []
133
+
134
+ for i, pi in enumerate(p):
135
+ b, a, gj, gi = indices[i]
136
+ idx = (b == batch_idx)
137
+ b, a, gj, gi = b[idx], a[idx], gj[idx], gi[idx]
138
+ all_b.append(b)
139
+ all_a.append(a)
140
+ all_gj.append(gj)
141
+ all_gi.append(gi)
142
+ all_anch.append(anch[i][idx])
143
+ from_which_layer.append(torch.ones(size=(len(b),)) * i)
144
+
145
+ fg_pred = pi[b, a, gj, gi]
146
+ p_obj.append(fg_pred[:, 4:5])
147
+ p_cls.append(fg_pred[:, 5:])
148
+
149
+ grid = torch.stack([gi, gj], dim=1)
150
+ pxy = fg_pred[:, :2] * torch.from_numpy(np.array([*image_size[::-1]]).astype(np.float32))
151
+ pwh = fg_pred[:, 2:4] * torch.from_numpy(np.array([*image_size[::-1]]).astype(np.float32))
152
+ pxywh = torch.cat([pxy, pwh], dim=-1)
153
+ pxyxy = torch.from_numpy(xywh_to_xyxy_format(pxywh).astype(np.float32))
154
+ pxyxys.append(pxyxy)
155
+
156
+ pxyxys_cat = torch.cat(pxyxys, dim=0)
157
+ if pxyxys_cat.shape[0] == 0:
158
+ continue
159
+ p_obj_cat = torch.cat(p_obj, dim=0)
160
+ p_cls_cat = torch.cat(p_cls, dim=0)
161
+ from_which_layer_cat = torch.cat(from_which_layer, dim=0)
162
+ all_b = torch.cat(all_b, dim=0)
163
+ all_a = torch.cat(all_a, dim=0)
164
+ all_gj = torch.cat(all_gj, dim=0)
165
+ all_gi = torch.cat(all_gi, dim=0)
166
+ all_anch = torch.cat(all_anch, dim=0)
167
+
168
+ pair_wise_iou = box_iou(txyxy, pxyxys_cat) # (BB-matched,4), BB-matches(scale)=#anchors-matches*3
169
+
170
+ pair_wise_iou_loss = -torch.log(pair_wise_iou + 1e-8)
171
+
172
+ top_k, _ = torch.topk(pair_wise_iou, min(10, pair_wise_iou.shape[1]), dim=1)
173
+ dynamic_ks = torch.clamp(top_k.sum(1).int(), min=1)
174
+
175
+ gt_cls_per_image = (
176
+ F.one_hot(this_target[:, 1].to(torch.int64), num_classes)
177
+ .float()
178
+ .unsqueeze(1)
179
+ .repeat(1, pxyxys_cat.shape[0], 1)
180
+ )
181
+
182
+ num_gt = this_target.shape[0]
183
+ cls_preds_ = (
184
+ p_cls_cat.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()
185
+ * p_obj_cat.unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()
186
+ )
187
+
188
+ y = cls_preds_.sqrt_()
189
+ pair_wise_cls_loss = F.binary_cross_entropy_with_logits(
190
+ torch.log(y / (1 - y)), gt_cls_per_image, reduction="none"
191
+ ).sum(-1)
192
+ del cls_preds_
193
+
194
+ cost = (
195
+ pair_wise_cls_loss
196
+ + 3.0 * pair_wise_iou_loss
197
+ )
198
+
199
+ matching_matrix = torch.zeros_like(cost)
200
+
201
+ for gt_idx in range(num_gt):
202
+ _, pos_idx = torch.topk(
203
+ cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False
204
+ )
205
+ matching_matrix[gt_idx][pos_idx] = 1.0
206
+
207
+ del top_k, dynamic_ks
208
+ anchor_matching_gt = matching_matrix.sum(0) # sum of the matches per anchor
209
+ if (anchor_matching_gt > 1).sum() > 0:
210
+ _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0)
211
+ matching_matrix[:, anchor_matching_gt > 1] *= 0.0
212
+ matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0
213
+ fg_mask_inboxes = matching_matrix.sum(0) > 0.0
214
+ matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0)
215
+
216
+ from_which_layer_cat = from_which_layer_cat[fg_mask_inboxes]
217
+ all_b = all_b[fg_mask_inboxes]
218
+ all_a = all_a[fg_mask_inboxes]
219
+ all_gj = all_gj[fg_mask_inboxes]
220
+ all_gi = all_gi[fg_mask_inboxes]
221
+ all_anch = all_anch[fg_mask_inboxes]
222
+
223
+ this_target = this_target[matched_gt_inds]
224
+
225
+ for i in range(nl):
226
+ layer_idx = from_which_layer_cat == i
227
+ matching_bs[i].append(all_b[layer_idx])
228
+ matching_as[i].append(all_a[layer_idx])
229
+ matching_gjs[i].append(all_gj[layer_idx])
230
+ matching_gis[i].append(all_gi[layer_idx])
231
+ matching_targets[i].append(this_target[layer_idx])
232
+ matching_anchs[i].append(all_anch[layer_idx])
233
+
234
+ for i in range(nl):
235
+ if matching_targets[i] != []:
236
+ matching_bs[i] = torch.cat(matching_bs[i], dim=0)
237
+ matching_as[i] = torch.cat(matching_as[i], dim=0)
238
+ matching_gjs[i] = torch.cat(matching_gjs[i], dim=0)
239
+ matching_gis[i] = torch.cat(matching_gis[i], dim=0)
240
+ matching_targets[i] = torch.cat(matching_targets[i], dim=0)
241
+ matching_anchs[i] = torch.cat(matching_anchs[i], dim=0)
242
+ else:
243
+ matching_bs[i] = torch.tensor([], dtype=torch.int64)
244
+ matching_as[i] = torch.tensor([], dtype=torch.int64)
245
+ matching_gjs[i] = torch.tensor([], dtype=torch.int64)
246
+ matching_gis[i] = torch.tensor([], dtype=torch.int64)
247
+ matching_targets[i] = torch.tensor([], dtype=torch.int64)
248
+ matching_anchs[i] = torch.tensor([], dtype=torch.int64)
249
+
250
+ return matching_bs, matching_as, matching_gjs, matching_gis, matching_targets, matching_anchs # SCALE*[B,anchor-idx, j, i, GTs, anchors]
@@ -1,9 +1,8 @@
1
1
  import importlib.util
2
2
  from functools import lru_cache
3
3
  from pathlib import Path
4
- from typing import Dict, List, Iterable, Any, Union, Optional
4
+ from typing import Dict, List, Iterable, Any, Union
5
5
  import sys
6
- import os
7
6
 
8
7
  import numpy as np
9
8
  import numpy.typing as npt
@@ -23,14 +22,12 @@ from code_loader.utils import get_root_exception_line_number, get_shape
23
22
 
24
23
  class LeapLoader:
25
24
  def __init__(self, dataset_script: str, code_path: str = '', code_entry_name: str = ''):
26
- print(f"Hello from LeapLoader init code_path: {code_path} code_entry_name: {code_entry_name}")
27
25
  self.dataset_script: str = dataset_script
28
26
  self.code_entry_name = code_entry_name
29
27
  self.code_path = code_path
30
28
 
31
29
  @lru_cache()
32
30
  def exec_script(self) -> None:
33
- print(f"Hello from LeapLoader exec_script")
34
31
  try:
35
32
  # TODO: make these fields mandatory and delete load_script method
36
33
  if self.code_path and self.code_entry_name:
@@ -42,17 +39,12 @@ class LeapLoader:
42
39
  raise DatasetScriptException(getattr(e, 'message', repr(e))) from e
43
40
 
44
41
  def load_script(self) -> None:
45
- print(f"Hello from LeapLoader load_script pre run")
46
42
  global_variables: Dict[Any, Any] = {}
47
43
  exec(self.dataset_script, global_variables)
48
- print(f"Hello from LeapLoader load_script done")
49
44
 
50
45
  def evaluate_module(self) -> None:
51
- print(f"Hello from LeapLoader evaluate_module")
52
-
53
46
  def append_path_recursively(full_path: str) -> None:
54
- if '/' not in full_path:
55
- print(f"Hello from LeapLoader append_path_recursively stop condition")
47
+ if '/' not in full_path or full_path == '/':
56
48
  return
57
49
 
58
50
  parent_path = str(Path(full_path).parent)
@@ -61,20 +53,16 @@ class LeapLoader:
61
53
 
62
54
  append_path_recursively(self.code_path)
63
55
 
64
- print(f"Hello from LeapLoader append_path_recursively done")
65
56
  file_path = Path(self.code_path, self.code_entry_name)
66
57
  spec = importlib.util.spec_from_file_location(self.code_path, file_path)
67
- print(f"Hello from LeapLoader spec_from_file_location done")
68
58
  if spec is None or spec.loader is None:
69
59
  raise DatasetScriptException(f'Something is went wrong with spec file from: {file_path}')
70
60
 
71
61
  file = importlib.util.module_from_spec(spec)
72
- print(f"Hello from LeapLoader module_from_spec done")
73
62
  if file is None:
74
63
  raise DatasetScriptException(f'Something is went wrong with import module from: {file_path}')
75
64
 
76
65
  spec.loader.exec_module(file)
77
- print(f"Hello from LeapLoader evaluate_module done")
78
66
 
79
67
  @lru_cache()
80
68
  def metric_by_name(self) -> Dict[str, MetricHandler]:
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "code-loader"
3
- version = "0.2.58.2-rc"
3
+ version = "0.2.59.dev2"
4
4
  description = ""
5
5
  authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
6
6
  license = "MIT"
@@ -0,0 +1,42 @@
1
+ # -*- coding: utf-8 -*-
2
+ from setuptools import setup
3
+
4
+ packages = \
5
+ ['code_loader',
6
+ 'code_loader.contract',
7
+ 'code_loader.helpers',
8
+ 'code_loader.helpers.detection',
9
+ 'code_loader.helpers.detection.yolo',
10
+ 'code_loader.leap_binder',
11
+ 'code_loader.metrics',
12
+ 'code_loader.visualizers']
13
+
14
+ package_data = \
15
+ {'': ['*']}
16
+
17
+ install_requires = \
18
+ ['numpy>=1.22.3,<2.0.0', 'typeguard>=2.13.3,<3.0.0']
19
+
20
+ extras_require = \
21
+ {':platform_machine == "arm64"': ['tensorflow-macos>=2.11.0,<3.0.0'],
22
+ ':platform_machine == "x86_64"': ['tensorflow>=2.11.0,<3.0.0']}
23
+
24
+ setup_kwargs = {
25
+ 'name': 'code-loader',
26
+ 'version': '0.2.59.dev2',
27
+ 'description': '',
28
+ 'long_description': '# tensorleap code loader\nUsed to load user code to tensorleap \n',
29
+ 'author': 'dorhar',
30
+ 'author_email': 'doron.harnoy@tensorleap.ai',
31
+ 'maintainer': 'None',
32
+ 'maintainer_email': 'None',
33
+ 'url': 'https://github.com/tensorleap/code-loader',
34
+ 'packages': packages,
35
+ 'package_data': package_data,
36
+ 'install_requires': install_requires,
37
+ 'extras_require': extras_require,
38
+ 'python_requires': '>=3.8,<3.11',
39
+ }
40
+
41
+
42
+ setup(**setup_kwargs)