oriented-det 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- export/__init__.py +9 -0
- export/ort_runtime.py +67 -0
- export/postprocess.py +151 -0
- export/scripts/__init__.py +6 -0
- export/scripts/build_faster_rcnn_savedmodel.py +104 -0
- export/scripts/export_onnx.py +210 -0
- export/scripts/onnx_to_savedmodel.py +35 -0
- export/scripts/predict_savedmodel.py +94 -0
- export/scripts/save_predictions_tf.py +447 -0
- export/scripts/to_tflite.py +32 -0
- export/tests/test_export_onnx_optional.py +82 -0
- export/tests/test_export_wrappers.py +105 -0
- export/tests/test_faster_rcnn_export_parity.py +201 -0
- export/tests/test_ort_runtime.py +41 -0
- export/tf_serving_model.py +96 -0
- export/val_dataset.py +116 -0
- export/wrappers.py +161 -0
- oriented_det/__init__.py +77 -0
- oriented_det/cli/__init__.py +92 -0
- oriented_det/cli/train.py +18 -0
- oriented_det/configs/_base_/augmentation.json +21 -0
- oriented_det/configs/_base_/datasets/dota_le90.json +21 -0
- oriented_det/configs/_base_/fp16.json +5 -0
- oriented_det/configs/_base_/models/oriented_rcnn_r50.json +26 -0
- oriented_det/configs/_base_/models/rotated_faster_rcnn_r50.json +53 -0
- oriented_det/configs/_base_/models/rotated_retinanet_r50.json +30 -0
- oriented_det/configs/_base_/preprocessing.json +8 -0
- oriented_det/configs/_base_/schedules/1x.json +33 -0
- oriented_det/configs/config.schema.json +404 -0
- oriented_det/configs/oriented_rcnn/dota_le90_1x.json +121 -0
- oriented_det/configs/oriented_rcnn/dota_le90_3x.json +11 -0
- oriented_det/configs/rotated_faster_rcnn/dota_le90_1x.json +119 -0
- oriented_det/configs/rotated_faster_rcnn/dota_le90_3x.json +9 -0
- oriented_det/configs/rotated_retinanet/dota_le90_1x.json +107 -0
- oriented_det/configs/rotated_retinanet/dota_le90_3x.json +30 -0
- oriented_det/data/__init__.py +89 -0
- oriented_det/data/airbus_playground.py +611 -0
- oriented_det/data/dota.py +741 -0
- oriented_det/data/dota_classes.py +26 -0
- oriented_det/data/evaluation.py +648 -0
- oriented_det/data/flips.py +115 -0
- oriented_det/data/preprocessing.py +335 -0
- oriented_det/data/tiling.py +399 -0
- oriented_det/data/transforms.py +377 -0
- oriented_det/geometry/__init__.py +8 -0
- oriented_det/geometry/poly.py +127 -0
- oriented_det/geometry/qbox.py +63 -0
- oriented_det/geometry/rbox.py +250 -0
- oriented_det/geometry/transforms.py +266 -0
- oriented_det/models/__init__.py +36 -0
- oriented_det/models/backbones/__init__.py +11 -0
- oriented_det/models/backbones/resnet_fpn.py +79 -0
- oriented_det/models/backbones/utils.py +81 -0
- oriented_det/models/bbox_coder.py +355 -0
- oriented_det/models/faster_rcnn_inference.py +494 -0
- oriented_det/models/horizontal_roi_coder.py +155 -0
- oriented_det/models/oriented_rcnn.py +1256 -0
- oriented_det/models/oriented_roi.py +1664 -0
- oriented_det/models/oriented_rpn.py +2104 -0
- oriented_det/models/rotated_retinanet.py +1030 -0
- oriented_det/models/utils.py +590 -0
- oriented_det/ops/__init__.py +58 -0
- oriented_det/ops/gpu_ops.py +1109 -0
- oriented_det/ops/iou.py +172 -0
- oriented_det/ops/kfiou.py +275 -0
- oriented_det/ops/nms.py +202 -0
- oriented_det/ops/probiou.py +165 -0
- oriented_det/ops/rotated_ops.py +122 -0
- oriented_det/ops/utils.py +257 -0
- oriented_det/pretrained/__init__.py +23 -0
- oriented_det/pretrained/hub.py +249 -0
- oriented_det/pretrained/manifest.json +46 -0
- oriented_det/runtime/__init__.py +29 -0
- oriented_det/runtime/checkpoint.py +274 -0
- oriented_det/runtime/collate.py +348 -0
- oriented_det/runtime/inference.py +1286 -0
- oriented_det/train/__init__.py +102 -0
- oriented_det/train/config.py +872 -0
- oriented_det/train/engine.py +2933 -0
- oriented_det/train/grouped_ce.py +139 -0
- oriented_det/train/piecewise_schedule.py +33 -0
- oriented_det/train/profiler.py +287 -0
- oriented_det/train/utils.py +999 -0
- oriented_det/utils/__init__.py +31 -0
- oriented_det/utils/config.py +376 -0
- oriented_det/utils/device.py +35 -0
- oriented_det/utils/logging.py +163 -0
- oriented_det/utils/progress.py +62 -0
- oriented_det/utils/viz.py +181 -0
- oriented_det-0.1.0.dist-info/METADATA +313 -0
- oriented_det-0.1.0.dist-info/RECORD +115 -0
- oriented_det-0.1.0.dist-info/WHEEL +5 -0
- oriented_det-0.1.0.dist-info/entry_points.txt +2 -0
- oriented_det-0.1.0.dist-info/licenses/LICENSE +202 -0
- oriented_det-0.1.0.dist-info/top_level.txt +3 -0
- tools/__init__.py +1 -0
- tools/app.py +1284 -0
- tools/dataset_stats.py +389 -0
- tools/dota_labels_to_comma.py +132 -0
- tools/free_gpu.py +142 -0
- tools/generate_airbus_playground_csv.py +154 -0
- tools/image_demo.py +200 -0
- tools/lr_finder.py +771 -0
- tools/measure_sampled_riou_error.py +483 -0
- tools/playground_to_dota.py +290 -0
- tools/pretrained_download.py +49 -0
- tools/preview_augmentation.py +510 -0
- tools/publish_checkpoint.py +96 -0
- tools/save_predictions.py +2350 -0
- tools/sync_vendored_configs.py +94 -0
- tools/tile_dota.py +447 -0
- tools/train.py +2324 -0
- tools/train_example.py +244 -0
- tools/train_multi_gpu.py +367 -0
- tools/visualize_boxes.py +183 -0
|
@@ -0,0 +1,741 @@
|
|
|
1
|
+
"""DOTA dataset loader with efficient polygon parsing and rbox conversion."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable, Iterator, Sequence
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Dict, List, Optional, Tuple
|
|
9
|
+
import re
|
|
10
|
+
import warnings
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
from PIL import Image
|
|
14
|
+
except ImportError:
|
|
15
|
+
Image = None # type: ignore
|
|
16
|
+
|
|
17
|
+
from ..geometry import Polygon, QBox, RBox, transforms
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def format_dota_line(
|
|
21
|
+
x1: float, y1: float, x2: float, y2: float,
|
|
22
|
+
x3: float, y3: float, x4: float, y4: float,
|
|
23
|
+
category: str,
|
|
24
|
+
difficult: int,
|
|
25
|
+
) -> str:
|
|
26
|
+
"""Format a single annotation line in official DOTA format (comma-separated).
|
|
27
|
+
|
|
28
|
+
See https://captain-whu.github.io/DOTA/dataset.html
|
|
29
|
+
"""
|
|
30
|
+
def _fmt(v: float) -> str:
|
|
31
|
+
# DOTA commonly stores integer pixel coords, but floats are also valid.
|
|
32
|
+
# Keep floats as-is, but drop the trailing ".0" when the value is integral.
|
|
33
|
+
try:
|
|
34
|
+
fv = float(v)
|
|
35
|
+
except Exception:
|
|
36
|
+
return str(v)
|
|
37
|
+
if fv.is_integer():
|
|
38
|
+
return str(int(fv))
|
|
39
|
+
return str(fv)
|
|
40
|
+
|
|
41
|
+
return (
|
|
42
|
+
f"{_fmt(x1)}, {_fmt(y1)}, {_fmt(x2)}, {_fmt(y2)}, "
|
|
43
|
+
f"{_fmt(x3)}, {_fmt(y3)}, {_fmt(x4)}, {_fmt(y4)}, {category}, {int(difficult)}"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class DOTAAnnotation:
|
|
49
|
+
"""Single DOTA annotation entry."""
|
|
50
|
+
|
|
51
|
+
class_name: str
|
|
52
|
+
difficult: int # 0 or 1
|
|
53
|
+
polygon: Polygon
|
|
54
|
+
rbox: RBox
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_line(cls, line: str, *, class_map: Optional[Dict[str, int]] = None) -> "DOTAAnnotation":
|
|
58
|
+
"""Parse a single DOTA annotation line.
|
|
59
|
+
|
|
60
|
+
DOTA Polygon Format (official default):
|
|
61
|
+
---------------------------------------
|
|
62
|
+
Official DOTA (captain-whu.github.io) uses comma-separated:
|
|
63
|
+
"x1, y1, x2, y2, x3, y3, x4, y4, category, difficult"
|
|
64
|
+
We produce this format by default (see format_dota_line, to_line).
|
|
65
|
+
For reading, we also accept space-separated:
|
|
66
|
+
"x1 y1 x2 y2 x3 y3 x4 y4 class_name difficult"
|
|
67
|
+
(class = parts[8:-1] for multi-word names).
|
|
68
|
+
|
|
69
|
+
Corner Order Convention:
|
|
70
|
+
- The 4 corners are ordered sequentially around the polygon perimeter
|
|
71
|
+
- Typically, corners follow a consistent winding order (clockwise or counter-clockwise)
|
|
72
|
+
- The first corner (x1, y1) is usually the top-left or top-most point
|
|
73
|
+
- Subsequent corners follow the polygon boundary
|
|
74
|
+
|
|
75
|
+
Conversion to QBox/RBox:
|
|
76
|
+
- The polygon is first converted to a QBox, which normalizes the point order
|
|
77
|
+
- QBox ensures counter-clockwise orientation and orders points starting from top-most
|
|
78
|
+
- RBox is then derived from QBox, computing center, dimensions, and angle
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
line: DOTA annotation line
|
|
82
|
+
class_map: Optional mapping from class names to IDs
|
|
83
|
+
|
|
84
|
+
Returns:
|
|
85
|
+
DOTAAnnotation with parsed polygon and RBox
|
|
86
|
+
|
|
87
|
+
Example:
|
|
88
|
+
>>> line = "100 200 300 200 300 400 100 400 plane 0"
|
|
89
|
+
>>> ann = DOTAAnnotation.from_line(line)
|
|
90
|
+
>>> # Creates a rectangle with corners at (100,200), (300,200), (300,400), (100,400)
|
|
91
|
+
"""
|
|
92
|
+
raw = line.strip()
|
|
93
|
+
if "," in raw:
|
|
94
|
+
# Official DOTA: x1, y1, x2, y2, x3, y3, x4, y4, category, difficult (10 fields)
|
|
95
|
+
parts = [p.strip() for p in raw.split(",")]
|
|
96
|
+
if len(parts) < 10:
|
|
97
|
+
raise ValueError(f"Invalid DOTA annotation line (comma): {line}")
|
|
98
|
+
coords = [float(parts[i]) for i in range(8)]
|
|
99
|
+
class_name = parts[8].strip()
|
|
100
|
+
difficult = int(parts[9])
|
|
101
|
+
else:
|
|
102
|
+
# Space-separated: allows multi-word class (e.g. "General Cargo")
|
|
103
|
+
parts = raw.split()
|
|
104
|
+
if len(parts) < 9:
|
|
105
|
+
raise ValueError(f"Invalid DOTA annotation line: {line}")
|
|
106
|
+
coords = [float(parts[i]) for i in range(8)]
|
|
107
|
+
class_name = " ".join(parts[8:-1]) if len(parts) > 9 else parts[8]
|
|
108
|
+
difficult = int(parts[-1])
|
|
109
|
+
|
|
110
|
+
# Extract 4 corner points: (x1,y1), (x2,y2), (x3,y3), (x4,y4)
|
|
111
|
+
points = [(coords[i], coords[i + 1]) for i in range(0, 8, 2)]
|
|
112
|
+
|
|
113
|
+
# Convert to Polygon (validates and normalizes orientation)
|
|
114
|
+
polygon = Polygon(points)
|
|
115
|
+
|
|
116
|
+
# Convert to RBox via QBox (QBox normalizes point order)
|
|
117
|
+
# QBox ensures counter-clockwise order and starts from top-most point
|
|
118
|
+
rbox = transforms.polygon_to_rbox(polygon)
|
|
119
|
+
|
|
120
|
+
return cls(class_name=class_name, difficult=difficult, polygon=polygon, rbox=rbox)
|
|
121
|
+
|
|
122
|
+
def to_line(self) -> str:
|
|
123
|
+
"""Serialize to official DOTA format (comma-separated)."""
|
|
124
|
+
coords = []
|
|
125
|
+
for p in self.polygon:
|
|
126
|
+
coords.extend(p)
|
|
127
|
+
return format_dota_line(*coords, self.class_name, self.difficult)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass(frozen=True)
|
|
131
|
+
class DOTASample:
|
|
132
|
+
"""Single DOTA image sample with annotations."""
|
|
133
|
+
|
|
134
|
+
image_path: Path
|
|
135
|
+
width: int
|
|
136
|
+
height: int
|
|
137
|
+
annotations: Tuple[DOTAAnnotation, ...]
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def num_objects(self) -> int:
|
|
141
|
+
return len(self.annotations)
|
|
142
|
+
|
|
143
|
+
def filter_by_class(
|
|
144
|
+
self,
|
|
145
|
+
allowed_classes: Optional[Sequence[str]] = None,
|
|
146
|
+
ignore_labels: Optional[Sequence[str]] = None,
|
|
147
|
+
drop_difficult: bool = False,
|
|
148
|
+
) -> "DOTASample":
|
|
149
|
+
"""Return a filtered copy of this sample."""
|
|
150
|
+
filtered = list(self.annotations)
|
|
151
|
+
|
|
152
|
+
if drop_difficult:
|
|
153
|
+
filtered = [ann for ann in filtered if ann.difficult == 0]
|
|
154
|
+
|
|
155
|
+
if allowed_classes is not None:
|
|
156
|
+
allowed_set = set(allowed_classes)
|
|
157
|
+
filtered = [ann for ann in filtered if ann.class_name in allowed_set]
|
|
158
|
+
|
|
159
|
+
if ignore_labels is not None:
|
|
160
|
+
ignore_set = set(ignore_labels)
|
|
161
|
+
filtered = [ann for ann in filtered if ann.class_name not in ignore_set]
|
|
162
|
+
|
|
163
|
+
return DOTASample(
|
|
164
|
+
image_path=self.image_path,
|
|
165
|
+
width=self.width,
|
|
166
|
+
height=self.height,
|
|
167
|
+
annotations=tuple(filtered)
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class DOTADataset:
|
|
172
|
+
"""Efficient DOTA dataset loader with lazy parsing."""
|
|
173
|
+
|
|
174
|
+
def __init__(
|
|
175
|
+
self,
|
|
176
|
+
root_dir: str | Path,
|
|
177
|
+
*,
|
|
178
|
+
split: str = "train",
|
|
179
|
+
split_file: Optional[str | Path] = None,
|
|
180
|
+
label_dir: Optional[str | Path] = None,
|
|
181
|
+
image_dir: Optional[str | Path] = None,
|
|
182
|
+
class_map: Optional[Dict[str, int]] = None,
|
|
183
|
+
allowed_classes: Optional[Sequence[str]] = None,
|
|
184
|
+
ignore_labels: Optional[Sequence[str]] = None,
|
|
185
|
+
# "drop" -> remove at read-time, "ignore"/"keep" -> keep in sample.annotations.
|
|
186
|
+
difficult_strategy: str = "drop",
|
|
187
|
+
filter_empty_gt: bool = False,
|
|
188
|
+
):
|
|
189
|
+
"""Initialize DOTA dataset.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
root_dir: Root directory of DOTA dataset (used as base if label_dir/image_dir not specified)
|
|
193
|
+
split: Split name ("train", "val", "test") - used for pattern matching
|
|
194
|
+
split_file: Optional path to a file listing image names (one per line).
|
|
195
|
+
If provided, only images listed in this file will be used.
|
|
196
|
+
This follows the official DOTA dataset convention where
|
|
197
|
+
train.txt, val.txt, test.txt list the image names.
|
|
198
|
+
label_dir: Optional custom path to label directory. If None, uses root_dir/labelTxt.
|
|
199
|
+
This allows having train/val/test in separate folders.
|
|
200
|
+
Can be the same as image_dir to read images and annotations from one folder.
|
|
201
|
+
image_dir: Optional custom path to image directory. If None, uses root_dir/images.
|
|
202
|
+
This allows having train/val/test in separate folders.
|
|
203
|
+
Can be the same as label_dir to read images and annotations from one folder.
|
|
204
|
+
class_map: Optional mapping from class names to numeric IDs
|
|
205
|
+
allowed_classes: Optional list of allowed class names (whitelist)
|
|
206
|
+
ignore_labels: Optional list of class names to exclude (blacklist)
|
|
207
|
+
difficult_strategy: How to handle difficult annotations: drop | ignore | keep.
|
|
208
|
+
filter_empty_gt: If True, drop tiles whose effective GT count is zero after
|
|
209
|
+
difficult_strategy, allowed_classes, and ignore_labels (MMRotate DOTADataset parity).
|
|
210
|
+
"""
|
|
211
|
+
self.root_dir = Path(root_dir)
|
|
212
|
+
self.filter_empty_gt = bool(filter_empty_gt)
|
|
213
|
+
self.split = split
|
|
214
|
+
self.split_file = Path(split_file) if split_file else None
|
|
215
|
+
self.class_map = class_map or {}
|
|
216
|
+
self.allowed_classes = allowed_classes
|
|
217
|
+
self.ignore_labels = list(ignore_labels) if ignore_labels else None
|
|
218
|
+
# Normalize difficult strategy
|
|
219
|
+
ds = (difficult_strategy or "drop").strip().lower()
|
|
220
|
+
if ds not in {"drop", "ignore", "keep"}:
|
|
221
|
+
raise ValueError(f"Invalid difficult_strategy={difficult_strategy!r}; expected 'drop', 'ignore', or 'keep'.")
|
|
222
|
+
self.difficult_strategy = ds
|
|
223
|
+
# Read-time drop of difficult=1 (only when strategy is "drop")
|
|
224
|
+
self._drop_difficult = self.difficult_strategy == "drop"
|
|
225
|
+
|
|
226
|
+
# Support custom label and image directories for split-specific folders
|
|
227
|
+
if label_dir is not None:
|
|
228
|
+
self.label_dir = Path(label_dir)
|
|
229
|
+
else:
|
|
230
|
+
self.label_dir = self.root_dir / "labelTxt"
|
|
231
|
+
|
|
232
|
+
if image_dir is not None:
|
|
233
|
+
self.image_dir = Path(image_dir)
|
|
234
|
+
else:
|
|
235
|
+
self.image_dir = self.root_dir / "images"
|
|
236
|
+
|
|
237
|
+
if not self.label_dir.exists():
|
|
238
|
+
raise FileNotFoundError(f"DOTA label directory not found: {self.label_dir}")
|
|
239
|
+
if not self.image_dir.exists():
|
|
240
|
+
raise FileNotFoundError(f"DOTA image directory not found: {self.image_dir}")
|
|
241
|
+
|
|
242
|
+
discovered = self._discover_annotation_files()
|
|
243
|
+
self._annotation_files_discovered_count = len(discovered)
|
|
244
|
+
if self.filter_empty_gt:
|
|
245
|
+
self._annotation_files = [
|
|
246
|
+
ann_path
|
|
247
|
+
for ann_path in discovered
|
|
248
|
+
if self._effective_gt_count(ann_path) > 0
|
|
249
|
+
]
|
|
250
|
+
self._empty_gt_filtered_count = (
|
|
251
|
+
self._annotation_files_discovered_count - len(self._annotation_files)
|
|
252
|
+
)
|
|
253
|
+
else:
|
|
254
|
+
self._annotation_files = discovered
|
|
255
|
+
self._empty_gt_filtered_count = 0
|
|
256
|
+
|
|
257
|
+
@property
|
|
258
|
+
def annotation_files_discovered_count(self) -> int:
|
|
259
|
+
"""Label files found before ``filter_empty_gt`` (if enabled)."""
|
|
260
|
+
return self._annotation_files_discovered_count
|
|
261
|
+
|
|
262
|
+
@property
|
|
263
|
+
def empty_gt_filtered_count(self) -> int:
|
|
264
|
+
"""Tiles removed by ``filter_empty_gt`` at init."""
|
|
265
|
+
return self._empty_gt_filtered_count
|
|
266
|
+
|
|
267
|
+
def _discover_annotation_files(self) -> List[Path]:
|
|
268
|
+
"""Discover annotation files for the split.
|
|
269
|
+
|
|
270
|
+
Two modes:
|
|
271
|
+
1. If split_file is provided: Read image names from file and find corresponding
|
|
272
|
+
annotation files (e.g., train.txt lists "P0001", finds "P0001_train.txt")
|
|
273
|
+
2. Otherwise: Pattern match on annotation filenames (e.g., "*_train.txt")
|
|
274
|
+
|
|
275
|
+
Returns:
|
|
276
|
+
Sorted list of annotation file paths
|
|
277
|
+
"""
|
|
278
|
+
if self.split_file is not None:
|
|
279
|
+
# Mode 1: Use split file (official DOTA convention)
|
|
280
|
+
split_path = Path(self.split_file)
|
|
281
|
+
if not split_path.is_absolute():
|
|
282
|
+
# Try relative to root_dir
|
|
283
|
+
split_path = self.root_dir / split_path
|
|
284
|
+
|
|
285
|
+
if not split_path.exists():
|
|
286
|
+
raise FileNotFoundError(
|
|
287
|
+
f"Split file not found: {self.split_file} "
|
|
288
|
+
f"(tried {split_path})"
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
# Read image names from split file
|
|
292
|
+
image_names = set()
|
|
293
|
+
with split_path.open("r", encoding="utf-8") as f:
|
|
294
|
+
for line in f:
|
|
295
|
+
line = line.strip()
|
|
296
|
+
if line:
|
|
297
|
+
# Remove extension if present
|
|
298
|
+
name = Path(line).stem
|
|
299
|
+
image_names.add(name)
|
|
300
|
+
|
|
301
|
+
# Find corresponding annotation files
|
|
302
|
+
annotation_files = []
|
|
303
|
+
for img_name in image_names:
|
|
304
|
+
# Try different annotation file naming conventions
|
|
305
|
+
for suffix in [f"_{self.split}.txt", ".txt"]:
|
|
306
|
+
ann_file = self.label_dir / f"{img_name}{suffix}"
|
|
307
|
+
if ann_file.exists():
|
|
308
|
+
annotation_files.append(ann_file)
|
|
309
|
+
break
|
|
310
|
+
|
|
311
|
+
return sorted(annotation_files)
|
|
312
|
+
else:
|
|
313
|
+
# Mode 2: Pattern matching (backward compatible)
|
|
314
|
+
# First try pattern with split suffix (e.g., "*_train.txt")
|
|
315
|
+
pattern = re.compile(rf".*_{self.split}\.txt$", re.IGNORECASE)
|
|
316
|
+
files = [f for f in self.label_dir.iterdir() if f.is_file() and pattern.match(f.name)]
|
|
317
|
+
|
|
318
|
+
# If no files found with split suffix, fall back to all .txt files
|
|
319
|
+
# This supports cases where annotation files have the same name as images
|
|
320
|
+
if not files:
|
|
321
|
+
files = [f for f in self.label_dir.iterdir() if f.is_file() and f.suffix.lower() == ".txt"]
|
|
322
|
+
|
|
323
|
+
return sorted(files)
|
|
324
|
+
|
|
325
|
+
def _load_image_size(self, image_path: Path) -> Tuple[int, int]:
|
|
326
|
+
"""Load image dimensions efficiently."""
|
|
327
|
+
if Image is None:
|
|
328
|
+
raise RuntimeError("PIL/Pillow is required to load image dimensions.")
|
|
329
|
+
with Image.open(image_path) as img:
|
|
330
|
+
return img.size # Returns (width, height)
|
|
331
|
+
|
|
332
|
+
def _parse_annotation_lines(self, ann_path: Path) -> Tuple[DOTAAnnotation, ...]:
|
|
333
|
+
"""Parse object lines from a DOTA label file (no image I/O)."""
|
|
334
|
+
annotations: List[DOTAAnnotation] = []
|
|
335
|
+
with ann_path.open("r", encoding="utf-8") as f:
|
|
336
|
+
for line in f:
|
|
337
|
+
line = line.strip()
|
|
338
|
+
if not line or line.startswith("imagesource") or line.startswith("gsd"):
|
|
339
|
+
continue
|
|
340
|
+
try:
|
|
341
|
+
annotations.append(
|
|
342
|
+
DOTAAnnotation.from_line(line, class_map=self.class_map)
|
|
343
|
+
)
|
|
344
|
+
except (ValueError, Exception):
|
|
345
|
+
continue
|
|
346
|
+
return tuple(annotations)
|
|
347
|
+
|
|
348
|
+
def _effective_gt_count(self, ann_path: Path) -> int:
|
|
349
|
+
"""GT count after the same filters applied when loading a sample."""
|
|
350
|
+
annotations = self._parse_annotation_lines(ann_path)
|
|
351
|
+
if not annotations:
|
|
352
|
+
return 0
|
|
353
|
+
if (
|
|
354
|
+
self.allowed_classes is not None
|
|
355
|
+
or self.ignore_labels is not None
|
|
356
|
+
or self._drop_difficult
|
|
357
|
+
):
|
|
358
|
+
sample = DOTASample(
|
|
359
|
+
image_path=ann_path,
|
|
360
|
+
width=0,
|
|
361
|
+
height=0,
|
|
362
|
+
annotations=annotations,
|
|
363
|
+
)
|
|
364
|
+
sample = sample.filter_by_class(
|
|
365
|
+
allowed_classes=self.allowed_classes,
|
|
366
|
+
ignore_labels=self.ignore_labels,
|
|
367
|
+
drop_difficult=self._drop_difficult,
|
|
368
|
+
)
|
|
369
|
+
return len(sample.annotations)
|
|
370
|
+
return len(annotations)
|
|
371
|
+
|
|
372
|
+
def _parse_annotation_file(self, ann_path: Path) -> Optional[DOTASample]:
|
|
373
|
+
"""Parse a single DOTA annotation file.
|
|
374
|
+
|
|
375
|
+
Returns:
|
|
376
|
+
DOTASample if image exists, None if image is missing (with warning printed)
|
|
377
|
+
"""
|
|
378
|
+
image_name = ann_path.stem + ".png"
|
|
379
|
+
image_path = self.image_dir / image_name
|
|
380
|
+
|
|
381
|
+
if not image_path.exists():
|
|
382
|
+
image_name = ann_path.stem + ".jpg"
|
|
383
|
+
image_path = self.image_dir / image_name
|
|
384
|
+
|
|
385
|
+
if not image_path.exists():
|
|
386
|
+
warnings.warn(f"Image not found for annotation: {ann_path}. Skipping this sample.", UserWarning)
|
|
387
|
+
return None
|
|
388
|
+
|
|
389
|
+
width, height = self._load_image_size(image_path)
|
|
390
|
+
annotations = self._parse_annotation_lines(ann_path)
|
|
391
|
+
|
|
392
|
+
sample = DOTASample(
|
|
393
|
+
image_path=image_path,
|
|
394
|
+
width=width,
|
|
395
|
+
height=height,
|
|
396
|
+
annotations=annotations,
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
if self.allowed_classes is not None or self.ignore_labels is not None or self._drop_difficult:
|
|
400
|
+
sample = sample.filter_by_class(
|
|
401
|
+
allowed_classes=self.allowed_classes,
|
|
402
|
+
ignore_labels=self.ignore_labels,
|
|
403
|
+
drop_difficult=self._drop_difficult,
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
return sample
|
|
407
|
+
|
|
408
|
+
def __len__(self) -> int:
|
|
409
|
+
return len(self._annotation_files)
|
|
410
|
+
|
|
411
|
+
def __getitem__(self, idx: int) -> DOTASample:
|
|
412
|
+
if idx < 0 or idx >= len(self._annotation_files):
|
|
413
|
+
raise IndexError(f"Index {idx} out of range for dataset of size {len(self)}")
|
|
414
|
+
ann_path = self._annotation_files[idx]
|
|
415
|
+
sample = self._parse_annotation_file(ann_path)
|
|
416
|
+
if sample is None:
|
|
417
|
+
# Image was missing (warning already printed)
|
|
418
|
+
# Raise error for indexed access since we can't return the requested item
|
|
419
|
+
raise FileNotFoundError(
|
|
420
|
+
f"Image not found for annotation at index {idx}: {ann_path}. "
|
|
421
|
+
f"Use iteration (for sample in dataset) to automatically skip missing images."
|
|
422
|
+
)
|
|
423
|
+
return sample
|
|
424
|
+
|
|
425
|
+
def __iter__(self) -> Iterator[DOTASample]:
|
|
426
|
+
for ann_path in self._annotation_files:
|
|
427
|
+
sample = self._parse_annotation_file(ann_path)
|
|
428
|
+
if sample is not None:
|
|
429
|
+
yield sample
|
|
430
|
+
|
|
431
|
+
def get_class_names(self) -> List[str]:
|
|
432
|
+
"""Extract unique class names from all annotations."""
|
|
433
|
+
classes = set()
|
|
434
|
+
for sample in self:
|
|
435
|
+
for ann in sample.annotations:
|
|
436
|
+
classes.add(ann.class_name)
|
|
437
|
+
return sorted(classes)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def resolve_dota_tile_roots(
|
|
441
|
+
*,
|
|
442
|
+
tiles_dirs: Optional[Sequence[str | Path]] = None,
|
|
443
|
+
tiles_dir: Optional[str | Path] = None,
|
|
444
|
+
split_label: str = "split",
|
|
445
|
+
) -> List[Path]:
|
|
446
|
+
"""Resolve one or more DOTA tile roots from singular/plural config fields."""
|
|
447
|
+
if tiles_dirs is not None:
|
|
448
|
+
roots = [Path(p) for p in tiles_dirs]
|
|
449
|
+
if not roots:
|
|
450
|
+
raise ValueError(f"dataset.*_tiles_dirs for {split_label} must be non-empty when set")
|
|
451
|
+
return roots
|
|
452
|
+
if tiles_dir is not None:
|
|
453
|
+
return [Path(tiles_dir)]
|
|
454
|
+
raise ValueError(
|
|
455
|
+
f"DOTA {split_label} requires dataset.*_tiles_dir or dataset.*_tiles_dirs"
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _dota_dirs_for_root(root: Path, *, same_folder: bool) -> tuple[Path, Path, Path]:
|
|
460
|
+
root = Path(root)
|
|
461
|
+
if same_folder:
|
|
462
|
+
return root, root, root
|
|
463
|
+
return root, root / "labels", root / "images"
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def iter_dota_datasets(dataset) -> Iterator["DOTADataset"]:
|
|
467
|
+
"""Yield ``DOTADataset`` instances from a dataset or ``ConcatDataset``."""
|
|
468
|
+
from torch.utils.data import ConcatDataset
|
|
469
|
+
|
|
470
|
+
if isinstance(dataset, ConcatDataset):
|
|
471
|
+
for sub in dataset.datasets:
|
|
472
|
+
yield from iter_dota_datasets(sub)
|
|
473
|
+
elif isinstance(dataset, DOTADataset):
|
|
474
|
+
yield dataset
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def dota_empty_gt_filter_summary(dataset) -> Tuple[int, int, int]:
|
|
478
|
+
"""Return (discovered, kept, filtered) tile counts for logging."""
|
|
479
|
+
discovered = kept = filtered = 0
|
|
480
|
+
for ds in iter_dota_datasets(dataset):
|
|
481
|
+
discovered += ds.annotation_files_discovered_count
|
|
482
|
+
kept += len(ds)
|
|
483
|
+
filtered += ds.empty_gt_filtered_count
|
|
484
|
+
return discovered, kept, filtered
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def format_dota_empty_gt_filter_log(dataset, *, split: str) -> str:
|
|
488
|
+
"""One-line summary for train logs when ``filter_empty_gt`` is enabled."""
|
|
489
|
+
discovered, kept, filtered = dota_empty_gt_filter_summary(dataset)
|
|
490
|
+
return (
|
|
491
|
+
f" {split}: filter_empty_gt dropped {filtered} / {discovered} tiles "
|
|
492
|
+
f"({kept} kept)"
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def build_dota_split_dataset(
|
|
497
|
+
tile_roots: Sequence[str | Path],
|
|
498
|
+
*,
|
|
499
|
+
split: str,
|
|
500
|
+
same_folder: bool = False,
|
|
501
|
+
difficult_strategy: str = "drop",
|
|
502
|
+
allowed_classes: Optional[Sequence[str]] = None,
|
|
503
|
+
ignore_labels: Optional[Sequence[str]] = None,
|
|
504
|
+
filter_empty_gt: bool = False,
|
|
505
|
+
):
|
|
506
|
+
"""Build one ``DOTADataset`` or ``ConcatDataset`` over multiple tile roots."""
|
|
507
|
+
from torch.utils.data import ConcatDataset
|
|
508
|
+
|
|
509
|
+
roots = [Path(r) for r in tile_roots]
|
|
510
|
+
if not roots:
|
|
511
|
+
raise ValueError("tile_roots must be non-empty")
|
|
512
|
+
|
|
513
|
+
datasets = [
|
|
514
|
+
DOTADataset(
|
|
515
|
+
root_dir=root,
|
|
516
|
+
split=split,
|
|
517
|
+
label_dir=label_dir,
|
|
518
|
+
image_dir=image_dir,
|
|
519
|
+
difficult_strategy=difficult_strategy,
|
|
520
|
+
allowed_classes=allowed_classes,
|
|
521
|
+
ignore_labels=ignore_labels,
|
|
522
|
+
filter_empty_gt=filter_empty_gt,
|
|
523
|
+
)
|
|
524
|
+
for root, label_dir, image_dir in (
|
|
525
|
+
_dota_dirs_for_root(r, same_folder=same_folder) for r in roots
|
|
526
|
+
)
|
|
527
|
+
]
|
|
528
|
+
if len(datasets) == 1:
|
|
529
|
+
return datasets[0]
|
|
530
|
+
return ConcatDataset(datasets)
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def collect_dota_image_paths(
|
|
534
|
+
tile_roots: Sequence[str | Path],
|
|
535
|
+
*,
|
|
536
|
+
same_folder: bool = False,
|
|
537
|
+
) -> List[Path]:
|
|
538
|
+
"""Collect image paths from one or more DOTA tile directories."""
|
|
539
|
+
paths: List[Path] = []
|
|
540
|
+
for root in tile_roots:
|
|
541
|
+
_, _, image_dir = _dota_dirs_for_root(Path(root), same_folder=same_folder)
|
|
542
|
+
if not image_dir.exists():
|
|
543
|
+
raise FileNotFoundError(f"DOTA image directory not found: {image_dir}")
|
|
544
|
+
paths.extend(sorted(image_dir.glob("*.jpg")))
|
|
545
|
+
paths.extend(sorted(image_dir.glob("*.png")))
|
|
546
|
+
return paths
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def _image_path_for_annotation(ann_path: Path, image_dir: Path) -> Optional[Path]:
|
|
550
|
+
"""Resolve the image file for a DOTA label path (png preferred, then jpg)."""
|
|
551
|
+
for ext in (".png", ".jpg"):
|
|
552
|
+
candidate = image_dir / f"{ann_path.stem}{ext}"
|
|
553
|
+
if candidate.exists():
|
|
554
|
+
return candidate
|
|
555
|
+
return None
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def collect_dota_split_image_paths(
|
|
559
|
+
tile_roots: Sequence[str | Path],
|
|
560
|
+
*,
|
|
561
|
+
split: str,
|
|
562
|
+
same_folder: bool = False,
|
|
563
|
+
difficult_strategy: str = "drop",
|
|
564
|
+
allowed_classes: Optional[Sequence[str]] = None,
|
|
565
|
+
ignore_labels: Optional[Sequence[str]] = None,
|
|
566
|
+
filter_empty_gt: bool = False,
|
|
567
|
+
log_filter_empty_gt: bool = False,
|
|
568
|
+
) -> List[Path]:
|
|
569
|
+
"""Collect image paths using the same tile set as :func:`build_dota_split_dataset`.
|
|
570
|
+
|
|
571
|
+
When ``filter_empty_gt`` is True, skips tiles with no effective GT after
|
|
572
|
+
``difficult_strategy``, ``allowed_classes``, and ``ignore_labels`` (training parity).
|
|
573
|
+
Set ``log_filter_empty_gt`` to print the same summary as training startup.
|
|
574
|
+
"""
|
|
575
|
+
dataset = build_dota_split_dataset(
|
|
576
|
+
tile_roots,
|
|
577
|
+
split=split,
|
|
578
|
+
same_folder=same_folder,
|
|
579
|
+
difficult_strategy=difficult_strategy,
|
|
580
|
+
allowed_classes=allowed_classes,
|
|
581
|
+
ignore_labels=ignore_labels,
|
|
582
|
+
filter_empty_gt=filter_empty_gt,
|
|
583
|
+
)
|
|
584
|
+
if log_filter_empty_gt and filter_empty_gt:
|
|
585
|
+
print("DOTA filter_empty_gt (preds/metrics, same as training):")
|
|
586
|
+
print(format_dota_empty_gt_filter_log(dataset, split=split))
|
|
587
|
+
paths: List[Path] = []
|
|
588
|
+
for ds in iter_dota_datasets(dataset):
|
|
589
|
+
for ann_path in ds._annotation_files:
|
|
590
|
+
image_path = _image_path_for_annotation(ann_path, ds.image_dir)
|
|
591
|
+
if image_path is not None:
|
|
592
|
+
paths.append(image_path)
|
|
593
|
+
return sorted(paths)
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def dota_label_path_for_image(image_path: Path, *, same_folder: bool = False) -> Path:
|
|
597
|
+
"""Resolve the DOTA annotation ``.txt`` path for an image file."""
|
|
598
|
+
image_path = Path(image_path)
|
|
599
|
+
if same_folder:
|
|
600
|
+
return image_path.with_suffix(".txt")
|
|
601
|
+
parent = image_path.parent
|
|
602
|
+
if parent.name == "images":
|
|
603
|
+
root = parent.parent
|
|
604
|
+
label_dir = root / "labels"
|
|
605
|
+
if not label_dir.exists():
|
|
606
|
+
label_dir = root / "labelTxt"
|
|
607
|
+
return label_dir / f"{image_path.stem}.txt"
|
|
608
|
+
label_dir = parent / "labels"
|
|
609
|
+
if label_dir.exists():
|
|
610
|
+
return label_dir / f"{image_path.stem}.txt"
|
|
611
|
+
label_txt = parent / "labelTxt"
|
|
612
|
+
if label_txt.exists():
|
|
613
|
+
return label_txt / f"{image_path.stem}.txt"
|
|
614
|
+
return parent / f"{image_path.stem}.txt"
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def dota_dataset_class_names(dataset) -> List[str]:
|
|
618
|
+
"""Union class names from ``DOTADataset``, ``ConcatDataset``, or ``Subset``."""
|
|
619
|
+
from torch.utils.data import ConcatDataset, Subset
|
|
620
|
+
|
|
621
|
+
if isinstance(dataset, Subset):
|
|
622
|
+
return dota_dataset_class_names(dataset.dataset)
|
|
623
|
+
if isinstance(dataset, ConcatDataset):
|
|
624
|
+
seen: set[str] = set()
|
|
625
|
+
names: List[str] = []
|
|
626
|
+
for sub in dataset.datasets:
|
|
627
|
+
for name in dota_dataset_class_names(sub):
|
|
628
|
+
if name not in seen:
|
|
629
|
+
seen.add(name)
|
|
630
|
+
names.append(name)
|
|
631
|
+
return sorted(names)
|
|
632
|
+
if isinstance(dataset, DOTADataset):
|
|
633
|
+
return dataset.get_class_names()
|
|
634
|
+
raise TypeError(f"Unsupported dataset type for class discovery: {type(dataset)!r}")
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def build_dota_loader(
|
|
638
|
+
root_dir: str | Path,
|
|
639
|
+
*,
|
|
640
|
+
split: str = "train",
|
|
641
|
+
split_file: Optional[str | Path] = None,
|
|
642
|
+
label_dir: Optional[str | Path] = None,
|
|
643
|
+
image_dir: Optional[str | Path] = None,
|
|
644
|
+
batch_size: int = 1,
|
|
645
|
+
shuffle: bool = False,
|
|
646
|
+
num_workers: int = 0,
|
|
647
|
+
class_map: Optional[Dict[str, int]] = None,
|
|
648
|
+
allowed_classes: Optional[Sequence[str]] = None,
|
|
649
|
+
ignore_labels: Optional[Sequence[str]] = None,
|
|
650
|
+
difficult_strategy: str = "drop",
|
|
651
|
+
filter_empty_gt: bool = False,
|
|
652
|
+
collate_fn: Optional[Callable] = None,
|
|
653
|
+
) -> "DataLoader":
|
|
654
|
+
"""Build a PyTorch DataLoader for DOTA dataset.
|
|
655
|
+
|
|
656
|
+
Args:
|
|
657
|
+
root_dir: Root directory of DOTA dataset (used as base if label_dir/image_dir not specified)
|
|
658
|
+
split: Split name ("train", "val", "test")
|
|
659
|
+
split_file: Optional path to a file listing image names (one per line).
|
|
660
|
+
If provided, only images listed in this file will be used.
|
|
661
|
+
This follows the official DOTA dataset convention.
|
|
662
|
+
label_dir: Optional custom path to label directory. If None, uses root_dir/labelTxt.
|
|
663
|
+
This allows having train/val/test in separate folders.
|
|
664
|
+
image_dir: Optional custom path to image directory. If None, uses root_dir/images.
|
|
665
|
+
This allows having train/val/test in separate folders.
|
|
666
|
+
batch_size: Batch size for DataLoader
|
|
667
|
+
shuffle: Whether to shuffle the dataset
|
|
668
|
+
num_workers: Number of worker processes for data loading
|
|
669
|
+
class_map: Optional mapping from class names to numeric IDs
|
|
670
|
+
allowed_classes: Optional list of allowed class names (whitelist)
|
|
671
|
+
ignore_labels: Optional list of class names to exclude (blacklist)
|
|
672
|
+
difficult_strategy: drop | ignore | keep (same as DOTADataset)
|
|
673
|
+
filter_empty_gt: Drop tiles with no effective GT (same as DOTADataset)
|
|
674
|
+
collate_fn: Optional custom collate function. If None, uses collate_dota_samples
|
|
675
|
+
from oriented_det.train.utils which converts DOTASample objects
|
|
676
|
+
to (images, targets) format expected by training engines.
|
|
677
|
+
|
|
678
|
+
Returns:
|
|
679
|
+
DataLoader that yields batches in format determined by collate_fn.
|
|
680
|
+
Default collate_fn returns (images, targets) tuple for training.
|
|
681
|
+
"""
|
|
682
|
+
try:
|
|
683
|
+
from torch.utils.data import DataLoader
|
|
684
|
+
except ImportError:
|
|
685
|
+
raise RuntimeError("PyTorch is required to build DataLoader.")
|
|
686
|
+
|
|
687
|
+
dataset = DOTADataset(
|
|
688
|
+
root_dir=root_dir,
|
|
689
|
+
split=split,
|
|
690
|
+
split_file=split_file,
|
|
691
|
+
label_dir=label_dir,
|
|
692
|
+
image_dir=image_dir,
|
|
693
|
+
class_map=class_map,
|
|
694
|
+
allowed_classes=allowed_classes,
|
|
695
|
+
ignore_labels=ignore_labels,
|
|
696
|
+
difficult_strategy=difficult_strategy,
|
|
697
|
+
filter_empty_gt=filter_empty_gt,
|
|
698
|
+
)
|
|
699
|
+
|
|
700
|
+
# Use provided collate_fn or default to collate_dota_samples
|
|
701
|
+
if collate_fn is None:
|
|
702
|
+
# Lazy import to avoid circular dependencies
|
|
703
|
+
try:
|
|
704
|
+
from ..train.utils import collate_dota_samples
|
|
705
|
+
collate_fn = collate_dota_samples
|
|
706
|
+
except ImportError:
|
|
707
|
+
# Fallback if train module not available
|
|
708
|
+
collate_fn = _default_collate_fn
|
|
709
|
+
|
|
710
|
+
return DataLoader(
|
|
711
|
+
dataset,
|
|
712
|
+
batch_size=batch_size,
|
|
713
|
+
shuffle=shuffle,
|
|
714
|
+
num_workers=num_workers,
|
|
715
|
+
collate_fn=collate_fn,
|
|
716
|
+
)
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
def _default_collate_fn(batch):
|
|
720
|
+
"""Default collate function that returns the batch as-is.
|
|
721
|
+
|
|
722
|
+
This is a module-level function (not a lambda) so it can be pickled
|
|
723
|
+
when using multiprocessing with num_workers > 0.
|
|
724
|
+
Note: This returns a list, not the (images, targets) tuple expected by training.
|
|
725
|
+
Use collate_dota_samples for training.
|
|
726
|
+
"""
|
|
727
|
+
return batch
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
__all__ = [
|
|
731
|
+
"DOTAAnnotation",
|
|
732
|
+
"DOTASample",
|
|
733
|
+
"DOTADataset",
|
|
734
|
+
"build_dota_loader",
|
|
735
|
+
"build_dota_split_dataset",
|
|
736
|
+
"collect_dota_image_paths",
|
|
737
|
+
"collect_dota_split_image_paths",
|
|
738
|
+
"dota_dataset_class_names",
|
|
739
|
+
"dota_label_path_for_image",
|
|
740
|
+
"resolve_dota_tile_roots",
|
|
741
|
+
]
|