easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Label inspection and bulk relabelling across Pascal VOC files."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import xml.etree.ElementTree as ET
|
|
6
|
+
from collections import Counter
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from .. import fs
|
|
11
|
+
from ..annotations import voc
|
|
12
|
+
from ..annotations.model import AnnotationError
|
|
13
|
+
from ..registry import Param, operation
|
|
14
|
+
from ..types import Action, Context, OpResult
|
|
15
|
+
|
|
16
|
+
LIST_PARAMS = (
|
|
17
|
+
Param("folder", "dir", "Folder of label files to summarise.", required=True, must_exist=True),
|
|
18
|
+
Param("recursive", "bool", "Include sub-folders.", default=True),
|
|
19
|
+
Param("label_ext", "str", "Label file extension.", default=".xml"),
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@operation(
|
|
24
|
+
"labels.list",
|
|
25
|
+
title="Summarise labels",
|
|
26
|
+
category="labels",
|
|
27
|
+
summary="Count classes and objects, flag unreadable files, and surface case-variant classes.",
|
|
28
|
+
params=LIST_PARAMS,
|
|
29
|
+
notes=(
|
|
30
|
+
"Read this before any export. The case_variants section is the one that\n"
|
|
31
|
+
"bites: labels that differ only in capitalisation become separate classes\n"
|
|
32
|
+
"in COCO and silently halve your per-class training data."
|
|
33
|
+
),
|
|
34
|
+
)
|
|
35
|
+
def list_labels(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
36
|
+
result = OpResult(op="labels.list", dry_run=ctx.dry_run)
|
|
37
|
+
folder: Path = params["folder"]
|
|
38
|
+
files = list(fs.iter_files(folder, recursive=params["recursive"], exts=[params["label_ext"]]))
|
|
39
|
+
|
|
40
|
+
counter: Counter[str] = Counter()
|
|
41
|
+
images_per_label: Counter[str] = Counter()
|
|
42
|
+
corrupt: list[dict[str, str]] = []
|
|
43
|
+
backgrounds = 0
|
|
44
|
+
polygon_objects = 0
|
|
45
|
+
|
|
46
|
+
for position, path in enumerate(files, start=1):
|
|
47
|
+
ctx.check_cancelled()
|
|
48
|
+
ctx.emit(position, len(files), path.name)
|
|
49
|
+
try:
|
|
50
|
+
annotation = voc.read(path)
|
|
51
|
+
except AnnotationError as exc:
|
|
52
|
+
corrupt.append({"file": str(path), "error": str(exc)})
|
|
53
|
+
continue
|
|
54
|
+
|
|
55
|
+
if annotation.is_background:
|
|
56
|
+
backgrounds += 1
|
|
57
|
+
for shape in annotation.shapes:
|
|
58
|
+
counter[shape.label] += 1
|
|
59
|
+
if shape.has_polygon:
|
|
60
|
+
polygon_objects += 1
|
|
61
|
+
for label in annotation.label_set:
|
|
62
|
+
images_per_label[label] += 1
|
|
63
|
+
|
|
64
|
+
# Group spellings that differ only by case, which is what breaks exports.
|
|
65
|
+
folded: dict[str, list[str]] = {}
|
|
66
|
+
for label in counter:
|
|
67
|
+
folded.setdefault(label.strip().casefold(), []).append(label)
|
|
68
|
+
variants = {key: sorted(names) for key, names in folded.items() if len(names) > 1}
|
|
69
|
+
|
|
70
|
+
result.counts.update(
|
|
71
|
+
{
|
|
72
|
+
"label_files": len(files),
|
|
73
|
+
"readable": len(files) - len(corrupt),
|
|
74
|
+
"corrupt": len(corrupt),
|
|
75
|
+
"objects": sum(counter.values()),
|
|
76
|
+
"classes": len(counter),
|
|
77
|
+
"background_images": backgrounds,
|
|
78
|
+
"polygon_objects": polygon_objects,
|
|
79
|
+
"bbox_only_objects": sum(counter.values()) - polygon_objects,
|
|
80
|
+
}
|
|
81
|
+
)
|
|
82
|
+
result.data["per_label"] = dict(sorted(counter.items()))
|
|
83
|
+
result.data["images_per_label"] = dict(sorted(images_per_label.items()))
|
|
84
|
+
result.data["case_variants"] = variants
|
|
85
|
+
result.data["corrupt_files"] = corrupt
|
|
86
|
+
if variants:
|
|
87
|
+
result.warnings.append(
|
|
88
|
+
"labels differing only by case: "
|
|
89
|
+
+ "; ".join(" / ".join(v) for v in variants.values())
|
|
90
|
+
)
|
|
91
|
+
return result
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
RENAME_PARAMS = (
|
|
95
|
+
Param("folder", "dir", "Folder of label files to edit.", required=True, must_exist=True),
|
|
96
|
+
Param("map", "mapping", "Renames as old=new pairs, e.g. planter_View=Plantar_View."),
|
|
97
|
+
Param("fold_case", "bool",
|
|
98
|
+
"Also unify labels that differ only by case onto their most common spelling.",
|
|
99
|
+
default=False),
|
|
100
|
+
Param("drop", "strs", "Class names to remove entirely from the labels."),
|
|
101
|
+
Param("recursive", "bool", "Include sub-folders.", default=True),
|
|
102
|
+
Param("label_ext", "str", "Label file extension.", default=".xml"),
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@operation(
|
|
107
|
+
"labels.rename",
|
|
108
|
+
title="Rename or drop label classes",
|
|
109
|
+
category="labels",
|
|
110
|
+
summary="Apply a class rename map across every label file, optionally unifying case variants.",
|
|
111
|
+
params=RENAME_PARAMS,
|
|
112
|
+
destructive=True,
|
|
113
|
+
notes=(
|
|
114
|
+
"fold_case fixes case drift at the source rather than papering over it at\n"
|
|
115
|
+
"export time: every spelling collapses onto whichever is most common, so\n"
|
|
116
|
+
"Open/open become one class in the XML itself.\n\n"
|
|
117
|
+
"drop removes objects of a class outright. An image left with no objects\n"
|
|
118
|
+
"stays as a valid background/hard-negative label rather than being deleted."
|
|
119
|
+
),
|
|
120
|
+
)
|
|
121
|
+
def rename_labels(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
122
|
+
result = OpResult(op="labels.rename", dry_run=ctx.dry_run)
|
|
123
|
+
folder: Path = params["folder"]
|
|
124
|
+
files = list(fs.iter_files(folder, recursive=params["recursive"], exts=[params["label_ext"]]))
|
|
125
|
+
mapping: dict[str, str] = dict(params["map"] or {})
|
|
126
|
+
drop = {d.strip() for d in (params["drop"] or []) if d.strip()}
|
|
127
|
+
|
|
128
|
+
if params["fold_case"]:
|
|
129
|
+
mapping = {**_case_folding_map(files), **mapping}
|
|
130
|
+
result.data["case_folding"] = {
|
|
131
|
+
old: new for old, new in mapping.items() if old.casefold() == new.casefold()
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if not mapping and not drop:
|
|
135
|
+
# Seed the counters even on the do-nothing path, so a consumer reading
|
|
136
|
+
# the JSON sees "0 changed" rather than an empty object it has to guess
|
|
137
|
+
# the meaning of -- the same contract every other operation keeps.
|
|
138
|
+
result.counts.update(
|
|
139
|
+
{"label_files": len(files), "files_changed": 0,
|
|
140
|
+
"objects_renamed": 0, "objects_dropped": 0}
|
|
141
|
+
)
|
|
142
|
+
result.warnings.append(
|
|
143
|
+
"every class name is already consistent, so nothing was changed"
|
|
144
|
+
if params["fold_case"]
|
|
145
|
+
else "nothing to do: supply --map, --drop, or --fold-case"
|
|
146
|
+
)
|
|
147
|
+
return result
|
|
148
|
+
|
|
149
|
+
renamed = Counter[str]()
|
|
150
|
+
dropped = Counter[str]()
|
|
151
|
+
|
|
152
|
+
for position, path in enumerate(files, start=1):
|
|
153
|
+
ctx.check_cancelled()
|
|
154
|
+
ctx.emit(position, len(files), path.name)
|
|
155
|
+
try:
|
|
156
|
+
tree = ET.parse(path)
|
|
157
|
+
except (ET.ParseError, OSError) as exc:
|
|
158
|
+
result.warnings.append(f"{path.name}: unreadable ({exc})")
|
|
159
|
+
result.bump("unreadable")
|
|
160
|
+
continue
|
|
161
|
+
|
|
162
|
+
root = tree.getroot()
|
|
163
|
+
changed = False
|
|
164
|
+
for obj in list(root.findall("object")):
|
|
165
|
+
node = obj.find("name")
|
|
166
|
+
if node is None or node.text is None:
|
|
167
|
+
continue
|
|
168
|
+
current = node.text.strip()
|
|
169
|
+
if current in drop:
|
|
170
|
+
root.remove(obj)
|
|
171
|
+
dropped[current] += 1
|
|
172
|
+
changed = True
|
|
173
|
+
continue
|
|
174
|
+
replacement = mapping.get(current)
|
|
175
|
+
if replacement and replacement != current:
|
|
176
|
+
node.text = replacement
|
|
177
|
+
renamed[f"{current} -> {replacement}"] += 1
|
|
178
|
+
changed = True
|
|
179
|
+
|
|
180
|
+
if changed:
|
|
181
|
+
if not ctx.dry_run:
|
|
182
|
+
tree.write(path, encoding="utf-8")
|
|
183
|
+
result.record(Action.WRITE, path, path, "labels updated")
|
|
184
|
+
result.bump("files_changed")
|
|
185
|
+
|
|
186
|
+
result.counts["label_files"] = len(files)
|
|
187
|
+
result.counts["objects_renamed"] = sum(renamed.values())
|
|
188
|
+
result.counts["objects_dropped"] = sum(dropped.values())
|
|
189
|
+
result.data["renamed"] = dict(renamed)
|
|
190
|
+
result.data["dropped"] = dict(dropped)
|
|
191
|
+
return result
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _case_folding_map(files: list[Path]) -> dict[str, str]:
|
|
195
|
+
"""Map every case-variant spelling onto the most common one."""
|
|
196
|
+
spellings: dict[str, Counter[str]] = {}
|
|
197
|
+
for path in files:
|
|
198
|
+
try:
|
|
199
|
+
annotation = voc.read(path)
|
|
200
|
+
except AnnotationError:
|
|
201
|
+
continue
|
|
202
|
+
for shape in annotation.shapes:
|
|
203
|
+
spellings.setdefault(shape.label.casefold(), Counter())[shape.label] += 1
|
|
204
|
+
|
|
205
|
+
mapping: dict[str, str] = {}
|
|
206
|
+
for variants in spellings.values():
|
|
207
|
+
if len(variants) < 2:
|
|
208
|
+
continue
|
|
209
|
+
# Most frequent spelling wins; alphabetical order breaks ties so the
|
|
210
|
+
# result does not depend on directory traversal order.
|
|
211
|
+
winner = sorted(variants.items(), key=lambda kv: (-kv[1], kv[0]))[0][0]
|
|
212
|
+
for spelling in variants:
|
|
213
|
+
if spelling != winner:
|
|
214
|
+
mapping[spelling] = winner
|
|
215
|
+
return mapping
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Orphan handling: images without labels, labels without images.
|
|
2
|
+
|
|
3
|
+
Three original scripts covered slices of this -- one deleted both kinds, one
|
|
4
|
+
quarantined both, one moved just the unannotated images to a "to annotate"
|
|
5
|
+
folder. They are the same operation with different ``scope`` and ``action``, so
|
|
6
|
+
they are one operation here.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from .. import fs
|
|
15
|
+
from ..pairs import PairIndex
|
|
16
|
+
from ..registry import Param, operation
|
|
17
|
+
from ..types import Action, Context, OpResult
|
|
18
|
+
|
|
19
|
+
PARAMS = (
|
|
20
|
+
Param("folder", "dir", "Folder to inspect.", required=True, must_exist=True),
|
|
21
|
+
Param("scope", "choice", "Which orphans to act on.",
|
|
22
|
+
default="both", choices=("both", "images", "labels")),
|
|
23
|
+
Param("action", "choice", "What to do with them.",
|
|
24
|
+
default="move", choices=("move", "delete", "report")),
|
|
25
|
+
Param("target", "dir", "Destination for moved orphans. Defaults to <folder>/_orphans."),
|
|
26
|
+
Param("recursive", "bool", "Include sub-folders.", default=False),
|
|
27
|
+
Param("label_ext", "str", "Sidecar label extension.", default=".xml"),
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@operation(
|
|
32
|
+
"clean.orphans",
|
|
33
|
+
title="Handle unpaired images and labels",
|
|
34
|
+
category="clean",
|
|
35
|
+
summary="Quarantine, delete, or just report images with no label and labels with no image.",
|
|
36
|
+
params=PARAMS,
|
|
37
|
+
destructive=True,
|
|
38
|
+
safe_when=lambda p: p.get("action") == "report",
|
|
39
|
+
notes=(
|
|
40
|
+
"An image with no label is not necessarily junk -- it is usually just work\n"
|
|
41
|
+
"not done yet. scope=images with action=move is the 'set aside what still\n"
|
|
42
|
+
"needs annotating' workflow.\n\n"
|
|
43
|
+
"A label with no image, by contrast, is always dead weight.\n\n"
|
|
44
|
+
"action=report changes nothing and is the safe way to see the damage first."
|
|
45
|
+
),
|
|
46
|
+
)
|
|
47
|
+
def orphans(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
48
|
+
result = OpResult(op="clean.orphans", dry_run=ctx.dry_run)
|
|
49
|
+
folder: Path = params["folder"]
|
|
50
|
+
action: str = params["action"]
|
|
51
|
+
scope: str = params["scope"]
|
|
52
|
+
target: Path = params["target"] or (folder / "_orphans")
|
|
53
|
+
|
|
54
|
+
index = PairIndex.scan(folder, recursive=params["recursive"], label_ext=params["label_ext"])
|
|
55
|
+
result.counts.update(index.summary())
|
|
56
|
+
# Present even at zero, so the shape of the result never varies.
|
|
57
|
+
result.counts["images_without_labels_handled"] = 0
|
|
58
|
+
result.counts["labels_without_images_handled"] = 0
|
|
59
|
+
|
|
60
|
+
jobs: list[tuple[Path, str]] = []
|
|
61
|
+
if scope in ("both", "images"):
|
|
62
|
+
jobs += [(p.image, "images_without_labels") for p in index.image_only() if p.image]
|
|
63
|
+
if scope in ("both", "labels"):
|
|
64
|
+
jobs += [(p.label, "labels_without_images") for p in index.label_only() if p.label]
|
|
65
|
+
|
|
66
|
+
taken: set[Path] = set()
|
|
67
|
+
for position, (path, bucket) in enumerate(sorted(jobs), start=1):
|
|
68
|
+
ctx.check_cancelled()
|
|
69
|
+
ctx.emit(position, len(jobs), f"orphan {path.name}")
|
|
70
|
+
|
|
71
|
+
if action == "report":
|
|
72
|
+
result.record(Action.SKIP, path, None, bucket)
|
|
73
|
+
elif action == "delete":
|
|
74
|
+
file_action = result.record(Action.DELETE, path, None, bucket)
|
|
75
|
+
fs.perform(file_action, ctx)
|
|
76
|
+
else:
|
|
77
|
+
relative = fs.relative_to_safe(path, folder)
|
|
78
|
+
destination = fs.unique_path(target / bucket / relative, taken=taken)
|
|
79
|
+
taken.add(destination)
|
|
80
|
+
file_action = result.record(Action.MOVE, path, destination, bucket)
|
|
81
|
+
fs.perform(file_action, ctx)
|
|
82
|
+
result.bump(bucket + "_handled")
|
|
83
|
+
|
|
84
|
+
result.data["action"] = action
|
|
85
|
+
if action == "move" and jobs:
|
|
86
|
+
result.data["target"] = target
|
|
87
|
+
return result
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
SPLIT_PARAMS = (
|
|
91
|
+
Param("folder", "dir", "Folder holding mixed annotated and unannotated images.",
|
|
92
|
+
required=True, must_exist=True),
|
|
93
|
+
Param("target", "dir", "Where unannotated images are moved.", required=True),
|
|
94
|
+
Param("mode", "choice", "Move them out, or copy them.",
|
|
95
|
+
default="move", choices=("move", "copy")),
|
|
96
|
+
Param("recursive", "bool", "Include sub-folders.", default=False),
|
|
97
|
+
Param("label_ext", "str", "Sidecar label extension.", default=".xml"),
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@operation(
|
|
102
|
+
"clean.unannotated",
|
|
103
|
+
title="Separate images still needing annotation",
|
|
104
|
+
category="clean",
|
|
105
|
+
summary="Move or copy every image that has no label file into its own folder.",
|
|
106
|
+
params=SPLIT_PARAMS,
|
|
107
|
+
destructive=True,
|
|
108
|
+
notes="The 'what is left to annotate' queue. Use mode=copy to leave the source intact.",
|
|
109
|
+
)
|
|
110
|
+
def unannotated(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
111
|
+
result = OpResult(op="clean.unannotated", dry_run=ctx.dry_run)
|
|
112
|
+
folder: Path = params["folder"]
|
|
113
|
+
target: Path = params["target"]
|
|
114
|
+
verb = Action.MOVE if params["mode"] == "move" else Action.COPY
|
|
115
|
+
|
|
116
|
+
index = PairIndex.scan(folder, recursive=params["recursive"], label_ext=params["label_ext"])
|
|
117
|
+
pending = [p for p in index.image_only() if p.image]
|
|
118
|
+
result.counts["scanned"] = len(index)
|
|
119
|
+
result.counts["already_annotated"] = len(index.complete())
|
|
120
|
+
|
|
121
|
+
taken: set[Path] = set()
|
|
122
|
+
for position, pair in enumerate(pending, start=1):
|
|
123
|
+
ctx.check_cancelled()
|
|
124
|
+
assert pair.image is not None
|
|
125
|
+
ctx.emit(position, len(pending), pair.image.name)
|
|
126
|
+
relative = fs.relative_to_safe(pair.image, folder)
|
|
127
|
+
destination = fs.unique_path(target / relative, taken=taken)
|
|
128
|
+
taken.add(destination)
|
|
129
|
+
fs.perform(result.record(verb, pair.image, destination, "no label file"), ctx)
|
|
130
|
+
result.bump("unannotated")
|
|
131
|
+
|
|
132
|
+
result.data["target"] = target
|
|
133
|
+
return result
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Sequential renaming that keeps image and label locked together.
|
|
2
|
+
|
|
3
|
+
Two fixes over the original script. It renamed each pair through a temporary
|
|
4
|
+
name individually, which still collides when the new name belongs to a file
|
|
5
|
+
later in the queue; renaming now happens in two global phases so any
|
|
6
|
+
permutation is safe. And it left the ``<filename>`` inside the XML pointing at
|
|
7
|
+
the old image, which quietly breaks every tool that trusts that field --
|
|
8
|
+
the label is now retargeted as part of the rename.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import xml.etree.ElementTree as ET
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from .. import fs
|
|
18
|
+
from ..pairs import PairIndex
|
|
19
|
+
from ..registry import Param, operation
|
|
20
|
+
from ..types import Action, Context, FileAction, OpResult
|
|
21
|
+
|
|
22
|
+
PARAMS = (
|
|
23
|
+
Param("folder", "dir", "Folder whose files are renamed in place.", required=True, must_exist=True),
|
|
24
|
+
Param("prefix", "str", "Name prefix, e.g. Door -> Door0001.jpg", required=True),
|
|
25
|
+
Param("start", "int", "First number in the sequence.", default=1, minimum=0),
|
|
26
|
+
Param("pad", "int", "Zero-pad numbers to this width (0 = no padding).",
|
|
27
|
+
default=4, minimum=0, maximum=12),
|
|
28
|
+
Param("include", "choice", "Rename complete pairs only, or every image.",
|
|
29
|
+
default="pairs", choices=("pairs", "all")),
|
|
30
|
+
Param("update_label_filename", "bool",
|
|
31
|
+
"Rewrite <filename>/<path> inside the label to match the new image name.",
|
|
32
|
+
default=True),
|
|
33
|
+
Param("label_ext", "str", "Sidecar label extension.", default=".xml"),
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@operation(
|
|
38
|
+
"clean.rename",
|
|
39
|
+
title="Rename images and labels in sequence",
|
|
40
|
+
category="clean",
|
|
41
|
+
summary="Give every pair a clean sequential name, renaming image and label together.",
|
|
42
|
+
params=PARAMS,
|
|
43
|
+
destructive=True,
|
|
44
|
+
notes=(
|
|
45
|
+
"Renaming happens in two passes (everything to temporary names, then to\n"
|
|
46
|
+
"final names), so a sequence that reuses names already in the folder --\n"
|
|
47
|
+
"renumbering Door5..Door9 as Door1..Door5, say -- cannot clobber anything.\n\n"
|
|
48
|
+
"Always preview with --dry-run first: this rewrites the whole folder."
|
|
49
|
+
),
|
|
50
|
+
)
|
|
51
|
+
def rename(params: dict[str, Any], ctx: Context) -> OpResult:
|
|
52
|
+
result = OpResult(op="clean.rename", dry_run=ctx.dry_run)
|
|
53
|
+
folder: Path = params["folder"]
|
|
54
|
+
prefix: str = params["prefix"]
|
|
55
|
+
pad: int = params["pad"]
|
|
56
|
+
label_ext: str = params["label_ext"]
|
|
57
|
+
|
|
58
|
+
index = PairIndex.scan(folder, label_ext=label_ext)
|
|
59
|
+
candidates = index.complete() if params["include"] == "pairs" else index.with_images()
|
|
60
|
+
result.counts["scanned"] = len(index)
|
|
61
|
+
result.counts["selected"] = len(candidates)
|
|
62
|
+
skipped = len(index) - len(candidates)
|
|
63
|
+
if skipped:
|
|
64
|
+
result.counts["skipped"] = skipped
|
|
65
|
+
|
|
66
|
+
if not candidates:
|
|
67
|
+
result.warnings.append("nothing to rename with the current include= setting")
|
|
68
|
+
return result
|
|
69
|
+
|
|
70
|
+
# Reserve every name we are about to free, so conflict detection only fires
|
|
71
|
+
# on files that are genuinely outside this rename.
|
|
72
|
+
involved = {path for pair in candidates for path in pair.files}
|
|
73
|
+
plan: list[tuple[Path, Path]] = []
|
|
74
|
+
renames: list[tuple[Path, Path, Path | None, Path | None]] = []
|
|
75
|
+
|
|
76
|
+
number = int(params["start"])
|
|
77
|
+
for pair in candidates:
|
|
78
|
+
ctx.check_cancelled()
|
|
79
|
+
base = f"{prefix}{number:0{pad}d}" if pad else f"{prefix}{number}"
|
|
80
|
+
number += 1
|
|
81
|
+
|
|
82
|
+
assert pair.image is not None
|
|
83
|
+
new_image = folder / f"{base}{pair.image.suffix.lower()}"
|
|
84
|
+
new_label = folder / f"{base}{label_ext}" if pair.label else None
|
|
85
|
+
|
|
86
|
+
conflict = next(
|
|
87
|
+
(p for p in (new_image, new_label) if p and p.exists() and p not in involved),
|
|
88
|
+
None,
|
|
89
|
+
)
|
|
90
|
+
if conflict is not None:
|
|
91
|
+
result.warnings.append(
|
|
92
|
+
f"{pair.stem}: target {conflict.name} already exists and is not part of this rename"
|
|
93
|
+
)
|
|
94
|
+
result.bump("conflicts")
|
|
95
|
+
continue
|
|
96
|
+
|
|
97
|
+
plan.append((pair.image, new_image))
|
|
98
|
+
if pair.label and new_label:
|
|
99
|
+
plan.append((pair.label, new_label))
|
|
100
|
+
renames.append((pair.image, new_image, pair.label, new_label))
|
|
101
|
+
|
|
102
|
+
# Phase 1: park everything under a temporary name so the final names are all
|
|
103
|
+
# free regardless of how the sequence overlaps the existing one.
|
|
104
|
+
staged: list[Path] = []
|
|
105
|
+
for position, (source, _) in enumerate(plan):
|
|
106
|
+
ctx.check_cancelled()
|
|
107
|
+
ctx.emit(position + 1, len(plan) * 2, f"staging {source.name}")
|
|
108
|
+
temporary = source.parent / f".dfx-rename-{position}{source.suffix}"
|
|
109
|
+
fs.perform(FileAction(Action.RENAME, source, temporary), ctx)
|
|
110
|
+
staged.append(temporary)
|
|
111
|
+
|
|
112
|
+
# Phase 2: move into place, and record the logical rename for the audit trail.
|
|
113
|
+
for position, ((source, destination), temporary) in enumerate(zip(plan, staged)):
|
|
114
|
+
ctx.check_cancelled()
|
|
115
|
+
ctx.emit(len(plan) + position + 1, len(plan) * 2, f"renaming {destination.name}")
|
|
116
|
+
fs.perform(FileAction(Action.RENAME, temporary, destination), ctx)
|
|
117
|
+
result.record(Action.RENAME, source, destination, "sequential rename")
|
|
118
|
+
result.bump("files_renamed")
|
|
119
|
+
|
|
120
|
+
if params["update_label_filename"]:
|
|
121
|
+
for _, new_image, old_label, new_label in renames:
|
|
122
|
+
if old_label is None or new_label is None:
|
|
123
|
+
continue
|
|
124
|
+
if _retarget_label(new_label, new_image.name, ctx):
|
|
125
|
+
result.bump("labels_retargeted")
|
|
126
|
+
|
|
127
|
+
result.counts["pairs_renamed"] = len(renames)
|
|
128
|
+
result.data["first"] = plan[0][1].name if plan else None
|
|
129
|
+
result.data["last"] = plan[-1][1].name if plan else None
|
|
130
|
+
return result
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _retarget_label(label_path: Path, image_name: str, ctx: Context) -> bool:
|
|
134
|
+
"""Point a VOC label's filename/path fields at the renamed image.
|
|
135
|
+
|
|
136
|
+
Edited in place with ElementTree rather than round-tripped through the model
|
|
137
|
+
so any extra fields a third-party tool wrote are preserved untouched.
|
|
138
|
+
"""
|
|
139
|
+
if ctx.dry_run:
|
|
140
|
+
return True
|
|
141
|
+
try:
|
|
142
|
+
tree = ET.parse(label_path)
|
|
143
|
+
except (ET.ParseError, OSError):
|
|
144
|
+
return False
|
|
145
|
+
root = tree.getroot()
|
|
146
|
+
changed = False
|
|
147
|
+
for tag in ("filename", "path"):
|
|
148
|
+
node = root.find(tag)
|
|
149
|
+
if node is not None:
|
|
150
|
+
node.text = image_name
|
|
151
|
+
changed = True
|
|
152
|
+
if changed:
|
|
153
|
+
tree.write(label_path, encoding="utf-8")
|
|
154
|
+
return changed
|