easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
data_factory/core/fs.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Filesystem helpers, and the single choke point through which every
|
|
2
|
+
mutation passes.
|
|
3
|
+
|
|
4
|
+
Operations never call ``shutil`` directly. They record a :class:`FileAction`
|
|
5
|
+
and hand it to :func:`perform`, which is a no-op under ``--dry-run``. That is
|
|
6
|
+
why a preview is guaranteed to match the real run: there is only one code path,
|
|
7
|
+
and the only thing dry-run changes is whether the last line executes.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import os
|
|
14
|
+
import shutil
|
|
15
|
+
import tempfile
|
|
16
|
+
import time
|
|
17
|
+
from collections.abc import Iterable, Iterator
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from .types import Action, Context, FileAction
|
|
21
|
+
|
|
22
|
+
IMAGE_EXTS: frozenset[str] = frozenset(
|
|
23
|
+
{".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff", ".webp"}
|
|
24
|
+
)
|
|
25
|
+
VIDEO_EXTS: frozenset[str] = frozenset({".mp4", ".avi", ".mov", ".mkv", ".wmv", ".m4v"})
|
|
26
|
+
DEFAULT_LABEL_EXT = ".xml"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def write_text_atomic(
|
|
30
|
+
path: Path, text: str, *, encoding: str = "utf-8", newline: str | None = None
|
|
31
|
+
) -> Path:
|
|
32
|
+
"""Write a file so a reader never sees it half-written.
|
|
33
|
+
|
|
34
|
+
The obvious ``path.write_text(...)`` truncates the file first and then
|
|
35
|
+
fills it. Two things go wrong with that, and both are silent:
|
|
36
|
+
|
|
37
|
+
* **A crash or a full disk mid-write leaves a truncated file.** For an XML
|
|
38
|
+
or JSON annotation that means an unparseable label -- and the original
|
|
39
|
+
contents are already gone, so there is nothing to fall back to.
|
|
40
|
+
* **Two concurrent writers interleave.** The annotation editor writes
|
|
41
|
+
through to sidecar files on every save; a second tab, or an export
|
|
42
|
+
running at the same time, would produce a mangled file rather than
|
|
43
|
+
one of the two intended ones.
|
|
44
|
+
|
|
45
|
+
Writing to a temporary file in the same directory and then renaming makes
|
|
46
|
+
the swap atomic on both POSIX and Windows: a reader sees either the whole
|
|
47
|
+
old file or the whole new one. Same directory matters -- ``os.replace``
|
|
48
|
+
across filesystems is not atomic.
|
|
49
|
+
"""
|
|
50
|
+
path = Path(path)
|
|
51
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
# ``newline`` defaults to None so this matches Path.write_text exactly:
|
|
53
|
+
# swapping in an atomic write must not also quietly change line endings in
|
|
54
|
+
# every label file the tool has ever produced. Callers that already emit
|
|
55
|
+
# their own terminators -- the csv module writes \r\n itself -- pass "",
|
|
56
|
+
# which is the difference between a valid CSV and one with a blank line
|
|
57
|
+
# between every row on Windows.
|
|
58
|
+
handle = tempfile.NamedTemporaryFile(
|
|
59
|
+
mode="w",
|
|
60
|
+
encoding=encoding,
|
|
61
|
+
newline=newline,
|
|
62
|
+
dir=path.parent,
|
|
63
|
+
prefix=f".{path.name}.",
|
|
64
|
+
suffix=".tmp",
|
|
65
|
+
delete=False,
|
|
66
|
+
)
|
|
67
|
+
temporary = Path(handle.name)
|
|
68
|
+
try:
|
|
69
|
+
with handle:
|
|
70
|
+
handle.write(text)
|
|
71
|
+
handle.flush()
|
|
72
|
+
# Without this the rename can land before the contents reach disk,
|
|
73
|
+
# which on a power loss gives an empty file where one used to be.
|
|
74
|
+
os.fsync(handle.fileno())
|
|
75
|
+
_replace_with_retry(temporary, path)
|
|
76
|
+
except BaseException:
|
|
77
|
+
temporary.unlink(missing_ok=True)
|
|
78
|
+
raise
|
|
79
|
+
return path
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
#: How long to keep retrying a rename that Windows refuses. Deliberately short:
|
|
83
|
+
#: this is for transient holds, and anything longer is a real problem that
|
|
84
|
+
#: should surface rather than hang a save.
|
|
85
|
+
_REPLACE_ATTEMPTS = 10
|
|
86
|
+
_REPLACE_PAUSE = 0.05
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _replace_with_retry(temporary: Path, path: Path) -> None:
|
|
90
|
+
"""``os.replace``, retried past Windows' transient sharing failures.
|
|
91
|
+
|
|
92
|
+
On POSIX the rename is unconditional. On Windows it goes through
|
|
93
|
+
``MoveFileEx``, which fails with ``PermissionError`` (``WinError 5``) if
|
|
94
|
+
*anything* currently has the destination open without ``FILE_SHARE_DELETE``
|
|
95
|
+
-- another thread reading it, an editor, a backup agent, or an antivirus
|
|
96
|
+
scanner that opened the file precisely because we just wrote it.
|
|
97
|
+
|
|
98
|
+
Found by the concurrency test: two threads writing and reading the same
|
|
99
|
+
file made this fail within a few hundred iterations. Left unhandled it
|
|
100
|
+
would surface as an annotation save that failed for no visible reason, on
|
|
101
|
+
a machine with antivirus, intermittently -- the worst kind of bug to be
|
|
102
|
+
told about second-hand.
|
|
103
|
+
"""
|
|
104
|
+
last: OSError | None = None
|
|
105
|
+
for attempt in range(_REPLACE_ATTEMPTS):
|
|
106
|
+
try:
|
|
107
|
+
os.replace(temporary, path)
|
|
108
|
+
return
|
|
109
|
+
except PermissionError as exc: # Windows only, and transient
|
|
110
|
+
last = exc
|
|
111
|
+
time.sleep(_REPLACE_PAUSE * (attempt + 1))
|
|
112
|
+
assert last is not None
|
|
113
|
+
raise last
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def is_image(path: Path, exts: Iterable[str] = IMAGE_EXTS) -> bool:
|
|
117
|
+
return path.suffix.lower() in set(exts)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def iter_images(
|
|
121
|
+
root: Path, *, recursive: bool = False, exts: Iterable[str] = IMAGE_EXTS
|
|
122
|
+
) -> Iterator[Path]:
|
|
123
|
+
"""Yield image files under ``root`` in a stable, reproducible order.
|
|
124
|
+
|
|
125
|
+
Sorting matters more than it looks: dedup keeps the *first* file it sees, so
|
|
126
|
+
an unstable directory order would make results differ between runs.
|
|
127
|
+
"""
|
|
128
|
+
extset = {e.lower() for e in exts}
|
|
129
|
+
walker = root.rglob("*") if recursive else root.glob("*")
|
|
130
|
+
for path in sorted(walker):
|
|
131
|
+
if path.is_file() and path.suffix.lower() in extset:
|
|
132
|
+
yield path
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class TooManyFiles(ValueError):
|
|
136
|
+
"""A folder walk passed its limit -- usually a whole drive, picked by mistake."""
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def walk_images(root: Path, *, limit: int, exts: Iterable[str] = IMAGE_EXTS) -> list[Path]:
|
|
140
|
+
"""Every image under a folder someone *chose*, sorted.
|
|
141
|
+
|
|
142
|
+
Unlike :func:`iter_images`, a directory link is never followed -- a
|
|
143
|
+
symlink, or on Windows a junction, can point back up the tree and walk
|
|
144
|
+
forever, or out of the chosen folder -- and the walk stops after
|
|
145
|
+
``limit`` entries rather than reading a whole drive.
|
|
146
|
+
"""
|
|
147
|
+
extset = {e.lower() for e in exts}
|
|
148
|
+
found: list[Path] = []
|
|
149
|
+
seen = 0
|
|
150
|
+
pending = [root]
|
|
151
|
+
while pending:
|
|
152
|
+
try:
|
|
153
|
+
entries = list(os.scandir(pending.pop()))
|
|
154
|
+
except OSError:
|
|
155
|
+
continue
|
|
156
|
+
for entry in entries:
|
|
157
|
+
seen += 1
|
|
158
|
+
if seen > limit:
|
|
159
|
+
raise TooManyFiles(f"more than {limit:,} files under {root} -- open a smaller folder")
|
|
160
|
+
try:
|
|
161
|
+
if entry.is_dir(follow_symlinks=False):
|
|
162
|
+
if not (entry.is_symlink() or entry.is_junction()):
|
|
163
|
+
pending.append(Path(entry.path))
|
|
164
|
+
elif entry.is_file() and os.path.splitext(entry.name)[1].lower() in extset:
|
|
165
|
+
found.append(Path(entry.path))
|
|
166
|
+
except OSError:
|
|
167
|
+
continue
|
|
168
|
+
return sorted(found)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def iter_files(
|
|
172
|
+
root: Path, *, recursive: bool = False, exts: Iterable[str] | None = None
|
|
173
|
+
) -> Iterator[Path]:
|
|
174
|
+
extset = {e.lower() for e in exts} if exts else None
|
|
175
|
+
walker = root.rglob("*") if recursive else root.glob("*")
|
|
176
|
+
for path in sorted(walker):
|
|
177
|
+
if not path.is_file():
|
|
178
|
+
continue
|
|
179
|
+
if extset is None or path.suffix.lower() in extset:
|
|
180
|
+
yield path
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def ensure_dir(path: Path, ctx: Context) -> Path:
|
|
184
|
+
if not ctx.dry_run:
|
|
185
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
186
|
+
return path
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def unique_path(dst: Path, *, taken: set[Path] | None = None) -> Path:
|
|
190
|
+
"""Return a destination that collides with nothing.
|
|
191
|
+
|
|
192
|
+
``taken`` lets a dry run reserve names it has not actually created yet, so
|
|
193
|
+
the preview does not show three files all landing on the same path.
|
|
194
|
+
"""
|
|
195
|
+
reserved = taken if taken is not None else set()
|
|
196
|
+
if not dst.exists() and dst not in reserved:
|
|
197
|
+
return dst
|
|
198
|
+
stem, suffix, parent = dst.stem, dst.suffix, dst.parent
|
|
199
|
+
counter = 1
|
|
200
|
+
while True:
|
|
201
|
+
candidate = parent / f"{stem}_{counter}{suffix}"
|
|
202
|
+
if not candidate.exists() and candidate not in reserved:
|
|
203
|
+
return candidate
|
|
204
|
+
counter += 1
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def perform(action: FileAction, ctx: Context) -> None:
|
|
208
|
+
"""Apply one recorded action. Does nothing when ``ctx.dry_run`` is set."""
|
|
209
|
+
if ctx.dry_run or action.action is Action.SKIP:
|
|
210
|
+
return
|
|
211
|
+
if action.action is Action.DELETE:
|
|
212
|
+
action.src.unlink(missing_ok=True)
|
|
213
|
+
return
|
|
214
|
+
if action.dst is None:
|
|
215
|
+
raise ValueError(f"{action.action.value} action requires a destination")
|
|
216
|
+
action.dst.parent.mkdir(parents=True, exist_ok=True)
|
|
217
|
+
if action.action in (Action.MOVE, Action.RENAME):
|
|
218
|
+
shutil.move(str(action.src), str(action.dst))
|
|
219
|
+
elif action.action is Action.COPY:
|
|
220
|
+
shutil.copy2(str(action.src), str(action.dst))
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def sha256_file(path: Path, chunk_size: int = 1 << 16) -> str:
|
|
224
|
+
"""Content fingerprint used for exact-duplicate detection."""
|
|
225
|
+
hasher = hashlib.sha256()
|
|
226
|
+
with path.open("rb") as handle:
|
|
227
|
+
while chunk := handle.read(chunk_size):
|
|
228
|
+
hasher.update(chunk)
|
|
229
|
+
return hasher.hexdigest()
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def human_bytes(size: float) -> str:
|
|
233
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
234
|
+
if size < 1024 or unit == "TB":
|
|
235
|
+
return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
|
|
236
|
+
size /= 1024
|
|
237
|
+
return f"{size:.1f} TB"
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def relative_to_safe(path: Path, root: Path) -> Path:
|
|
241
|
+
"""``path.relative_to(root)`` that degrades to the bare name off-tree."""
|
|
242
|
+
try:
|
|
243
|
+
return path.relative_to(root)
|
|
244
|
+
except ValueError:
|
|
245
|
+
return Path(path.name)
|