easyannotate 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. data_factory/__init__.py +82 -0
  2. data_factory/__main__.py +6 -0
  3. data_factory/annotate/__init__.py +15 -0
  4. data_factory/annotate/attach.py +398 -0
  5. data_factory/annotate/demo.py +161 -0
  6. data_factory/annotate/images.py +443 -0
  7. data_factory/annotate/labels.py +399 -0
  8. data_factory/annotate/loop.py +359 -0
  9. data_factory/annotate/projects.py +207 -0
  10. data_factory/annotate/review.py +471 -0
  11. data_factory/annotate/schema.sql +171 -0
  12. data_factory/annotate/shapes.py +488 -0
  13. data_factory/annotate/store.py +292 -0
  14. data_factory/annotate/tools/__init__.py +39 -0
  15. data_factory/annotate/tools/fake.py +170 -0
  16. data_factory/annotate/tools/install.py +254 -0
  17. data_factory/annotate/tools/local.py +389 -0
  18. data_factory/annotate/tools/postprocess.py +175 -0
  19. data_factory/annotate/tools/protocol.py +238 -0
  20. data_factory/annotate/tools/registry.py +196 -0
  21. data_factory/annotate/tools/results.py +83 -0
  22. data_factory/annotate/tools/serve.py +106 -0
  23. data_factory/cli/__init__.py +21 -0
  24. data_factory/cli/main.py +837 -0
  25. data_factory/cli/menu.py +173 -0
  26. data_factory/cli/render.py +158 -0
  27. data_factory/core/__init__.py +11 -0
  28. data_factory/core/annotations/__init__.py +5 -0
  29. data_factory/core/annotations/coco.py +394 -0
  30. data_factory/core/annotations/csvfmt.py +174 -0
  31. data_factory/core/annotations/dota.py +98 -0
  32. data_factory/core/annotations/formats.py +626 -0
  33. data_factory/core/annotations/labelme.py +153 -0
  34. data_factory/core/annotations/model.py +334 -0
  35. data_factory/core/annotations/voc.py +231 -0
  36. data_factory/core/annotations/yolo.py +283 -0
  37. data_factory/core/bktree.py +89 -0
  38. data_factory/core/fs.py +245 -0
  39. data_factory/core/imaging.py +626 -0
  40. data_factory/core/ops/__init__.py +44 -0
  41. data_factory/core/ops/analyze.py +383 -0
  42. data_factory/core/ops/annotate_export.py +326 -0
  43. data_factory/core/ops/audit.py +274 -0
  44. data_factory/core/ops/augment.py +454 -0
  45. data_factory/core/ops/collect.py +241 -0
  46. data_factory/core/ops/combine.py +582 -0
  47. data_factory/core/ops/convert.py +212 -0
  48. data_factory/core/ops/dataset.py +246 -0
  49. data_factory/core/ops/dedup.py +235 -0
  50. data_factory/core/ops/export.py +168 -0
  51. data_factory/core/ops/image.py +407 -0
  52. data_factory/core/ops/labels.py +215 -0
  53. data_factory/core/ops/pairing.py +133 -0
  54. data_factory/core/ops/rename.py +154 -0
  55. data_factory/core/ops/report.py +323 -0
  56. data_factory/core/ops/visualize.py +164 -0
  57. data_factory/core/pairs.py +144 -0
  58. data_factory/core/parallel.py +191 -0
  59. data_factory/core/provenance.py +136 -0
  60. data_factory/core/registry.py +339 -0
  61. data_factory/core/stats.py +403 -0
  62. data_factory/core/types.py +148 -0
  63. data_factory/core/writer.py +194 -0
  64. data_factory/desktop.py +207 -0
  65. data_factory/hardware.py +243 -0
  66. data_factory/pipeline.py +192 -0
  67. data_factory/py.typed +0 -0
  68. data_factory/service/__init__.py +26 -0
  69. data_factory/service/app.py +483 -0
  70. data_factory/service/binding.py +174 -0
  71. data_factory/service/deps.py +133 -0
  72. data_factory/service/jobs.py +177 -0
  73. data_factory/service/limits.py +71 -0
  74. data_factory/service/routers/__init__.py +11 -0
  75. data_factory/service/routers/annotate.py +582 -0
  76. data_factory/service/routers/local.py +464 -0
  77. data_factory/service/routers/media.py +154 -0
  78. data_factory/service/routers/predictions.py +116 -0
  79. data_factory/service/routers/promote.py +182 -0
  80. data_factory/service/routers/tools.py +308 -0
  81. data_factory/service/static/annotate/api.js +246 -0
  82. data_factory/service/static/annotate/app.js +1696 -0
  83. data_factory/service/static/annotate/assist.js +106 -0
  84. data_factory/service/static/annotate/filmstrip.js +286 -0
  85. data_factory/service/static/annotate/hit.js +146 -0
  86. data_factory/service/static/annotate/labels.js +197 -0
  87. data_factory/service/static/annotate/render.js +286 -0
  88. data_factory/service/static/annotate/scene.js +277 -0
  89. data_factory/service/static/annotate/shapelist.js +72 -0
  90. data_factory/service/static/annotate/tools/box.js +96 -0
  91. data_factory/service/static/annotate/tools/line.js +96 -0
  92. data_factory/service/static/annotate/tools/obb.js +88 -0
  93. data_factory/service/static/annotate/tools/point.js +49 -0
  94. data_factory/service/static/annotate/tools/polygon.js +110 -0
  95. data_factory/service/static/annotate/tools/select.js +227 -0
  96. data_factory/service/static/annotate/undo.js +176 -0
  97. data_factory/service/static/annotate/viewport.js +156 -0
  98. data_factory/service/static/annotate.html +390 -0
  99. data_factory/service/static/index.html +632 -0
  100. data_factory/service/workspace.py +244 -0
  101. easyannotate/__init__.py +64 -0
  102. easyannotate/__main__.py +5 -0
  103. easyannotate/project.py +181 -0
  104. easyannotate/py.typed +0 -0
  105. easyannotate-0.4.0.dist-info/METADATA +767 -0
  106. easyannotate-0.4.0.dist-info/RECORD +109 -0
  107. easyannotate-0.4.0.dist-info/WHEEL +4 -0
  108. easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
  109. easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,245 @@
1
+ """Filesystem helpers, and the single choke point through which every
2
+ mutation passes.
3
+
4
+ Operations never call ``shutil`` directly. They record a :class:`FileAction`
5
+ and hand it to :func:`perform`, which is a no-op under ``--dry-run``. That is
6
+ why a preview is guaranteed to match the real run: there is only one code path,
7
+ and the only thing dry-run changes is whether the last line executes.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import hashlib
13
+ import os
14
+ import shutil
15
+ import tempfile
16
+ import time
17
+ from collections.abc import Iterable, Iterator
18
+ from pathlib import Path
19
+
20
+ from .types import Action, Context, FileAction
21
+
22
+ IMAGE_EXTS: frozenset[str] = frozenset(
23
+ {".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff", ".webp"}
24
+ )
25
+ VIDEO_EXTS: frozenset[str] = frozenset({".mp4", ".avi", ".mov", ".mkv", ".wmv", ".m4v"})
26
+ DEFAULT_LABEL_EXT = ".xml"
27
+
28
+
29
+ def write_text_atomic(
30
+ path: Path, text: str, *, encoding: str = "utf-8", newline: str | None = None
31
+ ) -> Path:
32
+ """Write a file so a reader never sees it half-written.
33
+
34
+ The obvious ``path.write_text(...)`` truncates the file first and then
35
+ fills it. Two things go wrong with that, and both are silent:
36
+
37
+ * **A crash or a full disk mid-write leaves a truncated file.** For an XML
38
+ or JSON annotation that means an unparseable label -- and the original
39
+ contents are already gone, so there is nothing to fall back to.
40
+ * **Two concurrent writers interleave.** The annotation editor writes
41
+ through to sidecar files on every save; a second tab, or an export
42
+ running at the same time, would produce a mangled file rather than
43
+ one of the two intended ones.
44
+
45
+ Writing to a temporary file in the same directory and then renaming makes
46
+ the swap atomic on both POSIX and Windows: a reader sees either the whole
47
+ old file or the whole new one. Same directory matters -- ``os.replace``
48
+ across filesystems is not atomic.
49
+ """
50
+ path = Path(path)
51
+ path.parent.mkdir(parents=True, exist_ok=True)
52
+ # ``newline`` defaults to None so this matches Path.write_text exactly:
53
+ # swapping in an atomic write must not also quietly change line endings in
54
+ # every label file the tool has ever produced. Callers that already emit
55
+ # their own terminators -- the csv module writes \r\n itself -- pass "",
56
+ # which is the difference between a valid CSV and one with a blank line
57
+ # between every row on Windows.
58
+ handle = tempfile.NamedTemporaryFile(
59
+ mode="w",
60
+ encoding=encoding,
61
+ newline=newline,
62
+ dir=path.parent,
63
+ prefix=f".{path.name}.",
64
+ suffix=".tmp",
65
+ delete=False,
66
+ )
67
+ temporary = Path(handle.name)
68
+ try:
69
+ with handle:
70
+ handle.write(text)
71
+ handle.flush()
72
+ # Without this the rename can land before the contents reach disk,
73
+ # which on a power loss gives an empty file where one used to be.
74
+ os.fsync(handle.fileno())
75
+ _replace_with_retry(temporary, path)
76
+ except BaseException:
77
+ temporary.unlink(missing_ok=True)
78
+ raise
79
+ return path
80
+
81
+
82
+ #: How long to keep retrying a rename that Windows refuses. Deliberately short:
83
+ #: this is for transient holds, and anything longer is a real problem that
84
+ #: should surface rather than hang a save.
85
+ _REPLACE_ATTEMPTS = 10
86
+ _REPLACE_PAUSE = 0.05
87
+
88
+
89
+ def _replace_with_retry(temporary: Path, path: Path) -> None:
90
+ """``os.replace``, retried past Windows' transient sharing failures.
91
+
92
+ On POSIX the rename is unconditional. On Windows it goes through
93
+ ``MoveFileEx``, which fails with ``PermissionError`` (``WinError 5``) if
94
+ *anything* currently has the destination open without ``FILE_SHARE_DELETE``
95
+ -- another thread reading it, an editor, a backup agent, or an antivirus
96
+ scanner that opened the file precisely because we just wrote it.
97
+
98
+ Found by the concurrency test: two threads writing and reading the same
99
+ file made this fail within a few hundred iterations. Left unhandled it
100
+ would surface as an annotation save that failed for no visible reason, on
101
+ a machine with antivirus, intermittently -- the worst kind of bug to be
102
+ told about second-hand.
103
+ """
104
+ last: OSError | None = None
105
+ for attempt in range(_REPLACE_ATTEMPTS):
106
+ try:
107
+ os.replace(temporary, path)
108
+ return
109
+ except PermissionError as exc: # Windows only, and transient
110
+ last = exc
111
+ time.sleep(_REPLACE_PAUSE * (attempt + 1))
112
+ assert last is not None
113
+ raise last
114
+
115
+
116
+ def is_image(path: Path, exts: Iterable[str] = IMAGE_EXTS) -> bool:
117
+ return path.suffix.lower() in set(exts)
118
+
119
+
120
+ def iter_images(
121
+ root: Path, *, recursive: bool = False, exts: Iterable[str] = IMAGE_EXTS
122
+ ) -> Iterator[Path]:
123
+ """Yield image files under ``root`` in a stable, reproducible order.
124
+
125
+ Sorting matters more than it looks: dedup keeps the *first* file it sees, so
126
+ an unstable directory order would make results differ between runs.
127
+ """
128
+ extset = {e.lower() for e in exts}
129
+ walker = root.rglob("*") if recursive else root.glob("*")
130
+ for path in sorted(walker):
131
+ if path.is_file() and path.suffix.lower() in extset:
132
+ yield path
133
+
134
+
135
+ class TooManyFiles(ValueError):
136
+ """A folder walk passed its limit -- usually a whole drive, picked by mistake."""
137
+
138
+
139
+ def walk_images(root: Path, *, limit: int, exts: Iterable[str] = IMAGE_EXTS) -> list[Path]:
140
+ """Every image under a folder someone *chose*, sorted.
141
+
142
+ Unlike :func:`iter_images`, a directory link is never followed -- a
143
+ symlink, or on Windows a junction, can point back up the tree and walk
144
+ forever, or out of the chosen folder -- and the walk stops after
145
+ ``limit`` entries rather than reading a whole drive.
146
+ """
147
+ extset = {e.lower() for e in exts}
148
+ found: list[Path] = []
149
+ seen = 0
150
+ pending = [root]
151
+ while pending:
152
+ try:
153
+ entries = list(os.scandir(pending.pop()))
154
+ except OSError:
155
+ continue
156
+ for entry in entries:
157
+ seen += 1
158
+ if seen > limit:
159
+ raise TooManyFiles(f"more than {limit:,} files under {root} -- open a smaller folder")
160
+ try:
161
+ if entry.is_dir(follow_symlinks=False):
162
+ if not (entry.is_symlink() or entry.is_junction()):
163
+ pending.append(Path(entry.path))
164
+ elif entry.is_file() and os.path.splitext(entry.name)[1].lower() in extset:
165
+ found.append(Path(entry.path))
166
+ except OSError:
167
+ continue
168
+ return sorted(found)
169
+
170
+
171
+ def iter_files(
172
+ root: Path, *, recursive: bool = False, exts: Iterable[str] | None = None
173
+ ) -> Iterator[Path]:
174
+ extset = {e.lower() for e in exts} if exts else None
175
+ walker = root.rglob("*") if recursive else root.glob("*")
176
+ for path in sorted(walker):
177
+ if not path.is_file():
178
+ continue
179
+ if extset is None or path.suffix.lower() in extset:
180
+ yield path
181
+
182
+
183
+ def ensure_dir(path: Path, ctx: Context) -> Path:
184
+ if not ctx.dry_run:
185
+ path.mkdir(parents=True, exist_ok=True)
186
+ return path
187
+
188
+
189
+ def unique_path(dst: Path, *, taken: set[Path] | None = None) -> Path:
190
+ """Return a destination that collides with nothing.
191
+
192
+ ``taken`` lets a dry run reserve names it has not actually created yet, so
193
+ the preview does not show three files all landing on the same path.
194
+ """
195
+ reserved = taken if taken is not None else set()
196
+ if not dst.exists() and dst not in reserved:
197
+ return dst
198
+ stem, suffix, parent = dst.stem, dst.suffix, dst.parent
199
+ counter = 1
200
+ while True:
201
+ candidate = parent / f"{stem}_{counter}{suffix}"
202
+ if not candidate.exists() and candidate not in reserved:
203
+ return candidate
204
+ counter += 1
205
+
206
+
207
+ def perform(action: FileAction, ctx: Context) -> None:
208
+ """Apply one recorded action. Does nothing when ``ctx.dry_run`` is set."""
209
+ if ctx.dry_run or action.action is Action.SKIP:
210
+ return
211
+ if action.action is Action.DELETE:
212
+ action.src.unlink(missing_ok=True)
213
+ return
214
+ if action.dst is None:
215
+ raise ValueError(f"{action.action.value} action requires a destination")
216
+ action.dst.parent.mkdir(parents=True, exist_ok=True)
217
+ if action.action in (Action.MOVE, Action.RENAME):
218
+ shutil.move(str(action.src), str(action.dst))
219
+ elif action.action is Action.COPY:
220
+ shutil.copy2(str(action.src), str(action.dst))
221
+
222
+
223
+ def sha256_file(path: Path, chunk_size: int = 1 << 16) -> str:
224
+ """Content fingerprint used for exact-duplicate detection."""
225
+ hasher = hashlib.sha256()
226
+ with path.open("rb") as handle:
227
+ while chunk := handle.read(chunk_size):
228
+ hasher.update(chunk)
229
+ return hasher.hexdigest()
230
+
231
+
232
+ def human_bytes(size: float) -> str:
233
+ for unit in ("B", "KB", "MB", "GB", "TB"):
234
+ if size < 1024 or unit == "TB":
235
+ return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
236
+ size /= 1024
237
+ return f"{size:.1f} TB"
238
+
239
+
240
+ def relative_to_safe(path: Path, root: Path) -> Path:
241
+ """``path.relative_to(root)`` that degrades to the bare name off-tree."""
242
+ try:
243
+ return path.relative_to(root)
244
+ except ValueError:
245
+ return Path(path.name)