easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
data_factory/__init__.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Data Factory -- computer-vision dataset preparation, as one library and one CLI.
|
|
2
|
+
|
|
3
|
+
Importing this package registers every operation and gives you the whole tool
|
|
4
|
+
through three names::
|
|
5
|
+
|
|
6
|
+
from data_factory import get_operation, run, Context
|
|
7
|
+
|
|
8
|
+
result = run("inspect.dataset", {"folder": "./data"})
|
|
9
|
+
print(result.counts, result.data["issues"])
|
|
10
|
+
|
|
11
|
+
Every operation returns an :class:`~data_factory.core.types.OpResult`, which is
|
|
12
|
+
JSON-serialisable via ``result.as_dict()`` -- that is the whole contract a web
|
|
13
|
+
backend needs.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _detect_version() -> str:
|
|
22
|
+
"""Read the version from installed package metadata.
|
|
23
|
+
|
|
24
|
+
Declared once, in pyproject.toml. Hardcoding it here as well guarantees the
|
|
25
|
+
two drift -- which they had, with the CLI reporting 0.2.0 from a 0.3.0
|
|
26
|
+
wheel. The fallback covers running from a source checkout that was never
|
|
27
|
+
installed.
|
|
28
|
+
"""
|
|
29
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
30
|
+
|
|
31
|
+
# Published as "easyannotate"; "data-factory" is the name it had before,
|
|
32
|
+
# and an editable install made then still answers to it.
|
|
33
|
+
for distribution in ("easyannotate", "data-factory"):
|
|
34
|
+
try:
|
|
35
|
+
return version(distribution)
|
|
36
|
+
except PackageNotFoundError:
|
|
37
|
+
continue
|
|
38
|
+
return "0.0.0+source"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
__version__ = _detect_version()
|
|
42
|
+
|
|
43
|
+
from .core import ops as _ops # noqa: F401 -- registers the operations
|
|
44
|
+
from .core import registry
|
|
45
|
+
from .core.registry import Operation, ParamError
|
|
46
|
+
from .core.types import Action, Cancelled, Context, FileAction, OpResult
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def get_operation(name: str) -> Operation:
|
|
50
|
+
"""Look up one operation by name, e.g. ``clean.dedup``."""
|
|
51
|
+
return registry.get(name)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def list_operations() -> list[Operation]:
|
|
55
|
+
"""Every registered operation, ordered by category then name."""
|
|
56
|
+
return registry.all_operations()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def schema() -> dict[str, Any]:
|
|
60
|
+
"""Machine-readable description of every operation and its parameters."""
|
|
61
|
+
return registry.schema()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def run(name: str, params: dict[str, Any], ctx: Context | None = None) -> OpResult:
|
|
65
|
+
"""Run an operation by name. The one-liner entry point for library users."""
|
|
66
|
+
return registry.get(name).run(params, ctx)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
__all__ = [
|
|
70
|
+
"Action",
|
|
71
|
+
"Cancelled",
|
|
72
|
+
"Context",
|
|
73
|
+
"FileAction",
|
|
74
|
+
"OpResult",
|
|
75
|
+
"Operation",
|
|
76
|
+
"ParamError",
|
|
77
|
+
"__version__",
|
|
78
|
+
"get_operation",
|
|
79
|
+
"list_operations",
|
|
80
|
+
"run",
|
|
81
|
+
"schema",
|
|
82
|
+
]
|
data_factory/__main__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Interactive annotation: projects, images, shapes, review.
|
|
2
|
+
|
|
3
|
+
This package deliberately **does not import fastapi**. The batch operations in
|
|
4
|
+
``core/`` answer to one contract -- parameters in, an OpResult out -- which
|
|
5
|
+
cannot express a polygon or a per-shape edit. Annotation is stateful CRUD with
|
|
6
|
+
sub-100ms latency budgets, so it gets its own contract: plain domain objects
|
|
7
|
+
here, HTTP routers in ``service/routers/``.
|
|
8
|
+
|
|
9
|
+
Keeping FastAPI out means this layer is testable without a TestClient, and the
|
|
10
|
+
CLI can drive the same store as the browser.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
__all__ = ["attach", "images", "labels", "projects", "shapes", "store", "tools"]
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
"""Attached projects: annotate a folder where it lies.
|
|
2
|
+
|
|
3
|
+
The managed path copies a dataset into the workspace and treats the database as
|
|
4
|
+
the truth. That is right for an upload from a browser, and wrong for the case
|
|
5
|
+
this module exists for: a folder already on your disk, with sidecar files you
|
|
6
|
+
want *updated in place*. Copying a 2.5 GB dataset to annotate it, and then
|
|
7
|
+
having to export to get your edits back, is not what anyone means by "open this
|
|
8
|
+
folder".
|
|
9
|
+
|
|
10
|
+
So attached projects are:
|
|
11
|
+
|
|
12
|
+
* **Read where they are.** No copy. The image bytes are served from the folder.
|
|
13
|
+
* **Written through.** Every save writes the image's sidecar file as well as
|
|
14
|
+
the database row, so the ``.xml`` on disk is current the moment you stop
|
|
15
|
+
moving the mouse.
|
|
16
|
+
* **Local only.** A project pointing at a server-side path would be exactly the
|
|
17
|
+
arbitrary-path primitive the service layer exists to refuse, so there is no
|
|
18
|
+
HTTP route that creates one -- the CLI does, from a path the user typed
|
|
19
|
+
themselves. Hosted deployments set ``DATAFACTORY_ALLOW_ATTACHED=0`` as well.
|
|
20
|
+
|
|
21
|
+
**Write-through only works for per-image formats.** VOC, LabelMe and YOLO keep
|
|
22
|
+
one file per image, so a save touches one small file. COCO and CSV keep one
|
|
23
|
+
file for the whole dataset; rewriting a 200 MB ``instances.json`` on every
|
|
24
|
+
autosave would be absurd, so those folders open read-through with write-through
|
|
25
|
+
off, and ``annotate.export`` produces the output instead. That is recorded on
|
|
26
|
+
the project as ``sidecar_format`` and reported, never silently assumed.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from collections.abc import Iterable
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
from ..core import fs, imaging
|
|
36
|
+
from ..core.annotations import dota, formats, labelme, voc, yolo
|
|
37
|
+
from ..core.annotations.model import AnnotationError
|
|
38
|
+
from ..core.types import Context
|
|
39
|
+
from . import images as images_mod
|
|
40
|
+
from . import labels as labels_mod
|
|
41
|
+
from . import projects as projects_mod
|
|
42
|
+
from . import shapes as shapes_mod
|
|
43
|
+
from .store import LOCAL_OWNER, Store, StoreError
|
|
44
|
+
|
|
45
|
+
#: Formats that keep one label file per image, and can therefore be written
|
|
46
|
+
#: through on every save.
|
|
47
|
+
SIDECAR_FORMATS = ("voc", "labelme", "yolo")
|
|
48
|
+
|
|
49
|
+
#: What a folder of bare images becomes. VOC because it is the format the
|
|
50
|
+
#: existing browser annotator exports, so the two interoperate on day one.
|
|
51
|
+
DEFAULT_SIDECAR = "voc"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class AttachError(StoreError):
|
|
55
|
+
"""Raised when a folder cannot be opened for annotation."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def detect_sidecar_format(folder: Path) -> tuple[str, str]:
|
|
59
|
+
"""Work out what to write back, and why. Returns ``(format, note)``."""
|
|
60
|
+
try:
|
|
61
|
+
detected = formats.detect(folder)
|
|
62
|
+
except AnnotationError:
|
|
63
|
+
return DEFAULT_SIDECAR, (
|
|
64
|
+
f"no labels found, so new annotations will be written as "
|
|
65
|
+
f"{DEFAULT_SIDECAR.upper()} sidecar files"
|
|
66
|
+
)
|
|
67
|
+
if detected in SIDECAR_FORMATS:
|
|
68
|
+
return detected, f"detected {detected}; edits are written back to those files"
|
|
69
|
+
return "", (
|
|
70
|
+
f"detected {detected}, which keeps one file for the whole dataset. "
|
|
71
|
+
f"Edits are saved in the project but NOT written back to disk -- use "
|
|
72
|
+
f"'datafactory annotate.export' to produce an updated dataset."
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def sidecar_path(project: projects_mod.Project, image: images_mod.Image) -> Path | None:
|
|
77
|
+
"""Where this image's label file lives, or None if we do not write one."""
|
|
78
|
+
fmt = project.sidecar_format
|
|
79
|
+
if not fmt or not project.root:
|
|
80
|
+
return None
|
|
81
|
+
root = Path(project.root)
|
|
82
|
+
image_path = root / image.rel_path
|
|
83
|
+
if fmt == "voc":
|
|
84
|
+
return image_path.with_suffix(".xml")
|
|
85
|
+
if fmt == "labelme":
|
|
86
|
+
return image_path.with_suffix(".json")
|
|
87
|
+
if fmt in ("yolo", "dota"):
|
|
88
|
+
# Honours the images/ <-> labels/ convention when the folder has one,
|
|
89
|
+
# and falls back to a sidecar beside the image when it does not.
|
|
90
|
+
return yolo.label_path_for(image_path, root)
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def write_through(
|
|
95
|
+
store: Store, project: projects_mod.Project, image: images_mod.Image
|
|
96
|
+
) -> Path | None:
|
|
97
|
+
"""Write one image's annotations back to its sidecar file.
|
|
98
|
+
|
|
99
|
+
Returns the path written, or None when this project does not write through.
|
|
100
|
+
Every writer underneath is atomic, so a save landing while something else
|
|
101
|
+
reads the folder cannot produce a truncated label file.
|
|
102
|
+
"""
|
|
103
|
+
target = sidecar_path(project, image)
|
|
104
|
+
if target is None:
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
annotation = shapes_mod.annotation_for(store, project, image)
|
|
108
|
+
# The folder is the user's, not ours: an unwritable file is a normal thing
|
|
109
|
+
# to meet, and it must not lose the edit -- the database already has it.
|
|
110
|
+
try:
|
|
111
|
+
if project.sidecar_format == "voc":
|
|
112
|
+
voc.write(annotation, target)
|
|
113
|
+
elif project.sidecar_format == "labelme":
|
|
114
|
+
labelme.write(annotation, target)
|
|
115
|
+
elif project.sidecar_format == "yolo":
|
|
116
|
+
classes = yolo.ClassList(labels_mod.class_list(store, project))
|
|
117
|
+
yolo.write(annotation, target, classes)
|
|
118
|
+
_refresh_data_yaml(store, project, classes)
|
|
119
|
+
elif project.sidecar_format == "dota":
|
|
120
|
+
dota.write(annotation, target)
|
|
121
|
+
else:
|
|
122
|
+
return None
|
|
123
|
+
except OSError as exc:
|
|
124
|
+
raise AttachError(f"could not write {target.name}: {exc}") from exc
|
|
125
|
+
return target
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def sync(store: Store, project: projects_mod.Project, image_ids: Iterable[str]) -> list[str]:
|
|
129
|
+
"""Bring these images' label files into step with the database.
|
|
130
|
+
|
|
131
|
+
Every change that alters what an image holds -- a save, a prediction
|
|
132
|
+
import, a class renamed, merged or retired -- must end here, or the folder
|
|
133
|
+
drifts from the database and the next ordinary save writes the drift out
|
|
134
|
+
in one surprising lump. Returns problems rather than raising: the database
|
|
135
|
+
already has the change, so a read-only folder is a warning, not a loss.
|
|
136
|
+
"""
|
|
137
|
+
if not project.is_attached or not project.sidecar_format:
|
|
138
|
+
return []
|
|
139
|
+
problems: list[str] = []
|
|
140
|
+
for image_id in dict.fromkeys(image_ids):
|
|
141
|
+
try:
|
|
142
|
+
write_through(store, project, images_mod.get(store, project, image_id))
|
|
143
|
+
except StoreError as exc:
|
|
144
|
+
problems.append(str(exc))
|
|
145
|
+
if project.sidecar_format == "yolo":
|
|
146
|
+
# A rename changes no .txt -- indices are stable -- but it does change
|
|
147
|
+
# data.yaml, even when no image uses the class yet.
|
|
148
|
+
_refresh_data_yaml(store, project, yolo.ClassList(labels_mod.class_list(store, project)))
|
|
149
|
+
return problems
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def images_using(store: Store, project: projects_mod.Project, label_idx: int) -> list[str]:
|
|
153
|
+
"""Ids of the images holding at least one shape of this class."""
|
|
154
|
+
return [
|
|
155
|
+
row["image_id"]
|
|
156
|
+
for row in store.query(
|
|
157
|
+
"SELECT DISTINCT image_id FROM shape WHERE project_id = ? AND label_idx = ?",
|
|
158
|
+
(project.id, int(label_idx)),
|
|
159
|
+
)
|
|
160
|
+
]
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _refresh_data_yaml(store: Store, project: projects_mod.Project, classes) -> None:
|
|
164
|
+
"""Keep data.yaml in step with the label table.
|
|
165
|
+
|
|
166
|
+
A YOLO folder whose .txt files use index 3 while its data.yaml lists three
|
|
167
|
+
classes is a dataset that trains on nonsense, so the class list is rewritten
|
|
168
|
+
alongside. Cheap: it is a handful of lines, written atomically.
|
|
169
|
+
"""
|
|
170
|
+
root = Path(project.root)
|
|
171
|
+
existing = yolo.find_data_yaml(root)
|
|
172
|
+
target = existing or (root / yolo.DATA_YAML)
|
|
173
|
+
try:
|
|
174
|
+
yolo.write_data_yaml(target, classes)
|
|
175
|
+
except OSError:
|
|
176
|
+
# A read-only folder should not fail the save; the database is still
|
|
177
|
+
# authoritative and export will produce a correct data.yaml.
|
|
178
|
+
pass
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
#: Files and folders a folder walk will look at before giving up. Ten times
|
|
182
|
+
#: the 15,000-image door set with its label files, and a few seconds of walk;
|
|
183
|
+
#: a whole drive is millions.
|
|
184
|
+
MAX_FOLDER_ENTRIES = 1_000_000
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def open_folder(
|
|
188
|
+
store: Store,
|
|
189
|
+
folder: Path | str,
|
|
190
|
+
*,
|
|
191
|
+
owner_id: str = LOCAL_OWNER,
|
|
192
|
+
name: str = "",
|
|
193
|
+
import_annotations: bool = True,
|
|
194
|
+
ctx: Context | None = None,
|
|
195
|
+
) -> dict[str, Any]:
|
|
196
|
+
"""Open a folder for annotation, reusing the project if it is already open.
|
|
197
|
+
|
|
198
|
+
Returns a summary dict rather than just the project, because the things
|
|
199
|
+
worth telling the user -- how many images matched a label file, what will
|
|
200
|
+
be written back, what could not be read -- are exactly the things a silent
|
|
201
|
+
return value would drop.
|
|
202
|
+
"""
|
|
203
|
+
root = Path(folder).expanduser().resolve()
|
|
204
|
+
if not root.is_dir():
|
|
205
|
+
raise AttachError(f"{root} is not a folder")
|
|
206
|
+
if not projects_mod.attached_allowed():
|
|
207
|
+
raise AttachError("attached projects are disabled on this deployment")
|
|
208
|
+
|
|
209
|
+
existing = _find_open(store, root, owner_id)
|
|
210
|
+
reused = existing is not None
|
|
211
|
+
|
|
212
|
+
warnings: list[str] = []
|
|
213
|
+
sidecar_format = existing.sidecar_format if existing else ""
|
|
214
|
+
if not reused:
|
|
215
|
+
sidecar_format, note = detect_sidecar_format(root)
|
|
216
|
+
warnings.append(note)
|
|
217
|
+
|
|
218
|
+
# Found before the project exists, so a folder with no images -- or a
|
|
219
|
+
# whole drive, refused -- leaves no empty project behind.
|
|
220
|
+
try:
|
|
221
|
+
files = fs.walk_images(root, limit=MAX_FOLDER_ENTRIES)
|
|
222
|
+
except fs.TooManyFiles as exc:
|
|
223
|
+
raise AttachError(str(exc)) from None
|
|
224
|
+
if not files:
|
|
225
|
+
raise AttachError(f"{root} has no images in it")
|
|
226
|
+
|
|
227
|
+
project = existing or projects_mod.create(
|
|
228
|
+
store,
|
|
229
|
+
name or root.name,
|
|
230
|
+
owner_id=owner_id,
|
|
231
|
+
storage="attached",
|
|
232
|
+
root=str(root),
|
|
233
|
+
sidecar_format=sidecar_format,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
entries, unreadable = _scan_sizes(root, files, ctx)
|
|
237
|
+
if unreadable:
|
|
238
|
+
warnings.append(
|
|
239
|
+
f"{unreadable} image(s) could not be measured and were registered "
|
|
240
|
+
f"with an unknown size; they still open in the editor"
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
registered = images_mod.add_many(store, project, entries)
|
|
244
|
+
added = sum(1 for image in registered if image.version == 0 and image.state == "todo")
|
|
245
|
+
|
|
246
|
+
imported = 0
|
|
247
|
+
if import_annotations and not reused:
|
|
248
|
+
imported, notes = _import_existing(store, project, root, registered)
|
|
249
|
+
warnings.extend(notes)
|
|
250
|
+
|
|
251
|
+
# Case-variant classes are the defect the real door set actually has:
|
|
252
|
+
# Close/close/Open/open, from two people annotating in two tools. They are
|
|
253
|
+
# imported faithfully -- folding them on the way in would silently change
|
|
254
|
+
# the data -- but training a four-class model by accident is worse than
|
|
255
|
+
# being told, so say it.
|
|
256
|
+
duplicated = labels_mod.variants(store, project)
|
|
257
|
+
for name, found in duplicated.items():
|
|
258
|
+
warnings.append(
|
|
259
|
+
f"{len(found)} classes differ only by case or spacing "
|
|
260
|
+
f"({', '.join(repr(label.name) for label in found)}). "
|
|
261
|
+
f"Merge them in the editor, or run labels.rename --fold-case."
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
return {
|
|
265
|
+
"project": project,
|
|
266
|
+
"reused": reused,
|
|
267
|
+
"images": len(registered),
|
|
268
|
+
"new_images": added,
|
|
269
|
+
"shapes_imported": imported,
|
|
270
|
+
"sidecar_format": project.sidecar_format,
|
|
271
|
+
"writes_back": bool(project.sidecar_format),
|
|
272
|
+
"warnings": warnings,
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _scan_sizes(
|
|
277
|
+
root: Path, files: list[Path], ctx: Context | None = None
|
|
278
|
+
) -> tuple[list[tuple[str, int, int]], int]:
|
|
279
|
+
"""Measure every image, in parallel.
|
|
280
|
+
|
|
281
|
+
Serially this dominated opening a folder: 15ms per image on the real door
|
|
282
|
+
set, which is 56 seconds for 3,686 images and about four minutes for the
|
|
283
|
+
full 15,514 -- all of it spent waiting on the filesystem rather than the
|
|
284
|
+
CPU, which is exactly the shape thread pools are for.
|
|
285
|
+
|
|
286
|
+
``image_size`` reads the header only and honours EXIF, so no image is
|
|
287
|
+
decoded. An unreadable file is registered at size 0 rather than dropped:
|
|
288
|
+
a corrupt image is a normal thing to find in a dataset, and it should be
|
|
289
|
+
visible in the editor rather than quietly missing from it.
|
|
290
|
+
"""
|
|
291
|
+
from ..core import parallel
|
|
292
|
+
|
|
293
|
+
entries: list[tuple[str, int, int]] = []
|
|
294
|
+
unreadable = 0
|
|
295
|
+
|
|
296
|
+
def measure(path: Path) -> tuple[int, int]:
|
|
297
|
+
try:
|
|
298
|
+
return imaging.image_size(path)
|
|
299
|
+
except Exception:
|
|
300
|
+
return (0, 0)
|
|
301
|
+
|
|
302
|
+
for path, size in parallel.map_ordered(
|
|
303
|
+
files, measure, ctx=ctx or Context(), workers=0
|
|
304
|
+
):
|
|
305
|
+
width, height = size or (0, 0)
|
|
306
|
+
if not width:
|
|
307
|
+
unreadable += 1
|
|
308
|
+
entries.append((path.relative_to(root).as_posix(), width, height))
|
|
309
|
+
return entries, unreadable
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _find_open(store: Store, root: Path, owner_id: str) -> projects_mod.Project | None:
|
|
313
|
+
"""An attached project already pointing at this folder, if there is one.
|
|
314
|
+
|
|
315
|
+
Reopening the same folder must land you back in the same project, or a
|
|
316
|
+
second pass would start from zero and none of your review state would be
|
|
317
|
+
there. Compared resolved, so ``.\\data`` and the absolute path match.
|
|
318
|
+
"""
|
|
319
|
+
for project in projects_mod.listing(store, owner_id):
|
|
320
|
+
if not project.is_attached or not project.root:
|
|
321
|
+
continue
|
|
322
|
+
try:
|
|
323
|
+
if Path(project.root).resolve() == root:
|
|
324
|
+
return project
|
|
325
|
+
except OSError:
|
|
326
|
+
continue
|
|
327
|
+
return None
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _import_existing(
|
|
331
|
+
store: Store,
|
|
332
|
+
project: projects_mod.Project,
|
|
333
|
+
root: Path,
|
|
334
|
+
registered: list[images_mod.Image],
|
|
335
|
+
) -> tuple[int, list[str]]:
|
|
336
|
+
"""Read whatever labels are already in the folder into the project."""
|
|
337
|
+
notes: list[str] = []
|
|
338
|
+
try:
|
|
339
|
+
dataset = formats.read_dataset(root, "auto", recursive=True)
|
|
340
|
+
except AnnotationError:
|
|
341
|
+
return 0, ["no existing annotations were read"]
|
|
342
|
+
|
|
343
|
+
notes.extend(dataset.warnings[:10])
|
|
344
|
+
|
|
345
|
+
# An existing class order is already baked into files on disk. Inherit it
|
|
346
|
+
# rather than inventing one, or every .txt in the folder changes meaning.
|
|
347
|
+
if dataset.classes:
|
|
348
|
+
try:
|
|
349
|
+
labels_mod.adopt(store, project, list(dataset.classes))
|
|
350
|
+
except StoreError as exc:
|
|
351
|
+
notes.append(f"could not adopt the existing class order: {exc}")
|
|
352
|
+
|
|
353
|
+
# By path, not bare stem: train/001 and val/001 are different images, and
|
|
354
|
+
# in this mode a wrong match is written back into the user's own files.
|
|
355
|
+
found, unsure = images_mod.assign(
|
|
356
|
+
registered, ((images_mod.item_name(item, root), item.annotation) for item in dataset.items)
|
|
357
|
+
)
|
|
358
|
+
if unsure:
|
|
359
|
+
notes.append(
|
|
360
|
+
f"{len(unsure)} label file(s) could belong to more than one image and "
|
|
361
|
+
f"were not imported, for example: {', '.join(unsure[:5])}"
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
imported = 0
|
|
365
|
+
matched = 0
|
|
366
|
+
for image in registered:
|
|
367
|
+
annotation = found.get(image.id)
|
|
368
|
+
if annotation is None or not annotation.shapes:
|
|
369
|
+
continue
|
|
370
|
+
from_model = any(s.confidence is not None for s in annotation.shapes)
|
|
371
|
+
try:
|
|
372
|
+
shapes_mod.replace_all(
|
|
373
|
+
store, project, image, list(annotation.shapes),
|
|
374
|
+
origin="model" if from_model else "human",
|
|
375
|
+
)
|
|
376
|
+
except StoreError as exc:
|
|
377
|
+
notes.append(f"{image.rel_path}: {exc}")
|
|
378
|
+
continue
|
|
379
|
+
images_mod.set_state(store, project, image, "predicted" if from_model else "reviewed")
|
|
380
|
+
if from_model:
|
|
381
|
+
# Counted, so deleting one of these later registers as a
|
|
382
|
+
# correction in the agreement figure rather than vanishing.
|
|
383
|
+
with store.write() as conn:
|
|
384
|
+
conn.execute(
|
|
385
|
+
"UPDATE image SET predicted_count = ? WHERE id = ?",
|
|
386
|
+
(len(annotation.shapes), image.id),
|
|
387
|
+
)
|
|
388
|
+
imported += len(annotation.shapes)
|
|
389
|
+
matched += 1
|
|
390
|
+
|
|
391
|
+
if dataset.items and matched < len(dataset.items):
|
|
392
|
+
# Never silent: names not lining up is the usual reason an import
|
|
393
|
+
# appears to have done nothing.
|
|
394
|
+
notes.append(
|
|
395
|
+
f"{matched} image(s) matched a label file; "
|
|
396
|
+
f"{len(dataset.items) - matched} label file(s) matched no image"
|
|
397
|
+
)
|
|
398
|
+
return imported, notes
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""A demo project that shows the whole loop in the first minute.
|
|
2
|
+
|
|
3
|
+
``easyannotate demo`` builds a folder of generated scenes -- nothing from a
|
|
4
|
+
real dataset ships in the package -- and opens it with three kinds of image
|
|
5
|
+
already in it, because each is a different first impression:
|
|
6
|
+
|
|
7
|
+
* a few **labelled by hand**, so resume, export and the class list have
|
|
8
|
+
something to show;
|
|
9
|
+
* a run of **model predictions** waiting for review, slightly wrong on
|
|
10
|
+
purpose, so the dashed boxes, the confidence slider and ``A`` to accept make
|
|
11
|
+
sense on sight;
|
|
12
|
+
* the rest **untouched**, to draw on.
|
|
13
|
+
|
|
14
|
+
Deterministic: the same seed draws the same scenes, so a screenshot in the
|
|
15
|
+
README stays true.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
from ..core import imaging
|
|
26
|
+
from ..core.annotations import voc
|
|
27
|
+
from ..core.annotations.model import Annotation, Shape
|
|
28
|
+
|
|
29
|
+
CLASSES = ("crate", "ball", "cone")
|
|
30
|
+
SIZE = (960, 640) # width, height
|
|
31
|
+
MARKER = ".easyannotate-demo"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _scene(rng: np.random.Generator) -> tuple[np.ndarray, list[Shape]]:
|
|
35
|
+
"""One picture and the truth about it: crates, balls and cones on a floor."""
|
|
36
|
+
import cv2
|
|
37
|
+
|
|
38
|
+
width, height = SIZE
|
|
39
|
+
# A floor with a gentle gradient and grain, so a box edge is a judgement
|
|
40
|
+
# rather than a perfectly crisp line.
|
|
41
|
+
top, bottom = rng.integers(70, 120, 3), rng.integers(30, 70, 3)
|
|
42
|
+
ramp = np.linspace(0, 1, height)[:, None, None]
|
|
43
|
+
image = (top * (1 - ramp) + bottom * ramp).astype(np.float32)
|
|
44
|
+
image = np.repeat(image, width, axis=1)
|
|
45
|
+
image += rng.normal(0, 6, image.shape)
|
|
46
|
+
|
|
47
|
+
shapes: list[Shape] = []
|
|
48
|
+
for _ in range(int(rng.integers(2, 6))):
|
|
49
|
+
kind = CLASSES[int(rng.integers(0, len(CLASSES)))]
|
|
50
|
+
w = int(rng.integers(60, 200))
|
|
51
|
+
h = int(w * rng.uniform(0.6, 1.3))
|
|
52
|
+
x = int(rng.integers(10, width - w - 10))
|
|
53
|
+
y = int(rng.integers(10, height - h - 10))
|
|
54
|
+
colour = tuple(int(c) for c in rng.integers(90, 240, 3))
|
|
55
|
+
if kind == "crate":
|
|
56
|
+
cv2.rectangle(image, (x, y), (x + w, y + h), colour, -1)
|
|
57
|
+
cv2.rectangle(image, (x, y), (x + w, y + h), (25, 20, 15), 4)
|
|
58
|
+
cv2.line(image, (x, y), (x + w, y + h), (25, 20, 15), 3)
|
|
59
|
+
shapes.append(Shape(label=kind, bbox=(x, y, x + w, y + h)))
|
|
60
|
+
elif kind == "ball":
|
|
61
|
+
r = min(w, h) // 2
|
|
62
|
+
cx, cy = x + r, y + r
|
|
63
|
+
cv2.circle(image, (cx, cy), r, colour, -1)
|
|
64
|
+
cv2.circle(image, (cx - r // 3, cy - r // 3), max(2, r // 5), (245, 245, 245), -1)
|
|
65
|
+
shapes.append(Shape(label=kind, bbox=(cx - r, cy - r, cx + r, cy + r)))
|
|
66
|
+
else:
|
|
67
|
+
points = np.array([[x + w // 2, y], [x + w, y + h], [x, y + h]], np.int32)
|
|
68
|
+
cv2.fillPoly(image, [points], colour)
|
|
69
|
+
shapes.append(
|
|
70
|
+
Shape(
|
|
71
|
+
label=kind,
|
|
72
|
+
bbox=(x, y, x + w, y + h),
|
|
73
|
+
polygon=[(int(px), int(py)) for px, py in points],
|
|
74
|
+
)
|
|
75
|
+
)
|
|
76
|
+
return np.clip(image, 0, 255).astype(np.uint8), shapes
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _jitter(rng: np.random.Generator, shape: Shape) -> Shape:
|
|
80
|
+
"""What a half-trained model returns: close, not exact, with a score."""
|
|
81
|
+
x1, y1, x2, y2 = shape.bbox
|
|
82
|
+
w, h = x2 - x1, y2 - y1
|
|
83
|
+
dx, dy = rng.normal(0, w * 0.06), rng.normal(0, h * 0.06)
|
|
84
|
+
grow = rng.normal(1, 0.06)
|
|
85
|
+
nx1, ny1 = x1 + dx, y1 + dy
|
|
86
|
+
return Shape(
|
|
87
|
+
label=shape.label,
|
|
88
|
+
bbox=(
|
|
89
|
+
int(nx1),
|
|
90
|
+
int(ny1),
|
|
91
|
+
int(nx1 + w * grow),
|
|
92
|
+
int(ny1 + h * grow),
|
|
93
|
+
),
|
|
94
|
+
confidence=round(float(rng.uniform(0.35, 0.98)), 3),
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def build(folder: Path, *, images: int = 24, seed: int = 7) -> dict[str, Any]:
|
|
99
|
+
"""Write the demo into ``folder``; return what the caller must import.
|
|
100
|
+
|
|
101
|
+
Returns ``{"fresh": bool, "predictions": {name: [Shape]}}``. ``fresh`` is
|
|
102
|
+
False when the folder was already built, so a second ``demo`` reopens the
|
|
103
|
+
same project -- with whatever was done to it -- instead of rebuilding it.
|
|
104
|
+
"""
|
|
105
|
+
folder = Path(folder)
|
|
106
|
+
if (folder / MARKER).exists():
|
|
107
|
+
return {"fresh": False, "predictions": {}}
|
|
108
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
109
|
+
|
|
110
|
+
rng = np.random.default_rng(seed)
|
|
111
|
+
labelled = max(1, images // 6)
|
|
112
|
+
predicted_until = labelled + images // 2
|
|
113
|
+
predictions: dict[str, list[Shape]] = {}
|
|
114
|
+
|
|
115
|
+
for index in range(images):
|
|
116
|
+
name = f"scene_{index:03d}.jpg"
|
|
117
|
+
pixels, truth = _scene(rng)
|
|
118
|
+
imaging.save(pixels, folder / name, quality=90)
|
|
119
|
+
if index < labelled:
|
|
120
|
+
voc.write(
|
|
121
|
+
Annotation(filename=name, width=SIZE[0], height=SIZE[1], shapes=truth),
|
|
122
|
+
folder / f"scene_{index:03d}.xml",
|
|
123
|
+
)
|
|
124
|
+
elif index < predicted_until:
|
|
125
|
+
found = [_jitter(rng, shape) for shape in truth if rng.random() > 0.12]
|
|
126
|
+
# Now and then a false positive, which is most of what reviewing is.
|
|
127
|
+
if rng.random() < 0.3:
|
|
128
|
+
x, y = int(rng.integers(0, SIZE[0] - 90)), int(rng.integers(0, SIZE[1] - 90))
|
|
129
|
+
found.append(
|
|
130
|
+
Shape(
|
|
131
|
+
label=CLASSES[int(rng.integers(0, len(CLASSES)))],
|
|
132
|
+
bbox=(x, y, x + 80, y + 80),
|
|
133
|
+
confidence=round(float(rng.uniform(0.2, 0.5)), 3),
|
|
134
|
+
)
|
|
135
|
+
)
|
|
136
|
+
predictions[name] = found
|
|
137
|
+
|
|
138
|
+
(folder / MARKER).write_text("generated by easyannotate demo\n", encoding="utf-8")
|
|
139
|
+
return {"fresh": True, "predictions": predictions}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def write_predictions(predictions: dict[str, list[Shape]], target: Path) -> Path:
|
|
143
|
+
"""The predictions as Ultralytics writes them with ``--save-conf``: one
|
|
144
|
+
``.txt`` per image, ``class cx cy w h confidence`` normalised, beside a
|
|
145
|
+
``data.yaml`` naming the classes. So the demo imports through exactly the
|
|
146
|
+
path a real model run takes."""
|
|
147
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
148
|
+
width, height = SIZE
|
|
149
|
+
names = "".join(f" {i}: {name}\n" for i, name in enumerate(CLASSES))
|
|
150
|
+
(target / "data.yaml").write_text(f"names:\n{names}", encoding="utf-8")
|
|
151
|
+
for name, shapes in predictions.items():
|
|
152
|
+
lines = []
|
|
153
|
+
for shape in shapes:
|
|
154
|
+
x1, y1, x2, y2 = shape.bbox
|
|
155
|
+
lines.append(
|
|
156
|
+
f"{CLASSES.index(shape.label)} {(x1 + x2) / 2 / width:.6f} "
|
|
157
|
+
f"{(y1 + y2) / 2 / height:.6f} {(x2 - x1) / width:.6f} "
|
|
158
|
+
f"{(y2 - y1) / height:.6f} {shape.confidence:.3f}"
|
|
159
|
+
)
|
|
160
|
+
(target / f"{Path(name).stem}.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
161
|
+
return target
|