easyannotate 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_factory/__init__.py +82 -0
- data_factory/__main__.py +6 -0
- data_factory/annotate/__init__.py +15 -0
- data_factory/annotate/attach.py +398 -0
- data_factory/annotate/demo.py +161 -0
- data_factory/annotate/images.py +443 -0
- data_factory/annotate/labels.py +399 -0
- data_factory/annotate/loop.py +359 -0
- data_factory/annotate/projects.py +207 -0
- data_factory/annotate/review.py +471 -0
- data_factory/annotate/schema.sql +171 -0
- data_factory/annotate/shapes.py +488 -0
- data_factory/annotate/store.py +292 -0
- data_factory/annotate/tools/__init__.py +39 -0
- data_factory/annotate/tools/fake.py +170 -0
- data_factory/annotate/tools/install.py +254 -0
- data_factory/annotate/tools/local.py +389 -0
- data_factory/annotate/tools/postprocess.py +175 -0
- data_factory/annotate/tools/protocol.py +238 -0
- data_factory/annotate/tools/registry.py +196 -0
- data_factory/annotate/tools/results.py +83 -0
- data_factory/annotate/tools/serve.py +106 -0
- data_factory/cli/__init__.py +21 -0
- data_factory/cli/main.py +837 -0
- data_factory/cli/menu.py +173 -0
- data_factory/cli/render.py +158 -0
- data_factory/core/__init__.py +11 -0
- data_factory/core/annotations/__init__.py +5 -0
- data_factory/core/annotations/coco.py +394 -0
- data_factory/core/annotations/csvfmt.py +174 -0
- data_factory/core/annotations/dota.py +98 -0
- data_factory/core/annotations/formats.py +626 -0
- data_factory/core/annotations/labelme.py +153 -0
- data_factory/core/annotations/model.py +334 -0
- data_factory/core/annotations/voc.py +231 -0
- data_factory/core/annotations/yolo.py +283 -0
- data_factory/core/bktree.py +89 -0
- data_factory/core/fs.py +245 -0
- data_factory/core/imaging.py +626 -0
- data_factory/core/ops/__init__.py +44 -0
- data_factory/core/ops/analyze.py +383 -0
- data_factory/core/ops/annotate_export.py +326 -0
- data_factory/core/ops/audit.py +274 -0
- data_factory/core/ops/augment.py +454 -0
- data_factory/core/ops/collect.py +241 -0
- data_factory/core/ops/combine.py +582 -0
- data_factory/core/ops/convert.py +212 -0
- data_factory/core/ops/dataset.py +246 -0
- data_factory/core/ops/dedup.py +235 -0
- data_factory/core/ops/export.py +168 -0
- data_factory/core/ops/image.py +407 -0
- data_factory/core/ops/labels.py +215 -0
- data_factory/core/ops/pairing.py +133 -0
- data_factory/core/ops/rename.py +154 -0
- data_factory/core/ops/report.py +323 -0
- data_factory/core/ops/visualize.py +164 -0
- data_factory/core/pairs.py +144 -0
- data_factory/core/parallel.py +191 -0
- data_factory/core/provenance.py +136 -0
- data_factory/core/registry.py +339 -0
- data_factory/core/stats.py +403 -0
- data_factory/core/types.py +148 -0
- data_factory/core/writer.py +194 -0
- data_factory/desktop.py +207 -0
- data_factory/hardware.py +243 -0
- data_factory/pipeline.py +192 -0
- data_factory/py.typed +0 -0
- data_factory/service/__init__.py +26 -0
- data_factory/service/app.py +483 -0
- data_factory/service/binding.py +174 -0
- data_factory/service/deps.py +133 -0
- data_factory/service/jobs.py +177 -0
- data_factory/service/limits.py +71 -0
- data_factory/service/routers/__init__.py +11 -0
- data_factory/service/routers/annotate.py +582 -0
- data_factory/service/routers/local.py +464 -0
- data_factory/service/routers/media.py +154 -0
- data_factory/service/routers/predictions.py +116 -0
- data_factory/service/routers/promote.py +182 -0
- data_factory/service/routers/tools.py +308 -0
- data_factory/service/static/annotate/api.js +246 -0
- data_factory/service/static/annotate/app.js +1696 -0
- data_factory/service/static/annotate/assist.js +106 -0
- data_factory/service/static/annotate/filmstrip.js +286 -0
- data_factory/service/static/annotate/hit.js +146 -0
- data_factory/service/static/annotate/labels.js +197 -0
- data_factory/service/static/annotate/render.js +286 -0
- data_factory/service/static/annotate/scene.js +277 -0
- data_factory/service/static/annotate/shapelist.js +72 -0
- data_factory/service/static/annotate/tools/box.js +96 -0
- data_factory/service/static/annotate/tools/line.js +96 -0
- data_factory/service/static/annotate/tools/obb.js +88 -0
- data_factory/service/static/annotate/tools/point.js +49 -0
- data_factory/service/static/annotate/tools/polygon.js +110 -0
- data_factory/service/static/annotate/tools/select.js +227 -0
- data_factory/service/static/annotate/undo.js +176 -0
- data_factory/service/static/annotate/viewport.js +156 -0
- data_factory/service/static/annotate.html +390 -0
- data_factory/service/static/index.html +632 -0
- data_factory/service/workspace.py +244 -0
- easyannotate/__init__.py +64 -0
- easyannotate/__main__.py +5 -0
- easyannotate/project.py +181 -0
- easyannotate/py.typed +0 -0
- easyannotate-0.4.0.dist-info/METADATA +767 -0
- easyannotate-0.4.0.dist-info/RECORD +109 -0
- easyannotate-0.4.0.dist-info/WHEEL +4 -0
- easyannotate-0.4.0.dist-info/entry_points.txt +4 -0
- easyannotate-0.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Streaming dataset writer.
|
|
2
|
+
|
|
3
|
+
:func:`data_factory.core.annotations.formats.write_dataset` takes a whole
|
|
4
|
+
dataset in memory, which is fine for conversion -- the source already had to be
|
|
5
|
+
loaded. It is the wrong shape for the operations that *generate* images:
|
|
6
|
+
augmenting a 4,000-image set five ways would mean holding 20,000 decoded frames
|
|
7
|
+
before a single byte reached the disk.
|
|
8
|
+
|
|
9
|
+
This writer takes one item at a time and flushes as it goes. The formats that
|
|
10
|
+
genuinely cannot stream -- COCO and CSV are single files -- accumulate only the
|
|
11
|
+
annotations, never the pixels, so the memory cost is a few hundred bytes per
|
|
12
|
+
image rather than a few megabytes.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
|
|
22
|
+
from . import fs, imaging
|
|
23
|
+
from .annotations import coco as coco_fmt
|
|
24
|
+
from .annotations import csvfmt, formats, labelme, voc, yolo
|
|
25
|
+
from .annotations.model import Annotation
|
|
26
|
+
from .types import Action, Context, FileAction
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DatasetWriter:
|
|
30
|
+
"""Writes annotations and images incrementally in a chosen format.
|
|
31
|
+
|
|
32
|
+
``close()`` finalises whatever the format needs written once -- a
|
|
33
|
+
``data.yaml``, an ``instances_*.json``, a ``_annotations.csv`` -- and
|
|
34
|
+
returns a small report.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(
|
|
38
|
+
self,
|
|
39
|
+
output: Path,
|
|
40
|
+
fmt: str,
|
|
41
|
+
ctx: Context,
|
|
42
|
+
*,
|
|
43
|
+
classes: list[str] | None = None,
|
|
44
|
+
segmentation: bool = True,
|
|
45
|
+
quality: int = 95,
|
|
46
|
+
) -> None:
|
|
47
|
+
if fmt not in formats.FORMATS:
|
|
48
|
+
raise ValueError(f"unknown format {fmt!r}; known: {', '.join(formats.FORMATS)}")
|
|
49
|
+
self.output = Path(output)
|
|
50
|
+
self.fmt = fmt
|
|
51
|
+
self.ctx = ctx
|
|
52
|
+
self.segmentation = segmentation
|
|
53
|
+
self.quality = quality
|
|
54
|
+
# Seeding the class list up front keeps YOLO indices stable and
|
|
55
|
+
# independent of the order images happen to be processed in.
|
|
56
|
+
self.classes = yolo.ClassList(list(classes or []))
|
|
57
|
+
self.images_written = 0
|
|
58
|
+
self.labels_written = 0
|
|
59
|
+
self.warnings: list[str] = []
|
|
60
|
+
self.actions: list[FileAction] = []
|
|
61
|
+
self._splits: dict[str, str] = {}
|
|
62
|
+
self._collected: dict[str, list[tuple[Annotation, str]]] = {}
|
|
63
|
+
|
|
64
|
+
# -- one item ----------------------------------------------------------
|
|
65
|
+
def add(
|
|
66
|
+
self,
|
|
67
|
+
annotation: Annotation,
|
|
68
|
+
*,
|
|
69
|
+
stem: str,
|
|
70
|
+
image: np.ndarray | None = None,
|
|
71
|
+
encoded: bytes | None = None,
|
|
72
|
+
source: Path | None = None,
|
|
73
|
+
split: str = "",
|
|
74
|
+
suffix: str = ".jpg",
|
|
75
|
+
) -> None:
|
|
76
|
+
"""Write one image and its label.
|
|
77
|
+
|
|
78
|
+
Pass ``image`` for a freshly computed frame, ``encoded`` when the
|
|
79
|
+
caller already compressed it on a worker thread, or ``source`` alone to
|
|
80
|
+
copy an untouched file straight through -- re-encoding an image nothing
|
|
81
|
+
changed would cost quality for nothing.
|
|
82
|
+
"""
|
|
83
|
+
suffix = suffix or ".jpg"
|
|
84
|
+
image_dir, label_dir = self._directories(split)
|
|
85
|
+
name = f"{stem}{suffix}"
|
|
86
|
+
|
|
87
|
+
if encoded is not None or image is not None:
|
|
88
|
+
if not self.ctx.dry_run:
|
|
89
|
+
fs.ensure_dir(image_dir, self.ctx)
|
|
90
|
+
if encoded is not None:
|
|
91
|
+
imaging.write_bytes(encoded, image_dir / name)
|
|
92
|
+
else:
|
|
93
|
+
assert image is not None
|
|
94
|
+
imaging.save(image, image_dir / name, quality=self.quality)
|
|
95
|
+
self.actions.append(
|
|
96
|
+
FileAction(Action.WRITE, source or (image_dir / name), image_dir / name, "generated")
|
|
97
|
+
)
|
|
98
|
+
self.images_written += 1
|
|
99
|
+
elif source is not None:
|
|
100
|
+
fs.ensure_dir(image_dir, self.ctx)
|
|
101
|
+
action = FileAction(Action.COPY, source, image_dir / name, "unchanged")
|
|
102
|
+
fs.perform(action, self.ctx)
|
|
103
|
+
self.actions.append(action)
|
|
104
|
+
self.images_written += 1
|
|
105
|
+
|
|
106
|
+
stored = Annotation(
|
|
107
|
+
filename=name,
|
|
108
|
+
width=annotation.width,
|
|
109
|
+
height=annotation.height,
|
|
110
|
+
depth=annotation.depth,
|
|
111
|
+
shapes=annotation.shapes,
|
|
112
|
+
folder=annotation.folder,
|
|
113
|
+
source_path=annotation.source_path,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
if self.fmt in ("coco", "csv"):
|
|
117
|
+
self._collected.setdefault(split or "train", []).append((stored, name))
|
|
118
|
+
elif not self.ctx.dry_run:
|
|
119
|
+
fs.ensure_dir(label_dir, self.ctx)
|
|
120
|
+
self._write_sidecar(stored, label_dir, stem)
|
|
121
|
+
self.labels_written += 1
|
|
122
|
+
if split:
|
|
123
|
+
self._splits.setdefault(split, f"{yolo.IMAGE_DIR}/{split}")
|
|
124
|
+
|
|
125
|
+
def _write_sidecar(self, annotation: Annotation, label_dir: Path, stem: str) -> None:
|
|
126
|
+
if self.fmt == "voc":
|
|
127
|
+
voc.write(annotation, label_dir / f"{stem}.xml")
|
|
128
|
+
elif self.fmt == "labelme":
|
|
129
|
+
labelme.write(annotation, label_dir / f"{stem}.json")
|
|
130
|
+
elif self.fmt == "yolo":
|
|
131
|
+
yolo.write(
|
|
132
|
+
annotation, label_dir / f"{stem}.txt", self.classes, segmentation=self.segmentation
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def _directories(self, split: str) -> tuple[Path, Path]:
|
|
136
|
+
"""Where images and labels go, which is the one thing formats disagree on."""
|
|
137
|
+
if self.fmt == "yolo":
|
|
138
|
+
leaf = split or "train"
|
|
139
|
+
return (
|
|
140
|
+
self.output / yolo.IMAGE_DIR / leaf,
|
|
141
|
+
self.output / yolo.LABEL_DIR / leaf,
|
|
142
|
+
)
|
|
143
|
+
if self.fmt == "coco":
|
|
144
|
+
base = self.output / (split or "train")
|
|
145
|
+
return base, base
|
|
146
|
+
if self.fmt == "csv":
|
|
147
|
+
base = self.output / "images"
|
|
148
|
+
return base, base
|
|
149
|
+
base = self.output / split if split else self.output
|
|
150
|
+
return base, base
|
|
151
|
+
|
|
152
|
+
# -- finish ------------------------------------------------------------
|
|
153
|
+
def close(self) -> dict[str, Any]:
|
|
154
|
+
if not self.ctx.dry_run:
|
|
155
|
+
if self.fmt == "yolo":
|
|
156
|
+
yolo.write_data_yaml(
|
|
157
|
+
self.output / yolo.DATA_YAML,
|
|
158
|
+
self.classes,
|
|
159
|
+
splits=self._splits or {"train": f"{yolo.IMAGE_DIR}/train"},
|
|
160
|
+
root=".",
|
|
161
|
+
)
|
|
162
|
+
elif self.fmt == "coco":
|
|
163
|
+
self._write_coco()
|
|
164
|
+
elif self.fmt == "csv":
|
|
165
|
+
self._write_csv()
|
|
166
|
+
|
|
167
|
+
return {
|
|
168
|
+
"format": self.fmt,
|
|
169
|
+
"images": self.images_written,
|
|
170
|
+
"labels": self.labels_written,
|
|
171
|
+
"classes": list(self.classes.names),
|
|
172
|
+
"warnings": list(self.warnings),
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
def _write_coco(self) -> None:
|
|
176
|
+
for split, entries in sorted(self._collected.items()):
|
|
177
|
+
builder = coco_fmt.CocoBuilder(description=f"Data Factory ({split})")
|
|
178
|
+
for annotation, name in entries:
|
|
179
|
+
builder.add(annotation, file_name=name)
|
|
180
|
+
fs.ensure_dir(self.output / "annotations", self.ctx)
|
|
181
|
+
builder.write(self.output / "annotations" / f"instances_{split}.json")
|
|
182
|
+
merges = builder.categories.merges()
|
|
183
|
+
if merges:
|
|
184
|
+
self.warnings.append(
|
|
185
|
+
"case-variant classes merged: "
|
|
186
|
+
+ "; ".join(f"{k} <- {', '.join(v)}" for k, v in merges.items())
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
def _write_csv(self) -> None:
|
|
190
|
+
rows: list[Annotation] = []
|
|
191
|
+
for _, entries in sorted(self._collected.items()):
|
|
192
|
+
rows.extend(annotation for annotation, _ in entries)
|
|
193
|
+
fs.ensure_dir(self.output, self.ctx)
|
|
194
|
+
csvfmt.write(rows, self.output / "_annotations.csv")
|
data_factory/desktop.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""EasyAnnotate as a desktop application: a native window over the local server.
|
|
2
|
+
|
|
3
|
+
Same server, same pages, same code as ``easyannotate serve`` -- the window is
|
|
4
|
+
the only new part, and it is a window and nothing more. This is how the
|
|
5
|
+
binary launches, and what ``easyannotate app`` runs from a pip install.
|
|
6
|
+
|
|
7
|
+
Two things only the desktop can do, and this is where they live:
|
|
8
|
+
|
|
9
|
+
* **Open a folder with the system's own dialog.** The rule everywhere else is
|
|
10
|
+
that a client never sends a filesystem path over HTTP. Here the path comes
|
|
11
|
+
from the operating system's folder picker, inside this process, and never
|
|
12
|
+
crosses the network -- the one place that rule can be honoured and a
|
|
13
|
+
folder still opened by clicking.
|
|
14
|
+
* **Pick a port, not demand one.** A fixed port dies at startup when it is
|
|
15
|
+
taken, with a stack trace, which is a poor first impression for a
|
|
16
|
+
double-clicked icon.
|
|
17
|
+
|
|
18
|
+
When no window can be made -- pywebview not installed, no WebView2 runtime on
|
|
19
|
+
an old Windows, ``EASYANNOTATE_NO_WINDOW=1`` in CI -- it does what ``serve``
|
|
20
|
+
does: opens the browser. Degrading, not failing.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import os
|
|
26
|
+
import socket
|
|
27
|
+
import sys
|
|
28
|
+
import threading
|
|
29
|
+
import time
|
|
30
|
+
import webbrowser
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
PREFERRED_PORT = 8765
|
|
35
|
+
TITLE = "EasyAnnotate"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def free_port(preferred: int = PREFERRED_PORT) -> int:
|
|
39
|
+
"""The preferred port if it is free, otherwise whatever the OS hands us."""
|
|
40
|
+
for candidate in (preferred, 0):
|
|
41
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
|
42
|
+
try:
|
|
43
|
+
probe.bind(("127.0.0.1", candidate))
|
|
44
|
+
return probe.getsockname()[1]
|
|
45
|
+
except OSError:
|
|
46
|
+
continue
|
|
47
|
+
return preferred
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def default_workspace() -> Path:
|
|
51
|
+
"""Somewhere writable that survives an app update.
|
|
52
|
+
|
|
53
|
+
Never beside the executable: on Windows that is Program Files, which is
|
|
54
|
+
read-only for a normal user, and the first upload would fail.
|
|
55
|
+
"""
|
|
56
|
+
if sys.platform == "win32":
|
|
57
|
+
base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
|
|
58
|
+
elif sys.platform == "darwin":
|
|
59
|
+
base = Path.home() / "Library" / "Application Support"
|
|
60
|
+
else:
|
|
61
|
+
base = Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share"))
|
|
62
|
+
path = base / "EasyAnnotate" / "workspaces"
|
|
63
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
64
|
+
return path
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Bridge:
|
|
68
|
+
"""What the page may ask the desktop for.
|
|
69
|
+
|
|
70
|
+
Exposed to the page as ``window.pywebview.api``. Deliberately tiny: every
|
|
71
|
+
method here is a capability the browser version does not have, and each
|
|
72
|
+
one is a line the sandbox otherwise draws.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def __init__(self) -> None:
|
|
76
|
+
# Underscored on purpose. pywebview exposes every *public* attribute
|
|
77
|
+
# of this object to the page, walking into it recursively -- and the
|
|
78
|
+
# window holds native COM objects that may only be touched from the
|
|
79
|
+
# UI thread. Making it public flooded the log with errors and starved
|
|
80
|
+
# the server. Found by running the binary, not by reading it.
|
|
81
|
+
self._window: Any = None
|
|
82
|
+
|
|
83
|
+
def attach(self, window: Any) -> None:
|
|
84
|
+
self._window = window
|
|
85
|
+
|
|
86
|
+
def pick_folder(self) -> dict[str, Any]:
|
|
87
|
+
"""Native folder dialog, then open that folder as an attached project."""
|
|
88
|
+
import webview
|
|
89
|
+
|
|
90
|
+
if self._window is None:
|
|
91
|
+
return {"error": "the window is not ready yet"}
|
|
92
|
+
chosen = self._window.create_file_dialog(webview.FOLDER_DIALOG)
|
|
93
|
+
if not chosen:
|
|
94
|
+
return {"cancelled": True}
|
|
95
|
+
folder = Path(chosen[0] if isinstance(chosen, (list, tuple)) else chosen)
|
|
96
|
+
|
|
97
|
+
from .annotate import attach
|
|
98
|
+
from .annotate.store import StoreError
|
|
99
|
+
from .service.deps import get_store
|
|
100
|
+
|
|
101
|
+
try:
|
|
102
|
+
summary = attach.open_folder(get_store(), folder)
|
|
103
|
+
except StoreError as exc:
|
|
104
|
+
return {"error": str(exc)}
|
|
105
|
+
project = summary["project"]
|
|
106
|
+
return {
|
|
107
|
+
"project": project.id,
|
|
108
|
+
"name": project.name,
|
|
109
|
+
"images": summary["images"],
|
|
110
|
+
"imported": summary["shapes_imported"],
|
|
111
|
+
"writes_back": summary["writes_back"],
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
def version(self) -> str:
|
|
115
|
+
from . import __version__
|
|
116
|
+
|
|
117
|
+
return __version__
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def start_server(port: int) -> Any:
|
|
121
|
+
"""The service, on its own thread, listening before this returns."""
|
|
122
|
+
import uvicorn
|
|
123
|
+
|
|
124
|
+
from .service.app import app
|
|
125
|
+
|
|
126
|
+
# The app object rather than an import string: the frozen build has no
|
|
127
|
+
# importable module path for uvicorn's reloader to resolve.
|
|
128
|
+
server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="warning"))
|
|
129
|
+
thread = threading.Thread(target=server.run, daemon=True, name="easyannotate-server")
|
|
130
|
+
thread.start()
|
|
131
|
+
deadline = time.time() + 20
|
|
132
|
+
while not server.started and thread.is_alive() and time.time() < deadline:
|
|
133
|
+
time.sleep(0.05)
|
|
134
|
+
if not server.started:
|
|
135
|
+
raise RuntimeError("the server did not start")
|
|
136
|
+
server._thread = thread # noqa: SLF001 -- kept for the fallback's join
|
|
137
|
+
return server
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def open_window(url: str) -> bool:
|
|
141
|
+
"""Show the native window; return False if none could be made."""
|
|
142
|
+
if os.environ.get("EASYANNOTATE_NO_WINDOW"):
|
|
143
|
+
return False
|
|
144
|
+
try:
|
|
145
|
+
import webview
|
|
146
|
+
except ImportError:
|
|
147
|
+
return False
|
|
148
|
+
|
|
149
|
+
bridge = Bridge()
|
|
150
|
+
try:
|
|
151
|
+
window = webview.create_window(
|
|
152
|
+
TITLE,
|
|
153
|
+
url,
|
|
154
|
+
width=1440,
|
|
155
|
+
height=900,
|
|
156
|
+
min_size=(960, 600),
|
|
157
|
+
js_api=bridge,
|
|
158
|
+
text_select=False,
|
|
159
|
+
)
|
|
160
|
+
bridge.attach(window)
|
|
161
|
+
print(" window : native", flush=True)
|
|
162
|
+
webview.start() # blocks until the window closes
|
|
163
|
+
except Exception as exc: # no WebView2, no GTK, a display that is not there
|
|
164
|
+
print(f" window : unavailable ({exc}); using the browser", flush=True)
|
|
165
|
+
return False
|
|
166
|
+
return True
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def main(argv: list[str] | None = None) -> int:
|
|
170
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
171
|
+
os.environ.setdefault("DATAFACTORY_WORKSPACE", str(default_workspace()))
|
|
172
|
+
port = free_port()
|
|
173
|
+
from .annotate import projects
|
|
174
|
+
from .service.routers.local import LOOPBACK_ENV, TOKEN_ENV, mint_token
|
|
175
|
+
|
|
176
|
+
# The window is this machine's own; the token lets its page open folders
|
|
177
|
+
# by path too, not only through the dialog. Loopback: refuse any request
|
|
178
|
+
# that names another host (see LocalHostFence).
|
|
179
|
+
os.environ[LOOPBACK_ENV] = "1"
|
|
180
|
+
os.environ.pop(TOKEN_ENV, None)
|
|
181
|
+
url = f"http://127.0.0.1:{port}/"
|
|
182
|
+
if projects.attached_allowed():
|
|
183
|
+
url += f"#token={mint_token()}"
|
|
184
|
+
|
|
185
|
+
# Flushed: when the console is a pipe the banner would otherwise sit in a
|
|
186
|
+
# buffer until exit, and the one thing a launcher must do is say its URL.
|
|
187
|
+
print(TITLE, flush=True)
|
|
188
|
+
print(f" interface : {url}", flush=True)
|
|
189
|
+
print(f" datasets : {os.environ['DATAFACTORY_WORKSPACE']}", flush=True)
|
|
190
|
+
|
|
191
|
+
server = start_server(port)
|
|
192
|
+
|
|
193
|
+
if "--browser" not in argv and open_window(url):
|
|
194
|
+
server.should_exit = True
|
|
195
|
+
return 0
|
|
196
|
+
|
|
197
|
+
print(" close this window to quit", flush=True)
|
|
198
|
+
threading.Timer(1.0, lambda: webbrowser.open(url)).start()
|
|
199
|
+
try:
|
|
200
|
+
server._thread.join() # noqa: SLF001
|
|
201
|
+
except KeyboardInterrupt:
|
|
202
|
+
server.should_exit = True
|
|
203
|
+
return 0
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
if __name__ == "__main__":
|
|
207
|
+
raise SystemExit(main())
|
data_factory/hardware.py
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"""What this machine can run: GPU, VRAM, driver, RAM -- standard library only.
|
|
2
|
+
|
|
3
|
+
Used to decide, before anything heavy is downloaded, which PyTorch build to
|
|
4
|
+
install and which model sizes to use. Getting it wrong is expensive in a
|
|
5
|
+
specific way: a CUDA build newer than the driver installs fine (2 GB later)
|
|
6
|
+
and then silently runs on the CPU, and a model too big for the card fails
|
|
7
|
+
with an out-of-memory error on the first click.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import ctypes
|
|
13
|
+
import os
|
|
14
|
+
import platform
|
|
15
|
+
import re
|
|
16
|
+
import shutil
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
from dataclasses import asdict, dataclass, field
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class Gpu:
|
|
25
|
+
name: str
|
|
26
|
+
vram_gb: float
|
|
27
|
+
driver: str = ""
|
|
28
|
+
#: The newest CUDA the *driver* supports -- what nvidia-smi's header says.
|
|
29
|
+
cuda: str = ""
|
|
30
|
+
#: CUDA's index for it, so the models run on the card they were sized for.
|
|
31
|
+
index: int = 0
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class Hardware:
|
|
36
|
+
os: str
|
|
37
|
+
cpu_count: int
|
|
38
|
+
ram_gb: float
|
|
39
|
+
gpus: list[Gpu] = field(default_factory=list)
|
|
40
|
+
apple_silicon: bool = False
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def best_gpu(self) -> Gpu | None:
|
|
44
|
+
return max(self.gpus, key=lambda g: g.vram_gb) if self.gpus else None
|
|
45
|
+
|
|
46
|
+
def as_dict(self) -> dict[str, Any]:
|
|
47
|
+
out = asdict(self)
|
|
48
|
+
out["apple_silicon"] = self.apple_silicon
|
|
49
|
+
return out
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def detect() -> Hardware:
|
|
53
|
+
return Hardware(
|
|
54
|
+
os=platform.system(),
|
|
55
|
+
cpu_count=os.cpu_count() or 1,
|
|
56
|
+
ram_gb=round(_ram_bytes() / 2**30, 1),
|
|
57
|
+
gpus=_nvidia_gpus(),
|
|
58
|
+
apple_silicon=sys.platform == "darwin" and platform.machine() == "arm64",
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ---------------------------------------------------------------- the choice
|
|
63
|
+
|
|
64
|
+
#: PyTorch CUDA builds, newest first: the oldest driver each needs
|
|
65
|
+
#: (Windows, Linux), and the packages to ask for (empty: the newest).
|
|
66
|
+
#:
|
|
67
|
+
#: The minimums are each toolkit's *own* driver, not CUDA's minor-version
|
|
68
|
+
#: compatibility floor. Measured on an RTX 2050 with driver 529.04 (CUDA
|
|
69
|
+
#: 12.0): the cu126 build said torch.cuda.is_available() and then failed
|
|
70
|
+
#: every allocation ("device busy or unavailable"); PyTorch 2.7.1+cu118 ran.
|
|
71
|
+
#: 2.7 is the last PyTorch with a cu118 build, so older drivers get it.
|
|
72
|
+
CUDA_BUILDS = (
|
|
73
|
+
("cu128", ("570.65", "570.26"), ()),
|
|
74
|
+
("cu126", ("560.76", "560.28.03"), ()),
|
|
75
|
+
("cu118", ("522.06", "520.61.05"), ("torch==2.7.1", "torchvision==0.22.1")),
|
|
76
|
+
)
|
|
77
|
+
TORCH_INDEX = "https://download.pytorch.org/whl/{build}"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass
|
|
81
|
+
class Plan:
|
|
82
|
+
device: str # "cuda", "mps" or "cpu"
|
|
83
|
+
torch_index: str | None # None: PyPI's default build
|
|
84
|
+
sam: str # SAM weights file
|
|
85
|
+
world: str # YOLO-World weights file
|
|
86
|
+
download_gb: float
|
|
87
|
+
summary: str
|
|
88
|
+
warnings: list[str] = field(default_factory=list)
|
|
89
|
+
gpu_index: int = 0
|
|
90
|
+
#: What to install from ``torch_index``; empty means the newest torch.
|
|
91
|
+
torch_packages: tuple[str, ...] = ()
|
|
92
|
+
|
|
93
|
+
def as_dict(self) -> dict[str, Any]:
|
|
94
|
+
return asdict(self)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def plan(hardware: Hardware | None = None) -> Plan:
|
|
98
|
+
"""What to install for this machine, and why, in words a person can read."""
|
|
99
|
+
hw = hardware or detect()
|
|
100
|
+
warnings: list[str] = []
|
|
101
|
+
gpu = hw.best_gpu
|
|
102
|
+
|
|
103
|
+
if gpu is not None:
|
|
104
|
+
build = _cuda_build_for(gpu.driver, hw.os)
|
|
105
|
+
# Memory first: a 2 GB card is too small whatever its driver.
|
|
106
|
+
if gpu.vram_gb < 3.5:
|
|
107
|
+
warnings.append(f"{gpu.name} has {gpu.vram_gb:g} GB: too little for the models. Installing for CPU.")
|
|
108
|
+
build = None
|
|
109
|
+
elif build is None:
|
|
110
|
+
warnings.append(
|
|
111
|
+
f"{gpu.name}: driver {gpu.driver or 'unknown'} is too old for PyTorch's "
|
|
112
|
+
"CUDA builds -- update the NVIDIA driver to use the GPU. Installing for CPU."
|
|
113
|
+
)
|
|
114
|
+
elif build == "cu118":
|
|
115
|
+
warnings.append(
|
|
116
|
+
f"driver {gpu.driver} predates current PyTorch CUDA builds: installing PyTorch 2.7 "
|
|
117
|
+
"with CUDA 11.8. An NVIDIA driver update (560 or newer) gets the current one."
|
|
118
|
+
)
|
|
119
|
+
if build is not None:
|
|
120
|
+
large = gpu.vram_gb >= 8
|
|
121
|
+
return Plan(
|
|
122
|
+
device="cuda",
|
|
123
|
+
torch_index=TORCH_INDEX.format(build=build),
|
|
124
|
+
torch_packages=_packages_for(build),
|
|
125
|
+
sam="sam2.1_b.pt" if large else "sam2.1_s.pt",
|
|
126
|
+
world="yolov8l-worldv2.pt" if large else "yolov8m-worldv2.pt",
|
|
127
|
+
download_gb=3.4 if large else 3.1,
|
|
128
|
+
summary=(
|
|
129
|
+
f"{gpu.name}, {gpu.vram_gb:g} GB: PyTorch with CUDA ({build}), "
|
|
130
|
+
f"SAM 2.1 {'base' if large else 'small'}, YOLO-World {'large' if large else 'medium'}"
|
|
131
|
+
),
|
|
132
|
+
warnings=warnings,
|
|
133
|
+
gpu_index=gpu.index,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
if hw.apple_silicon:
|
|
137
|
+
return Plan(
|
|
138
|
+
device="mps", torch_index=None, sam="sam2.1_s.pt", world="yolov8m-worldv2.pt",
|
|
139
|
+
download_gb=1.1,
|
|
140
|
+
summary="Apple silicon: PyTorch with Metal, SAM 2.1 small, YOLO-World medium",
|
|
141
|
+
warnings=warnings,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
return Plan(
|
|
145
|
+
device="cpu",
|
|
146
|
+
torch_index="https://download.pytorch.org/whl/cpu",
|
|
147
|
+
sam="sam2.1_t.pt",
|
|
148
|
+
world="yolov8s-worldv2.pt",
|
|
149
|
+
download_gb=0.9,
|
|
150
|
+
summary=(
|
|
151
|
+
f"CPU, {hw.ram_gb:g} GB RAM: PyTorch for CPU, SAM 2.1 tiny, YOLO-World small "
|
|
152
|
+
"-- about 0.2 s per click once an image is encoded"
|
|
153
|
+
),
|
|
154
|
+
warnings=warnings + (
|
|
155
|
+
[f"{hw.ram_gb:g} GB RAM is tight for the models; close other apps while using them."]
|
|
156
|
+
if hw.ram_gb and hw.ram_gb < 8 else []
|
|
157
|
+
),
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _cuda_build_for(driver: str, os_name: str) -> str | None:
|
|
162
|
+
# As a tuple, never a float: 525.105 is newer than 525.60, but as decimal
|
|
163
|
+
# numbers it is smaller -- and a 12 GB card was sent to the CPU.
|
|
164
|
+
version = _version(driver)
|
|
165
|
+
if not version:
|
|
166
|
+
return None
|
|
167
|
+
column = 0 if os_name == "Windows" else 1
|
|
168
|
+
for build, minimums, _packages in CUDA_BUILDS:
|
|
169
|
+
if version >= _version(minimums[column]):
|
|
170
|
+
return build
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _packages_for(build: str) -> tuple[str, ...]:
|
|
175
|
+
return next((packages for name, _minimums, packages in CUDA_BUILDS if name == build), ())
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _version(text: str) -> tuple[int, ...]:
|
|
179
|
+
try:
|
|
180
|
+
return tuple(int(part) for part in text.strip().split("."))
|
|
181
|
+
except ValueError:
|
|
182
|
+
return ()
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---------------------------------------------------------------- probing
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _nvidia_gpus() -> list[Gpu]:
|
|
189
|
+
smi = shutil.which("nvidia-smi")
|
|
190
|
+
if not smi:
|
|
191
|
+
return []
|
|
192
|
+
try:
|
|
193
|
+
listing = subprocess.run(
|
|
194
|
+
[smi, "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader,nounits"],
|
|
195
|
+
capture_output=True, text=True, timeout=15,
|
|
196
|
+
)
|
|
197
|
+
header = subprocess.run([smi], capture_output=True, text=True, timeout=15)
|
|
198
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
199
|
+
return []
|
|
200
|
+
match = re.search(r"CUDA Version:\s*([\d.]+)", header.stdout or "")
|
|
201
|
+
cuda = match.group(1) if match else ""
|
|
202
|
+
gpus = []
|
|
203
|
+
# nvidia-smi numbers cards in PCI bus order. CUDA's default is fastest
|
|
204
|
+
# first, so every child we start sets CUDA_DEVICE_ORDER=PCI_BUS_ID
|
|
205
|
+
# (annotate/tools/local.child_environment) and the two agree.
|
|
206
|
+
for index, line in enumerate((listing.stdout or "").splitlines()):
|
|
207
|
+
parts = [p.strip() for p in line.split(",")]
|
|
208
|
+
if len(parts) < 3:
|
|
209
|
+
continue
|
|
210
|
+
try:
|
|
211
|
+
vram = round(float(parts[1]) / 1024, 1)
|
|
212
|
+
except ValueError:
|
|
213
|
+
continue
|
|
214
|
+
gpus.append(Gpu(name=parts[0], vram_gb=vram, driver=parts[2], cuda=cuda, index=index))
|
|
215
|
+
return gpus
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _ram_bytes() -> int:
|
|
219
|
+
try:
|
|
220
|
+
if sys.platform == "win32":
|
|
221
|
+
class Status(ctypes.Structure):
|
|
222
|
+
_fields_ = [
|
|
223
|
+
("dwLength", ctypes.c_ulong), ("dwMemoryLoad", ctypes.c_ulong),
|
|
224
|
+
("ullTotalPhys", ctypes.c_ulonglong), ("ullAvailPhys", ctypes.c_ulonglong),
|
|
225
|
+
("ullTotalPageFile", ctypes.c_ulonglong), ("ullAvailPageFile", ctypes.c_ulonglong),
|
|
226
|
+
("ullTotalVirtual", ctypes.c_ulonglong), ("ullAvailVirtual", ctypes.c_ulonglong),
|
|
227
|
+
("sullAvailExtendedVirtual", ctypes.c_ulonglong),
|
|
228
|
+
]
|
|
229
|
+
|
|
230
|
+
status = Status()
|
|
231
|
+
status.dwLength = ctypes.sizeof(Status)
|
|
232
|
+
ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status))
|
|
233
|
+
return int(status.ullTotalPhys)
|
|
234
|
+
if sys.platform == "darwin":
|
|
235
|
+
out = subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True, text=True, timeout=5)
|
|
236
|
+
return int(out.stdout.strip())
|
|
237
|
+
with open("/proc/meminfo", encoding="utf-8") as handle:
|
|
238
|
+
for line in handle:
|
|
239
|
+
if line.startswith("MemTotal:"):
|
|
240
|
+
return int(line.split()[1]) * 1024
|
|
241
|
+
except Exception:
|
|
242
|
+
return 0
|
|
243
|
+
return 0
|