flashstudio 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flashstudio might be problematic. Click here for more details.
- flashstudio/__init__.py +5 -0
- flashstudio/app.py +194 -0
- flashstudio/cli.py +18 -0
- flashstudio/components/__init__.py +0 -0
- flashstudio/components/project_manager.py +490 -0
- flashstudio/components/sidebar.py +87 -0
- flashstudio/components/styles.py +346 -0
- flashstudio/components/wizard.py +45 -0
- flashstudio/components/zone_drawer/__init__.py +177 -0
- flashstudio/constants.py +245 -0
- flashstudio/core/__init__.py +3 -0
- flashstudio/core/constants.py +3 -0
- flashstudio/launcher.py +288 -0
- flashstudio/pages/__init__.py +8 -0
- flashstudio/pages/dashboard/__init__.py +3 -0
- flashstudio/pages/dashboard/_common.py +10 -0
- flashstudio/pages/dashboard/overview/__init__.py +3 -0
- flashstudio/pages/dashboard/overview/metrics.py +33 -0
- flashstudio/pages/dashboard/overview/pipeline_status.py +35 -0
- flashstudio/pages/dashboard/overview/project_banner.py +23 -0
- flashstudio/pages/dashboard/overview/quick_actions.py +28 -0
- flashstudio/pages/dashboard/overview/recent_runs.py +99 -0
- flashstudio/pages/dashboard/overview/tab.py +35 -0
- flashstudio/pages/dashboard/page.py +17 -0
- flashstudio/pages/data/__init__.py +3 -0
- flashstudio/pages/data/_common.py +107 -0
- flashstudio/pages/data/download/__init__.py +3 -0
- flashstudio/pages/data/download/tab.py +75 -0
- flashstudio/pages/data/helpers.py +338 -0
- flashstudio/pages/data/page.py +50 -0
- flashstudio/pages/data/preview/__init__.py +3 -0
- flashstudio/pages/data/preview/tab.py +139 -0
- flashstudio/pages/data/upload/__init__.py +3 -0
- flashstudio/pages/data/upload/tab.py +156 -0
- flashstudio/pages/data/verify/__init__.py +3 -0
- flashstudio/pages/data/verify/tab.py +108 -0
- flashstudio/pages/export/__init__.py +3 -0
- flashstudio/pages/export/page.py +54 -0
- flashstudio/pages/export/weights/__init__.py +4 -0
- flashstudio/pages/export/weights/exporter.py +60 -0
- flashstudio/pages/export/weights/settings.py +86 -0
- flashstudio/pages/inference/__init__.py +3 -0
- flashstudio/pages/inference/_common.py +76 -0
- flashstudio/pages/inference/data/__init__.py +3 -0
- flashstudio/pages/inference/data/tab.py +121 -0
- flashstudio/pages/inference/model/__init__.py +3 -0
- flashstudio/pages/inference/model/tab.py +62 -0
- flashstudio/pages/inference/page.py +42 -0
- flashstudio/pages/inference/run/__init__.py +3 -0
- flashstudio/pages/inference/run/detection.py +133 -0
- flashstudio/pages/inference/run/tab.py +161 -0
- flashstudio/pages/inference/run/video.py +195 -0
- flashstudio/pages/inference/solution/__init__.py +3 -0
- flashstudio/pages/inference/solution/tab.py +121 -0
- flashstudio/pages/model/__init__.py +3 -0
- flashstudio/pages/model/_common.py +3 -0
- flashstudio/pages/model/advanced/__init__.py +3 -0
- flashstudio/pages/model/advanced/tab.py +54 -0
- flashstudio/pages/model/architecture/__init__.py +3 -0
- flashstudio/pages/model/architecture/tab.py +90 -0
- flashstudio/pages/model/augment/__init__.py +3 -0
- flashstudio/pages/model/augment/tab.py +18 -0
- flashstudio/pages/model/hyperparams/__init__.py +4 -0
- flashstudio/pages/model/hyperparams/tab.py +89 -0
- flashstudio/pages/model/page.py +30 -0
- flashstudio/pages/model/utils/__init__.py +3 -0
- flashstudio/pages/model/utils/config.py +27 -0
- flashstudio/pages/training/__init__.py +3 -0
- flashstudio/pages/training/_common.py +24 -0
- flashstudio/pages/training/launch/__init__.py +3 -0
- flashstudio/pages/training/launch/controls.py +84 -0
- flashstudio/pages/training/launch/dialogs.py +247 -0
- flashstudio/pages/training/launch/preflight.py +77 -0
- flashstudio/pages/training/launch/runner.py +440 -0
- flashstudio/pages/training/launch/tab.py +134 -0
- flashstudio/pages/training/monitor/__init__.py +3 -0
- flashstudio/pages/training/monitor/checkpoints.py +96 -0
- flashstudio/pages/training/monitor/curves.py +113 -0
- flashstudio/pages/training/monitor/dashboard.py +129 -0
- flashstudio/pages/training/monitor/gt_verification.py +122 -0
- flashstudio/pages/training/monitor/log_viewer.py +43 -0
- flashstudio/pages/training/monitor/parsers.py +253 -0
- flashstudio/pages/training/monitor/run_meta.py +165 -0
- flashstudio/pages/training/monitor/tab.py +142 -0
- flashstudio/pages/training/monitor/visualizations.py +52 -0
- flashstudio/pages/training/page.py +134 -0
- flashstudio/utils/__init__.py +152 -0
- flashstudio/utils/config.py +104 -0
- flashstudio/utils/device.py +67 -0
- flashstudio/utils/filesystem.py +67 -0
- flashstudio/utils/jobs.py +95 -0
- flashstudio/utils/training_hooks.py +266 -0
- flashstudio-0.0.1.dist-info/METADATA +107 -0
- flashstudio-0.0.1.dist-info/RECORD +97 -0
- flashstudio-0.0.1.dist-info/WHEEL +5 -0
- flashstudio-0.0.1.dist-info/entry_points.txt +2 -0
- flashstudio-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
"""Training runner — dataset validation, format conversion, and FlashDet subprocess launch."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import json
|
|
5
|
+
import streamlit as st
|
|
6
|
+
|
|
7
|
+
from flashstudio.constants import (
|
|
8
|
+
FLASHDET_MODELS, DEFAULT_DATA_DIR, TRAIN_WEIGHT_DECAY, TRAIN_VAL_INTERVAL, DEFAULT_OPTIMIZER,
|
|
9
|
+
LORA_RANK_DEFAULT, LORA_ALPHA_DEFAULT, LORA_DROPOUT_DEFAULT, LORA_TARGETS_DEFAULT,
|
|
10
|
+
QLORA_DTYPE_DEFAULT, CHUNKED_LOSS_CHUNK_SIZE, DEFAULT_FINETUNE_STRATEGY,
|
|
11
|
+
DEFAULT_PRETRAIN_OPTION, VIS_QUEUE_SIZE,
|
|
12
|
+
)
|
|
13
|
+
from flashstudio.pages.training._common import _get_save_dir
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _generate_run_name() -> str:
|
|
17
|
+
"""Generate a descriptive default run name."""
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
from flashstudio.utils import get_state
|
|
20
|
+
arch = get_state("model_arch")
|
|
21
|
+
size_code = arch.split("-")[-1].lower()[:4] if "-" in arch else "det"
|
|
22
|
+
dataset = st.session_state.get("dataset_name", "")
|
|
23
|
+
ds_code = dataset.split("(")[0].strip().replace(" ", "")[:8].lower() if dataset else "custom"
|
|
24
|
+
timestamp = datetime.now().strftime("%m%d_%H%M")
|
|
25
|
+
return f"{size_code}_{ds_code}_{timestamp}"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _start_training():
|
|
29
|
+
"""Start training using flashdet.Trainer Python API with pre-flight validation."""
|
|
30
|
+
from flashdet.data import detect_dataset_format, convert_dataset
|
|
31
|
+
|
|
32
|
+
train_images = st.session_state.get("train_img_path", "")
|
|
33
|
+
val_images = st.session_state.get("val_img_path", "")
|
|
34
|
+
|
|
35
|
+
# Validate dataset paths exist
|
|
36
|
+
if not train_images or not os.path.isdir(train_images):
|
|
37
|
+
dataset_name = st.session_state.get("dataset_name", "")
|
|
38
|
+
dataset_id = dataset_name.lower().replace(" ", "").replace("(demo)", "sample")
|
|
39
|
+
for candidate_id in ["sample", "coco2017", "coco2017-val", "voc2007", "voc2012"]:
|
|
40
|
+
if candidate_id in dataset_id or dataset_id in candidate_id:
|
|
41
|
+
candidate_dir = os.path.join(DEFAULT_DATA_DIR, candidate_id)
|
|
42
|
+
if os.path.isdir(os.path.join(candidate_dir, "train")):
|
|
43
|
+
train_images = os.path.join(candidate_dir, "train")
|
|
44
|
+
val_images = os.path.join(candidate_dir, "valid")
|
|
45
|
+
break
|
|
46
|
+
|
|
47
|
+
# Convert to absolute paths to avoid relative path issues
|
|
48
|
+
if train_images:
|
|
49
|
+
train_images = os.path.abspath(train_images)
|
|
50
|
+
if val_images:
|
|
51
|
+
val_images = os.path.abspath(val_images)
|
|
52
|
+
|
|
53
|
+
# Auto-find val path if missing but train exists
|
|
54
|
+
if train_images and os.path.isdir(train_images) and (not val_images or not os.path.isdir(val_images)):
|
|
55
|
+
parent = os.path.dirname(train_images)
|
|
56
|
+
for vname in ("valid", "val", "test"):
|
|
57
|
+
vpath = os.path.join(parent, vname)
|
|
58
|
+
if os.path.isdir(vpath):
|
|
59
|
+
val_images = vpath
|
|
60
|
+
break
|
|
61
|
+
if not val_images or not os.path.isdir(val_images):
|
|
62
|
+
val_images = train_images
|
|
63
|
+
|
|
64
|
+
if not train_images:
|
|
65
|
+
st.session_state["training_active"] = False
|
|
66
|
+
st.session_state["training_status"] = "No dataset"
|
|
67
|
+
st.session_state["training_error"] = (
|
|
68
|
+
"No dataset paths configured. Go to the Data page and either "
|
|
69
|
+
"enter train/val image paths manually, or download a dataset first."
|
|
70
|
+
)
|
|
71
|
+
return
|
|
72
|
+
|
|
73
|
+
# Auto-detect classes from annotation if not already set
|
|
74
|
+
_cls_raw = st.session_state.get("class_names", "")
|
|
75
|
+
if isinstance(_cls_raw, list):
|
|
76
|
+
_cls_raw = "\n".join(_cls_raw)
|
|
77
|
+
if not _cls_raw.strip():
|
|
78
|
+
ann_file = os.path.join(train_images, "_annotations.coco.json")
|
|
79
|
+
if not os.path.isfile(ann_file):
|
|
80
|
+
json_files = [f for f in os.listdir(train_images) if f.endswith(".json")] if os.path.isdir(train_images) else []
|
|
81
|
+
if json_files:
|
|
82
|
+
ann_file = os.path.join(train_images, json_files[0])
|
|
83
|
+
if os.path.isfile(ann_file):
|
|
84
|
+
try:
|
|
85
|
+
with open(ann_file, encoding="utf-8") as _af:
|
|
86
|
+
ann_data = json.load(_af)
|
|
87
|
+
cats = ann_data.get("categories", [])
|
|
88
|
+
if cats:
|
|
89
|
+
sorted_cats = sorted(cats, key=lambda c: c.get("id", 0))
|
|
90
|
+
names = [c["name"] for c in sorted_cats]
|
|
91
|
+
st.session_state["class_names"] = "\n".join(names)
|
|
92
|
+
st.session_state["num_classes"] = len(names)
|
|
93
|
+
except (json.JSONDecodeError, KeyError, TypeError):
|
|
94
|
+
pass
|
|
95
|
+
|
|
96
|
+
# Auto-detect and convert format if not COCO
|
|
97
|
+
parent_dir = os.path.dirname(train_images) if os.path.basename(train_images) in ("train", "images") else train_images
|
|
98
|
+
detected_fmt = detect_dataset_format(parent_dir)
|
|
99
|
+
|
|
100
|
+
if detected_fmt in ("txt", "voc"):
|
|
101
|
+
st.warning(f"Dataset format detected: **{detected_fmt}**. Converting to COCO JSON...")
|
|
102
|
+
try:
|
|
103
|
+
output_dir = parent_dir + "_coco"
|
|
104
|
+
class_names = None
|
|
105
|
+
raw_names = st.session_state.get("class_names", "")
|
|
106
|
+
if isinstance(raw_names, list):
|
|
107
|
+
raw_names = "\n".join(raw_names)
|
|
108
|
+
if raw_names.strip():
|
|
109
|
+
class_names = [c.strip() for c in raw_names.strip().split("\n") if c.strip()]
|
|
110
|
+
|
|
111
|
+
convert_dataset(source_dir=parent_dir, output_dir=output_dir,
|
|
112
|
+
target_format="coco", class_names=class_names)
|
|
113
|
+
|
|
114
|
+
train_images = os.path.join(output_dir, "train")
|
|
115
|
+
val_dir = os.path.join(output_dir, "valid")
|
|
116
|
+
if not os.path.isdir(val_dir):
|
|
117
|
+
val_dir = os.path.join(output_dir, "val")
|
|
118
|
+
val_images = val_dir
|
|
119
|
+
st.session_state["train_img_path"] = train_images
|
|
120
|
+
st.session_state["val_img_path"] = val_images
|
|
121
|
+
st.success("Dataset converted to COCO format.")
|
|
122
|
+
except Exception as e:
|
|
123
|
+
st.session_state["training_active"] = False
|
|
124
|
+
st.session_state["training_status"] = "Conversion failed"
|
|
125
|
+
st.session_state["training_error"] = f"Format conversion failed: {e}"
|
|
126
|
+
return
|
|
127
|
+
|
|
128
|
+
# Verify COCO annotation file exists (check train dir, then parent)
|
|
129
|
+
train_ann = os.path.join(train_images, "_annotations.coco.json")
|
|
130
|
+
if not os.path.isfile(train_ann):
|
|
131
|
+
ann_candidates = [f for f in os.listdir(train_images) if f.endswith(".json")] if os.path.isdir(train_images) else []
|
|
132
|
+
if ann_candidates:
|
|
133
|
+
st.info(f"Found annotation file: `{ann_candidates[0]}` (expected `_annotations.coco.json`)")
|
|
134
|
+
else:
|
|
135
|
+
# Check parent directory (layout: train/images/ with train/_annotations.coco.json)
|
|
136
|
+
parent_dir = os.path.dirname(train_images)
|
|
137
|
+
parent_ann = os.path.join(parent_dir, "_annotations.coco.json")
|
|
138
|
+
parent_jsons = [f for f in os.listdir(parent_dir) if f.endswith(".json")] if os.path.isdir(parent_dir) else []
|
|
139
|
+
if os.path.isfile(parent_ann) or parent_jsons:
|
|
140
|
+
import shutil
|
|
141
|
+
src_ann = parent_ann if os.path.isfile(parent_ann) else os.path.join(parent_dir, parent_jsons[0])
|
|
142
|
+
dst_ann = os.path.join(train_images, os.path.basename(src_ann))
|
|
143
|
+
shutil.copy2(src_ann, dst_ann)
|
|
144
|
+
st.info(f"Copied annotation from parent: `{os.path.basename(src_ann)}`")
|
|
145
|
+
else:
|
|
146
|
+
st.session_state["training_active"] = False
|
|
147
|
+
st.session_state["training_status"] = "Missing annotations"
|
|
148
|
+
st.session_state["training_error"] = (
|
|
149
|
+
f"Missing annotations! FlashDet expects _annotations.coco.json in: {train_images}"
|
|
150
|
+
)
|
|
151
|
+
return
|
|
152
|
+
|
|
153
|
+
# Normalize dataset layout for FlashDet:
|
|
154
|
+
# FlashDet computes data_root = dirname(train_images) and expects
|
|
155
|
+
# data_root/valid/_annotations.coco.json to exist.
|
|
156
|
+
data_root = os.path.dirname(os.path.normpath(train_images))
|
|
157
|
+
valid_dir_in_root = os.path.join(data_root, "valid")
|
|
158
|
+
val_ann_check = os.path.join(valid_dir_in_root, "_annotations.coco.json")
|
|
159
|
+
|
|
160
|
+
if not os.path.exists(val_ann_check) and val_images and os.path.isdir(val_images):
|
|
161
|
+
val_ann_file = os.path.join(val_images, "_annotations.coco.json")
|
|
162
|
+
if os.path.isfile(val_ann_file):
|
|
163
|
+
try:
|
|
164
|
+
if os.path.exists(valid_dir_in_root):
|
|
165
|
+
if os.path.islink(valid_dir_in_root):
|
|
166
|
+
os.unlink(valid_dir_in_root)
|
|
167
|
+
else:
|
|
168
|
+
pass
|
|
169
|
+
if not os.path.exists(valid_dir_in_root):
|
|
170
|
+
os.symlink(os.path.abspath(val_images), valid_dir_in_root)
|
|
171
|
+
val_images = valid_dir_in_root
|
|
172
|
+
st.session_state["val_img_path"] = val_images
|
|
173
|
+
except OSError:
|
|
174
|
+
pass
|
|
175
|
+
|
|
176
|
+
if not os.path.exists(val_ann_check):
|
|
177
|
+
try:
|
|
178
|
+
os.makedirs(valid_dir_in_root, exist_ok=True)
|
|
179
|
+
import shutil
|
|
180
|
+
shutil.copytree(train_images, valid_dir_in_root, dirs_exist_ok=True)
|
|
181
|
+
val_images = valid_dir_in_root
|
|
182
|
+
st.session_state["val_img_path"] = val_images
|
|
183
|
+
except Exception:
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
st.session_state["train_img_path"] = train_images
|
|
187
|
+
st.session_state["val_img_path"] = val_images
|
|
188
|
+
|
|
189
|
+
# Run training
|
|
190
|
+
_run_flashdet_training()
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _run_flashdet_training():
|
|
194
|
+
"""Run actual FlashDet training via CLI (non-blocking subprocess)."""
|
|
195
|
+
import subprocess
|
|
196
|
+
import sys
|
|
197
|
+
|
|
198
|
+
size_map = {name: info["size"] for name, info in FLASHDET_MODELS.items()}
|
|
199
|
+
|
|
200
|
+
from flashstudio.utils import get_state as _gs
|
|
201
|
+
arch_family = _gs("arch_family")
|
|
202
|
+
model_arch = _gs("model_arch")
|
|
203
|
+
save_dir = st.session_state.get("active_run_path", os.path.join(_get_save_dir(), st.session_state.get("run_name", "untitled_run")))
|
|
204
|
+
|
|
205
|
+
device = "cpu"
|
|
206
|
+
try:
|
|
207
|
+
import torch
|
|
208
|
+
if torch.cuda.is_available():
|
|
209
|
+
device = "cuda"
|
|
210
|
+
except ImportError:
|
|
211
|
+
pass
|
|
212
|
+
|
|
213
|
+
train_images = st.session_state.get("train_img_path", "")
|
|
214
|
+
val_images = st.session_state.get("val_img_path", "")
|
|
215
|
+
|
|
216
|
+
architecture = "flashdet"
|
|
217
|
+
if "YOLOv8" in arch_family:
|
|
218
|
+
architecture = "yolov8"
|
|
219
|
+
elif "YOLOv9" in arch_family:
|
|
220
|
+
architecture = "yolov9"
|
|
221
|
+
elif "YOLOv10" in arch_family:
|
|
222
|
+
architecture = "yolov10"
|
|
223
|
+
elif "YOLOv11" in arch_family:
|
|
224
|
+
architecture = "yolov11"
|
|
225
|
+
elif "YOLOX" in arch_family:
|
|
226
|
+
architecture = "yolox"
|
|
227
|
+
|
|
228
|
+
from flashstudio.utils import get_state
|
|
229
|
+
model_size = size_map.get(model_arch, "n")
|
|
230
|
+
epochs = get_state("epochs")
|
|
231
|
+
batch_size = get_state("batch_size")
|
|
232
|
+
lr = get_state("lr")
|
|
233
|
+
workers = get_state("num_workers")
|
|
234
|
+
amp = st.session_state.get("amp", True) and device == "cuda"
|
|
235
|
+
mosaic = st.session_state.get("aug_mosaic", True)
|
|
236
|
+
mixup = st.session_state.get("aug_mixup", False)
|
|
237
|
+
copy_paste = st.session_state.get("aug_copypaste", False)
|
|
238
|
+
warmup_epochs = get_state("warmup_epochs")
|
|
239
|
+
grad_accum = get_state("grad_accum")
|
|
240
|
+
patience = get_state("patience")
|
|
241
|
+
input_size = get_state("img_size")
|
|
242
|
+
activation_checkpointing = st.session_state.get("activation_checkpointing", False)
|
|
243
|
+
activation_offloading = st.session_state.get("activation_offloading", False)
|
|
244
|
+
optimizer_in_bwd = st.session_state.get("optimizer_in_bwd", False)
|
|
245
|
+
use_8bit_optimizer = st.session_state.get("use_8bit_optimizer", False)
|
|
246
|
+
compile_model = st.session_state.get("compile_model", False)
|
|
247
|
+
multi_gpu = st.session_state.get("ddp", False)
|
|
248
|
+
|
|
249
|
+
# LoRA settings
|
|
250
|
+
finetune_strategy = st.session_state.get("finetune_strategy", DEFAULT_FINETUNE_STRATEGY)
|
|
251
|
+
lora = finetune_strategy == "LoRA"
|
|
252
|
+
lora_rank = st.session_state.get("lora_rank", LORA_RANK_DEFAULT)
|
|
253
|
+
lora_alpha = st.session_state.get("lora_alpha", LORA_ALPHA_DEFAULT)
|
|
254
|
+
lora_dropout = st.session_state.get("lora_dropout", LORA_DROPOUT_DEFAULT)
|
|
255
|
+
lora_variant = st.session_state.get("lora_variant", "standard")
|
|
256
|
+
lora_targets = st.session_state.get("lora_targets", LORA_TARGETS_DEFAULT)
|
|
257
|
+
|
|
258
|
+
# QLoRA
|
|
259
|
+
qlora = st.session_state.get("qlora", False)
|
|
260
|
+
qlora_dtype = st.session_state.get("qlora_dtype", QLORA_DTYPE_DEFAULT)
|
|
261
|
+
|
|
262
|
+
# Chunked loss
|
|
263
|
+
chunked_loss = st.session_state.get("chunked_loss", False)
|
|
264
|
+
chunk_size = st.session_state.get("chunk_size", CHUNKED_LOSS_CHUNK_SIZE)
|
|
265
|
+
|
|
266
|
+
# Pretrained / resume
|
|
267
|
+
pretrain_option = st.session_state.get("pretrain_option", DEFAULT_PRETRAIN_OPTION)
|
|
268
|
+
finetune_path = None
|
|
269
|
+
if pretrain_option == "Custom":
|
|
270
|
+
finetune_path = st.session_state.get("custom_weights", "")
|
|
271
|
+
|
|
272
|
+
resume_ckpt = None
|
|
273
|
+
if st.session_state.get("resume_training", False):
|
|
274
|
+
resume_ckpt = st.session_state.get("resume_path", "")
|
|
275
|
+
st.session_state["resume_training"] = False
|
|
276
|
+
|
|
277
|
+
# Class file
|
|
278
|
+
class_file = st.session_state.get("class_file", None)
|
|
279
|
+
from flashstudio.utils import get_class_names_str
|
|
280
|
+
class_names_raw = get_class_names_str()
|
|
281
|
+
if class_names_raw.strip() and not class_file:
|
|
282
|
+
import tempfile
|
|
283
|
+
cls_tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False, prefix="classes_")
|
|
284
|
+
for name in class_names_raw.strip().split("\n"):
|
|
285
|
+
if name.strip():
|
|
286
|
+
cls_tmp.write(name.strip() + "\n")
|
|
287
|
+
cls_tmp.close()
|
|
288
|
+
class_file = cls_tmp.name
|
|
289
|
+
|
|
290
|
+
backbone_type = "lite"
|
|
291
|
+
if model_arch == "FlashDet-Pico":
|
|
292
|
+
pico_bb = st.session_state.get("pico_backbone", "")
|
|
293
|
+
if "PicoBackbone" in pico_bb or "RepNeXt" in pico_bb:
|
|
294
|
+
backbone_type = "pico_v2"
|
|
295
|
+
|
|
296
|
+
# Build command-line arguments for FlashDet's full train.py
|
|
297
|
+
# Locate train.py: try sibling FlashDet repo, then package location
|
|
298
|
+
import flashdet
|
|
299
|
+
flashdet_pkg_dir = os.path.dirname(flashdet.__file__)
|
|
300
|
+
train_script_candidates = [
|
|
301
|
+
os.path.join(flashdet_pkg_dir, "..", "train.py"), # editable install
|
|
302
|
+
os.path.join(flashdet_pkg_dir, "train.py"), # inside package
|
|
303
|
+
os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
304
|
+
"..", "..", "..", "..", "..", "FlashDet", "train.py"), # sibling project
|
|
305
|
+
]
|
|
306
|
+
train_script = None
|
|
307
|
+
for candidate in train_script_candidates:
|
|
308
|
+
if os.path.isfile(candidate):
|
|
309
|
+
train_script = os.path.abspath(candidate)
|
|
310
|
+
break
|
|
311
|
+
|
|
312
|
+
if not train_script:
|
|
313
|
+
st.session_state["training_active"] = False
|
|
314
|
+
st.session_state["training_status"] = "Failed"
|
|
315
|
+
st.session_state["training_error"] = (
|
|
316
|
+
"Could not find FlashDet train.py. Ensure FlashDet is installed "
|
|
317
|
+
"or the FlashDet repo is a sibling directory."
|
|
318
|
+
)
|
|
319
|
+
return
|
|
320
|
+
|
|
321
|
+
cmd = [
|
|
322
|
+
sys.executable, "-u", train_script,
|
|
323
|
+
"--model-size", model_size,
|
|
324
|
+
"--architecture", architecture,
|
|
325
|
+
"--epochs", str(epochs),
|
|
326
|
+
"--batch-size", str(batch_size),
|
|
327
|
+
"--lr", str(lr),
|
|
328
|
+
"--workers", str(workers),
|
|
329
|
+
"--device", device,
|
|
330
|
+
"--save-dir", save_dir,
|
|
331
|
+
"--train-images", train_images,
|
|
332
|
+
"--val-images", val_images,
|
|
333
|
+
"--warmup-epochs", str(warmup_epochs),
|
|
334
|
+
"--grad-accum", str(grad_accum),
|
|
335
|
+
"--patience", str(patience),
|
|
336
|
+
"--input-size", str(input_size),
|
|
337
|
+
"--weight-decay", str(st.session_state.get("weight_decay", TRAIN_WEIGHT_DECAY)),
|
|
338
|
+
"--optimizer", st.session_state.get("optimizer", DEFAULT_OPTIMIZER).lower(),
|
|
339
|
+
]
|
|
340
|
+
|
|
341
|
+
val_interval = st.session_state.get("val_interval", TRAIN_VAL_INTERVAL)
|
|
342
|
+
if val_interval:
|
|
343
|
+
cmd += ["--val-interval", str(val_interval)]
|
|
344
|
+
|
|
345
|
+
if amp:
|
|
346
|
+
cmd.append("--amp")
|
|
347
|
+
if mosaic:
|
|
348
|
+
cmd.append("--mosaic")
|
|
349
|
+
if mixup:
|
|
350
|
+
cmd.append("--mixup")
|
|
351
|
+
if copy_paste:
|
|
352
|
+
cmd.append("--copy-paste")
|
|
353
|
+
if st.session_state.get("multiscale", False):
|
|
354
|
+
cmd.append("--multi-scale")
|
|
355
|
+
if multi_gpu:
|
|
356
|
+
cmd.append("--multi-gpu")
|
|
357
|
+
|
|
358
|
+
if lora:
|
|
359
|
+
cmd += ["--lora",
|
|
360
|
+
"--lora-variant", lora_variant,
|
|
361
|
+
"--lora-rank", str(lora_rank),
|
|
362
|
+
"--lora-alpha", str(lora_alpha),
|
|
363
|
+
"--lora-dropout", str(lora_dropout),
|
|
364
|
+
"--lora-targets"] + lora_targets
|
|
365
|
+
if qlora:
|
|
366
|
+
cmd += ["--qlora", "--qlora-dtype", qlora_dtype]
|
|
367
|
+
|
|
368
|
+
if activation_checkpointing:
|
|
369
|
+
cmd.append("--activation-checkpointing")
|
|
370
|
+
if activation_offloading:
|
|
371
|
+
cmd.append("--activation-offloading")
|
|
372
|
+
if optimizer_in_bwd:
|
|
373
|
+
cmd.append("--optimizer-in-bwd")
|
|
374
|
+
if use_8bit_optimizer:
|
|
375
|
+
cmd.append("--use-8bit-optimizer")
|
|
376
|
+
if compile_model:
|
|
377
|
+
cmd.append("--compile")
|
|
378
|
+
if chunked_loss:
|
|
379
|
+
cmd += ["--chunked-loss", "--chunk-size", str(chunk_size)]
|
|
380
|
+
|
|
381
|
+
if finetune_path:
|
|
382
|
+
cmd += ["--finetune", finetune_path]
|
|
383
|
+
if resume_ckpt:
|
|
384
|
+
cmd += ["--resume", resume_ckpt]
|
|
385
|
+
if class_file:
|
|
386
|
+
cmd += ["--class-file", class_file]
|
|
387
|
+
if backbone_type != "lite":
|
|
388
|
+
cmd += ["--backbone", backbone_type]
|
|
389
|
+
|
|
390
|
+
os.makedirs(save_dir, exist_ok=True)
|
|
391
|
+
|
|
392
|
+
import time as _time
|
|
393
|
+
log_filename = f"train_{_time.strftime('%Y%m%d_%H%M%S')}.log"
|
|
394
|
+
log_path = os.path.join(save_dir, log_filename)
|
|
395
|
+
|
|
396
|
+
log_file_handle = open(log_path, "w")
|
|
397
|
+
env = os.environ.copy()
|
|
398
|
+
env["FLASHDET_VIS_QUEUE_SIZE"] = str(VIS_QUEUE_SIZE)
|
|
399
|
+
process = subprocess.Popen(
|
|
400
|
+
cmd,
|
|
401
|
+
stdout=log_file_handle, stderr=subprocess.STDOUT,
|
|
402
|
+
start_new_session=True,
|
|
403
|
+
env=env,
|
|
404
|
+
)
|
|
405
|
+
# Keep the file handle in session state so it stays open while the subprocess runs
|
|
406
|
+
st.session_state["_log_file_handle"] = log_file_handle
|
|
407
|
+
|
|
408
|
+
st.session_state["training_pid"] = process.pid
|
|
409
|
+
st.session_state["training_log_file"] = log_path
|
|
410
|
+
|
|
411
|
+
# Wait briefly to catch immediate crashes (import errors, bad args, etc.)
|
|
412
|
+
import time as _tw
|
|
413
|
+
_tw.sleep(2)
|
|
414
|
+
exit_code = process.poll()
|
|
415
|
+
if exit_code is not None:
|
|
416
|
+
fh = st.session_state.pop("_log_file_handle", None)
|
|
417
|
+
if fh and not fh.closed:
|
|
418
|
+
try:
|
|
419
|
+
fh.close()
|
|
420
|
+
except Exception:
|
|
421
|
+
pass
|
|
422
|
+
st.session_state["training_active"] = False
|
|
423
|
+
st.session_state["training_pid"] = None
|
|
424
|
+
st.session_state["training_status"] = "Failed"
|
|
425
|
+
try:
|
|
426
|
+
with open(log_path, "r", encoding="utf-8", errors="replace") as _lf:
|
|
427
|
+
log_content = _lf.read()
|
|
428
|
+
except OSError:
|
|
429
|
+
log_content = "(could not read log)"
|
|
430
|
+
from flashstudio.utils import flash
|
|
431
|
+
flash("Training failed to start — see error below", "error")
|
|
432
|
+
st.session_state["training_error"] = log_content[-2000:] if len(log_content) > 2000 else log_content
|
|
433
|
+
st.rerun()
|
|
434
|
+
return
|
|
435
|
+
|
|
436
|
+
st.session_state["training_active"] = True
|
|
437
|
+
st.session_state["training_status"] = "Running"
|
|
438
|
+
from flashstudio.utils import flash
|
|
439
|
+
flash(f"Training started (PID {process.pid})", "success")
|
|
440
|
+
st.rerun()
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Launch tab — create run, configure, and start training."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import streamlit as st
|
|
5
|
+
|
|
6
|
+
from flashstudio.utils.device import has_cuda
|
|
7
|
+
from flashstudio.pages.training._common import _get_save_dir
|
|
8
|
+
from flashstudio.pages.training.launch.preflight import _run_preflight_checks
|
|
9
|
+
from flashstudio.pages.training.launch.runner import _start_training, _generate_run_name
|
|
10
|
+
from flashstudio.pages.training.launch.controls import _stop_training, _pause_training, _resume_active_training
|
|
11
|
+
from flashstudio.pages.training.launch.dialogs import _render_clean_workspace_dialog, _render_resume_dialog
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _render_start_tab():
|
|
15
|
+
"""Tab: launch a new training run — professional step-based layout."""
|
|
16
|
+
|
|
17
|
+
runs_root = _get_save_dir()
|
|
18
|
+
if runs_root:
|
|
19
|
+
os.makedirs(runs_root, exist_ok=True)
|
|
20
|
+
|
|
21
|
+
if "run_name" not in st.session_state or not st.session_state.get("run_name", "").strip():
|
|
22
|
+
st.session_state["run_name"] = _generate_run_name()
|
|
23
|
+
|
|
24
|
+
prev_name = st.session_state.get("_run_name_at_create", "")
|
|
25
|
+
current_name = st.session_state.get("run_name", "")
|
|
26
|
+
if prev_name and current_name != prev_name:
|
|
27
|
+
st.session_state["run_created"] = False
|
|
28
|
+
|
|
29
|
+
run_created = st.session_state.get("run_created", False)
|
|
30
|
+
nc = st.session_state.get("num_classes", 0)
|
|
31
|
+
|
|
32
|
+
with st.container(border=True):
|
|
33
|
+
st.markdown("#### 1. Create Run")
|
|
34
|
+
st.text_input("Run Name", key="run_name")
|
|
35
|
+
run_name = st.session_state.get("run_name", "untitled_run")
|
|
36
|
+
full_run_path = os.path.join(runs_root, run_name)
|
|
37
|
+
st.caption(f"Output: `{full_run_path}`")
|
|
38
|
+
|
|
39
|
+
if not run_created:
|
|
40
|
+
if st.button("Create Run", key="btn_create_run", use_container_width=True, type="primary"):
|
|
41
|
+
from flashstudio.utils import flash
|
|
42
|
+
if not run_name.strip():
|
|
43
|
+
flash("Run name cannot be empty", "error")
|
|
44
|
+
st.rerun()
|
|
45
|
+
else:
|
|
46
|
+
os.makedirs(full_run_path, exist_ok=True)
|
|
47
|
+
st.session_state["run_created"] = True
|
|
48
|
+
st.session_state["_run_name_at_create"] = run_name
|
|
49
|
+
st.session_state["save_dir_run"] = full_run_path
|
|
50
|
+
flash(f"Run `{run_name}` created", "success")
|
|
51
|
+
st.rerun()
|
|
52
|
+
else:
|
|
53
|
+
st.success(f"Run: **{run_name}**")
|
|
54
|
+
|
|
55
|
+
run_name = st.session_state.get("run_name", "untitled_run")
|
|
56
|
+
full_run_path = os.path.join(runs_root, run_name)
|
|
57
|
+
|
|
58
|
+
if not run_created:
|
|
59
|
+
st.info("Create a run first to configure and start training.")
|
|
60
|
+
return
|
|
61
|
+
|
|
62
|
+
with st.container(border=True):
|
|
63
|
+
st.markdown("#### 2. Config")
|
|
64
|
+
from flashstudio.utils import get_state
|
|
65
|
+
cc1, cc2, cc3, cc4 = st.columns(4)
|
|
66
|
+
with cc1:
|
|
67
|
+
st.metric("Model", get_state("model_arch").replace("FlashDet-", ""))
|
|
68
|
+
with cc2:
|
|
69
|
+
st.metric("Dataset", (st.session_state.get("dataset_name") or "—")[:12])
|
|
70
|
+
with cc3:
|
|
71
|
+
st.metric("Classes", nc if nc else "—")
|
|
72
|
+
with cc4:
|
|
73
|
+
st.metric("Epochs", get_state("epochs"))
|
|
74
|
+
cc5, cc6, cc7, cc8 = st.columns(4)
|
|
75
|
+
with cc5:
|
|
76
|
+
st.metric("Batch", get_state("batch_size"))
|
|
77
|
+
with cc6:
|
|
78
|
+
st.metric("Img Size", get_state("img_size"))
|
|
79
|
+
with cc7:
|
|
80
|
+
st.metric("LR", f"{get_state('lr'):.1e}")
|
|
81
|
+
with cc8:
|
|
82
|
+
st.metric("Device", "GPU" if has_cuda() else "CPU")
|
|
83
|
+
|
|
84
|
+
with st.container(border=True):
|
|
85
|
+
st.markdown("#### 3. Launch")
|
|
86
|
+
checks = _run_preflight_checks()
|
|
87
|
+
all_ok = True
|
|
88
|
+
check_html = []
|
|
89
|
+
for label, ok, msg in checks:
|
|
90
|
+
status = "OK" if ok else "Failed"
|
|
91
|
+
check_html.append(f'{label}: {status}')
|
|
92
|
+
if not ok:
|
|
93
|
+
all_ok = False
|
|
94
|
+
st.markdown(
|
|
95
|
+
'<div class="ds-card-stats">' + ' '.join(f'<span>{c}</span>' for c in check_html) + '</div>',
|
|
96
|
+
unsafe_allow_html=True,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
b1, b2, b3, b4 = st.columns(4)
|
|
100
|
+
with b1:
|
|
101
|
+
if not st.session_state.get("training_active"):
|
|
102
|
+
if st.button("Start", use_container_width=True, type="primary",
|
|
103
|
+
key="btn_start_training", disabled=not all_ok):
|
|
104
|
+
import time as _time
|
|
105
|
+
st.session_state.update({"training_paused": False,
|
|
106
|
+
"active_run_path": full_run_path,
|
|
107
|
+
"training_start_time": _time.time(),
|
|
108
|
+
"_goto_monitor": True})
|
|
109
|
+
_start_training()
|
|
110
|
+
else:
|
|
111
|
+
if st.button("Stop", use_container_width=True, key="btn_stop_training"):
|
|
112
|
+
_stop_training()
|
|
113
|
+
with b2:
|
|
114
|
+
if st.session_state.get("training_active"):
|
|
115
|
+
if st.session_state.get("training_paused"):
|
|
116
|
+
if st.button("Resume", use_container_width=True, key="btn_resume_active"):
|
|
117
|
+
_resume_active_training()
|
|
118
|
+
else:
|
|
119
|
+
if st.button("Pause", use_container_width=True, key="btn_pause_training"):
|
|
120
|
+
_pause_training()
|
|
121
|
+
else:
|
|
122
|
+
st.button("Pause", use_container_width=True, key="btn_pause_disabled", disabled=True)
|
|
123
|
+
with b3:
|
|
124
|
+
if st.button("Resume Ckpt", use_container_width=True, key="btn_resume_training", help="Resume from checkpoint"):
|
|
125
|
+
st.session_state["show_resume_dialog"] = True
|
|
126
|
+
with b4:
|
|
127
|
+
if st.button("Clean", use_container_width=True, key="btn_clean_workspace"):
|
|
128
|
+
st.session_state["show_clean_dialog"] = True
|
|
129
|
+
|
|
130
|
+
if st.session_state.get("show_clean_dialog"):
|
|
131
|
+
_render_clean_workspace_dialog()
|
|
132
|
+
|
|
133
|
+
if st.session_state.get("show_resume_dialog"):
|
|
134
|
+
_render_resume_dialog()
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Checkpoint and file listing for training runs."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import streamlit as st
|
|
6
|
+
|
|
7
|
+
from flashstudio.pages.training.monitor.parsers import _find_log_file
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _render_checkpoints(run_dir):
|
|
11
|
+
"""Files tab — list all artifacts from a FlashDet training run."""
|
|
12
|
+
from flashstudio.constants import ONNX_MODEL_FILE, ONNX_DATA_FILE
|
|
13
|
+
files_info = []
|
|
14
|
+
EXTENSIONS = (".pth", ".json", ".csv", ".log", ".onnx", ".onnx.data", ".txt")
|
|
15
|
+
for f in sorted(os.listdir(run_dir)):
|
|
16
|
+
fpath = os.path.join(run_dir, f)
|
|
17
|
+
if os.path.isfile(fpath) and (f.endswith(EXTENSIONS) or f in (ONNX_MODEL_FILE, ONNX_DATA_FILE)):
|
|
18
|
+
size = os.path.getsize(fpath)
|
|
19
|
+
if size > 1_073_741_824:
|
|
20
|
+
size_str = f"{size / (1024**3):.1f}GB"
|
|
21
|
+
elif size > 1_048_576:
|
|
22
|
+
size_str = f"{size / (1024*1024):.1f}MB"
|
|
23
|
+
else:
|
|
24
|
+
size_str = f"{size / 1024:.0f}KB"
|
|
25
|
+
files_info.append({"File": f, "Size": size_str, "Type": _file_type(f)})
|
|
26
|
+
|
|
27
|
+
# Also list subdirectories with their file counts
|
|
28
|
+
for d in sorted(os.listdir(run_dir)):
|
|
29
|
+
dpath = os.path.join(run_dir, d)
|
|
30
|
+
if os.path.isdir(dpath):
|
|
31
|
+
n_files = sum(1 for _ in os.listdir(dpath) if os.path.isfile(os.path.join(dpath, _)))
|
|
32
|
+
files_info.append({"File": f"{d}/", "Size": f"{n_files} files", "Type": "Directory"})
|
|
33
|
+
|
|
34
|
+
if files_info:
|
|
35
|
+
st.dataframe(files_info, use_container_width=True, hide_index=True,
|
|
36
|
+
height=min(35 * len(files_info) + 40, 400))
|
|
37
|
+
else:
|
|
38
|
+
st.info("No files.")
|
|
39
|
+
|
|
40
|
+
# Show training summary from log if complete
|
|
41
|
+
log_file = _find_log_file(run_dir)
|
|
42
|
+
if log_file:
|
|
43
|
+
try:
|
|
44
|
+
with open(log_file, "r", encoding="utf-8", errors="replace") as _lf:
|
|
45
|
+
_tail = _lf.readlines()[-10:]
|
|
46
|
+
for _ln in _tail:
|
|
47
|
+
bm = re.search(r"Best mAP@0\.5:\s*([\d.]+)\s*\|\s*Best Loss:\s*([\d.]+)", _ln)
|
|
48
|
+
if bm:
|
|
49
|
+
rc1, rc2 = st.columns(2)
|
|
50
|
+
with rc1:
|
|
51
|
+
st.metric("Best mAP@0.5", bm.group(1))
|
|
52
|
+
with rc2:
|
|
53
|
+
st.metric("Best Val Loss", bm.group(2))
|
|
54
|
+
break
|
|
55
|
+
except OSError:
|
|
56
|
+
pass
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _file_type(filename):
|
|
60
|
+
"""Categorize a file by its name — aligned with FlashDet output."""
|
|
61
|
+
if "final" in filename and "inference" in filename:
|
|
62
|
+
return "Final inference weights"
|
|
63
|
+
if "final" in filename and "fp16" in filename:
|
|
64
|
+
return "Final FP16 weights"
|
|
65
|
+
if "best" in filename and "inference" in filename:
|
|
66
|
+
return "Best inference weights"
|
|
67
|
+
if "best" in filename and "fp16" in filename:
|
|
68
|
+
return "Best FP16 weights"
|
|
69
|
+
if "best" in filename:
|
|
70
|
+
return "Best checkpoint"
|
|
71
|
+
if "last" in filename and "inference" in filename:
|
|
72
|
+
return "Latest inference weights"
|
|
73
|
+
if "last" in filename and "fp16" in filename:
|
|
74
|
+
return "Latest FP16 weights"
|
|
75
|
+
if "checkpoint_last" in filename:
|
|
76
|
+
return "Latest checkpoint (full)"
|
|
77
|
+
if "inference" in filename:
|
|
78
|
+
return "Inference weights"
|
|
79
|
+
if "fp16" in filename:
|
|
80
|
+
return "FP16 weights"
|
|
81
|
+
from flashstudio.constants import ONNX_MODEL_FILE, ONNX_DATA_FILE, RESULTS_JSON_FILE
|
|
82
|
+
if filename == ONNX_MODEL_FILE:
|
|
83
|
+
return "ONNX model"
|
|
84
|
+
if filename == ONNX_DATA_FILE:
|
|
85
|
+
return "ONNX weights data"
|
|
86
|
+
if filename == RESULTS_JSON_FILE:
|
|
87
|
+
return "Training results"
|
|
88
|
+
if filename.endswith(".json"):
|
|
89
|
+
return "Report/Config"
|
|
90
|
+
if filename.endswith(".csv"):
|
|
91
|
+
return "Training metrics CSV"
|
|
92
|
+
if filename.endswith(".log"):
|
|
93
|
+
return "Training log"
|
|
94
|
+
if filename.endswith(".txt"):
|
|
95
|
+
return "Summary"
|
|
96
|
+
return "Other"
|