flashstudio 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flashstudio might be problematic. Click here for more details.
- flashstudio/__init__.py +5 -0
- flashstudio/app.py +194 -0
- flashstudio/cli.py +18 -0
- flashstudio/components/__init__.py +0 -0
- flashstudio/components/project_manager.py +490 -0
- flashstudio/components/sidebar.py +87 -0
- flashstudio/components/styles.py +346 -0
- flashstudio/components/wizard.py +45 -0
- flashstudio/components/zone_drawer/__init__.py +177 -0
- flashstudio/constants.py +245 -0
- flashstudio/core/__init__.py +3 -0
- flashstudio/core/constants.py +3 -0
- flashstudio/launcher.py +288 -0
- flashstudio/pages/__init__.py +8 -0
- flashstudio/pages/dashboard/__init__.py +3 -0
- flashstudio/pages/dashboard/_common.py +10 -0
- flashstudio/pages/dashboard/overview/__init__.py +3 -0
- flashstudio/pages/dashboard/overview/metrics.py +33 -0
- flashstudio/pages/dashboard/overview/pipeline_status.py +35 -0
- flashstudio/pages/dashboard/overview/project_banner.py +23 -0
- flashstudio/pages/dashboard/overview/quick_actions.py +28 -0
- flashstudio/pages/dashboard/overview/recent_runs.py +99 -0
- flashstudio/pages/dashboard/overview/tab.py +35 -0
- flashstudio/pages/dashboard/page.py +17 -0
- flashstudio/pages/data/__init__.py +3 -0
- flashstudio/pages/data/_common.py +107 -0
- flashstudio/pages/data/download/__init__.py +3 -0
- flashstudio/pages/data/download/tab.py +75 -0
- flashstudio/pages/data/helpers.py +338 -0
- flashstudio/pages/data/page.py +50 -0
- flashstudio/pages/data/preview/__init__.py +3 -0
- flashstudio/pages/data/preview/tab.py +139 -0
- flashstudio/pages/data/upload/__init__.py +3 -0
- flashstudio/pages/data/upload/tab.py +156 -0
- flashstudio/pages/data/verify/__init__.py +3 -0
- flashstudio/pages/data/verify/tab.py +108 -0
- flashstudio/pages/export/__init__.py +3 -0
- flashstudio/pages/export/page.py +54 -0
- flashstudio/pages/export/weights/__init__.py +4 -0
- flashstudio/pages/export/weights/exporter.py +60 -0
- flashstudio/pages/export/weights/settings.py +86 -0
- flashstudio/pages/inference/__init__.py +3 -0
- flashstudio/pages/inference/_common.py +76 -0
- flashstudio/pages/inference/data/__init__.py +3 -0
- flashstudio/pages/inference/data/tab.py +121 -0
- flashstudio/pages/inference/model/__init__.py +3 -0
- flashstudio/pages/inference/model/tab.py +62 -0
- flashstudio/pages/inference/page.py +42 -0
- flashstudio/pages/inference/run/__init__.py +3 -0
- flashstudio/pages/inference/run/detection.py +133 -0
- flashstudio/pages/inference/run/tab.py +161 -0
- flashstudio/pages/inference/run/video.py +195 -0
- flashstudio/pages/inference/solution/__init__.py +3 -0
- flashstudio/pages/inference/solution/tab.py +121 -0
- flashstudio/pages/model/__init__.py +3 -0
- flashstudio/pages/model/_common.py +3 -0
- flashstudio/pages/model/advanced/__init__.py +3 -0
- flashstudio/pages/model/advanced/tab.py +54 -0
- flashstudio/pages/model/architecture/__init__.py +3 -0
- flashstudio/pages/model/architecture/tab.py +90 -0
- flashstudio/pages/model/augment/__init__.py +3 -0
- flashstudio/pages/model/augment/tab.py +18 -0
- flashstudio/pages/model/hyperparams/__init__.py +4 -0
- flashstudio/pages/model/hyperparams/tab.py +89 -0
- flashstudio/pages/model/page.py +30 -0
- flashstudio/pages/model/utils/__init__.py +3 -0
- flashstudio/pages/model/utils/config.py +27 -0
- flashstudio/pages/training/__init__.py +3 -0
- flashstudio/pages/training/_common.py +24 -0
- flashstudio/pages/training/launch/__init__.py +3 -0
- flashstudio/pages/training/launch/controls.py +84 -0
- flashstudio/pages/training/launch/dialogs.py +247 -0
- flashstudio/pages/training/launch/preflight.py +77 -0
- flashstudio/pages/training/launch/runner.py +440 -0
- flashstudio/pages/training/launch/tab.py +134 -0
- flashstudio/pages/training/monitor/__init__.py +3 -0
- flashstudio/pages/training/monitor/checkpoints.py +96 -0
- flashstudio/pages/training/monitor/curves.py +113 -0
- flashstudio/pages/training/monitor/dashboard.py +129 -0
- flashstudio/pages/training/monitor/gt_verification.py +122 -0
- flashstudio/pages/training/monitor/log_viewer.py +43 -0
- flashstudio/pages/training/monitor/parsers.py +253 -0
- flashstudio/pages/training/monitor/run_meta.py +165 -0
- flashstudio/pages/training/monitor/tab.py +142 -0
- flashstudio/pages/training/monitor/visualizations.py +52 -0
- flashstudio/pages/training/page.py +134 -0
- flashstudio/utils/__init__.py +152 -0
- flashstudio/utils/config.py +104 -0
- flashstudio/utils/device.py +67 -0
- flashstudio/utils/filesystem.py +67 -0
- flashstudio/utils/jobs.py +95 -0
- flashstudio/utils/training_hooks.py +266 -0
- flashstudio-0.0.1.dist-info/METADATA +107 -0
- flashstudio-0.0.1.dist-info/RECORD +97 -0
- flashstudio-0.0.1.dist-info/WHEEL +5 -0
- flashstudio-0.0.1.dist-info/entry_points.txt +2 -0
- flashstudio-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Unified training config build/apply — single source of truth for YAML configs."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import yaml
|
|
5
|
+
import streamlit as st
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# Keys to persist in config grouped by section
|
|
9
|
+
CONFIG_SCHEMA = {
|
|
10
|
+
"project": ["run_name"],
|
|
11
|
+
"dataset": [
|
|
12
|
+
"dataset_name", "train_img_path", "val_img_path",
|
|
13
|
+
"ann_format", "upload_num_classes", "class_names", "num_classes",
|
|
14
|
+
],
|
|
15
|
+
"model": [
|
|
16
|
+
"arch_family", "model_arch", "pico_backbone",
|
|
17
|
+
"pretrain_option", "custom_weights",
|
|
18
|
+
],
|
|
19
|
+
"training": [
|
|
20
|
+
"epochs", "batch_size", "lr", "img_size", "input_size",
|
|
21
|
+
"warmup_epochs", "patience", "grad_accum", "num_workers",
|
|
22
|
+
"val_interval", "lr_final_ratio", "grad_clip", "multiscale",
|
|
23
|
+
],
|
|
24
|
+
"optimizer": ["optimizer", "weight_decay", "use_8bit_optimizer"],
|
|
25
|
+
"augmentation": ["aug_mosaic", "aug_mixup", "aug_copypaste"],
|
|
26
|
+
"finetune": [
|
|
27
|
+
"finetune_strategy", "lora_variant", "lora_rank",
|
|
28
|
+
"lora_alpha", "lora_dropout", "lora_targets",
|
|
29
|
+
"qlora", "qlora_dtype",
|
|
30
|
+
],
|
|
31
|
+
"advanced": [
|
|
32
|
+
"amp", "compile_model", "ddp",
|
|
33
|
+
"activation_checkpointing", "activation_offloading",
|
|
34
|
+
"optimizer_in_bwd", "chunked_loss", "chunk_size",
|
|
35
|
+
"save_best", "resume_training",
|
|
36
|
+
],
|
|
37
|
+
"output": ["save_dir"],
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def build_training_config() -> dict:
|
|
42
|
+
"""Build a complete training config dictionary from current session state.
|
|
43
|
+
|
|
44
|
+
Uses mirror-aware reads so values survive page navigation.
|
|
45
|
+
"""
|
|
46
|
+
from flashstudio.utils import get_state
|
|
47
|
+
config = {}
|
|
48
|
+
for section, keys in CONFIG_SCHEMA.items():
|
|
49
|
+
config[section] = {}
|
|
50
|
+
for key in keys:
|
|
51
|
+
val = get_state(key)
|
|
52
|
+
if val is not None:
|
|
53
|
+
config[section][key] = val
|
|
54
|
+
return config
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def apply_training_config(config: dict):
|
|
58
|
+
"""Apply a loaded config dictionary to session state."""
|
|
59
|
+
for section, keys in CONFIG_SCHEMA.items():
|
|
60
|
+
if section not in config:
|
|
61
|
+
continue
|
|
62
|
+
section_data = config[section]
|
|
63
|
+
for key in keys:
|
|
64
|
+
if key in section_data:
|
|
65
|
+
val = section_data[key]
|
|
66
|
+
if val is not None:
|
|
67
|
+
st.session_state[key] = val
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def save_config_yaml(config: dict, path: str):
|
|
71
|
+
"""Save config dict to a YAML file."""
|
|
72
|
+
os.makedirs(os.path.dirname(path) if os.path.dirname(path) else ".", exist_ok=True)
|
|
73
|
+
with open(path, "w") as f:
|
|
74
|
+
yaml.dump(config, f, default_flow_style=False, sort_keys=False)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def load_config_yaml(path: str) -> dict:
|
|
78
|
+
"""Load config from a YAML file. Returns empty dict on failure."""
|
|
79
|
+
try:
|
|
80
|
+
with open(path) as f:
|
|
81
|
+
return yaml.safe_load(f.read()) or {}
|
|
82
|
+
except (OSError, yaml.YAMLError):
|
|
83
|
+
return {}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def config_to_yaml_str(config: dict) -> str:
|
|
87
|
+
"""Convert config dict to YAML string for display/download."""
|
|
88
|
+
return yaml.dump(config, default_flow_style=False, sort_keys=False)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def get_pipeline_steps() -> list:
|
|
92
|
+
"""Generate pipeline step descriptions based on current config."""
|
|
93
|
+
from flashstudio.utils import get_state
|
|
94
|
+
model = get_state("model_arch")
|
|
95
|
+
epochs = get_state("epochs")
|
|
96
|
+
return [
|
|
97
|
+
{"step": 1, "action": "load_dataset", "description": "Load and validate dataset"},
|
|
98
|
+
{"step": 2, "action": "convert_format", "description": "Convert to COCO JSON if needed"},
|
|
99
|
+
{"step": 3, "action": "configure_model", "description": f"Initialize {model}"},
|
|
100
|
+
{"step": 4, "action": "apply_augmentations", "description": "Apply data augmentation pipeline"},
|
|
101
|
+
{"step": 5, "action": "train", "description": f"Train for {epochs} epochs"},
|
|
102
|
+
{"step": 6, "action": "evaluate", "description": "Evaluate on validation set"},
|
|
103
|
+
{"step": 7, "action": "export", "description": "Export best model (ONNX)"},
|
|
104
|
+
]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Device and environment utilities."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def is_colab() -> bool:
|
|
5
|
+
"""Check if running inside Google Colab."""
|
|
6
|
+
try:
|
|
7
|
+
import google.colab # noqa: F401
|
|
8
|
+
return True
|
|
9
|
+
except ImportError:
|
|
10
|
+
return False
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def has_cuda() -> bool:
|
|
14
|
+
"""Check if CUDA GPU is available."""
|
|
15
|
+
try:
|
|
16
|
+
import torch
|
|
17
|
+
return torch.cuda.is_available()
|
|
18
|
+
except ImportError:
|
|
19
|
+
return False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_device() -> str:
|
|
23
|
+
"""Get the best available device string."""
|
|
24
|
+
try:
|
|
25
|
+
import torch
|
|
26
|
+
if torch.cuda.is_available():
|
|
27
|
+
return "cuda"
|
|
28
|
+
elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
|
|
29
|
+
return "mps"
|
|
30
|
+
except ImportError:
|
|
31
|
+
pass
|
|
32
|
+
return "cpu"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_gpu_info() -> dict:
|
|
36
|
+
"""Get GPU information if available."""
|
|
37
|
+
info = {"available": False, "name": "CPU", "memory_total": 0, "memory_used": 0}
|
|
38
|
+
try:
|
|
39
|
+
import torch
|
|
40
|
+
if torch.cuda.is_available():
|
|
41
|
+
info["available"] = True
|
|
42
|
+
info["name"] = torch.cuda.get_device_name(0)
|
|
43
|
+
info["memory_total"] = torch.cuda.get_device_properties(0).total_memory / 1e9
|
|
44
|
+
info["memory_used"] = torch.cuda.memory_allocated(0) / 1e9
|
|
45
|
+
except Exception:
|
|
46
|
+
pass
|
|
47
|
+
return info
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def get_colab_runtime_type() -> str:
|
|
51
|
+
"""Detect Colab runtime type."""
|
|
52
|
+
if not is_colab():
|
|
53
|
+
return "local"
|
|
54
|
+
try:
|
|
55
|
+
import torch
|
|
56
|
+
if torch.cuda.is_available():
|
|
57
|
+
name = torch.cuda.get_device_name(0)
|
|
58
|
+
if "T4" in name:
|
|
59
|
+
return "T4"
|
|
60
|
+
elif "A100" in name:
|
|
61
|
+
return "A100"
|
|
62
|
+
elif "V100" in name:
|
|
63
|
+
return "V100"
|
|
64
|
+
return f"GPU ({name})"
|
|
65
|
+
except ImportError:
|
|
66
|
+
pass
|
|
67
|
+
return "CPU"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Filesystem utilities — shared helpers for directory operations."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def dir_size_bytes(path: str) -> int:
|
|
8
|
+
"""Calculate total size of a directory in bytes."""
|
|
9
|
+
total = 0
|
|
10
|
+
try:
|
|
11
|
+
for dirpath, _dirs, files in os.walk(path):
|
|
12
|
+
for f in files:
|
|
13
|
+
try:
|
|
14
|
+
total += os.path.getsize(os.path.join(dirpath, f))
|
|
15
|
+
except OSError:
|
|
16
|
+
pass
|
|
17
|
+
except OSError:
|
|
18
|
+
pass
|
|
19
|
+
return total
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def dir_size_str(path: str) -> str:
|
|
23
|
+
"""Get human-readable directory size string."""
|
|
24
|
+
total = dir_size_bytes(path)
|
|
25
|
+
if total > 1_073_741_824:
|
|
26
|
+
return f"{total / 1_073_741_824:.1f} GB"
|
|
27
|
+
elif total > 1_048_576:
|
|
28
|
+
return f"{total / 1_048_576:.0f} MB"
|
|
29
|
+
elif total > 1024:
|
|
30
|
+
return f"{total / 1024:.0f} KB"
|
|
31
|
+
return f"{total} B"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def ensure_dir(path: str) -> str:
|
|
35
|
+
"""Create directory if it doesn't exist. Returns the path."""
|
|
36
|
+
os.makedirs(path, exist_ok=True)
|
|
37
|
+
return path
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def safe_rmtree(path: str) -> bool:
|
|
41
|
+
"""Safely remove a directory tree. Returns True if removed."""
|
|
42
|
+
try:
|
|
43
|
+
if os.path.isdir(path):
|
|
44
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
45
|
+
return True
|
|
46
|
+
except OSError:
|
|
47
|
+
pass
|
|
48
|
+
return False
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def list_subdirs(path: str, sort_by_mtime: bool = True) -> list:
|
|
52
|
+
"""List immediate subdirectories, optionally sorted by modification time (newest first)."""
|
|
53
|
+
if not os.path.isdir(path):
|
|
54
|
+
return []
|
|
55
|
+
dirs = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d))]
|
|
56
|
+
if sort_by_mtime:
|
|
57
|
+
dirs.sort(key=lambda d: os.path.getmtime(os.path.join(path, d)), reverse=True)
|
|
58
|
+
return dirs
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def count_files(path: str, extensions: tuple = None) -> int:
|
|
62
|
+
"""Count files in a directory, optionally filtered by extension."""
|
|
63
|
+
if not os.path.isdir(path):
|
|
64
|
+
return 0
|
|
65
|
+
if extensions:
|
|
66
|
+
return sum(1 for f in os.scandir(path) if f.is_file() and f.name.endswith(extensions))
|
|
67
|
+
return sum(1 for f in os.scandir(path) if f.is_file())
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Background job tracker — tracks training and inference processes across page navigation."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import signal
|
|
5
|
+
import time
|
|
6
|
+
import streamlit as st
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _get_jobs() -> list:
|
|
10
|
+
"""Get the list of active jobs from session state."""
|
|
11
|
+
if "active_jobs" not in st.session_state:
|
|
12
|
+
st.session_state["active_jobs"] = []
|
|
13
|
+
return st.session_state["active_jobs"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def add_job(pid: int, job_type: str, name: str, details: dict | None = None):
|
|
17
|
+
"""Register a new background job."""
|
|
18
|
+
jobs = _get_jobs()
|
|
19
|
+
jobs.append({
|
|
20
|
+
"pid": pid,
|
|
21
|
+
"type": job_type,
|
|
22
|
+
"name": name,
|
|
23
|
+
"started": time.time(),
|
|
24
|
+
"details": details or {},
|
|
25
|
+
})
|
|
26
|
+
st.session_state["active_jobs"] = jobs
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def remove_job(pid: int):
|
|
30
|
+
"""Remove a job by PID."""
|
|
31
|
+
jobs = _get_jobs()
|
|
32
|
+
st.session_state["active_jobs"] = [j for j in jobs if j["pid"] != pid]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def get_jobs_by_type(job_type: str) -> list:
|
|
36
|
+
"""Get all jobs of a given type (e.g., 'training', 'inference')."""
|
|
37
|
+
return [j for j in _get_jobs() if j["type"] == job_type]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def is_pid_alive(pid: int) -> bool:
|
|
41
|
+
"""Check if a process is still running."""
|
|
42
|
+
try:
|
|
43
|
+
os.kill(pid, 0)
|
|
44
|
+
return True
|
|
45
|
+
except (ProcessLookupError, PermissionError):
|
|
46
|
+
return False
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def cleanup_dead_jobs():
|
|
50
|
+
"""Remove jobs whose PIDs are no longer running."""
|
|
51
|
+
jobs = _get_jobs()
|
|
52
|
+
alive = [j for j in jobs if is_pid_alive(j["pid"])]
|
|
53
|
+
if len(alive) != len(jobs):
|
|
54
|
+
st.session_state["active_jobs"] = alive
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def stop_job(pid: int) -> bool:
|
|
58
|
+
"""Stop a job by sending SIGTERM, then SIGKILL if needed."""
|
|
59
|
+
try:
|
|
60
|
+
os.kill(pid, signal.SIGTERM)
|
|
61
|
+
time.sleep(0.5)
|
|
62
|
+
if is_pid_alive(pid):
|
|
63
|
+
os.kill(pid, signal.SIGKILL)
|
|
64
|
+
remove_job(pid)
|
|
65
|
+
return True
|
|
66
|
+
except (ProcessLookupError, PermissionError):
|
|
67
|
+
remove_job(pid)
|
|
68
|
+
return False
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def stop_all_by_type(job_type: str):
|
|
72
|
+
"""Stop all jobs of a given type."""
|
|
73
|
+
for job in get_jobs_by_type(job_type):
|
|
74
|
+
stop_job(job["pid"])
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def get_active_count() -> dict:
|
|
78
|
+
"""Return count of active jobs by type."""
|
|
79
|
+
cleanup_dead_jobs()
|
|
80
|
+
jobs = _get_jobs()
|
|
81
|
+
counts = {}
|
|
82
|
+
for j in jobs:
|
|
83
|
+
counts[j["type"]] = counts.get(j["type"], 0) + 1
|
|
84
|
+
return counts
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def format_job_duration(started: float) -> str:
|
|
88
|
+
"""Format elapsed time since job started."""
|
|
89
|
+
elapsed = time.time() - started
|
|
90
|
+
if elapsed < 60:
|
|
91
|
+
return f"{elapsed:.0f}s"
|
|
92
|
+
elif elapsed < 3600:
|
|
93
|
+
return f"{elapsed / 60:.0f}m"
|
|
94
|
+
else:
|
|
95
|
+
return f"{elapsed / 3600:.1f}h"
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""FlashStudio training hooks — callbacks and pre-training verification.
|
|
2
|
+
|
|
3
|
+
Used by the generated training subprocess to produce outputs that the
|
|
4
|
+
Training dashboard (Visualizations, Ground Truth, Curves tabs) expects.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import json
|
|
9
|
+
import csv
|
|
10
|
+
import numpy as np
|
|
11
|
+
from typing import Any, Dict, List
|
|
12
|
+
|
|
13
|
+
from flashdet.engine.core.callbacks import Callback
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def run_gt_verification(
|
|
17
|
+
train_images: str,
|
|
18
|
+
val_images: str,
|
|
19
|
+
save_dir: str,
|
|
20
|
+
class_names: List[str],
|
|
21
|
+
input_size: tuple = (320, 320),
|
|
22
|
+
num_images: int = 3,
|
|
23
|
+
):
|
|
24
|
+
"""Generate GT verification images and a report JSON.
|
|
25
|
+
|
|
26
|
+
Creates:
|
|
27
|
+
save_dir/gt_verification/verification_report.json
|
|
28
|
+
save_dir/gt_verification/verification_summary.txt
|
|
29
|
+
save_dir/gt_verification/images/raw/*.jpg
|
|
30
|
+
"""
|
|
31
|
+
from flashstudio.constants import GT_VERIFICATION_DIR
|
|
32
|
+
gt_dir = os.path.join(save_dir, GT_VERIFICATION_DIR)
|
|
33
|
+
raw_dir = os.path.join(gt_dir, "images", "raw")
|
|
34
|
+
os.makedirs(raw_dir, exist_ok=True)
|
|
35
|
+
|
|
36
|
+
train_ann = os.path.join(train_images, "_annotations.coco.json")
|
|
37
|
+
val_ann = os.path.join(val_images, "_annotations.coco.json") if val_images else None
|
|
38
|
+
|
|
39
|
+
report = {"passed": True, "num_classes": len(class_names), "splits": {}}
|
|
40
|
+
summary_lines = []
|
|
41
|
+
|
|
42
|
+
for split_name, img_dir, ann_file in [
|
|
43
|
+
("train", train_images, train_ann),
|
|
44
|
+
("val", val_images, val_ann),
|
|
45
|
+
]:
|
|
46
|
+
if not ann_file or not os.path.isfile(ann_file):
|
|
47
|
+
continue
|
|
48
|
+
try:
|
|
49
|
+
with open(ann_file, encoding="utf-8") as f:
|
|
50
|
+
ann_data = json.load(f)
|
|
51
|
+
|
|
52
|
+
num_imgs = len(ann_data.get("images", []))
|
|
53
|
+
num_anns = len(ann_data.get("annotations", []))
|
|
54
|
+
num_cats = len(ann_data.get("categories", []))
|
|
55
|
+
|
|
56
|
+
report["splits"][split_name] = {
|
|
57
|
+
"coco": {
|
|
58
|
+
"num_images": num_imgs,
|
|
59
|
+
"num_annotations": num_anns,
|
|
60
|
+
"num_categories": num_cats,
|
|
61
|
+
},
|
|
62
|
+
"dataloader": {
|
|
63
|
+
"avg_boxes_per_sample": num_anns / max(num_imgs, 1),
|
|
64
|
+
},
|
|
65
|
+
}
|
|
66
|
+
summary_lines.append(
|
|
67
|
+
f"{split_name}: {num_imgs} images, {num_anns} annotations, {num_cats} categories"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
if split_name == "train" and num_imgs > 0:
|
|
71
|
+
_save_gt_images(
|
|
72
|
+
ann_data, img_dir, raw_dir, class_names,
|
|
73
|
+
num_images=min(num_images, num_imgs),
|
|
74
|
+
)
|
|
75
|
+
except Exception as e:
|
|
76
|
+
summary_lines.append(f"{split_name}: ERROR - {e}")
|
|
77
|
+
report["passed"] = False
|
|
78
|
+
|
|
79
|
+
from flashstudio.constants import GT_REPORT_FILE, GT_SUMMARY_FILE
|
|
80
|
+
with open(os.path.join(gt_dir, GT_REPORT_FILE), "w") as f:
|
|
81
|
+
json.dump(report, f, indent=2)
|
|
82
|
+
|
|
83
|
+
with open(os.path.join(gt_dir, GT_SUMMARY_FILE), "w") as f:
|
|
84
|
+
f.write("\n".join(summary_lines))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _save_gt_images(ann_data, img_dir, out_dir, class_names, num_images=8):
|
|
88
|
+
"""Draw GT boxes on sample images and save them."""
|
|
89
|
+
try:
|
|
90
|
+
import cv2
|
|
91
|
+
except ImportError:
|
|
92
|
+
return
|
|
93
|
+
|
|
94
|
+
images_info = ann_data.get("images", [])[:num_images]
|
|
95
|
+
annotations = ann_data.get("annotations", [])
|
|
96
|
+
|
|
97
|
+
img_id_to_anns = {}
|
|
98
|
+
for ann in annotations:
|
|
99
|
+
img_id_to_anns.setdefault(ann["image_id"], []).append(ann)
|
|
100
|
+
|
|
101
|
+
cat_id_to_name = {}
|
|
102
|
+
for cat in ann_data.get("categories", []):
|
|
103
|
+
cat_id_to_name[cat["id"]] = cat["name"]
|
|
104
|
+
|
|
105
|
+
colors = _make_palette(len(class_names))
|
|
106
|
+
|
|
107
|
+
for img_info in images_info:
|
|
108
|
+
img_path = os.path.join(img_dir, img_info["file_name"])
|
|
109
|
+
if not os.path.isfile(img_path):
|
|
110
|
+
continue
|
|
111
|
+
|
|
112
|
+
img = cv2.imread(img_path)
|
|
113
|
+
if img is None:
|
|
114
|
+
continue
|
|
115
|
+
|
|
116
|
+
anns = img_id_to_anns.get(img_info["id"], [])
|
|
117
|
+
for ann in anns:
|
|
118
|
+
bbox = ann.get("bbox", [])
|
|
119
|
+
if len(bbox) != 4:
|
|
120
|
+
continue
|
|
121
|
+
x, y, w, h = [int(v) for v in bbox]
|
|
122
|
+
cat_id = ann.get("category_id", 0)
|
|
123
|
+
cat_name = cat_id_to_name.get(cat_id, str(cat_id))
|
|
124
|
+
color = colors.get(cat_id % len(colors), (0, 255, 0))
|
|
125
|
+
|
|
126
|
+
cv2.rectangle(img, (x, y), (x + w, y + h), color, 2)
|
|
127
|
+
label = f"{cat_name}"
|
|
128
|
+
(tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
|
|
129
|
+
cv2.rectangle(img, (x, y - th - 4), (x + tw, y), color, -1)
|
|
130
|
+
cv2.putText(img, label, (x, y - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
|
|
131
|
+
|
|
132
|
+
out_path = os.path.join(out_dir, f"gt_{img_info['id']:06d}.jpg")
|
|
133
|
+
cv2.imwrite(out_path, img)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _make_palette(n):
|
|
137
|
+
"""Generate a BGR color palette."""
|
|
138
|
+
import colorsys
|
|
139
|
+
palette = {}
|
|
140
|
+
for i in range(max(n, 1)):
|
|
141
|
+
hue = i / max(n, 1)
|
|
142
|
+
r, g, b = colorsys.hsv_to_rgb(hue, 0.85, 0.9)
|
|
143
|
+
palette[i] = (int(b * 255), int(g * 255), int(r * 255))
|
|
144
|
+
return palette
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class StudioCSVLogger(Callback):
|
|
148
|
+
"""Callback that saves training_log.csv compatible with the dashboard parser."""
|
|
149
|
+
|
|
150
|
+
FIELDNAMES = ["epoch", "train_loss", "lr", "val_loss", "mAP@0.5"]
|
|
151
|
+
|
|
152
|
+
def __init__(self, save_dir: str):
|
|
153
|
+
from flashstudio.constants import TRAINING_LOG_CSV
|
|
154
|
+
self.csv_path = os.path.join(save_dir, TRAINING_LOG_CSV)
|
|
155
|
+
self._initialized = False
|
|
156
|
+
|
|
157
|
+
def on_epoch_end(self, trainer: Any, epoch: int, metrics: Dict) -> None:
|
|
158
|
+
row = {"epoch": epoch}
|
|
159
|
+
if "train_loss" in metrics:
|
|
160
|
+
row["train_loss"] = f"{metrics['train_loss']:.6f}"
|
|
161
|
+
if "lr" in metrics:
|
|
162
|
+
row["lr"] = f"{metrics['lr']:.8f}"
|
|
163
|
+
if "val_loss" in metrics:
|
|
164
|
+
row["val_loss"] = f"{metrics['val_loss']:.6f}"
|
|
165
|
+
if "val_mAP" in metrics:
|
|
166
|
+
row["mAP@0.5"] = f"{metrics['val_mAP']:.6f}"
|
|
167
|
+
|
|
168
|
+
if not self._initialized:
|
|
169
|
+
with open(self.csv_path, "w", newline="") as f:
|
|
170
|
+
writer = csv.DictWriter(f, fieldnames=self.FIELDNAMES)
|
|
171
|
+
writer.writeheader()
|
|
172
|
+
writer.writerow(row)
|
|
173
|
+
self._initialized = True
|
|
174
|
+
else:
|
|
175
|
+
with open(self.csv_path, "a", newline="") as f:
|
|
176
|
+
writer = csv.DictWriter(f, fieldnames=self.FIELDNAMES)
|
|
177
|
+
writer.writerow(row)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class StudioVisualizationCallback(Callback):
|
|
181
|
+
"""Callback that saves GT-vs-Pred visualization images during validation."""
|
|
182
|
+
|
|
183
|
+
def __init__(self, save_dir: str, max_kept: int = 10):
|
|
184
|
+
from flashstudio.constants import VIS_DIR_NAMES
|
|
185
|
+
self.vis_dir = os.path.join(save_dir, VIS_DIR_NAMES[0])
|
|
186
|
+
self.max_kept = max_kept
|
|
187
|
+
os.makedirs(self.vis_dir, exist_ok=True)
|
|
188
|
+
self._epoch = 0
|
|
189
|
+
|
|
190
|
+
def on_epoch_start(self, trainer: Any, epoch: int) -> None:
|
|
191
|
+
self._epoch = epoch
|
|
192
|
+
|
|
193
|
+
def on_val_end(self, trainer: Any, metrics: Dict) -> None:
|
|
194
|
+
"""After validation, save a visualization from the model."""
|
|
195
|
+
try:
|
|
196
|
+
self._save_vis(trainer)
|
|
197
|
+
except Exception:
|
|
198
|
+
pass
|
|
199
|
+
|
|
200
|
+
def _save_vis(self, trainer):
|
|
201
|
+
import torch
|
|
202
|
+
import cv2
|
|
203
|
+
from flashdet.utils.visualization import make_gt_pred_panel
|
|
204
|
+
|
|
205
|
+
model = trainer._get_eval_model() if hasattr(trainer, '_get_eval_model') else None
|
|
206
|
+
if model is None:
|
|
207
|
+
return
|
|
208
|
+
|
|
209
|
+
val_loader = getattr(trainer, '_val_loader', None)
|
|
210
|
+
if val_loader is None:
|
|
211
|
+
return
|
|
212
|
+
|
|
213
|
+
model.eval()
|
|
214
|
+
device = trainer.device
|
|
215
|
+
|
|
216
|
+
for images, gt_meta in val_loader:
|
|
217
|
+
images = images.to(device)
|
|
218
|
+
with torch.no_grad():
|
|
219
|
+
results = model.predict(images, None, score_thr=0.3)
|
|
220
|
+
|
|
221
|
+
img = images[0].cpu().numpy().transpose(1, 2, 0)
|
|
222
|
+
mean = np.array([123.675, 116.28, 103.53])
|
|
223
|
+
std = np.array([58.395, 57.12, 57.375])
|
|
224
|
+
img = np.clip(img * std + mean, 0, 255).astype(np.uint8)
|
|
225
|
+
img_bgr = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
|
|
226
|
+
|
|
227
|
+
gt_boxes = gt_meta["gt_bboxes"][0] if gt_meta.get("gt_bboxes") else np.empty((0, 4))
|
|
228
|
+
gt_labels = gt_meta["gt_labels"][0] if gt_meta.get("gt_labels") else np.empty(0)
|
|
229
|
+
|
|
230
|
+
pred_boxes = np.empty((0, 4))
|
|
231
|
+
pred_labels = np.empty(0, dtype=int)
|
|
232
|
+
pred_scores = np.empty(0)
|
|
233
|
+
if results and len(results) > 0:
|
|
234
|
+
dets, lbs = results[0]
|
|
235
|
+
if dets is not None and dets.numel() > 0:
|
|
236
|
+
dets_np = dets.cpu().numpy()
|
|
237
|
+
pred_boxes = dets_np[:, :4]
|
|
238
|
+
pred_scores = dets_np[:, 4]
|
|
239
|
+
pred_labels = lbs.cpu().numpy().astype(int)
|
|
240
|
+
|
|
241
|
+
panel = make_gt_pred_panel(
|
|
242
|
+
img_bgr, gt_boxes,
|
|
243
|
+
gt_labels.astype(int) if len(gt_labels) else gt_labels,
|
|
244
|
+
pred_boxes, pred_labels, pred_scores,
|
|
245
|
+
title_extra=f"| Epoch {self._epoch}",
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
out_path = os.path.join(self.vis_dir, f"epoch{self._epoch:04d}.jpg")
|
|
249
|
+
cv2.imwrite(out_path, panel)
|
|
250
|
+
|
|
251
|
+
self._cleanup()
|
|
252
|
+
break
|
|
253
|
+
|
|
254
|
+
model.train()
|
|
255
|
+
|
|
256
|
+
def _cleanup(self):
|
|
257
|
+
"""Keep only the latest N visualization images."""
|
|
258
|
+
files = sorted(
|
|
259
|
+
[f for f in os.listdir(self.vis_dir) if f.endswith(".jpg")],
|
|
260
|
+
key=lambda f: os.path.getmtime(os.path.join(self.vis_dir, f)),
|
|
261
|
+
)
|
|
262
|
+
while len(files) > self.max_kept:
|
|
263
|
+
try:
|
|
264
|
+
os.remove(os.path.join(self.vis_dir, files.pop(0)))
|
|
265
|
+
except OSError:
|
|
266
|
+
break
|