sinan-captcha 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sinan_captcha-0.1.0/PKG-INFO +89 -0
- sinan_captcha-0.1.0/README.md +72 -0
- sinan_captcha-0.1.0/core/__init__.py +5 -0
- sinan_captcha-0.1.0/core/autolabel/__init__.py +2 -0
- sinan_captcha-0.1.0/core/autolabel/cli.py +42 -0
- sinan_captcha-0.1.0/core/autolabel/service.py +191 -0
- sinan_captcha-0.1.0/core/cli.py +75 -0
- sinan_captcha-0.1.0/core/common/__init__.py +2 -0
- sinan_captcha-0.1.0/core/common/images.py +76 -0
- sinan_captcha-0.1.0/core/common/jsonl.py +40 -0
- sinan_captcha-0.1.0/core/common/paths.py +29 -0
- sinan_captcha-0.1.0/core/convert/__init__.py +2 -0
- sinan_captcha-0.1.0/core/convert/cli.py +37 -0
- sinan_captcha-0.1.0/core/convert/service.py +150 -0
- sinan_captcha-0.1.0/core/dataset/__init__.py +2 -0
- sinan_captcha-0.1.0/core/dataset/cli.py +28 -0
- sinan_captcha-0.1.0/core/dataset/contracts.py +76 -0
- sinan_captcha-0.1.0/core/dataset/validation.py +123 -0
- sinan_captcha-0.1.0/core/evaluate/__init__.py +2 -0
- sinan_captcha-0.1.0/core/evaluate/cli.py +42 -0
- sinan_captcha-0.1.0/core/evaluate/service.py +262 -0
- sinan_captcha-0.1.0/core/inference/__init__.py +2 -0
- sinan_captcha-0.1.0/core/inference/service.py +19 -0
- sinan_captcha-0.1.0/core/materials/__init__.py +2 -0
- sinan_captcha-0.1.0/core/materials/cli.py +30 -0
- sinan_captcha-0.1.0/core/materials/service.py +872 -0
- sinan_captcha-0.1.0/core/ops/__init__.py +1 -0
- sinan_captcha-0.1.0/core/ops/env.py +117 -0
- sinan_captcha-0.1.0/core/ops/setup_train.py +264 -0
- sinan_captcha-0.1.0/core/py.typed +1 -0
- sinan_captcha-0.1.0/core/release/__init__.py +2 -0
- sinan_captcha-0.1.0/core/release/cli.py +76 -0
- sinan_captcha-0.1.0/core/release/service.py +116 -0
- sinan_captcha-0.1.0/core/train/__init__.py +2 -0
- sinan_captcha-0.1.0/core/train/base.py +81 -0
- sinan_captcha-0.1.0/core/train/group1/__init__.py +2 -0
- sinan_captcha-0.1.0/core/train/group1/cli.py +49 -0
- sinan_captcha-0.1.0/core/train/group1/service.py +31 -0
- sinan_captcha-0.1.0/core/train/group2/__init__.py +2 -0
- sinan_captcha-0.1.0/core/train/group2/cli.py +49 -0
- sinan_captcha-0.1.0/core/train/group2/service.py +31 -0
- sinan_captcha-0.1.0/pyproject.toml +55 -0
- sinan_captcha-0.1.0/setup.cfg +4 -0
- sinan_captcha-0.1.0/sinan_captcha.egg-info/PKG-INFO +89 -0
- sinan_captcha-0.1.0/sinan_captcha.egg-info/SOURCES.txt +47 -0
- sinan_captcha-0.1.0/sinan_captcha.egg-info/dependency_links.txt +1 -0
- sinan_captcha-0.1.0/sinan_captcha.egg-info/entry_points.txt +2 -0
- sinan_captcha-0.1.0/sinan_captcha.egg-info/requires.txt +11 -0
- sinan_captcha-0.1.0/sinan_captcha.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sinan-captcha
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Training and data engineering workspace for graphical captcha models.
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Provides-Extra: train
|
|
8
|
+
Requires-Dist: matplotlib; extra == "train"
|
|
9
|
+
Requires-Dist: numpy; extra == "train"
|
|
10
|
+
Requires-Dist: opencv-python; extra == "train"
|
|
11
|
+
Requires-Dist: pandas; extra == "train"
|
|
12
|
+
Requires-Dist: pillow; extra == "train"
|
|
13
|
+
Requires-Dist: pyyaml; extra == "train"
|
|
14
|
+
Requires-Dist: scikit-image; extra == "train"
|
|
15
|
+
Requires-Dist: tqdm; extra == "train"
|
|
16
|
+
Requires-Dist: ultralytics; extra == "train"
|
|
17
|
+
|
|
18
|
+
# sinan-captcha
|
|
19
|
+
|
|
20
|
+
`sinan-captcha` 是一个本地训练工程,用来生成和训练两类行为验证码模型:
|
|
21
|
+
|
|
22
|
+
- `group1`:图形点选
|
|
23
|
+
- `group2`:滑块缺口定位
|
|
24
|
+
|
|
25
|
+
项目对外只保留两个正式入口:
|
|
26
|
+
|
|
27
|
+
- `sinan-generator`
|
|
28
|
+
- Go CLI
|
|
29
|
+
- 负责固定工作区、素材导入/同步、原始样本生成、批次 QA、YOLO 数据集导出
|
|
30
|
+
- `sinan`
|
|
31
|
+
- Python CLI
|
|
32
|
+
- 负责训练目录初始化、素材包构建、JSONL 校验、YOLO 转换、自动标注、评估、训练、发布交付
|
|
33
|
+
|
|
34
|
+
运行时目录建议始终分开:
|
|
35
|
+
|
|
36
|
+
- 生成器安装目录:保存 `sinan-generator(.exe)` 和生成器配置
|
|
37
|
+
- 生成器工作区:保存 `workspace.json`、`presets/`、`materials/`、`cache/`、`jobs/`、`logs/`
|
|
38
|
+
- 训练目录:保存 `pyproject.toml`、`.venv/`、`datasets/`、`runs/`、`reports/`
|
|
39
|
+
|
|
40
|
+
## 仓库结构
|
|
41
|
+
|
|
42
|
+
下面是源码仓库结构,不是 Windows 训练机的推荐运行目录。训练机目录请看用户指南。
|
|
43
|
+
|
|
44
|
+
```text
|
|
45
|
+
sinan-captcha/
|
|
46
|
+
generator/ # Go 生成器工程
|
|
47
|
+
core/ # Python 训练与数据工程 CLI
|
|
48
|
+
configs/ # 配置文件
|
|
49
|
+
materials/ # 背景图、图标和素材 manifest
|
|
50
|
+
datasets/ # 原始样本、reviewed 数据和 YOLO 数据集
|
|
51
|
+
reports/ # QA 与评估输出
|
|
52
|
+
docs/ # 正式文档
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## 正式工作流
|
|
56
|
+
|
|
57
|
+
1. 用 `uvx --from sinan-captcha sinan env setup-train` 创建独立训练目录
|
|
58
|
+
2. 用 `sinan-generator workspace init --workspace <generator-workspace>` 初始化生成器固定工作区
|
|
59
|
+
3. 用 `sinan-generator materials import|fetch --workspace <generator-workspace>` 准备素材
|
|
60
|
+
4. 用 `sinan-generator make-dataset --workspace <generator-workspace>` 直接产出可交给训练 CLI 的 YOLO 数据集目录
|
|
61
|
+
5. 用 `uv run sinan train group1` 或 `uv run sinan train group2` 启动训练
|
|
62
|
+
6. 用 `uv run sinan evaluate` 做 JSONL 对比评估
|
|
63
|
+
|
|
64
|
+
## 典型命令
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
uvx --from sinan-captcha sinan env setup-train --train-root D:\sinan-captcha-work
|
|
68
|
+
uv run sinan release build --project-dir .
|
|
69
|
+
sinan-generator workspace init --workspace D:\sinan-captcha-generator\workspace
|
|
70
|
+
sinan-generator materials import --workspace D:\sinan-captcha-generator\workspace --from D:\materials-pack
|
|
71
|
+
sinan-generator make-dataset --workspace D:\sinan-captcha-generator\workspace --task group1 --dataset-dir D:\sinan-captcha-work\datasets\group1\firstpass\yolo
|
|
72
|
+
uv run sinan train group1 --dataset-yaml D:\sinan-captcha-work\datasets\group1\firstpass\yolo\dataset.yaml --project D:\sinan-captcha-work\runs\group1
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## 文档入口
|
|
76
|
+
|
|
77
|
+
- [用户指南总览](docs/02-user-guide/user-guide.md)
|
|
78
|
+
- [使用交付物与正式 CLI](docs/02-user-guide/use-build-artifacts.md)
|
|
79
|
+
- [Windows 训练机安装与模型训练完整指南](docs/02-user-guide/from-base-model-to-training-guide.md)
|
|
80
|
+
- [训练完成后的模型使用与测试](docs/02-user-guide/use-and-test-trained-models.md)
|
|
81
|
+
|
|
82
|
+
## 当前事实
|
|
83
|
+
|
|
84
|
+
- 生成器模式:`group1 -> click/native`、`group2 -> slide/native`
|
|
85
|
+
- 训练标签:`gold` 真值必须通过一致性校验、重放校验和负样本校验
|
|
86
|
+
- 生成器交付给训练 CLI 的正式接口:YOLO 数据集目录 + `dataset.yaml`
|
|
87
|
+
- 如果生成器命令不显式传 `--workspace`,Windows 默认工作区会落在 `%LOCALAPPDATA%\\SinanGenerator`
|
|
88
|
+
|
|
89
|
+
这个项目不是 HTTP 服务,也不是单一可执行程序。它是一个围绕本地 CLI、训练数据和模型训练流程组织的工程。
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# sinan-captcha
|
|
2
|
+
|
|
3
|
+
`sinan-captcha` 是一个本地训练工程,用来生成和训练两类行为验证码模型:
|
|
4
|
+
|
|
5
|
+
- `group1`:图形点选
|
|
6
|
+
- `group2`:滑块缺口定位
|
|
7
|
+
|
|
8
|
+
项目对外只保留两个正式入口:
|
|
9
|
+
|
|
10
|
+
- `sinan-generator`
|
|
11
|
+
- Go CLI
|
|
12
|
+
- 负责固定工作区、素材导入/同步、原始样本生成、批次 QA、YOLO 数据集导出
|
|
13
|
+
- `sinan`
|
|
14
|
+
- Python CLI
|
|
15
|
+
- 负责训练目录初始化、素材包构建、JSONL 校验、YOLO 转换、自动标注、评估、训练、发布交付
|
|
16
|
+
|
|
17
|
+
运行时目录建议始终分开:
|
|
18
|
+
|
|
19
|
+
- 生成器安装目录:保存 `sinan-generator(.exe)` 和生成器配置
|
|
20
|
+
- 生成器工作区:保存 `workspace.json`、`presets/`、`materials/`、`cache/`、`jobs/`、`logs/`
|
|
21
|
+
- 训练目录:保存 `pyproject.toml`、`.venv/`、`datasets/`、`runs/`、`reports/`
|
|
22
|
+
|
|
23
|
+
## 仓库结构
|
|
24
|
+
|
|
25
|
+
下面是源码仓库结构,不是 Windows 训练机的推荐运行目录。训练机目录请看用户指南。
|
|
26
|
+
|
|
27
|
+
```text
|
|
28
|
+
sinan-captcha/
|
|
29
|
+
generator/ # Go 生成器工程
|
|
30
|
+
core/ # Python 训练与数据工程 CLI
|
|
31
|
+
configs/ # 配置文件
|
|
32
|
+
materials/ # 背景图、图标和素材 manifest
|
|
33
|
+
datasets/ # 原始样本、reviewed 数据和 YOLO 数据集
|
|
34
|
+
reports/ # QA 与评估输出
|
|
35
|
+
docs/ # 正式文档
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## 正式工作流
|
|
39
|
+
|
|
40
|
+
1. 用 `uvx --from sinan-captcha sinan env setup-train` 创建独立训练目录
|
|
41
|
+
2. 用 `sinan-generator workspace init --workspace <generator-workspace>` 初始化生成器固定工作区
|
|
42
|
+
3. 用 `sinan-generator materials import|fetch --workspace <generator-workspace>` 准备素材
|
|
43
|
+
4. 用 `sinan-generator make-dataset --workspace <generator-workspace>` 直接产出可交给训练 CLI 的 YOLO 数据集目录
|
|
44
|
+
5. 用 `uv run sinan train group1` 或 `uv run sinan train group2` 启动训练
|
|
45
|
+
6. 用 `uv run sinan evaluate` 做 JSONL 对比评估
|
|
46
|
+
|
|
47
|
+
## 典型命令
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
uvx --from sinan-captcha sinan env setup-train --train-root D:\sinan-captcha-work
|
|
51
|
+
uv run sinan release build --project-dir .
|
|
52
|
+
sinan-generator workspace init --workspace D:\sinan-captcha-generator\workspace
|
|
53
|
+
sinan-generator materials import --workspace D:\sinan-captcha-generator\workspace --from D:\materials-pack
|
|
54
|
+
sinan-generator make-dataset --workspace D:\sinan-captcha-generator\workspace --task group1 --dataset-dir D:\sinan-captcha-work\datasets\group1\firstpass\yolo
|
|
55
|
+
uv run sinan train group1 --dataset-yaml D:\sinan-captcha-work\datasets\group1\firstpass\yolo\dataset.yaml --project D:\sinan-captcha-work\runs\group1
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## 文档入口
|
|
59
|
+
|
|
60
|
+
- [用户指南总览](docs/02-user-guide/user-guide.md)
|
|
61
|
+
- [使用交付物与正式 CLI](docs/02-user-guide/use-build-artifacts.md)
|
|
62
|
+
- [Windows 训练机安装与模型训练完整指南](docs/02-user-guide/from-base-model-to-training-guide.md)
|
|
63
|
+
- [训练完成后的模型使用与测试](docs/02-user-guide/use-and-test-trained-models.md)
|
|
64
|
+
|
|
65
|
+
## 当前事实
|
|
66
|
+
|
|
67
|
+
- 生成器模式:`group1 -> click/native`、`group2 -> slide/native`
|
|
68
|
+
- 训练标签:`gold` 真值必须通过一致性校验、重放校验和负样本校验
|
|
69
|
+
- 生成器交付给训练 CLI 的正式接口:YOLO 数据集目录 + `dataset.yaml`
|
|
70
|
+
- 如果生成器命令不显式传 `--workspace`,Windows 默认工作区会落在 `%LOCALAPPDATA%\\SinanGenerator`
|
|
71
|
+
|
|
72
|
+
这个项目不是 HTTP 服务,也不是单一可执行程序。它是一个围绕本地 CLI、训练数据和模型训练流程组织的工程。
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""CLI for offline autolabel flows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from core.autolabel.service import AutolabelRequest, run_autolabel
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
13
|
+
parser = argparse.ArgumentParser(description="Run offline autolabel flows.")
|
|
14
|
+
parser.add_argument("--task", choices=["group1", "group2"], required=True)
|
|
15
|
+
parser.add_argument("--mode", required=True)
|
|
16
|
+
parser.add_argument("--input-dir", type=Path, required=True)
|
|
17
|
+
parser.add_argument("--output-dir", type=Path, required=True)
|
|
18
|
+
parser.add_argument("--limit", type=int, default=None)
|
|
19
|
+
parser.add_argument("--jitter-pixels", type=int, default=4)
|
|
20
|
+
return parser
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def main(argv: list[str] | None = None) -> int:
|
|
24
|
+
parser = build_parser()
|
|
25
|
+
args = parser.parse_args(argv)
|
|
26
|
+
|
|
27
|
+
result = run_autolabel(
|
|
28
|
+
AutolabelRequest(
|
|
29
|
+
task=args.task,
|
|
30
|
+
mode=args.mode,
|
|
31
|
+
input_dir=args.input_dir,
|
|
32
|
+
output_dir=args.output_dir,
|
|
33
|
+
limit=args.limit,
|
|
34
|
+
jitter_pixels=args.jitter_pixels,
|
|
35
|
+
)
|
|
36
|
+
)
|
|
37
|
+
print(json.dumps(result.to_dict(), ensure_ascii=False, indent=2))
|
|
38
|
+
return 0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
if __name__ == "__main__":
|
|
42
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Offline autolabel flows for reviewed-seed and pseudo-auto datasets."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import asdict, dataclass
|
|
6
|
+
import hashlib
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import shutil
|
|
9
|
+
|
|
10
|
+
from core.common.images import get_image_size
|
|
11
|
+
from core.common.jsonl import read_jsonl, write_jsonl
|
|
12
|
+
from core.dataset.validation import (
|
|
13
|
+
get_group2_query_image,
|
|
14
|
+
get_group2_scene_image,
|
|
15
|
+
get_group2_target,
|
|
16
|
+
set_group2_target,
|
|
17
|
+
validate_group1_row,
|
|
18
|
+
validate_group2_row,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class AutolabelRequest:
|
|
24
|
+
task: str
|
|
25
|
+
mode: str
|
|
26
|
+
input_dir: Path
|
|
27
|
+
output_dir: Path
|
|
28
|
+
limit: int | None = None
|
|
29
|
+
jitter_pixels: int = 4
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class AutolabelResult:
|
|
34
|
+
task: str
|
|
35
|
+
mode: str
|
|
36
|
+
output_dir: Path
|
|
37
|
+
labels_path: Path
|
|
38
|
+
processed_count: int
|
|
39
|
+
|
|
40
|
+
def to_dict(self) -> dict[str, object]:
|
|
41
|
+
payload = asdict(self)
|
|
42
|
+
payload["output_dir"] = str(self.output_dir)
|
|
43
|
+
payload["labels_path"] = str(self.labels_path)
|
|
44
|
+
return payload
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def run_autolabel(request: AutolabelRequest) -> AutolabelResult:
|
|
48
|
+
rows = read_jsonl(request.input_dir / "labels.jsonl")
|
|
49
|
+
selected_rows = _apply_limit(rows, request.limit)
|
|
50
|
+
output_rows: list[dict[str, object]] = []
|
|
51
|
+
|
|
52
|
+
for raw_row in selected_rows:
|
|
53
|
+
copied_row = dict(raw_row)
|
|
54
|
+
if request.task == "group1":
|
|
55
|
+
row = validate_group1_row(copied_row)
|
|
56
|
+
transformed = _transform_group1_row(row, request)
|
|
57
|
+
elif request.task == "group2":
|
|
58
|
+
row = validate_group2_row(copied_row)
|
|
59
|
+
transformed = _transform_group2_row(row, request)
|
|
60
|
+
else:
|
|
61
|
+
raise ValueError(f"unsupported autolabel task: {request.task}")
|
|
62
|
+
|
|
63
|
+
if request.task == "group1":
|
|
64
|
+
_copy_asset(request.input_dir, request.output_dir, Path(str(transformed["query_image"])))
|
|
65
|
+
_copy_asset(request.input_dir, request.output_dir, Path(str(transformed["scene_image"])))
|
|
66
|
+
else:
|
|
67
|
+
_copy_asset(request.input_dir, request.output_dir, Path(get_group2_query_image(transformed)))
|
|
68
|
+
_copy_asset(request.input_dir, request.output_dir, Path(get_group2_scene_image(transformed)))
|
|
69
|
+
output_rows.append(transformed)
|
|
70
|
+
|
|
71
|
+
labels_path = request.output_dir / "labels.jsonl"
|
|
72
|
+
write_jsonl(labels_path, output_rows)
|
|
73
|
+
return AutolabelResult(
|
|
74
|
+
task=request.task,
|
|
75
|
+
mode=request.mode,
|
|
76
|
+
output_dir=request.output_dir,
|
|
77
|
+
labels_path=labels_path,
|
|
78
|
+
processed_count=len(output_rows),
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _apply_limit(rows: list[dict[str, object]], limit: int | None) -> list[dict[str, object]]:
|
|
83
|
+
ordered = sorted(rows, key=lambda row: str(row["sample_id"]))
|
|
84
|
+
if limit is None or limit >= len(ordered):
|
|
85
|
+
return ordered
|
|
86
|
+
return ordered[:limit]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _transform_group1_row(row: dict[str, object], request: AutolabelRequest) -> dict[str, object]:
|
|
90
|
+
if request.mode == "seed-review":
|
|
91
|
+
result = dict(row)
|
|
92
|
+
result["label_source"] = "reviewed"
|
|
93
|
+
return result
|
|
94
|
+
if request.mode != "warmup-auto":
|
|
95
|
+
raise ValueError(f"unsupported group1 autolabel mode: {request.mode}")
|
|
96
|
+
|
|
97
|
+
scene_path = request.input_dir / str(row["scene_image"])
|
|
98
|
+
image_width, image_height = get_image_size(scene_path)
|
|
99
|
+
result = dict(row)
|
|
100
|
+
result["targets"] = [
|
|
101
|
+
_perturb_object(
|
|
102
|
+
obj,
|
|
103
|
+
sample_id=str(row["sample_id"]),
|
|
104
|
+
image_width=image_width,
|
|
105
|
+
image_height=image_height,
|
|
106
|
+
jitter_pixels=request.jitter_pixels,
|
|
107
|
+
salt=f"target:{index}",
|
|
108
|
+
)
|
|
109
|
+
for index, obj in enumerate(row["targets"])
|
|
110
|
+
]
|
|
111
|
+
result["distractors"] = [
|
|
112
|
+
_perturb_object(
|
|
113
|
+
obj,
|
|
114
|
+
sample_id=str(row["sample_id"]),
|
|
115
|
+
image_width=image_width,
|
|
116
|
+
image_height=image_height,
|
|
117
|
+
jitter_pixels=request.jitter_pixels,
|
|
118
|
+
salt=f"distractor:{index}",
|
|
119
|
+
)
|
|
120
|
+
for index, obj in enumerate(row["distractors"])
|
|
121
|
+
]
|
|
122
|
+
result["label_source"] = "auto"
|
|
123
|
+
return result
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _transform_group2_row(row: dict[str, object], request: AutolabelRequest) -> dict[str, object]:
|
|
127
|
+
if request.mode != "rule-auto":
|
|
128
|
+
raise ValueError(f"unsupported group2 autolabel mode: {request.mode}")
|
|
129
|
+
|
|
130
|
+
scene_path = request.input_dir / get_group2_scene_image(row)
|
|
131
|
+
image_width, image_height = get_image_size(scene_path)
|
|
132
|
+
result = set_group2_target(
|
|
133
|
+
row,
|
|
134
|
+
_perturb_object(
|
|
135
|
+
get_group2_target(row),
|
|
136
|
+
sample_id=str(row["sample_id"]),
|
|
137
|
+
image_width=image_width,
|
|
138
|
+
image_height=image_height,
|
|
139
|
+
jitter_pixels=request.jitter_pixels,
|
|
140
|
+
salt="group2-target",
|
|
141
|
+
),
|
|
142
|
+
)
|
|
143
|
+
result["label_source"] = "auto"
|
|
144
|
+
return result
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _perturb_object(
|
|
148
|
+
obj: dict[str, object],
|
|
149
|
+
*,
|
|
150
|
+
sample_id: str,
|
|
151
|
+
image_width: int,
|
|
152
|
+
image_height: int,
|
|
153
|
+
jitter_pixels: int,
|
|
154
|
+
salt: str,
|
|
155
|
+
) -> dict[str, object]:
|
|
156
|
+
x1, y1, x2, y2 = [int(value) for value in obj["bbox"]]
|
|
157
|
+
width = max(1, x2 - x1)
|
|
158
|
+
height = max(1, y2 - y1)
|
|
159
|
+
|
|
160
|
+
dx = _stable_offset(sample_id, salt, "dx", jitter_pixels)
|
|
161
|
+
dy = _stable_offset(sample_id, salt, "dy", jitter_pixels)
|
|
162
|
+
grow = _stable_offset(sample_id, salt, "grow", max(1, jitter_pixels // 2))
|
|
163
|
+
|
|
164
|
+
new_x1 = _clamp(x1 + dx - grow, 0, image_width - 2)
|
|
165
|
+
new_y1 = _clamp(y1 + dy - grow, 0, image_height - 2)
|
|
166
|
+
new_x2 = _clamp(x1 + dx + width + grow, new_x1 + 1, image_width)
|
|
167
|
+
new_y2 = _clamp(y1 + dy + height + grow, new_y1 + 1, image_height)
|
|
168
|
+
|
|
169
|
+
updated = dict(obj)
|
|
170
|
+
updated["bbox"] = [new_x1, new_y1, new_x2, new_y2]
|
|
171
|
+
updated["center"] = [(new_x1 + new_x2) // 2, (new_y1 + new_y2) // 2]
|
|
172
|
+
return updated
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _stable_offset(sample_id: str, salt: str, axis: str, magnitude: int) -> int:
|
|
176
|
+
if magnitude <= 0:
|
|
177
|
+
return 0
|
|
178
|
+
digest = hashlib.sha256(f"{sample_id}:{salt}:{axis}".encode("utf-8")).digest()
|
|
179
|
+
span = magnitude * 2 + 1
|
|
180
|
+
return int(digest[0] % span) - magnitude
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _copy_asset(input_dir: Path, output_dir: Path, relative_path: Path) -> None:
|
|
184
|
+
source = input_dir / relative_path
|
|
185
|
+
destination = output_dir / relative_path
|
|
186
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
187
|
+
shutil.copy2(source, destination)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _clamp(value: int, lower: int, upper: int) -> int:
|
|
191
|
+
return max(lower, min(value, upper))
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Unified Python CLI for training and data-engineering flows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
|
|
8
|
+
from core.autolabel import cli as autolabel_cli
|
|
9
|
+
from core.convert import cli as convert_cli
|
|
10
|
+
from core.dataset import cli as dataset_cli
|
|
11
|
+
from core.evaluate import cli as evaluate_cli
|
|
12
|
+
from core.materials import cli as materials_cli
|
|
13
|
+
from core.ops import env as env_cli
|
|
14
|
+
from core.ops import setup_train as setup_train_cli
|
|
15
|
+
from core.release import cli as release_cli
|
|
16
|
+
from core.train.group1 import cli as train_group1_cli
|
|
17
|
+
from core.train.group2 import cli as train_group2_cli
|
|
18
|
+
|
|
19
|
+
CommandHandler = tuple[tuple[str, ...], Callable[[list[str] | None], int]]
|
|
20
|
+
|
|
21
|
+
COMMANDS: list[CommandHandler] = [
|
|
22
|
+
(("env", "check"), lambda argv: env_cli.main(argv)),
|
|
23
|
+
(("env", "setup-train"), lambda argv: setup_train_cli.main(argv)),
|
|
24
|
+
(("materials", "build"), lambda argv: materials_cli.main(argv)),
|
|
25
|
+
(("dataset", "validate"), lambda argv: dataset_cli.main(argv)),
|
|
26
|
+
(("dataset", "build-yolo"), lambda argv: convert_cli.main(argv)),
|
|
27
|
+
(("autolabel",), lambda argv: autolabel_cli.main(argv)),
|
|
28
|
+
(("evaluate",), lambda argv: evaluate_cli.main(argv)),
|
|
29
|
+
(("train", "group1"), lambda argv: train_group1_cli.main(argv)),
|
|
30
|
+
(("train", "group2"), lambda argv: train_group2_cli.main(argv)),
|
|
31
|
+
(("release",), lambda argv: release_cli.main(argv)),
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def main(argv: list[str] | None = None) -> int:
|
|
36
|
+
args = list(sys.argv[1:] if argv is None else argv)
|
|
37
|
+
if not args or args[0] in {"-h", "--help", "help"}:
|
|
38
|
+
print(_usage())
|
|
39
|
+
return 0
|
|
40
|
+
|
|
41
|
+
for tokens, handler in COMMANDS:
|
|
42
|
+
if args[: len(tokens)] == list(tokens):
|
|
43
|
+
return handler(args[len(tokens) :])
|
|
44
|
+
|
|
45
|
+
print(f"unknown command: {' '.join(args)}", file=sys.stderr)
|
|
46
|
+
print(_usage(), file=sys.stderr)
|
|
47
|
+
return 1
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _usage() -> str:
|
|
51
|
+
return "\n".join(
|
|
52
|
+
[
|
|
53
|
+
"uv run sinan <command>",
|
|
54
|
+
"",
|
|
55
|
+
"Commands:",
|
|
56
|
+
" env check Check whether the training host is ready.",
|
|
57
|
+
" env setup-train Create a dedicated training root and bootstrap its uv environment.",
|
|
58
|
+
" materials build Build a local offline materials pack.",
|
|
59
|
+
" dataset validate Validate a JSONL dataset file.",
|
|
60
|
+
" dataset build-yolo Convert a labeled batch into a YOLO dataset.",
|
|
61
|
+
" autolabel Run offline autolabel flows.",
|
|
62
|
+
" evaluate Evaluate prediction JSONL files against gold data.",
|
|
63
|
+
" train group1 Run group1 YOLO training.",
|
|
64
|
+
" train group2 Run group2 YOLO training.",
|
|
65
|
+
" release <subcommand> Build, publish, or package delivery artifacts.",
|
|
66
|
+
"",
|
|
67
|
+
"Examples:",
|
|
68
|
+
" uv run sinan env check",
|
|
69
|
+
" uv run sinan env setup-train --train-root D:\\sinan-captcha-work",
|
|
70
|
+
" uv run sinan materials build --spec configs/materials-pack.toml --output-root materials",
|
|
71
|
+
" uv run sinan dataset build-yolo --task group1 --version v1 --source-dir batch --output-dir yolo",
|
|
72
|
+
" uv run sinan train group1 --dataset-yaml datasets/group1/v1/yolo/dataset.yaml --project runs/group1",
|
|
73
|
+
" uv run sinan release build --project-dir .",
|
|
74
|
+
]
|
|
75
|
+
)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Image metadata helpers using the Python standard library only."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import struct
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def get_image_size(path: Path) -> tuple[int, int]:
|
|
10
|
+
"""Return `(width, height)` for PNG and JPEG images."""
|
|
11
|
+
|
|
12
|
+
with path.open("rb") as handle:
|
|
13
|
+
header = handle.read(24)
|
|
14
|
+
if len(header) < 24:
|
|
15
|
+
raise ValueError(f"Image file is too small: {path}")
|
|
16
|
+
|
|
17
|
+
if header.startswith(b"\x89PNG\r\n\x1a\n"):
|
|
18
|
+
width, height = struct.unpack(">II", header[16:24])
|
|
19
|
+
return width, height
|
|
20
|
+
|
|
21
|
+
if header[:2] == b"\xff\xd8":
|
|
22
|
+
handle.seek(2)
|
|
23
|
+
return _read_jpeg_size(handle, path)
|
|
24
|
+
|
|
25
|
+
raise ValueError(f"Unsupported image format: {path}")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _read_jpeg_size(handle, path: Path) -> tuple[int, int]:
|
|
29
|
+
while True:
|
|
30
|
+
marker_prefix = handle.read(1)
|
|
31
|
+
if not marker_prefix:
|
|
32
|
+
break
|
|
33
|
+
if marker_prefix != b"\xff":
|
|
34
|
+
continue
|
|
35
|
+
|
|
36
|
+
marker = handle.read(1)
|
|
37
|
+
while marker == b"\xff":
|
|
38
|
+
marker = handle.read(1)
|
|
39
|
+
if not marker:
|
|
40
|
+
break
|
|
41
|
+
|
|
42
|
+
marker_value = marker[0]
|
|
43
|
+
if marker_value in {0xD8, 0xD9}:
|
|
44
|
+
continue
|
|
45
|
+
|
|
46
|
+
segment_length_raw = handle.read(2)
|
|
47
|
+
if len(segment_length_raw) != 2:
|
|
48
|
+
break
|
|
49
|
+
segment_length = struct.unpack(">H", segment_length_raw)[0]
|
|
50
|
+
if segment_length < 2:
|
|
51
|
+
raise ValueError(f"Invalid JPEG segment length: {path}")
|
|
52
|
+
|
|
53
|
+
if marker_value in {
|
|
54
|
+
0xC0,
|
|
55
|
+
0xC1,
|
|
56
|
+
0xC2,
|
|
57
|
+
0xC3,
|
|
58
|
+
0xC5,
|
|
59
|
+
0xC6,
|
|
60
|
+
0xC7,
|
|
61
|
+
0xC9,
|
|
62
|
+
0xCA,
|
|
63
|
+
0xCB,
|
|
64
|
+
0xCD,
|
|
65
|
+
0xCE,
|
|
66
|
+
0xCF,
|
|
67
|
+
}:
|
|
68
|
+
segment = handle.read(segment_length - 2)
|
|
69
|
+
if len(segment) < 5:
|
|
70
|
+
break
|
|
71
|
+
height, width = struct.unpack(">HH", segment[1:5])
|
|
72
|
+
return width, height
|
|
73
|
+
|
|
74
|
+
handle.seek(segment_length - 2, 1)
|
|
75
|
+
|
|
76
|
+
raise ValueError(f"Could not determine JPEG size: {path}")
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Small JSONL helpers used across dataset and report flows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
JsonMapping = dict[str, Any]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def read_jsonl(path: Path) -> list[JsonMapping]:
|
|
13
|
+
"""Read a JSONL file into a list of dictionaries."""
|
|
14
|
+
|
|
15
|
+
rows: list[JsonMapping] = []
|
|
16
|
+
with path.open("r", encoding="utf-8") as handle:
|
|
17
|
+
for line_number, line in enumerate(handle, start=1):
|
|
18
|
+
raw = line.strip()
|
|
19
|
+
if not raw:
|
|
20
|
+
continue
|
|
21
|
+
try:
|
|
22
|
+
payload = json.loads(raw)
|
|
23
|
+
except json.JSONDecodeError as exc: # pragma: no cover - direct error path
|
|
24
|
+
message = f"Invalid JSONL at {path}:{line_number}: {exc.msg}"
|
|
25
|
+
raise ValueError(message) from exc
|
|
26
|
+
if not isinstance(payload, dict):
|
|
27
|
+
message = f"Expected object at {path}:{line_number}, got {type(payload).__name__}"
|
|
28
|
+
raise ValueError(message)
|
|
29
|
+
rows.append(payload)
|
|
30
|
+
return rows
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def write_jsonl(path: Path, rows: list[JsonMapping]) -> None:
|
|
34
|
+
"""Write dictionaries to a UTF-8 JSONL file."""
|
|
35
|
+
|
|
36
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
37
|
+
with path.open("w", encoding="utf-8") as handle:
|
|
38
|
+
for row in rows:
|
|
39
|
+
handle.write(json.dumps(row, ensure_ascii=False))
|
|
40
|
+
handle.write("\n")
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Repository and work-directory path helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class WorkspacePaths:
|
|
11
|
+
"""Well-known project paths used by scripts and CLIs."""
|
|
12
|
+
|
|
13
|
+
repo_root: Path
|
|
14
|
+
work_root: Path
|
|
15
|
+
datasets_dir: Path
|
|
16
|
+
reports_dir: Path
|
|
17
|
+
dist_dir: Path
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def from_roots(cls, repo_root: Path, work_root: Path | None = None) -> "WorkspacePaths":
|
|
21
|
+
resolved_repo_root = repo_root.resolve()
|
|
22
|
+
resolved_work_root = (work_root or resolved_repo_root).resolve()
|
|
23
|
+
return cls(
|
|
24
|
+
repo_root=resolved_repo_root,
|
|
25
|
+
work_root=resolved_work_root,
|
|
26
|
+
datasets_dir=resolved_work_root / "datasets",
|
|
27
|
+
reports_dir=resolved_work_root / "reports",
|
|
28
|
+
dist_dir=resolved_repo_root / "dist",
|
|
29
|
+
)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""CLI for JSONL to YOLO conversion flows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from core.convert.service import ConversionRequest, build_yolo_dataset
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
12
|
+
parser = argparse.ArgumentParser(description="Convert JSONL source-of-truth files into YOLO datasets.")
|
|
13
|
+
parser.add_argument("--task", choices=["group1", "group2"], required=True)
|
|
14
|
+
parser.add_argument("--version", required=True)
|
|
15
|
+
parser.add_argument("--source-dir", type=Path, required=True)
|
|
16
|
+
parser.add_argument("--output-dir", type=Path, required=True)
|
|
17
|
+
return parser
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def main(argv: list[str] | None = None) -> int:
|
|
21
|
+
parser = build_parser()
|
|
22
|
+
args = parser.parse_args(argv)
|
|
23
|
+
|
|
24
|
+
build_yolo_dataset(
|
|
25
|
+
ConversionRequest(
|
|
26
|
+
task=args.task,
|
|
27
|
+
version=args.version,
|
|
28
|
+
source_dir=args.source_dir,
|
|
29
|
+
output_dir=args.output_dir,
|
|
30
|
+
)
|
|
31
|
+
)
|
|
32
|
+
print(f"built YOLO dataset at {args.output_dir}")
|
|
33
|
+
return 0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
if __name__ == "__main__":
|
|
37
|
+
raise SystemExit(main())
|