mblt-vision-python 0.0.0__py3-none-any.whl

mblt_vision/cli/val.py

@@ -0,0 +1,562 @@
+"""Vision validation CLI command."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from mblt_vision._tasks import normalize_vision_task
+from mblt_vision.benchmark.argparse_utils import parse_positive_int
+from mblt_vision.datasets import get_dataset_config, get_dataset_config_for_task
+from mblt_vision.utils.datasets.readiness import dataset_ready
+from mblt_vision.wrapper import get_mobilint_cache_dir
+
+from ._vision import (
+    add_e2e_arg,
+    add_threshold_args,
+    create_vision_engine,
+    parse_target_clusters,
+    parse_target_cores,
+)
+
+DEFAULT_IMAGENET_IMAGE_SOURCE = get_dataset_config("imagenet")["download"]["images"]
+DEFAULT_IMAGENET_XML_SOURCE = get_dataset_config("imagenet")["download"]["annotations"]
+DEFAULT_COCO_IMAGE_SOURCE = get_dataset_config("coco")["download"]["images"]
+DEFAULT_COCO_ANNOTATION_SOURCE = get_dataset_config("coco")["download"]["annotations"]
+DEFAULT_WIDERFACE_IMAGE_SOURCE = get_dataset_config("widerface")["download"]["images"]
+DEFAULT_WIDERFACE_ANNOTATION_SOURCE = get_dataset_config("widerface")["download"][
+    "annotations"
+]
+DEFAULT_DOTAV1_SOURCE = get_dataset_config("dotav1")["download"]["url"]
+DEFAULT_NYU_DEPTH_SOURCE = get_dataset_config("nyu-depth")["download"]["url"]
+DEFAULT_ADE20K_SOURCE = get_dataset_config("ade20k")["download"]["url"]
+CITYSCAPES_DOWNLOAD_CONFIG = get_dataset_config("cityscapes")["download"]
+CITYSCAPES_IMAGE_ARCHIVE = CITYSCAPES_DOWNLOAD_CONFIG["images_archive"]
+CITYSCAPES_ANNOTATION_ARCHIVE = CITYSCAPES_DOWNLOAD_CONFIG["annotations_archive"]
+
+
+def _candidate_search_roots(data_path: str) -> list[Path]:
+    """Returns directories to inspect for existing raw dataset sources."""
+
+    root = Path(data_path).expanduser()
+    candidates = [root, root.parent, Path.cwd()]
+    ordered: list[Path] = []
+    seen: set[Path] = set()
+    for candidate in candidates:
+        resolved = candidate.resolve()
+        if resolved in seen:
+            continue
+        seen.add(resolved)
+        ordered.append(resolved)
+    return ordered
+
+
+def _find_existing_source(data_path: str, candidate_names: list[str]) -> str | None:
+    """Finds a nearby raw archive or extracted dataset directory."""
+
+    for root in _candidate_search_roots(data_path):
+        for name in candidate_names:
+            candidate = root / name
+            if candidate.exists():
+                return str(candidate)
+    return None
+
+
+def _normalize_coco_annotation_source(annotation_dir: str | None) -> str | None:
+    """Normalizes a COCO annotation source for the organizer contract.
+
+    The COCO organizer expects either the annotation archive or the extracted
+    parent directory that contains an ``annotations`` subdirectory. When source
+    discovery finds the extracted leaf ``annotations`` directory directly,
+    return its parent so downstream code does not resolve ``annotations``
+    twice.
+    """
+
+    if annotation_dir is None:
+        return None
+
+    candidate = Path(annotation_dir).expanduser()
+    if candidate.is_dir() and candidate.name == "annotations":
+        return str(candidate.parent)
+    return annotation_dir
+
+
+def _resolve_imagenet_sources(
+    args: argparse.Namespace, data_path: str
+) -> tuple[str, str]:
+    """Resolves local or remote sources for ImageNet organization."""
+
+    image_dir = args.image_dir
+    xml_dir = args.xml_dir
+    if not args.force_organize:
+        image_dir = image_dir or _find_existing_source(
+            data_path, ["ILSVRC2012_img_val.tar", "ILSVRC2012_img_val"]
+        )
+        xml_dir = xml_dir or _find_existing_source(
+            data_path, ["ILSVRC2012_bbox_val_v3.tgz", "ILSVRC2012_bbox_val_v3"]
+        )
+    return (
+        image_dir or DEFAULT_IMAGENET_IMAGE_SOURCE,
+        xml_dir or DEFAULT_IMAGENET_XML_SOURCE,
+    )
+
+
+def _resolve_coco_sources(args: argparse.Namespace, data_path: str) -> tuple[str, str]:
+    """Resolves local or remote sources for COCO organization."""
+
+    image_dir = args.image_dir
+    annotation_dir = _normalize_coco_annotation_source(args.annotation_dir)
+    if not args.force_organize:
+        image_dir = image_dir or _find_existing_source(
+            data_path, ["val2017.zip", "val2017"]
+        )
+        annotation_dir = annotation_dir or _normalize_coco_annotation_source(
+            _find_existing_source(
+                data_path,
+                [
+                    "annotations_trainval2017.zip",
+                    "annotations_trainval2017",
+                    "annotations",
+                ],
+            )
+        )
+    return (
+        image_dir or DEFAULT_COCO_IMAGE_SOURCE,
+        annotation_dir or DEFAULT_COCO_ANNOTATION_SOURCE,
+    )
+
+
+def _resolve_widerface_sources(
+    args: argparse.Namespace, data_path: str
+) -> tuple[str, str]:
+    """Resolves local or remote sources for WiderFace organization."""
+
+    image_dir = args.image_dir
+    annotation_dir = args.annotation_dir
+    if not args.force_organize:
+        image_dir = image_dir or _find_existing_source(
+            data_path, ["WIDER_val.zip", "WIDER_val"]
+        )
+        annotation_dir = annotation_dir or _find_existing_source(
+            data_path,
+            ["wider_face_split.zip", "wider_face_split"],
+        )
+    return (
+        image_dir or DEFAULT_WIDERFACE_IMAGE_SOURCE,
+        annotation_dir or DEFAULT_WIDERFACE_ANNOTATION_SOURCE,
+    )
+
+
+def _resolve_dotav1_source(args: argparse.Namespace, data_path: str) -> str:
+    """Resolves a local or remote source for DOTAv1 organization."""
+
+    dataset_path = args.annotation_dir or args.image_dir
+    if not args.force_organize:
+        dataset_path = dataset_path or _find_existing_source(
+            data_path, ["DOTAv1.zip", "DOTAv1"]
+        )
+    return dataset_path or DEFAULT_DOTAV1_SOURCE
+
+
+def _resolve_nyu_depth_source(args: argparse.Namespace, data_path: str) -> str:
+    """Resolve a local archive or URL for NYU Depth organization."""
+
+    dataset_path = args.annotation_dir or args.image_dir
+    if not args.force_organize:
+        dataset_path = dataset_path or _find_existing_source(
+            data_path, ["nyu-depth.zip", "nyu-depth"]
+        )
+    return dataset_path or DEFAULT_NYU_DEPTH_SOURCE
+
+
+def _resolve_ade20k_source(args: argparse.Namespace, data_path: str) -> str:
+    """Resolve a local archive, extracted directory, or URL for ADE20K organization."""
+
+    dataset_path = args.annotation_dir or args.image_dir
+    if not args.force_organize:
+        dataset_path = dataset_path or _find_existing_source(
+            data_path,
+            ["ADEChallengeData2016.zip", "ADEChallengeData2016"],
+        )
+    return dataset_path or DEFAULT_ADE20K_SOURCE
+
+
+def _resolve_cityscapes_sources(
+    args: argparse.Namespace, data_path: str
+) -> tuple[str, str]:
+    """Resolve the two manually downloaded official Cityscapes archives.
+
+    Args:
+        args: Parsed validation CLI arguments.
+        data_path: Organized Cityscapes output path used as a discovery anchor.
+
+    Returns:
+        Image and annotation ZIP paths.
+
+    Raises:
+        SystemExit: If either required archive cannot be found.
+    """
+
+    image_dir = args.image_dir or _find_existing_source(
+        data_path, [CITYSCAPES_IMAGE_ARCHIVE]
+    )
+    annotation_dir = args.annotation_dir or _find_existing_source(
+        data_path, [CITYSCAPES_ANNOTATION_ARCHIVE]
+    )
+    if image_dir is None or annotation_dir is None:
+        raise SystemExit(
+            "Cityscapes organization requires the official image and annotation ZIP archives. "
+            "Register at https://www.cityscapes-dataset.com/, then download them with:\n"
+            "  csDownload -d <download-dir> gtFine_trainvaltest.zip leftImg8bit_trainvaltest.zip\n"
+            "Pass the resulting files with --image-dir and --annotation-dir, or place them near the dataset path."
+        )
+    return image_dir, annotation_dir
+
+
+def _default_data_path_for_task(task: str, dataset: str | None = None) -> str:
+    """Returns the default organized dataset path for a vision task."""
+
+    try:
+        configured_path = Path(get_dataset_config_for_task(task, dataset)["path"])
+    except ValueError as exc:
+        raise SystemExit(f"Unsupported vision task for validation: {task}") from exc
+
+    configured_path = configured_path.expanduser()
+    default_cache_root = Path.home() / ".mblt_model_zoo"
+    try:
+        relative_path = configured_path.relative_to(default_cache_root)
+    except ValueError:
+        return str(configured_path)
+    return str(Path(get_mobilint_cache_dir()) / relative_path)
+
+
+def _dataset_ready(task: str, data_path: str, dataset: str | None = None) -> bool:
+    """Checks whether the organized dataset appears ready for validation."""
+
+    return dataset_ready(data_path, task, dataset)
+
+
+def _ensure_dataset(
+    args: argparse.Namespace, task: str, dataset: str | None = None
+) -> str:
+    """Organizes the dataset automatically when the expected layout is missing."""
+
+    task = normalize_vision_task(task)
+    data_path = os.path.expanduser(
+        args.data_path or _default_data_path_for_task(task, dataset)
+    )
+    if _dataset_ready(task, data_path, dataset) and not args.force_organize:
+        print(f"Using organized dataset at {data_path}")
+        return data_path
+
+    try:
+        from mblt_vision.utils.datasets import (
+            organize_ade20k,
+            organize_cityscapes,
+            organize_coco,
+            organize_dotav1,
+            organize_imagenet,
+            organize_nyu_depth,
+            organize_widerface,
+        )
+    except ImportError as exc:
+        print(
+            f"Missing dependencies for vision dataset organization: {exc}",
+            file=sys.stderr,
+        )
+        raise SystemExit(2) from exc
+
+    print(f"Preparing validation dataset for task `{task}` at {data_path}...")
+    if task == "image_classification":
+        image_dir, xml_dir = _resolve_imagenet_sources(args, data_path)
+        organize_imagenet(
+            image_dir=image_dir,
+            xml_dir=xml_dir,
+            output_dir=data_path,
+        )
+    elif task in {"object_detection", "instance_segmentation", "pose_estimation"}:
+        image_dir, annotation_dir = _resolve_coco_sources(args, data_path)
+        organize_coco(
+            image_dir=image_dir,
+            annotation_dir=annotation_dir,
+            output_dir=data_path,
+        )
+    elif task == "face_detection":
+        image_dir, annotation_dir = _resolve_widerface_sources(args, data_path)
+        organize_widerface(
+            image_dir=image_dir,
+            annotation_dir=annotation_dir,
+            output_dir=data_path,
+        )
+    elif task == "obb":
+        organize_dotav1(
+            dataset_path=_resolve_dotav1_source(args, data_path),
+            output_dir=data_path,
+        )
+    elif task == "depth_estimation":
+        organize_nyu_depth(
+            dataset_path=_resolve_nyu_depth_source(args, data_path),
+            output_dir=data_path,
+        )
+    elif task == "semantic_segmentation":
+        if dataset == "cityscapes":
+            image_dir, annotation_dir = _resolve_cityscapes_sources(args, data_path)
+            organize_cityscapes(
+                image_dir=image_dir,
+                annotation_dir=annotation_dir,
+                output_dir=data_path,
+            )
+        else:
+            organize_ade20k(
+                dataset_path=_resolve_ade20k_source(args, data_path),
+                output_dir=data_path,
+            )
+    else:
+        raise SystemExit(f"Unsupported vision task for validation: {task}")
+
+    if not _dataset_ready(task, data_path, dataset):
+        raise SystemExit(
+            f"Organized validation dataset at {data_path} is incomplete or does not match "
+            f"the expected {dataset or task} dataset."
+        )
+    return data_path
+
+
+def _run_validation(args: argparse.Namespace) -> float:
+    """Runs model validation on the dataset associated with the model task."""
+
+    try:
+        from mblt_vision.utils.evaluation import (
+            eval_ade20k,
+            eval_cityscapes,
+            eval_coco_metrics,
+            eval_dota,
+            eval_imagenet_metrics,
+            eval_nyu_depth,
+            eval_widerface,
+        )
+    except ImportError as exc:
+        print(f"Missing dependencies for vision CLI: {exc}", file=sys.stderr)
+        raise SystemExit(2) from exc
+
+    model = create_vision_engine(args)
+    try:
+        if not getattr(getattr(model, "postprocessor", None), "e2e", True):
+            raise SystemExit(
+                "Validation requires end-to-end YOLO postprocessing. Use `--e2e true` or omit the option."
+            )
+
+        task = normalize_vision_task(model.post_cfg.get("task", ""))
+        dataset = model.post_cfg.get("dataset")
+        taxonomy = str(dataset).lower() if isinstance(dataset, str) else None
+        if task == "semantic_segmentation" and taxonomy not in {"ade20k", "cityscapes"}:
+            raise SystemExit(
+                f"Unsupported semantic segmentation taxonomy for validation: {taxonomy!r}. "
+                "Expected `ade20k` or `cityscapes`."
+            )
+        data_path = _ensure_dataset(args, task, taxonomy)
+
+        if task == "image_classification":
+            imagenet_result = eval_imagenet_metrics(
+                model=model, data_path=data_path, batch_size=args.batch_size
+            )
+            print(
+                "Validation score "
+                f"(Top-1 accuracy): {imagenet_result.top1:.5f}, "
+                f"(Top-5 accuracy): {imagenet_result.top5:.5f}"
+            )
+            return imagenet_result.primary_score
+
+        if task == "depth_estimation":
+            depth_result = eval_nyu_depth(
+                model=model, data_path=data_path, batch_size=args.batch_size
+            )
+            print(
+                "Validation score "
+                f"(delta1): {depth_result.delta1:.5f}, "
+                f"(abs_rel): {depth_result.abs_rel:.5f}, "
+                f"(rmse): {depth_result.rmse:.5f}"
+            )
+            return depth_result.primary_score
+
+        if task == "semantic_segmentation":
+            if taxonomy == "cityscapes":
+                semantic_result = eval_cityscapes(
+                    model=model, data_path=data_path, batch_size=args.batch_size
+                )
+            elif taxonomy == "ade20k":
+                semantic_result = eval_ade20k(
+                    model=model, data_path=data_path, batch_size=args.batch_size
+                )
+            else:
+                raise AssertionError(
+                    f"Unexpected validated semantic taxonomy: {taxonomy!r}"
+                )
+            print(
+                "Validation score "
+                f"(mIoU): {semantic_result.miou:.5f}, "
+                f"(pixel accuracy): {semantic_result.pixel_accuracy:.5f}"
+            )
+            return semantic_result.primary_score
+
+        if task in {"object_detection", "instance_segmentation", "pose_estimation"}:
+            coco_result = eval_coco_metrics(
+                model=model,
+                data_path=data_path,
+                batch_size=args.batch_size,
+                conf_thres=args.conf_thres,
+                iou_thres=args.iou_thres,
+            )
+            print(
+                f"Validation score (mAP50-95): {coco_result.map5095:.5f}, (mAP50): {coco_result.map50:.5f}"
+            )
+            return coco_result.primary_score
+
+        if task == "obb":
+            dota_result = eval_dota(
+                model=model,
+                data_path=data_path,
+                batch_size=args.batch_size,
+                conf_thres=args.conf_thres,
+                iou_thres=args.iou_thres,
+            )
+            print(
+                "Validation score "
+                f"(rotated mAP50-95): {dota_result.map5095:.5f}, "
+                f"(rotated mAP50): {dota_result.map50:.5f}"
+            )
+            return dota_result.primary_score
+
+        if task == "face_detection":
+            widerface_result = eval_widerface(
+                model=model,
+                data_path=data_path,
+                batch_size=args.batch_size,
+                conf_thres=args.conf_thres,
+                iou_thres=args.iou_thres,
+            )
+            print(
+                "Validation score "
+                f"(Easy AP): {widerface_result.easy_ap:.5f}, "
+                f"(Medium AP): {widerface_result.medium_ap:.5f}, "
+                f"(Hard AP): {widerface_result.hard_ap:.5f}, "
+                f"(Mean AP): {widerface_result.mean_ap:.5f}"
+            )
+            return widerface_result.primary_score
+
+        raise SystemExit(f"Unsupported vision task for validation: {task}")
+    finally:
+        model.dispose()
+
+
+def _cmd_val(args: argparse.Namespace) -> int:
+    """Runs vision validation on the task-appropriate benchmark dataset."""
+
+    _run_validation(args)
+    return 0
+
+
+def add_val_parser(
+    subparsers: argparse._SubParsersAction[argparse.ArgumentParser],
+) -> None:
+    """Registers the unified vision validation CLI command."""
+
+    parser = subparsers.add_parser(
+        "val", help="Validate a vision model on its benchmark dataset."
+    )
+    parser.set_defaults(_handler=_cmd_val)
+    parser.add_argument(
+        "--model",
+        required=True,
+        help="Vision model name, for example `resnet50` or `yolo11m`.",
+    )
+    parser.add_argument(
+        "--framework",
+        default=None,
+        choices=["mxq", "onnx"],
+        help="Inference framework to use. When omitted, `--model-path` suffix is used first, then `mxq`.",
+    )
+    parser.add_argument(
+        "--model-path",
+        dest="model_path",
+        default="",
+        help="Optional generic local model path for MXQ or ONNX inference.",
+    )
+    parser.add_argument(
+        "--mxq-path",
+        dest="mxq_path",
+        default="",
+        help="Optional local MXQ model path. Preserved as a compatibility alias.",
+    )
+    parser.add_argument(
+        "--onnx-path",
+        dest="onnx_path",
+        default="",
+        help="Optional local ONNX model path.",
+    )
+    parser.add_argument(
+        "--model-type",
+        default="DEFAULT",
+        help="Model variant from the YAML configuration.",
+    )
+    parser.add_argument(
+        "--core-mode",
+        default=None,
+        choices=["single", "multi", "global4", "global8"],
+        help="NPU core execution mode. Defaults to global8 on Aries and single on Regulus.",
+    )
+    parser.add_argument("--dev-no", type=int, default=0, help="NPU device number.")
+    parser.add_argument(
+        "--target-device",
+        default="aries-rb",
+        choices=["aries-rb", "regulus-ra", "regulus-rb"],
+        help="NPU board target. Determines the backend implementation.",
+    )
+    parser.add_argument(
+        "--target-cores",
+        type=parse_target_cores,
+        help="Optional semicolon-separated core list for single-core mode, for example `0:0;0:1`.",
+    )
+    parser.add_argument(
+        "--target-clusters",
+        type=parse_target_clusters,
+        help="Optional semicolon-separated cluster list for multi/global modes, for example `0;1`.",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=parse_positive_int,
+        default=1,
+        help="Positive batch size for validation.",
+    )
+    parser.add_argument(
+        "--data-path",
+        help="Path to an already organized validation dataset. If omitted, the default cache path is used.",
+    )
+    parser.add_argument(
+        "--force-organize",
+        "--force",
+        "--reload",
+        action="store_true",
+        dest="force_organize",
+        help="Rebuild the organized dataset even when the target directory already looks ready.",
+    )
+    parser.add_argument(
+        "--image-dir",
+        help=(
+            "Local archive path or download URL for dataset images. Cityscapes requires leftImg8bit_trainvaltest.zip."
+        ),
+    )
+    parser.add_argument(
+        "--xml-dir",
+        help="Local archive path or download URL for ImageNet annotations used by automatic organization.",
+    )
+    parser.add_argument(
+        "--annotation-dir",
+        help=(
+            "Local archive path or download URL for dataset annotations. Cityscapes requires gtFine_trainvaltest.zip."
+        ),
+    )
+    add_threshold_args(parser, conf_default=None, iou_default=None)
+    add_e2e_arg(parser)

mblt_vision/compile/__init__.py

@@ -0,0 +1,21 @@
+"""Vision model compilation and calibration-data preparation."""
+
+from .vision import (
+    compile_vision_model,
+    copy_calibration_subset,
+    ensure_calibration_dataset,
+    make_calibration_subset,
+    prepare_calibration_arrays,
+    resolve_quantization_values,
+    select_calibration_images,
+)
+
+__all__ = [
+    "compile_vision_model",
+    "copy_calibration_subset",
+    "ensure_calibration_dataset",
+    "make_calibration_subset",
+    "prepare_calibration_arrays",
+    "resolve_quantization_values",
+    "select_calibration_images",
+]