visionforge-studio 0.1.0__py3-none-any.whl

visionforge/__init__.py

@@ -0,0 +1,11 @@
+"""VisionForge — local-first computer-vision experimentation platform."""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _version
+
+try:
+    __version__ = _version("visionforge-studio")
+except PackageNotFoundError:  # pragma: no cover - source tree without an install
+    __version__ = "0.0.0+unknown"
+
+__all__ = ["__version__"]

visionforge/__main__.py

@@ -0,0 +1,281 @@
+"""VisionForge CLI entry point."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+from visionforge import __version__
+
+# Map ExperimentConfig.block snake_case literals to BlockRegistry class names.
+_BLOCK_ALIASES: dict[str, str] = {
+    "classification": "ClassificationBlock",
+    "cross_validation": "CrossValidationBlock",
+    "grid_search": "GridSearchBlock",
+    "random_search": "RandomSearchBlock",
+    "transfer_learning": "TransferLearningBlock",
+    "model_comparison": "ModelComparisonBlock",
+    "batch_prediction": "BatchPredictionBlock",
+    "export_onnx": "ExportONNXBlock",
+}
+
+
+def _peek_task(config_path: Path) -> str:
+    """Read just the ``task`` field from a YAML config (defaults to multiclass)."""
+    import yaml
+
+    with config_path.open(encoding="utf-8") as f:
+        raw: Any = yaml.safe_load(f)
+    if not isinstance(raw, dict):
+        return "multiclass"
+    return str(raw.get("task", "multiclass"))
+
+
+def build_task_block(config_path: Path) -> tuple[Any, Any]:
+    """Load a config and return ``(config, block)`` ready to run, dispatched by task.
+
+    The standalone tasks (detection/regression/segmentation/anomaly) go to their
+    own config loader + block; the classification family goes to
+    ``ExperimentConfig`` + the ``BlockRegistry`` block named by ``config.block``.
+    The block is ``setup()``-d but not run.
+
+    Raises:
+        ValueError: if the config's ``task`` is not recognized.
+    """
+    task = _peek_task(config_path)
+
+    if task == "detection":
+        from visionforge.blocks.detection import DetectionBlock
+        from visionforge.utils.detection_config import load_detection_config
+
+        config: Any = load_detection_config(config_path)
+        block: Any = DetectionBlock()
+    elif task == "regression":
+        from visionforge.blocks.regression import RegressionBlock
+        from visionforge.utils.regression_config import load_regression_config
+
+        config = load_regression_config(config_path)
+        block = RegressionBlock()
+    elif task == "segmentation":
+        from visionforge.blocks.segmentation import SegmentationBlock
+        from visionforge.utils.segmentation_config import load_segmentation_config
+
+        config = load_segmentation_config(config_path)
+        block = SegmentationBlock()
+    elif task == "anomaly":
+        from visionforge.blocks.anomaly import AnomalyBlock
+        from visionforge.utils.anomaly_config import load_anomaly_config
+
+        config = load_anomaly_config(config_path)
+        block = AnomalyBlock()
+    elif task in ("binary", "multiclass"):
+        import visionforge.blocks.batch_prediction  # noqa: F401
+        import visionforge.blocks.classification  # noqa: F401
+        import visionforge.blocks.cross_validation  # noqa: F401
+        import visionforge.blocks.export_onnx  # noqa: F401
+        import visionforge.blocks.grid_search  # noqa: F401
+        import visionforge.blocks.model_comparison  # noqa: F401
+        import visionforge.blocks.random_search  # noqa: F401
+        import visionforge.blocks.transfer_learning  # noqa: F401
+        from visionforge.blocks.registry import BlockRegistry
+        from visionforge.utils.config import load_config
+
+        config = load_config(config_path)
+        registry = BlockRegistry.discover()
+        block = registry[_BLOCK_ALIASES[config.block]]()
+    else:
+        raise ValueError(
+            f"Unknown task '{task}'. Expected one of: binary, multiclass, "
+            f"detection, regression, segmentation, anomaly."
+        )
+
+    block.setup(config)
+    return config, block
+
+
+def main() -> None:
+    """Run a VisionForge experiment or start the GUI from the command line."""
+    from visionforge.utils.logger import setup_logger
+
+    parser = argparse.ArgumentParser(
+        prog="visionforge",
+        description="VisionForge — Computer Vision experimentation platform.",
+    )
+    # A bug report without a version is a bug report nobody can act on.
+    parser.add_argument(
+        "--version", action="version", version=f"visionforge {__version__}"
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    run_parser = subparsers.add_parser("run", help="Run an experiment from YAML.")
+    run_parser.add_argument("config", type=Path, help="path to .yaml config")
+
+    gui_parser = subparsers.add_parser("gui", help="Start the VisionForge web GUI.")
+    gui_parser.add_argument(
+        "--host", default="127.0.0.1", help="bind host (default 127.0.0.1)"
+    )
+    gui_parser.add_argument(
+        "--port", type=int, default=8000, help="bind port (default 8000)"
+    )
+
+    doctor_parser = subparsers.add_parser(
+        "doctor", help="Check the environment and recommend the correct torch wheel."
+    )
+    doctor_parser.add_argument(
+        "--fix",
+        action="store_true",
+        help="Run the recommended install after prompting for confirmation.",
+    )
+
+    selftest_parser = subparsers.add_parser(
+        "selftest",
+        help="Train every task through the real GUI API on synthetic data (ADR-060).",
+    )
+    selftest_parser.add_argument(
+        "--tasks",
+        default="all",
+        help="comma-separated task keys, or 'all' (default): "
+        "classification,regression,segmentation,anomaly,detection,custom",
+    )
+    selftest_parser.add_argument(
+        "--strategies",
+        default="all",
+        help="comma-separated strategies, or 'all' (default): simple,cv,sweep,replicates",
+    )
+    selftest_parser.add_argument(
+        "--quick",
+        action="store_true",
+        help="only the 'simple' strategy — a fast is-my-install-sane check",
+    )
+    selftest_parser.add_argument(
+        "--workdir",
+        type=Path,
+        default=None,
+        help="where datasets and outputs land (default: a temp dir, kept on failure)",
+    )
+    selftest_parser.add_argument(
+        "--json", type=Path, default=None, help="also write the outcomes as JSON here"
+    )
+
+    newtask_parser = subparsers.add_parser(
+        "new-task",
+        help="Scaffold a custom task under user_tasks/ (ADR-058).",
+    )
+    newtask_parser.add_argument(
+        "key", help="task key: lowercase letters/digits/underscores"
+    )
+    newtask_parser.add_argument(
+        "--package",
+        action="store_true",
+        help="create user_tasks/<key>/task.py (room for assets) instead of a flat file",
+    )
+    newtask_parser.add_argument(
+        "--force", action="store_true", help="overwrite an existing file"
+    )
+
+    args = parser.parse_args()
+
+    setup_logger()
+
+    if args.command == "selftest":
+        import sys as _sys
+        import tempfile
+
+        from visionforge.utils.logger import logger
+        from visionforge.utils.selftest import (
+            STRATEGIES,
+            TASKS,
+            format_report,
+            run_selftest,
+        )
+
+        tasks = TASKS if args.tasks == "all" else tuple(args.tasks.split(","))
+        if args.quick:
+            strategies: tuple[str, ...] = ("simple",)
+        else:
+            strategies = (
+                STRATEGIES
+                if args.strategies == "all"
+                else tuple(args.strategies.split(","))
+            )
+        unknown = [t for t in tasks if t not in TASKS] + [
+            s for s in strategies if s not in STRATEGIES
+        ]
+        if unknown:
+            logger.error("Unknown task/strategy: {}", ", ".join(unknown))
+            _sys.exit(2)
+
+        workdir = args.workdir or Path(tempfile.mkdtemp(prefix="visionforge_selftest_"))
+        logger.info("Self-test workdir: {}", workdir)
+        outcomes = run_selftest(workdir, tasks=tasks, strategies=strategies)
+        report = format_report(outcomes)
+        print(f"\n{report}\n")  # noqa: T201 — the table IS the command's output
+
+        if args.json is not None:
+            args.json.parent.mkdir(parents=True, exist_ok=True)
+            args.json.write_text(
+                json.dumps([vars(o) for o in outcomes], indent=2), encoding="utf-8"
+            )
+            logger.info("Wrote {}", args.json)
+
+        _sys.exit(0 if all(o.status == "passed" for o in outcomes) else 1)
+
+    if args.command == "new-task":
+        import sys as _sys
+
+        from visionforge.tasks.scaffold import scaffold_task
+        from visionforge.utils.logger import logger
+
+        try:
+            target = scaffold_task(args.key, package=args.package, force=args.force)
+        except (ValueError, FileExistsError) as exc:
+            logger.error("{}", exc)
+            _sys.exit(1)
+        logger.success("Task template created: {}", target)
+        logger.info(
+            "Next: edit the TODOs in {}, then `visionforge gui` — the "
+            "'{}' tab appears automatically. Guide: user_tasks/README.md",
+            target,
+            args.key,
+        )
+        return
+
+    if args.command == "doctor":
+        import sys as _sys
+
+        from visionforge.utils.doctor import _default_confirm, run_doctor
+
+        _sys.exit(run_doctor(fix=args.fix, confirm_fn=_default_confirm))
+
+    if args.command == "gui":
+        from visionforge.gui.server import start_server
+
+        start_server(host=args.host, port=args.port)
+        return
+
+    if args.command == "run":
+        from visionforge.utils.logger import logger
+
+        logger.info("Loading config: {}", args.config)
+        config, block = build_task_block(args.config)
+
+        logger.info(
+            "Experiment: {} | task: {} | block: {}",
+            config.name,
+            config.task,
+            type(block).__name__,
+        )
+        logger.info("Running {}...", type(block).__name__)
+        block.run()
+
+        report = block.report()
+        logger.success("Done. Report: {}", report)
+
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

visionforge/blocks/__init__.py

@@ -0,0 +1,23 @@
+from visionforge.blocks.base import ExperimentBlock
+from visionforge.blocks.batch_prediction import BatchPredictionBlock
+from visionforge.blocks.classification import ClassificationBlock
+from visionforge.blocks.cross_validation import CrossValidationBlock
+from visionforge.blocks.export_onnx import ExportONNXBlock
+from visionforge.blocks.grid_search import GridSearchBlock
+from visionforge.blocks.model_comparison import ModelComparisonBlock
+from visionforge.blocks.random_search import RandomSearchBlock
+from visionforge.blocks.registry import BlockRegistry
+from visionforge.blocks.transfer_learning import TransferLearningBlock
+
+__all__ = [
+    "ExperimentBlock",
+    "BatchPredictionBlock",
+    "ClassificationBlock",
+    "CrossValidationBlock",
+    "ExportONNXBlock",
+    "GridSearchBlock",
+    "ModelComparisonBlock",
+    "RandomSearchBlock",
+    "BlockRegistry",
+    "TransferLearningBlock",
+]

visionforge/blocks/_search_utils.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import csv
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+import torch
+import yaml
+from loguru import logger
+
+from visionforge.blocks.classification import ClassificationBlock
+from visionforge.utils.config import ExperimentConfig
+
+
+def set_nested(d: dict[str, Any], dot_key: str, value: Any) -> None:
+    """Set a value in a nested dict using a dot-notation key.
+
+    Raises:
+        ValueError: if any intermediate key does not exist in the dict.
+    """
+    keys = dot_key.split(".")
+    node = d
+    for k in keys[:-1]:
+        if k not in node or not isinstance(node[k], dict):
+            raise ValueError(
+                f"Unknown hyperparameter path: '{dot_key}' (failed at '{k}')"
+            )
+        node = node[k]
+    if keys[-1] not in node:
+        raise ValueError(
+            f"Unknown hyperparameter path: '{dot_key}' (failed at '{keys[-1]}')"
+        )
+    node[keys[-1]] = value
+
+
+def validate_dot_keys(base_raw: dict[str, Any], search_space: dict[str, Any]) -> None:
+    """Raise ValueError for any dot-key that doesn't exist in the base config dict."""
+    for dot_key in search_space:
+        probe = dict(base_raw.items())
+        set_nested(probe, dot_key, None)
+
+
+def best_trial(trials: list[dict[str, Any]]) -> dict[str, Any] | None:
+    """Return the successful trial with the lowest best_val_loss, or None."""
+    successful = [t for t in trials if t["status"] == "success"]
+    if not successful:
+        return None
+    return min(
+        successful,
+        key=lambda t: (t["best_val_loss"] is None, t["best_val_loss"] or float("inf")),
+    )
+
+
+def write_trials_csv(csv_path: Path, trials: list[dict[str, Any]]) -> None:
+    """Write a list of trial records to a CSV file."""
+    if not trials:
+        return
+    fieldnames = list(trials[0].keys())
+    with csv_path.open("w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(trials)
+
+
+def write_best_config_yaml(
+    yaml_path: Path,
+    base_config: ExperimentConfig,
+    best_idx: int,
+    best_overrides: dict[str, Any],
+) -> None:
+    """Write the best trial's config as a YAML file."""
+    best_raw: dict[str, Any] = base_config.model_dump(mode="json")
+    best_raw["training"]["seed"] = base_config.training.seed + best_idx
+    for dot_key, val in best_overrides.items():
+        set_nested(best_raw, dot_key, val)
+    with yaml_path.open("w", encoding="utf-8") as f:
+        yaml.dump(best_raw, f, default_flow_style=False, allow_unicode=True)
+
+
+def make_trial_progress_wrapper(
+    progress_callback: Callable[[dict[str, Any]], None] | None,
+    trial_idx: int,
+    total_trials: int,
+) -> Callable[[dict[str, Any]], None] | None:
+    """Wrap a sweep-level callback so a trial's inner Trainer events carry trial
+    context and don't masquerade as the terminal 'end' event.
+
+    The GUI closes the SSE stream on a bare 'end'; since each trial's Trainer
+    emits its own 'end', the inner one is rewritten to 'trial_end' to keep the
+    stream open across trials. The inner 'start' is dropped because the sweep
+    emits its own 'trial_start'. Returns None when there is nothing to forward.
+    """
+    if progress_callback is None:
+        return None
+
+    def _wrapped(event: dict[str, Any]) -> None:
+        kind = event.get("event")
+        if kind == "start":
+            return
+        annotated = {**event, "trial_index": trial_idx, "total_trials": total_trials}
+        if kind == "end":
+            annotated["event"] = "trial_end"
+        progress_callback(annotated)
+
+    return _wrapped
+
+
+def run_trial(
+    base_config: ExperimentConfig,
+    trial_idx: int,
+    total_trials: int,
+    trial_seed: int,
+    trial_overrides: dict[str, Any],
+    trial_record: dict[str, Any],
+    progress_callback: Callable[[dict[str, Any]], None] | None = None,
+) -> None:
+    """Execute one trial and mutate trial_record in place with results or error.
+
+    Always calls torch.cuda.empty_cache() after the trial regardless of outcome.
+
+    Args:
+        base_config: the base experiment config to derive the trial config from.
+        trial_idx: 0-based trial index (used for logging).
+        total_trials: total number of trials in the sweep (used for logging).
+        trial_seed: seed to set for this trial's training config.
+        trial_overrides: dot-notation key→value pairs to apply on top of base_config.
+        trial_record: mutable dict that receives status, metrics, and error fields.
+        progress_callback: injected onto the inner block so the trial's Trainer
+            streams epoch progress to the GUI; None disables streaming.
+    """
+    base_raw: dict[str, Any] = base_config.model_dump(mode="json")
+    base_raw["training"]["seed"] = trial_seed
+    for dot_key, val in trial_overrides.items():
+        set_nested(base_raw, dot_key, val)
+
+    block = ClassificationBlock()
+    try:
+        trial_config = ExperimentConfig.model_validate(base_raw)
+        block.setup(trial_config)
+        block._progress_callback = progress_callback
+        block.run()
+        report = block.report()
+
+        trial_record["status"] = "success"
+        trial_record["best_val_loss"] = report.get("train", {}).get("best_val_loss")
+        trial_record["test_accuracy"] = report.get("eval", {}).get("accuracy")
+        trial_record["test_f1"] = report.get("eval", {}).get("f1")
+
+        logger.info(
+            "Trial {}/{} succeeded: val_loss={} accuracy={}",
+            trial_idx + 1,
+            total_trials,
+            trial_record["best_val_loss"],
+            trial_record["test_accuracy"],
+        )
+
+    except Exception as exc:  # noqa: BLE001
+        trial_record["error"] = str(exc)
+        logger.warning("Trial {}/{} failed: {}", trial_idx + 1, total_trials, exc)
+
+    finally:
+        del block
+        torch.cuda.empty_cache()
+
+
+__all__ = [
+    "set_nested",
+    "validate_dot_keys",
+    "best_trial",
+    "write_trials_csv",
+    "write_best_config_yaml",
+    "make_trial_progress_wrapper",
+    "run_trial",
+]

visionforge/blocks/anomaly.py

@@ -0,0 +1,118 @@
+"""Anomaly-detection experiment block (Phase 9).
+
+Mirrors the `ExperimentBlock` setup/run/report contract but over the standalone
+`AnomalyConfig` tree rather than `ExperimentConfig` (see ADR-038). It is not an
+`ExperimentBlock` subclass — that ABC is bound to the classification config — so
+it is dispatched directly by the anomaly run endpoint, not the block registry.
+Same rationale as ADR-033 (detection) / ADR-036 (regression) / ADR-037 (segmentation).
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+import torch
+
+from visionforge.core.anomaly_data import AnomalyDataModule
+from visionforge.core.anomaly_trainer import AnomalyTrainer, AnomalyTrainResult
+from visionforge.core.plotter import MetricsPlotter
+from visionforge.models.anomaly_factory import AnomalyModelFactory
+from visionforge.utils.anomaly_config import AnomalyConfig
+
+
+class AnomalyBlock:
+    """End-to-end anomaly-detection training block over `AnomalyConfig`."""
+
+    def setup(self, config: AnomalyConfig) -> None:
+        self._config = config
+        self._train_result: AnomalyTrainResult | None = None
+        self._test_metrics: tuple[float, float, float] | None = None
+        # Injected by the GUI layer to stream live progress via SSE.
+        self._progress_callback: Callable[[dict[str, Any]], None] | None = None
+
+    def run(self) -> None:
+        model = AnomalyModelFactory.create(self._config.model)
+        data = AnomalyDataModule(self._config)
+        trainer = AnomalyTrainer(self._config)
+
+        self._train_result = trainer.fit(
+            model, data, progress_callback=self._progress_callback
+        )
+
+        # Reload the best checkpoint before final scoring.
+        state_dict = torch.load(
+            str(self._train_result.model_path), map_location="cpu", weights_only=True
+        )
+        model.load_state_dict(state_dict)  # type: ignore[arg-type]
+
+        self._test_metrics = trainer.evaluate(
+            model, data.train_loader(), data.test_loader()
+        )
+
+        run_dir = self._train_result.model_path.parent
+        graphics = self._render_plots(run_dir)
+        self._update_run_json(run_dir, graphics)
+
+    def report(self) -> dict[str, Any]:
+        """Return a summary of the run for logging and GUI display."""
+        result: dict[str, Any] = {}
+        if self._train_result is not None:
+            r = self._train_result
+            result["train"] = {
+                "best_epoch": r.best_epoch,
+                "best_auroc": r.best_auroc,
+                "total_epochs": r.total_epochs,
+                "device_used": r.device_used,
+                "run_dir": str(r.model_path.parent),
+            }
+        if self._test_metrics is not None:
+            auroc, threshold, image_f1 = self._test_metrics
+            result["test"] = {
+                "auroc": auroc,
+                "threshold": threshold,
+                "image_f1": image_f1,
+            }
+        return result
+
+    # ── private ───────────────────────────────────────────────────────────────
+
+    def _render_plots(self, run_dir: Path) -> list[Path]:
+        """Render the AUROC-over-epochs curve (reuses the generic plotter curve)."""
+        assert self._train_result is not None
+        auroc_path = run_dir / "auroc.png"
+        epochs = [r.epoch for r in self._train_result.history]
+        values = [r.val_auroc for r in self._train_result.history]
+        MetricsPlotter.metric_curve(
+            epochs,
+            values,
+            auroc_path,
+            label="AUROC",
+            ylabel="Image AUROC",
+            title="Image-level AUROC over epochs",
+        )
+        return [auroc_path]
+
+    def _update_run_json(self, run_dir: Path, graphics: list[Path]) -> None:
+        """Rewrite run.json with test metrics and artifact paths."""
+        run_json_path = run_dir / "run.json"
+        if not run_json_path.exists():
+            return
+
+        data: dict[str, Any] = json.loads(run_json_path.read_text(encoding="utf-8"))
+        if self._test_metrics is not None:
+            auroc, threshold, image_f1 = self._test_metrics
+            data["metrics"].update(
+                {
+                    "test_auroc": auroc,
+                    "test_threshold": threshold,
+                    "test_image_f1": image_f1,
+                }
+            )
+        data["artifacts"]["graphics"] = [str(p) for p in graphics]
+        run_json_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+
+
+__all__ = ["AnomalyBlock"]

visionforge/blocks/anomaly_runner.py

@@ -0,0 +1,55 @@
+"""TaskRunner adapter for anomaly detection (ADR-041 / ADR-044 / ADR-045)."""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from visionforge.blocks.anomaly import AnomalyBlock
+from visionforge.core.task_runner import RunResult
+from visionforge.utils.anomaly_config import AnomalyConfig
+
+# Anomaly's test metrics; auroc (higher is better) is the ranking default.
+_METRIC_KEYS = ("auroc", "image_f1")
+
+
+class AnomalyRunner:
+    """Drives AnomalyBlock for one training run, exposing the uniform handle.
+
+    GPU cleanup is the caller's responsibility (the generic comparison/sweep
+    runners flush between trials).
+    """
+
+    config_type = AnomalyConfig
+
+    def run(self, cfg: Any) -> RunResult:
+        """Run a single anomaly trial and return a uniform RunResult."""
+        block = AnomalyBlock()
+        try:
+            block.setup(cfg)
+            t0 = time.monotonic()
+            block.run()
+            elapsed = time.monotonic() - t0
+
+            test: dict[str, Any] = block.report().get("test", {})
+            metrics = {
+                k: float(test[k]) for k in _METRIC_KEYS if test.get(k) is not None
+            }
+            return RunResult(
+                metrics=metrics, status="success", training_time_s=elapsed, error=""
+            )
+        except Exception as exc:  # noqa: BLE001
+            return RunResult(
+                metrics={}, status="failed", training_time_s=None, error=str(exc)
+            )
+
+    def metrics(self, result: RunResult) -> dict[str, float]:
+        """Return result.metrics unchanged."""
+        return result.metrics
+
+    def primary_metric(self) -> str:
+        """Return 'auroc' — the anomaly default ranking metric."""
+        return "auroc"
+
+
+__all__ = ["AnomalyRunner"]

visionforge/blocks/base.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from visionforge.utils.config import ExperimentConfig
+
+
+class ExperimentBlock(ABC):
+    """Base class for all experiment strategies."""
+
+    @abstractmethod
+    def setup(self, config: ExperimentConfig) -> None:
+        """Receive the experiment config and prepare internal state."""
+
+    @abstractmethod
+    def run(self) -> None:
+        """Execute the experiment strategy."""
+
+    @abstractmethod
+    def report(self) -> dict[str, Any]:
+        """Return a dict of results for logging and GUI display."""
+
+
+__all__ = ["ExperimentBlock"]