Justin-Williams-CV-Benchmarking 1.0.0__py3-none-any.whl

justin_williams_cv_benchmarking/__init__.py

@@ -0,0 +1,21 @@
+"""Justin_Williams_CV_Benchmarking.
+
+Benchmark six classical and neural image classifiers on any labelled image
+dataset with a single function call::
+
+    from justin_williams_cv_benchmarking import benchmark_image_classification
+
+    results = benchmark_image_classification(
+        dataset="./animals",
+        dataset_type="folder",
+        target_labels=["cat", "dog", "horse"],
+        color_mode="rgb",
+    )
+"""
+
+__version__ = "1.0.0"
+
+from .benchmark import benchmark_image_classification  # noqa: E402
+from .data_loader import DatasetError  # noqa: E402
+
+__all__ = ["benchmark_image_classification", "DatasetError", "__version__"]

justin_williams_cv_benchmarking/benchmark.py

@@ -0,0 +1,416 @@
+"""The public entry point: :func:`benchmark_image_classification`.
+
+One call loads any of the four supported dataset organisations, standardises
+it, creates a single stratified split, trains all six required models on that
+one split, evaluates them with identical metrics, writes every required
+artefact to ``benchmark_results/`` and returns the whole comparison.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import random
+import time
+from typing import Any, Dict
+
+import numpy as np
+
+from . import __version__
+from .classical_models import build_classical_models, describe
+from .config import (
+    CLASSIFICATION_REPORT_DIR,
+    CNN_BATCH_SIZE,
+    CNN_EARLY_STOPPING_PATIENCE,
+    CNN_MAX_EPOCHS,
+    CONFUSION_MATRIX_DIR,
+    IMAGE_SIZE,
+    MODEL_FILE_STEMS,
+    MODEL_ORDER,
+    OUTPUT_DIR,
+    PACKAGE_NAME,
+    RANDOM_SEED,
+    TEST_SIZE,
+)
+from .data_loader import DatasetError, load_dataset
+from .evaluation import (
+    build_summary,
+    best_model_name,
+    classification_report_frame,
+    compute_metrics,
+    confusion_matrix_array,
+    prediction_examples,
+)
+from .neural_models import build_cnn_model, build_mlp_model
+from .preprocessing import distribution, prepare_dataset, stratified_split
+from .visualization import (
+    ensure_directory,
+    plot_class_distribution,
+    plot_confusion_matrix,
+    plot_model_comparison,
+    plot_prediction_examples,
+)
+
+__all__ = ["benchmark_image_classification", "DatasetError"]
+
+# Models fed the flattened feature vectors rather than the image tensor.
+_FLAT_INPUT_MODELS = ("Logistic Regression", "Decision Tree", "Random Forest", "SVM", "Neural Network")
+
+
+def _seed_everything(seed: int = RANDOM_SEED) -> None:
+    random.seed(seed)
+    np.random.seed(seed)
+    try:
+        import torch
+
+        torch.manual_seed(seed)
+    except Exception:  # pragma: no cover - torch is a hard dependency in practice
+        pass
+
+
+def _json_safe(value: Any) -> Any:
+    """Recursively convert NumPy scalars/arrays into JSON-serialisable values."""
+    if isinstance(value, dict):
+        return {str(key): _json_safe(item) for key, item in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_json_safe(item) for item in value]
+    if isinstance(value, np.ndarray):
+        return value.tolist()
+    if isinstance(value, (np.integer,)):
+        return int(value)
+    if isinstance(value, (np.floating,)):
+        number = float(value)
+        return None if np.isnan(number) else number
+    if isinstance(value, (np.bool_,)):
+        return bool(value)
+    if isinstance(value, float) and np.isnan(value):
+        return None
+    return value
+
+
+def _build_models() -> Dict[str, object]:
+    """All six models, in reporting order."""
+    models = build_classical_models()
+    models["Neural Network"] = build_mlp_model()
+    models["Simple CNN"] = build_cnn_model()
+    return {name: models[name] for name in MODEL_ORDER}
+
+
+def _write_json(path: str, payload: Any) -> None:
+    with open(path, "w", encoding="utf-8") as handle:
+        json.dump(_json_safe(payload), handle, indent=2, sort_keys=False)
+        handle.write("\n")
+
+
+def benchmark_image_classification(
+    dataset,
+    dataset_type: str,
+    target_labels,
+    color_mode: str,
+) -> dict:
+    """Load an image dataset, train all required classifiers,
+    and return a complete benchmark comparison.
+
+    Parameters
+    ----------
+    dataset:
+        Dataset root directory, CSV/JSON/JSONL manifest path, Pandas DataFrame,
+        or NumPy image array.
+    dataset_type:
+        One of ``"folder"``, ``"csv"``, ``"json"`` or ``"array"``.
+    target_labels:
+        Class-folder names, manifest label-field name, DataFrame label column,
+        or a label vector.
+    color_mode:
+        ``"grayscale"`` for one channel or ``"rgb"`` for three.
+
+    Returns
+    -------
+    dict
+        Benchmark table, best model, dataset summary, split information,
+        per-model results, confusion matrices, class-level reports and
+        warnings.  The same artefacts are written to ``benchmark_results/``.
+    """
+    _seed_everything()
+
+    # -- 1. load and validate ------------------------------------------------
+    loaded = load_dataset(dataset, dataset_type, target_labels)
+
+    # -- 2. standardise ------------------------------------------------------
+    prepared = prepare_dataset(loaded, color_mode)
+    warnings = list(prepared.warnings)
+    class_names = prepared.class_names
+    n_classes = len(class_names)
+
+    # -- 3. one stratified split, reused by every model ----------------------
+    train_index, test_index = stratified_split(prepared.y)
+    y_train = prepared.y[train_index]
+    y_test = prepared.y[test_index]
+
+    features_train = prepared.features[train_index]
+    features_test = prepared.features[test_index]
+    images_train = prepared.images[train_index]
+    images_test = prepared.images[test_index]
+
+    # -- 4. train and evaluate every model on that one split -----------------
+    model_results: Dict[str, dict] = {}
+    confusion_matrices: Dict[str, np.ndarray] = {}
+    classification_reports: Dict[str, Any] = {}
+
+    for name, model in _build_models().items():
+        uses_flat = name in _FLAT_INPUT_MODELS
+        x_train = features_train if uses_flat else images_train
+        x_test = features_test if uses_flat else images_test
+
+        record: Dict[str, Any] = {
+            "status": "success",
+            "error": None,
+            "input_representation": "flattened features" if uses_flat else "image tensor",
+            "training_samples": int(len(train_index)),
+            "testing_samples": int(len(test_index)),
+        }
+
+        try:
+            started = time.perf_counter()
+            model.fit(x_train, y_train)
+            record["training_time_seconds"] = float(time.perf_counter() - started)
+
+            started = time.perf_counter()
+            predictions = np.asarray(model.predict(x_test))
+            inference_seconds = float(time.perf_counter() - started)
+
+            if predictions.shape[0] != y_test.shape[0]:
+                raise RuntimeError(
+                    "model returned %d predictions for %d test images"
+                    % (predictions.shape[0], y_test.shape[0])
+                )
+
+            record["inference_time_seconds_total"] = inference_seconds
+            record["inference_time_ms_per_image"] = (
+                inference_seconds / max(len(y_test), 1) * 1000.0
+            )
+            record["metrics"] = compute_metrics(y_test, predictions, n_classes)
+            record["predictions"] = predictions
+            record["configuration"] = describe(model)
+
+            confusion_matrices[name] = confusion_matrix_array(y_test, predictions, n_classes)
+            classification_reports[name] = classification_report_frame(
+                y_test, predictions, class_names
+            )
+            record["prediction_examples"] = prediction_examples(
+                y_test, predictions, test_index, class_names
+            )
+            if name == "Simple CNN":
+                record["epochs_run"] = int(getattr(model, "epochs_run_", 0))
+                record["best_epoch"] = int(getattr(model, "best_epoch_", 0))
+                record["stopped_early"] = bool(getattr(model, "stopped_early_", False))
+
+        except Exception as exc:  # noqa: BLE001 - a failed model is reported, not hidden
+            record["status"] = "failed"
+            record["error"] = "%s: %s" % (type(exc).__name__, exc)
+            record.setdefault("training_time_seconds", float("nan"))
+            record["inference_time_ms_per_image"] = float("nan")
+            record["metrics"] = {}
+            record["predictions"] = np.array([], dtype=np.int64)
+            record["configuration"] = describe(model)
+            warnings.append("Model %r failed and is reported without metrics: %s" % (name, record["error"]))
+
+        model_results[name] = record
+
+    # -- 5. rank -------------------------------------------------------------
+    summary = build_summary(model_results)
+    best_model = best_model_name(summary)
+    if not best_model:
+        warnings.append("Every model failed; there is no best model to report.")
+
+    # -- 6. distributions ----------------------------------------------------
+    split_information = {
+        "training_samples": int(len(train_index)),
+        "testing_samples": int(len(test_index)),
+        "test_size": TEST_SIZE,
+        "random_seed": RANDOM_SEED,
+        "stratified": True,
+        "shared_by_all_models": True,
+        "full_class_distribution": distribution(prepared.y, class_names),
+        "training_class_distribution": distribution(y_train, class_names),
+        "testing_class_distribution": distribution(y_test, class_names),
+    }
+
+    dataset_information = {
+        "dataset_type": loaded.dataset_type,
+        "number_of_images": int(prepared.images.shape[0]),
+        "number_of_classes": n_classes,
+        "class_names": list(class_names),
+        "color_mode": prepared.color_mode,
+        "image_shape": list(prepared.image_shape),
+        "flattened_features": int(prepared.features.shape[1]),
+        "skipped_samples": len(prepared.skipped),
+        "skipped_details": prepared.skipped,
+    }
+
+    # -- 7. write every required artefact ------------------------------------
+    output_directory = _write_outputs(
+        summary=summary,
+        model_results=model_results,
+        confusion_matrices=confusion_matrices,
+        classification_reports=classification_reports,
+        class_names=class_names,
+        dataset_information=dataset_information,
+        split_information=split_information,
+        best_model=best_model,
+        warnings=warnings,
+        images_test=images_test,
+        y_test=y_test,
+    )
+
+    return {
+        "package_information": {"name": PACKAGE_NAME, "version": __version__},
+        "summary": summary,
+        "best_model": best_model,
+        "dataset_information": dataset_information,
+        "split_information": split_information,
+        "model_results": model_results,
+        "confusion_matrices": confusion_matrices,
+        "classification_reports": classification_reports,
+        "warnings": warnings,
+        "output_directory": output_directory,
+        "class_names": list(class_names),
+        "test_indices": test_index,
+        "train_indices": train_index,
+    }
+
+
+def _write_outputs(
+    summary,
+    model_results,
+    confusion_matrices,
+    classification_reports,
+    class_names,
+    dataset_information,
+    split_information,
+    best_model,
+    warnings,
+    images_test,
+    y_test,
+) -> str:
+    """Create ``benchmark_results/`` and populate it with every required file."""
+    import sklearn
+
+    root = ensure_directory(os.path.abspath(OUTPUT_DIR))
+    matrices_dir = ensure_directory(os.path.join(root, CONFUSION_MATRIX_DIR))
+    reports_dir = ensure_directory(os.path.join(root, CLASSIFICATION_REPORT_DIR))
+
+    summary.to_csv(os.path.join(root, "benchmark_summary.csv"), index=False)
+
+    metrics_payload = {
+        "package_information": {"name": PACKAGE_NAME, "version": __version__},
+        "best_model": best_model,
+        "dataset_information": dataset_information,
+        "split_information": split_information,
+        "summary": summary.to_dict(orient="records"),
+        "model_results": {
+            name: {
+                key: value
+                for key, value in record.items()
+                if key not in ("predictions", "configuration")
+            }
+            for name, record in model_results.items()
+        },
+        "confusion_matrices": {
+            name: matrix.tolist() for name, matrix in confusion_matrices.items()
+        },
+        "class_names": list(class_names),
+        "warnings": warnings,
+    }
+    _write_json(os.path.join(root, "benchmark_metrics.json"), metrics_payload)
+
+    try:
+        torch_version = __import__("torch").__version__
+    except Exception:  # pragma: no cover
+        torch_version = "unavailable"
+
+    run_configuration = {
+        "package_information": {"name": PACKAGE_NAME, "version": __version__},
+        "internal_constants": {
+            "image_size": [IMAGE_SIZE, IMAGE_SIZE],
+            "aspect_ratio_preserved": True,
+            "padding": "centred zero padding",
+            "pixel_range": [0, 1],
+            "random_seed": RANDOM_SEED,
+            "test_size": TEST_SIZE,
+            "stratified": True,
+            "output_directory": OUTPUT_DIR,
+            "cnn_max_epochs": CNN_MAX_EPOCHS,
+            "cnn_batch_size": CNN_BATCH_SIZE,
+            "cnn_early_stopping_patience": CNN_EARLY_STOPPING_PATIENCE,
+            "data_augmentation": False,
+            "pretrained_weights": False,
+        },
+        "dataset_information": dataset_information,
+        "split_information": split_information,
+        "model_configuration": {
+            name: record.get("configuration", {}) for name, record in model_results.items()
+        },
+        "model_status": {
+            name: {"status": record.get("status"), "error": record.get("error")}
+            for name, record in model_results.items()
+        },
+        "environment": {
+            "python": platform.python_version(),
+            "platform": platform.platform(),
+            "numpy": np.__version__,
+            "scikit_learn": sklearn.__version__,
+            "torch": torch_version,
+        },
+    }
+    _write_json(os.path.join(root, "run_configuration.json"), run_configuration)
+
+    plot_class_distribution(
+        {
+            "Full dataset": split_information["full_class_distribution"],
+            "Training": split_information["training_class_distribution"],
+            "Testing": split_information["testing_class_distribution"],
+        },
+        os.path.join(root, "class_distribution.png"),
+    )
+    plot_model_comparison(summary, os.path.join(root, "model_comparison.png"))
+
+    for name in MODEL_ORDER:
+        stem = MODEL_FILE_STEMS[name]
+        matrix = confusion_matrices.get(name)
+        if matrix is None:
+            matrix = np.zeros((len(class_names), len(class_names)), dtype=int)
+        plot_confusion_matrix(
+            matrix, class_names, name, os.path.join(matrices_dir, "%s.png" % stem)
+        )
+
+        report = classification_reports.get(name)
+        report_path = os.path.join(reports_dir, "%s.csv" % stem)
+        if report is None:
+            import pandas as pd
+
+            pd.DataFrame(
+                {
+                    "class": list(class_names),
+                    "precision": [float("nan")] * len(class_names),
+                    "recall": [float("nan")] * len(class_names),
+                    "f1-score": [float("nan")] * len(class_names),
+                    "support": [float("nan")] * len(class_names),
+                    "status": ["model failed"] * len(class_names),
+                }
+            ).to_csv(report_path, index=False)
+        else:
+            report.to_csv(report_path, index=False)
+
+    if best_model and model_results.get(best_model, {}).get("status") == "success":
+        plot_prediction_examples(
+            images_test,
+            y_test,
+            model_results[best_model]["predictions"],
+            class_names,
+            best_model,
+            os.path.join(root, "prediction_examples.png"),
+        )
+
+    return root

justin_williams_cv_benchmarking/classical_models.py

@@ -0,0 +1,58 @@
+"""The four classical Scikit-learn classifiers.
+
+Logistic Regression and the SVM are wrapped in a ``Pipeline`` whose
+``StandardScaler`` is fitted inside ``fit`` and therefore only ever sees
+training data.  Tree-based models are scale invariant and are used bare.
+"""
+
+from __future__ import annotations
+
+from typing import Dict
+
+from .config import RANDOM_SEED
+
+
+def build_classical_models() -> Dict[str, object]:
+    """Return {display name: unfitted estimator} for the classical models."""
+    from sklearn.ensemble import RandomForestClassifier
+    from sklearn.linear_model import LogisticRegression
+    from sklearn.pipeline import Pipeline
+    from sklearn.preprocessing import StandardScaler
+    from sklearn.svm import SVC
+    from sklearn.tree import DecisionTreeClassifier
+
+    return {
+        "Logistic Regression": Pipeline(
+            [
+                ("scaler", StandardScaler()),
+                ("classifier", LogisticRegression(max_iter=1000, random_state=RANDOM_SEED)),
+            ]
+        ),
+        "Decision Tree": DecisionTreeClassifier(random_state=RANDOM_SEED),
+        "Random Forest": RandomForestClassifier(
+            n_estimators=200, random_state=RANDOM_SEED, n_jobs=-1
+        ),
+        "SVM": Pipeline(
+            [
+                ("scaler", StandardScaler()),
+                ("classifier", SVC(kernel="rbf", random_state=RANDOM_SEED)),
+            ]
+        ),
+    }
+
+
+def describe(estimator) -> dict:
+    """Flat, JSON-safe view of an estimator's effective parameters."""
+    try:
+        params = estimator.get_params(deep=True)
+    except AttributeError:
+        return {"repr": repr(estimator)}
+
+    described = {}
+    for key, value in sorted(params.items()):
+        if isinstance(value, (str, int, float, bool)) or value is None:
+            described[key] = value
+        else:
+            described[key] = repr(value)
+    described["estimator"] = type(estimator).__name__
+    return described

justin_williams_cv_benchmarking/config.py

@@ -0,0 +1,56 @@
+"""Internal, non-configurable constants for the benchmark.
+
+The assignment requires a four-input public API, so image size, random seed,
+training epochs and output location are fixed here rather than exposed as
+function parameters.  Every value in this module is copied verbatim into
+``benchmark_results/run_configuration.json`` on each run.
+"""
+
+PACKAGE_NAME = "Justin_Williams_CV_Benchmarking"
+IMPORT_NAME = "justin_williams_cv_benchmarking"
+
+# --- image standardisation -------------------------------------------------
+IMAGE_SIZE = 64                 # every image becomes IMAGE_SIZE x IMAGE_SIZE
+PAD_COLOR = 0                   # aspect ratio is preserved by zero padding
+PIXEL_SCALE = 255.0             # normalise raw 8-bit pixels into [0, 1]
+
+# --- reproducibility -------------------------------------------------------
+RANDOM_SEED = 42
+
+# --- stratified split ------------------------------------------------------
+TEST_SIZE = 0.20                # 80% train / 20% test, stratified by label
+CNN_VALIDATION_SIZE = 0.10      # carved out of the TRAINING portion only
+
+# --- neural network training ----------------------------------------------
+CNN_MAX_EPOCHS = 20
+CNN_BATCH_SIZE = 32
+CNN_EARLY_STOPPING_PATIENCE = 3
+CNN_LEARNING_RATE = 1e-3
+CNN_DROPOUT = 0.30
+
+MLP_HIDDEN_LAYERS = (128, 64)
+MLP_MAX_ITER = 100
+
+# --- dataset validation ----------------------------------------------------
+MIN_IMAGES_PER_CLASS = 5        # recommended minimum; below this we warn
+MIN_CLASSES = 2                 # below this we raise
+SUPPORTED_EXTENSIONS = (".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff")
+IMAGE_PATH_COLUMN = "image_path"
+
+# --- output ----------------------------------------------------------------
+OUTPUT_DIR = "benchmark_results"
+CONFUSION_MATRIX_DIR = "confusion_matrices"
+CLASSIFICATION_REPORT_DIR = "classification_reports"
+
+# --- model registry --------------------------------------------------------
+# Display name -> output file stem.  Order is the reporting order used before
+# the summary table is sorted by Macro F1-score.
+MODEL_FILE_STEMS = {
+    "Logistic Regression": "logistic_regression",
+    "Decision Tree": "decision_tree",
+    "Random Forest": "random_forest",
+    "SVM": "svm",
+    "Neural Network": "neural_network",
+    "Simple CNN": "simple_cnn",
+}
+MODEL_ORDER = tuple(MODEL_FILE_STEMS)