oversampleqa 0.5.1__py3-none-any.whl

oversampleqa/__init__.py

@@ -0,0 +1,210 @@
+"""
+oversampleqa: A diagnostic toolkit for validating oversampling methods.
+
+This package implements validation methods for synthetic data generated by
+oversampling techniques like SMOTE, ADASYN, and their variants.
+"""
+
+__version__ = "0.5.1"
+__author__ = "Diogo Ribeiro"
+__email__ = "dfr@esmad.ipp.pt"
+__orcid__ = "0009-0001-2022-7072"
+
+from .advanced_benchmark import (
+    DatasetRepository,
+    StatisticalBenchmark,
+    create_benchmark_report,
+    format_statistical_summary,
+)
+from .benchmark import (
+    compute_ranking,
+    export_benchmark_results,
+    load_standard_datasets,
+    run_benchmark,
+)
+from .caching import ValidationCache
+from .cli import main as cli_main
+from .clustering import cluster_based_diagnostics
+from .deprecation import deprecated
+from .distance import (
+    braycurtis_distance,
+    canberra_distance,
+    chebyshev_distance,
+    correlation_distance,
+    distance_matrix,
+    energy_distance,
+    hamming_distance,
+    hassanat_distance,
+    hellinger_distance,
+    jaccard_distance,
+    jensen_shannon_distance,
+    mahalanobis_distance,
+    minkowski_distance,
+    wasserstein_1d_distance,
+)
+from .estimator import OversamplingValidator, validation_scorer
+from .fidelity import (
+    BoundaryReport,
+    FidelityReport,
+    ManifoldMetrics,
+    MemorisationReport,
+    boundary_violation_rate,
+    fidelity_report,
+    memorisation_report,
+    precision_recall_density_coverage,
+)
+from .inference import (
+    NullCalibration,
+    TwoSampleTestResult,
+    cross_match_test,
+    mst_two_sample_test,
+    nn_two_sample_test,
+    null_error_rate,
+)
+from .memory_efficient_validator import MemoryEfficientValidator
+from .metrics import (
+    calculate_error_rate,
+    check_model_fairness,
+    confidence_ratio,
+    duplication_rate,
+    local_density_divergence,
+    minority_recall_loss,
+    noise_sensitivity_diagnostic,
+    umap_manifold_distance,
+)
+from .optimized_distance import OptimizedDistanceMatrix
+from .plotting import (
+    plot_class_balance,
+    plot_distance_histogram,
+    plot_error_boxplot,
+    plot_error_comparison,
+    plot_error_heatmap,
+    plot_error_ranking,
+    plot_noise_sensitivity,
+    plot_sample_distribution,
+)
+from .plugin_contract import (
+    METRIC_DOMAINS,
+    AxiomReport,
+    MetricPlugin,
+    check_metric_axioms,
+)
+from .plugin_system import plugin_manager, register_metric, register_validator
+from .report import generate_report
+from .reports import SCHEMA_VERSION, RunMetadata, ValidationReport
+from .surrogate import evaluate_surrogate_models
+from .typed_validator import (
+    PydanticValidationConfig,
+    TypedValidator,
+    registry,
+    validation_session,
+)
+from .types import (
+    BenchmarkConfig,
+    ConfigurationError,
+    MetricError,
+    OversampleQAError,
+    ReferenceSet,
+    ValidationConfig,
+    ValidationDetails,
+    ValidationError,
+    ValidationMode,
+    ValidationResult,
+)
+from .validator import (
+    extract_synthetic_samples,
+    validate_multiclass_oversampling,
+    validate_oversampling,
+)
+
+__all__ = [
+    "METRIC_DOMAINS",
+    "SCHEMA_VERSION",
+    "AxiomReport",
+    "BenchmarkConfig",
+    "BoundaryReport",
+    "ConfigurationError",
+    "DatasetRepository",
+    "FidelityReport",
+    "ManifoldMetrics",
+    "MemorisationReport",
+    "MemoryEfficientValidator",
+    "MetricError",
+    "MetricPlugin",
+    "NullCalibration",
+    "OptimizedDistanceMatrix",
+    "OversampleQAError",
+    "OversamplingValidator",
+    "PydanticValidationConfig",
+    "ReferenceSet",
+    "RunMetadata",
+    "StatisticalBenchmark",
+    "TwoSampleTestResult",
+    "TypedValidator",
+    "ValidationCache",
+    "ValidationConfig",
+    "ValidationDetails",
+    "ValidationError",
+    "ValidationMode",
+    "ValidationReport",
+    "ValidationResult",
+    "boundary_violation_rate",
+    "braycurtis_distance",
+    "calculate_error_rate",
+    "canberra_distance",
+    "chebyshev_distance",
+    "check_metric_axioms",
+    "check_model_fairness",
+    "cli_main",
+    "cluster_based_diagnostics",
+    "compute_ranking",
+    "confidence_ratio",
+    "correlation_distance",
+    "create_benchmark_report",
+    "cross_match_test",
+    "deprecated",
+    "distance_matrix",
+    "duplication_rate",
+    "energy_distance",
+    "evaluate_surrogate_models",
+    "export_benchmark_results",
+    "extract_synthetic_samples",
+    "fidelity_report",
+    "format_statistical_summary",
+    "generate_report",
+    "hamming_distance",
+    "hassanat_distance",
+    "hellinger_distance",
+    "jaccard_distance",
+    "jensen_shannon_distance",
+    "load_standard_datasets",
+    "local_density_divergence",
+    "mahalanobis_distance",
+    "memorisation_report",
+    "minkowski_distance",
+    "minority_recall_loss",
+    "mst_two_sample_test",
+    "nn_two_sample_test",
+    "noise_sensitivity_diagnostic",
+    "null_error_rate",
+    "plot_class_balance",
+    "plot_distance_histogram",
+    "plot_error_boxplot",
+    "plot_error_comparison",
+    "plot_error_heatmap",
+    "plot_error_ranking",
+    "plot_noise_sensitivity",
+    "plot_sample_distribution",
+    "plugin_manager",
+    "precision_recall_density_coverage",
+    "register_metric",
+    "register_validator",
+    "registry",
+    "run_benchmark",
+    "umap_manifold_distance",
+    "validate_multiclass_oversampling",
+    "validate_oversampling",
+    "validation_scorer",
+    "validation_session",
+    "wasserstein_1d_distance",
+]

oversampleqa/_provenance.py

@@ -0,0 +1,131 @@
+"""Provenance metadata for benchmark datasets.
+
+A benchmark result is only interpretable if you can say where its data came
+from. These helpers build one consistent record for that, so the two catalogs --
+:func:`~oversampleqa.benchmark.load_standard_datasets` and
+:class:`~oversampleqa.advanced_benchmark.DatasetRepository` -- describe their
+datasets the same way rather than each inventing a shape.
+
+Every record carries the same six keys:
+
+``source``
+    ``"synthetic"``, ``"bundled"`` or ``"OpenML"``. The coarse question of
+    whether the data is generated, shipped with scikit-learn, or downloaded.
+``generator``
+    The fully-qualified callable that produced or fetched it.
+``params``
+    Arguments passed, including any ``random_state``. For synthetic data this
+    is sufficient to regenerate the dataset exactly.
+``url``
+    Where a human can read about it.
+``license``
+    Terms. "Unknown" is a legitimate value and is better than omitting the key,
+    because omission reads as "no restrictions" to a hurried reader.
+``notes``
+    Anything a reader needs in order not to misread the numbers -- applied
+    preprocessing, truncation, known caveats.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+__all__ = [
+    "SKLEARN_LICENSE",
+    "bundled_provenance",
+    "openml_provenance",
+    "synthetic_provenance",
+]
+
+SKLEARN_LICENSE = "BSD-3-Clause (scikit-learn synthetic generator)"
+
+_SKLEARN_GENERATORS_URL = (
+    "https://scikit-learn.org/stable/datasets/sample_generators.html"
+)
+
+
+def synthetic_provenance(generator: str, **params: Any) -> dict[str, Any]:
+    """Build provenance for a scikit-learn synthetic dataset.
+
+    Args:
+        generator: Fully-qualified name of the generator used.
+        **params: Generation parameters, including ``random_state``.
+
+    Returns:
+        A provenance record.
+    """
+    return {
+        "source": "synthetic",
+        "generator": generator,
+        "params": params,
+        "url": _SKLEARN_GENERATORS_URL,
+        "license": SKLEARN_LICENSE,
+        "notes": (
+            "Generated deterministically from the fixed random_state; "
+            "not real-world data."
+        ),
+    }
+
+
+def bundled_provenance(
+    generator: str,
+    *,
+    url: str,
+    license: str,
+    notes: str = "",
+    **params: Any,
+) -> dict[str, Any]:
+    """Build provenance for a dataset shipped inside scikit-learn.
+
+    Bundled data is reproducible in the sense that it does not change between
+    runs, but it is real-world data with its own citation and terms, so it is
+    recorded separately from synthetic data rather than lumped in with it.
+
+    Args:
+        generator: Fully-qualified loader, e.g. ``sklearn.datasets.load_breast_cancer``.
+        url: Where the dataset is documented.
+        license: Terms of use.
+        notes: Caveats a reader needs, such as applied truncation.
+        **params: Arguments passed to the loader.
+
+    Returns:
+        A provenance record.
+    """
+    return {
+        "source": "bundled",
+        "generator": generator,
+        "params": params,
+        "url": url,
+        "license": license,
+        "notes": notes,
+    }
+
+
+def openml_provenance(
+    name: str,
+    version: int,
+    *,
+    notes: str = "",
+) -> dict[str, Any]:
+    """Build provenance for a dataset fetched from OpenML.
+
+    The version is part of the record because it is the only thing standing
+    between a "reproducible" benchmark and an upstream dataset being silently
+    replaced under it.
+
+    Args:
+        name: OpenML dataset name.
+        version: Pinned OpenML version.
+        notes: Preprocessing or caveats.
+
+    Returns:
+        A provenance record.
+    """
+    return {
+        "source": "OpenML",
+        "generator": "sklearn.datasets.fetch_openml",
+        "params": {"name": name, "version": version},
+        "url": f"https://www.openml.org/search?type=data&q={name}",
+        "license": "Varies per dataset; see the OpenML page.",
+        "notes": notes or f"Downloaded from OpenML with the version pinned to {version}.",
+    }

oversampleqa/_render.py

@@ -0,0 +1,81 @@
+"""Frame rendering shared by the report and export paths.
+
+Lives here rather than in ``report`` because ``report`` imports from
+``benchmark``, so ``benchmark`` importing back from ``report`` would be a
+cycle -- and both need to render a frame as Markdown. Two copies is how the
+``to_csv(sep="|")`` bug survived in one of them after being fixed in the
+other.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+__all__ = ["frame_to_html", "frame_to_markdown"]
+
+
+def frame_to_markdown(frame: pd.DataFrame, *, float_format: str = "{:.4f}") -> str:
+    """Render a DataFrame as a GitHub-flavoured Markdown table.
+
+    Written out rather than delegated to ``DataFrame.to_markdown``, which needs
+    ``tabulate``. That is installed here only as a transitive dependency of
+    something else, and depending on a package nobody declared is how a working
+    install becomes a broken one after an unrelated upgrade.
+
+    The previous implementation used ``to_csv(sep="|")``, which is not Markdown:
+    it has no header separator row and no leading or trailing pipes, so it
+    rendered as one run-on paragraph rather than a table.
+
+    Args:
+        frame: Frame to render. The index becomes the first column when it is
+            named, since ``compute_ranking`` returns the oversampler there.
+        float_format: Format applied to floating-point cells. Raw repr leaks
+            values like ``0.21000000000000002`` into a document meant to be read.
+
+    Returns:
+        A Markdown table, or a note when the frame is empty.
+    """
+    if frame.empty:
+        return "_No results._"
+
+    display = frame.reset_index() if frame.index.name else frame.copy()
+
+    def render(value: Any) -> str:
+        if isinstance(value, float):
+            return float_format.format(value)
+        return str(value)
+
+    headers = [str(c) for c in display.columns]
+    rows = [[render(v) for v in row] for row in display.itertuples(index=False)]
+
+    widths = [
+        max(len(headers[i]), *(len(r[i]) for r in rows)) if rows else len(headers[i])
+        for i in range(len(headers))
+    ]
+
+    def line(cells: list[str]) -> str:
+        padded = [c.ljust(w) for c, w in zip(cells, widths, strict=True)]
+        return "| " + " | ".join(padded) + " |"
+
+    separator = "| " + " | ".join("-" * w for w in widths) + " |"
+    return "\n".join([line(headers), separator, *(line(r) for r in rows)])
+
+
+def frame_to_html(frame: pd.DataFrame, *, float_format: str = "{:.4f}") -> str:
+    """Render a DataFrame as an HTML table.
+
+    Args:
+        frame: Frame to render. A named index becomes the first column.
+        float_format: Format applied to floating-point cells.
+
+    Returns:
+        An HTML table, or a note when the frame is empty.
+    """
+    if frame.empty:
+        return "<p><em>No results.</em></p>"
+    display = frame.reset_index() if frame.index.name else frame
+    # pandas ships no type information, so to_html is typed Any. str() makes
+    # the declared return type honest instead of suppressing the error.
+    return str(display.to_html(index=False, float_format=float_format.format))

oversampleqa/_rng.py

@@ -0,0 +1,83 @@
+"""Random-number normalisation shared by every validator.
+
+Seeding behaviour lives here so it cannot drift between the validators again.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+__all__ = ["RandomStateLike", "as_generator", "spawn_generators"]
+
+
+RandomStateLike = int | np.random.Generator | np.random.SeedSequence | None
+"""Anything accepted as a seed by the validators."""
+
+
+def as_generator(random_state: RandomStateLike) -> np.random.Generator:
+    """Normalise a seed-like value to a :class:`numpy.random.Generator`.
+
+    Parameters
+    ----------
+    random_state : int, Generator, SeedSequence or None
+        ``None`` draws from fresh entropy, meaning results are not
+        reproducible. An ``int`` seeds a new generator. A ``Generator`` is
+        returned unchanged, so callers can thread one generator through a whole
+        pipeline and keep a single stream.
+
+    Returns
+    -------
+    numpy.random.Generator
+    """
+    if isinstance(random_state, np.random.Generator):
+        return random_state
+    if isinstance(random_state, np.random.SeedSequence):
+        return np.random.default_rng(random_state)
+    return np.random.default_rng(random_state)
+
+
+def spawn_generators(
+    random_state: RandomStateLike, n: int
+) -> list[np.random.Generator]:
+    """Return ``n`` independent generators derived from ``random_state``.
+
+    Uses :class:`numpy.random.SeedSequence` spawning, which guarantees the
+    streams are statistically independent.
+
+    Deriving repeat seeds as ``seed + i`` would **not** be safe: consecutive
+    integer seeds produce correlated streams, so repeats built that way share
+    structure and the resulting dispersion is understated.
+
+    Parameters
+    ----------
+    random_state : int, Generator, SeedSequence or None
+        Parent seed.
+    n : int
+        Number of child generators.
+
+    Returns
+    -------
+    list of numpy.random.Generator
+    """
+    if n < 1:
+        raise ValueError(f"n must be at least 1; got {n}")
+
+    if isinstance(random_state, np.random.SeedSequence):
+        parent = random_state
+    elif isinstance(random_state, np.random.Generator):
+        # Draw a fresh entropy value from the supplied generator so repeated
+        # calls on the same generator do not replay the same children.
+        parent = np.random.SeedSequence(int(random_state.integers(0, 2**63 - 1)))
+    else:
+        parent = np.random.SeedSequence(random_state)
+
+    return [np.random.default_rng(child) for child in parent.spawn(n)]
+
+
+def integer_seed(rng: np.random.Generator) -> int:
+    """Draw an integer seed from ``rng``.
+
+    For interoperability with scikit-learn estimators, which accept
+    ``int``/``RandomState`` but not ``Generator``.
+    """
+    return int(rng.integers(0, 2**31 - 1))