evalsuite-python 0.1.0a1__tar.gz

evalsuite_python-0.1.0a1/.gitignore

@@ -0,0 +1,19 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.venv/
+venv/
+.env
+.coverage
+coverage.xml
+htmlcov/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.hypothesis/
+.DS_Store
+.idea/
+.vscode/

evalsuite_python-0.1.0a1/CHANGELOG.md

@@ -0,0 +1,23 @@
+# Changelog
+
+All notable changes to this project are documented here. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
+[Semantic Versioning](https://semver.org/).
+
+## [Unreleased]
+
+## [0.1.0a1]
+
+First alpha, published to reserve the name and test the release pipeline. Distribution name
+`evalsuite-python` (`pip install --pre evalsuite-python`), imported as `evalsuite`.
+
+### Added
+
+- Core: input validation with actionable errors, exception hierarchy, immutable result objects with
+  JSON/pandas/Markdown/LaTeX export, metric registry (`list_metrics`, `metric_info`), and an evaluation
+  context that computes the confusion matrix once per evaluation.
+- Classification metrics for binary, multiclass and multilabel targets with all averaging modes and
+  sample weights.
+- Regression metrics for single- and multi-output targets with sample weights.
+- `evaluate()` high-level API with task inference and default metric sets.
+- Reference tests against scikit-learn and property-based tests; Python 3.9 to 3.13.

evalsuite_python-0.1.0a1/CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# Contributing to EvalSuite
+
+Thank you for helping. EvalSuite values correctness and clarity over breadth.
+
+## Adding or changing a metric
+
+1. Implement it with `@register(...)`: definition, formula, range, input requirements and at least one
+   reference.
+2. Validate inputs through `evalsuite.core.validation`; raise `InputValidationError` for bad inputs and
+   `MetricInputError` for values outside the metric's domain, with a message saying how to fix it.
+3. Test it: against an established implementation where definitions coincide (scikit-learn, SciPy), plus
+   hand-computed cases, edge cases and, where useful, property-based tests.
+4. Document conventions explicitly (averaging, label order, zero division) and update `CHANGELOG.md`.
+
+## Checks
+
+```bash
+pytest --cov=evalsuite      # coverage must stay at or above 95% for core modules
+ruff check . && ruff format --check .
+mypy                        # strict
+```
+
+## Reporting issues
+
+Include the EvalSuite, Python and NumPy versions, a minimal example, the expected result (with a reference)
+and the actual result.

evalsuite_python-0.1.0a1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Manoj Kumar C S
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalsuite_python-0.1.0a1/PKG-INFO

@@ -0,0 +1,150 @@
+Metadata-Version: 2.5
+Name: evalsuite-python
+Version: 0.1.0a1
+Summary: A unified Python framework for machine-learning, clinical, statistical, segmentation, object-detection, uncertainty, and model evaluation.
+Project-URL: Homepage, https://evalsuite-nine.vercel.app
+Project-URL: Documentation, https://evalsuite-nine.vercel.app/docs
+Project-URL: Source, https://github.com/mkcs28/evalsuite-python
+Project-URL: Issues, https://github.com/mkcs28/evalsuite-python/issues
+Project-URL: Changelog, https://github.com/mkcs28/evalsuite-python/blob/main/CHANGELOG.md
+Author: Manoj Kumar C S
+Maintainer: Manoj Kumar C S
+License-Expression: MIT
+License-File: LICENSE
+Keywords: classification,evaluation,machine learning,metrics,regression,reproducibility,statistics
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: numpy>=1.22
+Requires-Dist: pandas>=1.4
+Requires-Dist: scipy>=1.8
+Provides-Extra: all
+Requires-Dist: matplotlib>=3.5; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: hypothesis>=6.80; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pandas-stubs; extra == 'dev'
+Requires-Dist: pip-audit; extra == 'dev'
+Requires-Dist: pytest-cov>=4; extra == 'dev'
+Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Requires-Dist: scikit-learn>=1.2; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Provides-Extra: plot
+Requires-Dist: matplotlib>=3.5; extra == 'plot'
+Description-Content-Type: text/markdown
+
+# EvalSuite
+
+[![CI](https://github.com/mkcs28/evalsuite-python/actions/workflows/ci.yml/badge.svg)](https://github.com/mkcs28/evalsuite-python/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/evalsuite-python)](https://pypi.org/project/evalsuite-python/)
+[![Python](https://img.shields.io/pypi/pyversions/evalsuite-python)](https://pypi.org/project/evalsuite-python/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+**Unified, reproducible evaluation for machine learning and research.**
+
+EvalSuite brings classification and regression metrics (with clinical, statistical, segmentation and
+object-detection evaluation on the roadmap) into one consistent, validated, documented framework.
+
+> **Status: in development (v0.1.0 in progress).** The API may change before 0.1.0.
+
+## Installation
+
+```bash
+pip install evalsuite-python
+```
+
+The package is installed as `evalsuite-python` and imported as `evalsuite`:
+
+```python
+import evalsuite as es
+```
+
+## Why EvalSuite
+
+- **One consistent API.** Every metric returns a result object that behaves like a number and exports to
+  JSON, pandas, Markdown and LaTeX.
+- **Explicit conventions.** Averaging, label order, the positive class and zero-division behaviour are stated
+  and recorded in every result, never silently assumed.
+- **Validated.** Each metric is tested against scikit-learn where definitions coincide, plus property-based
+  tests and edge cases.
+- **Documented.** Every metric carries its definition, formula, range, input requirements and references,
+  available programmatically through `metric_info()`.
+- **Efficient.** `evaluate()` validates inputs once and computes the confusion matrix once for all metrics.
+- **Lightweight.** Requires only NumPy, SciPy and pandas.
+
+## Quick start
+
+```python
+import evalsuite as es
+
+y_true = [0, 1, 1, 0, 1, 0]
+y_pred = [0, 1, 0, 0, 1, 1]
+y_prob = [0.1, 0.9, 0.4, 0.2, 0.8, 0.6]
+
+result = es.evaluate(y_true, y_pred, y_prob=y_prob)
+print(result.summary())
+
+result["f1"]  # MetricResult(f1=0.666667)
+f"{result['mcc']:.3f}"  # '0.333'
+result.to_latex(caption="Test-set performance")
+result.to_dataframe()
+
+es.f1(y_true, y_pred)  # individual metrics
+es.roc_auc(y_true, y_prob)
+es.metric_info("classification.mcc").formula  # documentation
+es.list_metrics("regression")
+```
+
+## Metrics in this release
+
+**Classification** (binary, multiclass, multilabel; micro/macro/weighted/samples/per-class averaging;
+sample weights): accuracy, balanced accuracy, precision, recall, specificity, NPV, F1, F-beta, Jaccard,
+MCC, Cohen's kappa (unweighted, linear, quadratic), Hamming loss, confusion matrix, ROC AUC (binary,
+one-vs-rest, one-vs-one), average precision, ROC and PR curves, log loss, Brier score, top-k accuracy.
+
+**Regression** (single and multi-output; sample weights): MAE, MSE, RMSE, R², adjusted R², MAPE, sMAPE,
+MSLE, RMSLE, median absolute error, explained variance, max error, mean bias error, quantile (pinball)
+loss, Huber loss, relative absolute error, relative squared error.
+
+## Conventions
+
+- `average="auto"` resolves to `"binary"` for binary targets and `"macro"` otherwise; the resolved value
+  is stored in `result.params["average"]`.
+- Labels are sorted unless you pass `labels=[...]`; that order defines per-class outputs and the columns
+  of 2-D `y_prob`.
+- Undefined ratios (zero denominators) return 0 **with an `UndefinedMetricWarning`**; pass
+  `zero_division=np.nan` to propagate NaN, or `0`/`1` to choose silently.
+- Domain violations raise clear errors instead of being patched over (for example MAPE with zero targets).
+
+## Development
+
+```bash
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev]"
+pytest --cov=evalsuite
+ruff check . && ruff format --check . && mypy
+```
+
+## Links
+
+- PyPI: https://pypi.org/project/evalsuite-python/
+- Website and documentation: https://evalsuite-nine.vercel.app
+- Website source: https://github.com/mkcs28/evalsuite
+
+## License
+
+MIT. See [LICENSE](LICENSE).

evalsuite_python-0.1.0a1/README.md

@@ -0,0 +1,101 @@
+# EvalSuite
+
+[![CI](https://github.com/mkcs28/evalsuite-python/actions/workflows/ci.yml/badge.svg)](https://github.com/mkcs28/evalsuite-python/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/evalsuite-python)](https://pypi.org/project/evalsuite-python/)
+[![Python](https://img.shields.io/pypi/pyversions/evalsuite-python)](https://pypi.org/project/evalsuite-python/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+**Unified, reproducible evaluation for machine learning and research.**
+
+EvalSuite brings classification and regression metrics (with clinical, statistical, segmentation and
+object-detection evaluation on the roadmap) into one consistent, validated, documented framework.
+
+> **Status: in development (v0.1.0 in progress).** The API may change before 0.1.0.
+
+## Installation
+
+```bash
+pip install evalsuite-python
+```
+
+The package is installed as `evalsuite-python` and imported as `evalsuite`:
+
+```python
+import evalsuite as es
+```
+
+## Why EvalSuite
+
+- **One consistent API.** Every metric returns a result object that behaves like a number and exports to
+  JSON, pandas, Markdown and LaTeX.
+- **Explicit conventions.** Averaging, label order, the positive class and zero-division behaviour are stated
+  and recorded in every result, never silently assumed.
+- **Validated.** Each metric is tested against scikit-learn where definitions coincide, plus property-based
+  tests and edge cases.
+- **Documented.** Every metric carries its definition, formula, range, input requirements and references,
+  available programmatically through `metric_info()`.
+- **Efficient.** `evaluate()` validates inputs once and computes the confusion matrix once for all metrics.
+- **Lightweight.** Requires only NumPy, SciPy and pandas.
+
+## Quick start
+
+```python
+import evalsuite as es
+
+y_true = [0, 1, 1, 0, 1, 0]
+y_pred = [0, 1, 0, 0, 1, 1]
+y_prob = [0.1, 0.9, 0.4, 0.2, 0.8, 0.6]
+
+result = es.evaluate(y_true, y_pred, y_prob=y_prob)
+print(result.summary())
+
+result["f1"]  # MetricResult(f1=0.666667)
+f"{result['mcc']:.3f}"  # '0.333'
+result.to_latex(caption="Test-set performance")
+result.to_dataframe()
+
+es.f1(y_true, y_pred)  # individual metrics
+es.roc_auc(y_true, y_prob)
+es.metric_info("classification.mcc").formula  # documentation
+es.list_metrics("regression")
+```
+
+## Metrics in this release
+
+**Classification** (binary, multiclass, multilabel; micro/macro/weighted/samples/per-class averaging;
+sample weights): accuracy, balanced accuracy, precision, recall, specificity, NPV, F1, F-beta, Jaccard,
+MCC, Cohen's kappa (unweighted, linear, quadratic), Hamming loss, confusion matrix, ROC AUC (binary,
+one-vs-rest, one-vs-one), average precision, ROC and PR curves, log loss, Brier score, top-k accuracy.
+
+**Regression** (single and multi-output; sample weights): MAE, MSE, RMSE, R², adjusted R², MAPE, sMAPE,
+MSLE, RMSLE, median absolute error, explained variance, max error, mean bias error, quantile (pinball)
+loss, Huber loss, relative absolute error, relative squared error.
+
+## Conventions
+
+- `average="auto"` resolves to `"binary"` for binary targets and `"macro"` otherwise; the resolved value
+  is stored in `result.params["average"]`.
+- Labels are sorted unless you pass `labels=[...]`; that order defines per-class outputs and the columns
+  of 2-D `y_prob`.
+- Undefined ratios (zero denominators) return 0 **with an `UndefinedMetricWarning`**; pass
+  `zero_division=np.nan` to propagate NaN, or `0`/`1` to choose silently.
+- Domain violations raise clear errors instead of being patched over (for example MAPE with zero targets).
+
+## Development
+
+```bash
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev]"
+pytest --cov=evalsuite
+ruff check . && ruff format --check . && mypy
+```
+
+## Links
+
+- PyPI: https://pypi.org/project/evalsuite-python/
+- Website and documentation: https://evalsuite-nine.vercel.app
+- Website source: https://github.com/mkcs28/evalsuite
+
+## License
+
+MIT. See [LICENSE](LICENSE).

evalsuite_python-0.1.0a1/pyproject.toml

@@ -0,0 +1,96 @@
+[build-system]
+requires = ["hatchling>=1.24"]
+build-backend = "hatchling.build"
+
+[project]
+name = "evalsuite-python"
+dynamic = ["version"]
+description = "A unified Python framework for machine-learning, clinical, statistical, segmentation, object-detection, uncertainty, and model evaluation."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.9"
+authors = [{ name = "Manoj Kumar C S" }]
+maintainers = [{ name = "Manoj Kumar C S" }]
+keywords = ["evaluation", "metrics", "machine learning", "statistics", "classification", "regression", "reproducibility"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Science/Research",
+  "Intended Audience :: Developers",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Scientific/Engineering",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Typing :: Typed",
+]
+dependencies = ["numpy>=1.22", "scipy>=1.8", "pandas>=1.4"]
+
+[project.optional-dependencies]
+plot = ["matplotlib>=3.5"]
+all = ["matplotlib>=3.5"]
+dev = [
+  "pytest>=7",
+  "pytest-cov>=4",
+  "hypothesis>=6.80",
+  "scikit-learn>=1.2",
+  "ruff>=0.6",
+  "mypy>=1.10",
+  "pandas-stubs",
+  "build",
+  "twine",
+  "pip-audit",
+]
+
+[project.urls]
+Homepage = "https://evalsuite-nine.vercel.app"
+Documentation = "https://evalsuite-nine.vercel.app/docs"
+Source = "https://github.com/mkcs28/evalsuite-python"
+Issues = "https://github.com/mkcs28/evalsuite-python/issues"
+Changelog = "https://github.com/mkcs28/evalsuite-python/blob/main/CHANGELOG.md"
+
+[project.scripts]
+evalsuite = "evalsuite.cli.main:main"
+
+[tool.hatch.version]
+path = "src/evalsuite/version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/evalsuite"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/evalsuite", "tests", "README.md", "LICENSE", "CHANGELOG.md", "CONTRIBUTING.md", "pyproject.toml"]
+
+[tool.ruff]
+line-length = 115
+target-version = "py39"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "N", "SIM", "NPY", "RUF", "S"]
+# UP007/UP045: keep Optional[...] and Union[...] for clarity alongside Python 3.9 support.
+ignore = ["N803", "N806", "RUF022", "UP007", "UP045", "RUF001", "RUF002", "RUF003"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S101", "N802", "E501"]
+# Metric modules carry full literature references, which are not wrapped.
+"src/evalsuite/*/metrics.py" = ["E501"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+files = ["src/evalsuite"]
+
+[[tool.mypy.overrides]]
+module = ["scipy.*", "matplotlib.*"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q --strict-markers"
+filterwarnings = ["error::RuntimeWarning"]

evalsuite_python-0.1.0a1/src/evalsuite/__init__.py

@@ -0,0 +1,118 @@
+"""EvalSuite: unified, reproducible evaluation for machine learning and research.
+
+>>> import evalsuite as es
+>>> result = es.evaluate([0, 1, 1, 0], [0, 1, 0, 0])
+>>> print(result.summary())  # doctest: +SKIP
+"""
+
+from . import classification, regression
+from .api import evaluate
+from .classification import (
+    accuracy,
+    average_precision,
+    balanced_accuracy,
+    brier_score,
+    cohen_kappa,
+    confusion_matrix,
+    f1,
+    fbeta,
+    hamming_loss,
+    jaccard,
+    log_loss,
+    mcc,
+    npv,
+    pr_curve,
+    precision,
+    recall,
+    roc_auc,
+    roc_curve,
+    specificity,
+    top_k_accuracy,
+)
+from .core.exceptions import (
+    EvalSuiteError,
+    InputValidationError,
+    MetricInputError,
+    OptionalDependencyError,
+    StatisticalTestError,
+    UndefinedMetricWarning,
+    UnsupportedTaskError,
+)
+from .core.registry import MetricInfo, list_metrics, metric_info
+from .core.result import EvaluationResult, MetricResult
+from .regression import (
+    adjusted_r2,
+    explained_variance,
+    huber_loss,
+    mae,
+    mape,
+    max_error,
+    mean_bias_error,
+    median_absolute_error,
+    mse,
+    msle,
+    quantile_loss,
+    r2,
+    rae,
+    rmse,
+    rmsle,
+    rse,
+    smape,
+)
+from .version import __version__
+
+__all__ = [
+    "EvalSuiteError",
+    "EvaluationResult",
+    "InputValidationError",
+    "MetricInfo",
+    "MetricInputError",
+    "MetricResult",
+    "OptionalDependencyError",
+    "StatisticalTestError",
+    "UndefinedMetricWarning",
+    "UnsupportedTaskError",
+    "__version__",
+    "accuracy",
+    "adjusted_r2",
+    "average_precision",
+    "balanced_accuracy",
+    "brier_score",
+    "classification",
+    "cohen_kappa",
+    "confusion_matrix",
+    "evaluate",
+    "explained_variance",
+    "f1",
+    "fbeta",
+    "hamming_loss",
+    "huber_loss",
+    "jaccard",
+    "list_metrics",
+    "log_loss",
+    "mae",
+    "mape",
+    "max_error",
+    "mcc",
+    "mean_bias_error",
+    "median_absolute_error",
+    "metric_info",
+    "mse",
+    "msle",
+    "npv",
+    "pr_curve",
+    "precision",
+    "quantile_loss",
+    "r2",
+    "rae",
+    "recall",
+    "regression",
+    "rmse",
+    "rmsle",
+    "roc_auc",
+    "roc_curve",
+    "rse",
+    "smape",
+    "specificity",
+    "top_k_accuracy",
+]