mattergraph-benchmarks 0.1.0__tar.gz

mattergraph_benchmarks-0.1.0/.gitignore

@@ -0,0 +1,48 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+dist/
+build/
+.venv/
+venv/
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.hypothesis/
+.coverage
+coverage.xml
+htmlcov/
+
+# Docs build output
+site/
+
+# Env
+.env
+.env.local
+*.local
+
+# Node
+node_modules/
+apps/web/dist/
+apps/web/playwright-report/
+apps/web/test-results/
+*.tsbuildinfo
+.next/
+out/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+
+# OS
+.DS_Store
+
+# Data artifacts (keep demo/ tracked)
+data/cache/
+*.sqlite3
+
+.claude
+apps/private-platform-ui/

mattergraph_benchmarks-0.1.0/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: mattergraph-benchmarks
+Version: 0.1.0
+Summary: Benchmark adapters and evaluation utilities for MatterGraph.
+Project-URL: Homepage, https://github.com/cyrusmo/MatterGraph
+Project-URL: Repository, https://github.com/cyrusmo/MatterGraph
+Project-URL: Issues, https://github.com/cyrusmo/MatterGraph/issues
+Project-URL: Changelog, https://github.com/cyrusmo/MatterGraph/blob/main/CHANGELOG.md
+Author: MatterGraph contributors
+License-Expression: Apache-2.0
+Keywords: benchmarks,machine-learning,matbench,materials-science,uncertainty
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Chemistry
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Python: >=3.10
+Requires-Dist: httpx
+Requires-Dist: mattergraph-core~=0.1.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: pandas>=2.0
+Requires-Dist: pymatgen>=2024.1.1
+Requires-Dist: scikit-learn>=1.3
+Description-Content-Type: text/markdown
+
+# mattergraph-benchmarks
+
+Benchmark adapters and evaluation utilities for [MatterGraph](https://github.com/cyrusmo/MatterGraph).
+
+## What's in here
+
+- **Discovery metrics** — ranking-quality measures (nDCG and friends) for screening workflows, where what matters is whether good candidates surface near the top.
+- **Uncertainty** — `coverage_at_target` for checking whether predicted intervals are calibrated.
+- **Validation splits** — stratified splitting helpers that respect composition and structure grouping.
+- **Matbench adapter** — optional bridge to [Matbench](https://matbench.materialsproject.org/) tasks. Install `matbench` separately to enable it.
+
+## Install
+
+```bash
+pip install mattergraph-benchmarks
+```
+
+## License
+
+Apache-2.0

mattergraph_benchmarks-0.1.0/README.md

@@ -0,0 +1,20 @@
+# mattergraph-benchmarks
+
+Benchmark adapters and evaluation utilities for [MatterGraph](https://github.com/cyrusmo/MatterGraph).
+
+## What's in here
+
+- **Discovery metrics** — ranking-quality measures (nDCG and friends) for screening workflows, where what matters is whether good candidates surface near the top.
+- **Uncertainty** — `coverage_at_target` for checking whether predicted intervals are calibrated.
+- **Validation splits** — stratified splitting helpers that respect composition and structure grouping.
+- **Matbench adapter** — optional bridge to [Matbench](https://matbench.materialsproject.org/) tasks. Install `matbench` separately to enable it.
+
+## Install
+
+```bash
+pip install mattergraph-benchmarks
+```
+
+## License
+
+Apache-2.0

mattergraph_benchmarks-0.1.0/mattergraph_benchmarks/__init__.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+  from mattergraph_benchmarks.discovery_metrics import dcg, ndcg_at_k
+  from mattergraph_benchmarks.matbench_adapter import matbench_dataframe, matbench_regression
+  from mattergraph_benchmarks.uncertainty import coverage_at_target
+  from mattergraph_benchmarks.validation_split import stratified_regression_split
+
+_EXPORTS: dict[str, tuple[str, str, str | None]] = {
+  "matbench_dataframe": (
+    "mattergraph_benchmarks.matbench_adapter",
+    "matbench_dataframe",
+    None,
+  ),
+  "matbench_regression": (
+    "mattergraph_benchmarks.matbench_adapter",
+    "matbench_regression",
+    None,
+  ),
+  "ndcg_at_k": (
+    "mattergraph_benchmarks.discovery_metrics",
+    "ndcg_at_k",
+    None,
+  ),
+  "dcg": (
+    "mattergraph_benchmarks.discovery_metrics",
+    "dcg",
+    None,
+  ),
+  "coverage_at_target": (
+    "mattergraph_benchmarks.uncertainty",
+    "coverage_at_target",
+    None,
+  ),
+  "stratified_regression_split": (
+    "mattergraph_benchmarks.validation_split",
+    "stratified_regression_split",
+    (
+      "Install the optional `scikit-learn` dependency or run "
+      "`uv sync --all-packages --group dev` to use stratified_regression_split."
+    ),
+  ),
+}
+
+__all__ = [
+  "matbench_dataframe",
+  "matbench_regression",
+  "ndcg_at_k",
+  "dcg",
+  "coverage_at_target",
+  "stratified_regression_split",
+]
+
+
+def __getattr__(name: str) -> Any:
+  if name not in _EXPORTS:
+    msg = f"module {__name__!r} has no attribute {name!r}"
+    raise AttributeError(msg)
+  module_name, attr_name, hint = _EXPORTS[name]
+  try:
+    module = import_module(module_name)
+  except ImportError as e:
+    if hint is None:
+      raise
+    raise ImportError(hint) from e
+  value = getattr(module, attr_name)
+  globals()[name] = value
+  return value
+
+
+def __dir__() -> list[str]:
+  return sorted(set(globals()) | set(__all__))

mattergraph_benchmarks-0.1.0/mattergraph_benchmarks/discovery_metrics.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike
+
+
+def dcg(relevances: ArrayLike) -> float:
+  r = np.asarray(relevances, dtype=float)
+  if r.size == 0:
+    return 0.0
+  g = 2.0**r - 1.0
+  i = np.arange(1, len(r) + 1, dtype=float)
+  return float(np.sum(g / np.log2(i + 1)))
+
+
+def ndcg_at_k(relevances: ArrayLike, k: int = 10) -> float:
+  r = np.asarray(relevances, dtype=float)[:k]
+  if r.size == 0:
+    return 0.0
+  ideal = np.sort(r)[::-1]
+  d = dcg(r)
+  i = dcg(ideal)
+  return d / i if i > 0 else 0.0

mattergraph_benchmarks-0.1.0/mattergraph_benchmarks/matbench_adapter.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+def _load_matbench_task(name: str) -> Any:
+  try:
+    from matbench.task import MatbenchTask  # type: ignore[import-not-found, import-untyped]
+  except ImportError as e:  # pragma: no cover
+    msg = "Install the optional `matbench` package to use the Matbench adapter."
+    raise ImportError(msg) from e
+  t = MatbenchTask(name)  # type: ignore[operator]
+  t.load()  # type: ignore[no-untyped-call]
+  return t
+
+
+def matbench_dataframe(task: str = "matbench_v0.1_log_gvrh") -> pd.DataFrame:
+  """
+  Load a Matbench task into a :class:`pandas.DataFrame` with a ``target`` and ``split`` column.
+
+  The ``matbench`` package is an optional install.
+  """
+  t = _load_matbench_task(task)
+  f = t.get_train_and_val()  # type: ignore[no-untyped-call]
+  train = f[0]  # type: ignore[index]
+  test = f[1]  # type: ignore[index]
+  tr_x, tr_y, _ = train  # type: ignore[misc]
+  te_x, te_y, _ = test  # type: ignore[misc]
+
+  def _as_df(x: object, y: object, s: str) -> pd.DataFrame:
+    d = x.to_pandas() if hasattr(x, "to_pandas") else pd.DataFrame(x)  # type: ignore[attr-defined, arg-type, call-overload]  # noqa: E501
+    d = d.copy()
+    d["target"] = np.asarray(y).ravel()
+    d["split"] = s
+    return d
+
+  tr = _as_df(tr_x, tr_y, "train")
+  te = _as_df(te_x, te_y, "test")
+  return pd.concat([tr, te], ignore_index=True)
+
+
+def matbench_regression(task: str = "matbench_v0.1_log_gvrh") -> dict[str, Any]:
+  """Return simple numpy arrays for train/test splits of a Matbench regression task."""
+  df = matbench_dataframe(task=task)
+  train = df[df["split"] == "train"]
+  test = df[df["split"] == "test"]
+  y_tr = train["target"].to_numpy()
+  y_te = test["target"].to_numpy()
+  x_tr = train.drop(columns=["target", "split"], errors="ignore")
+  x_te = test.drop(columns=["target", "split"], errors="ignore")
+  return {
+    "train_X": x_tr,
+    "train_y": y_tr,
+    "test_X": x_te,
+    "test_y": y_te,
+  }

mattergraph_benchmarks-0.1.0/mattergraph_benchmarks/uncertainty.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike
+
+
+def coverage_at_target(
+  y_true: ArrayLike,
+  y_lo: ArrayLike,
+  y_hi: ArrayLike,
+) -> float:
+  """Fraction of points where the interval ``[y_lo, y_hi]`` contains ``y_true``."""
+  yt = np.asarray(y_true, dtype=float).ravel()
+  lo = np.asarray(y_lo, dtype=float).ravel()
+  hi = np.asarray(y_hi, dtype=float).ravel()
+  m = (yt >= lo) & (yt <= hi)
+  return float(np.mean(m)) if m.size else 0.0

mattergraph_benchmarks-0.1.0/mattergraph_benchmarks/validation_split.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from numpy.typing import ArrayLike, NDArray
+
+
+def _load_train_test_split() -> Any:
+  try:
+    from sklearn.model_selection import train_test_split
+  except ImportError as e:
+    msg = "Install the optional `scikit-learn` dependency to use stratified_regression_split."
+    raise ImportError(msg) from e
+  return train_test_split
+
+
+def stratified_regression_split(
+  y: ArrayLike,
+  n_bins: int = 8,
+  test_size: float = 0.2,
+  random_state: int = 0,
+) -> tuple[NDArray[np.int_], NDArray[np.int_]]:
+  """Bin continuous ``y`` and return train/test index arrays (stratified by bin)."""
+  train_test_split = _load_train_test_split()
+  yv = np.asarray(y, dtype=float).ravel()
+  yq, _ = pd.qcut(yv, n_bins, labels=False, retbins=True, duplicates="drop")
+  yq = np.nan_to_num(yq, nan=0.0)
+  i = np.arange(len(yv))
+  i_train, i_test = train_test_split(
+    i, test_size=test_size, random_state=random_state, stratify=yq
+  )
+  return i_train, i_test

mattergraph_benchmarks-0.1.0/pyproject.toml

@@ -0,0 +1,48 @@
+[project]
+name = "mattergraph-benchmarks"
+version = "0.1.0"
+description = "Benchmark adapters and evaluation utilities for MatterGraph."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [{ name = "MatterGraph contributors" }]
+keywords = ["materials-science", "benchmarks", "matbench", "machine-learning", "uncertainty"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Science/Research",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Scientific/Engineering :: Chemistry",
+  "Topic :: Scientific/Engineering :: Physics",
+]
+dependencies = [
+  "mattergraph-core~=0.1.0",
+  "numpy>=1.24",
+  "pandas>=2.0",
+  "scikit-learn>=1.3",
+  "pymatgen>=2024.1.1",
+  "httpx",
+]
+
+[project.urls]
+Homepage = "https://github.com/cyrusmo/MatterGraph"
+Repository = "https://github.com/cyrusmo/MatterGraph"
+Issues = "https://github.com/cyrusmo/MatterGraph/issues"
+Changelog = "https://github.com/cyrusmo/MatterGraph/blob/main/CHANGELOG.md"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.uv.sources]
+mattergraph-core = { workspace = true }
+
+[tool.hatch.build.targets.wheel]
+packages = ["mattergraph_benchmarks"]
+core-metadata-version = "2.4"
+
+[tool.hatch.build.targets.sdist]
+core-metadata-version = "2.4"

mattergraph_benchmarks-0.1.0/tests/test_benchmarks_optional_imports.py

@@ -0,0 +1,21 @@
+import importlib
+
+import mattergraph_benchmarks
+import pytest
+
+
+def test_stratified_split_has_helpful_optional_dependency_error(
+  monkeypatch: pytest.MonkeyPatch,
+) -> None:
+  module = importlib.reload(mattergraph_benchmarks)
+  real_import_module = module.import_module
+
+  def fake_import_module(name: str, package: str | None = None) -> object:
+    if name == "mattergraph_benchmarks.validation_split":
+      raise ImportError("No module named 'sklearn'")
+    return real_import_module(name, package)
+
+  monkeypatch.setattr(module, "import_module", fake_import_module)
+
+  with pytest.raises(ImportError, match="scikit-learn"):
+    _ = module.stratified_regression_split

mattergraph_benchmarks-0.1.0/tests/test_discovery_metrics.py

@@ -0,0 +1,9 @@
+from mattergraph_benchmarks import dcg, ndcg_at_k
+
+
+def test_dcg_and_ndcg_bounds() -> None:
+  rel = [3, 2, 1]
+  assert dcg(rel) > 0
+  score = ndcg_at_k(rel, k=3)
+  assert 0.0 <= score <= 1.0
+  assert ndcg_at_k([], k=3) == 0.0