mattergraph-benchmarks 0.1.0__py3-none-any.whl

mattergraph_benchmarks/__init__.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+  from mattergraph_benchmarks.discovery_metrics import dcg, ndcg_at_k
+  from mattergraph_benchmarks.matbench_adapter import matbench_dataframe, matbench_regression
+  from mattergraph_benchmarks.uncertainty import coverage_at_target
+  from mattergraph_benchmarks.validation_split import stratified_regression_split
+
+_EXPORTS: dict[str, tuple[str, str, str | None]] = {
+  "matbench_dataframe": (
+    "mattergraph_benchmarks.matbench_adapter",
+    "matbench_dataframe",
+    None,
+  ),
+  "matbench_regression": (
+    "mattergraph_benchmarks.matbench_adapter",
+    "matbench_regression",
+    None,
+  ),
+  "ndcg_at_k": (
+    "mattergraph_benchmarks.discovery_metrics",
+    "ndcg_at_k",
+    None,
+  ),
+  "dcg": (
+    "mattergraph_benchmarks.discovery_metrics",
+    "dcg",
+    None,
+  ),
+  "coverage_at_target": (
+    "mattergraph_benchmarks.uncertainty",
+    "coverage_at_target",
+    None,
+  ),
+  "stratified_regression_split": (
+    "mattergraph_benchmarks.validation_split",
+    "stratified_regression_split",
+    (
+      "Install the optional `scikit-learn` dependency or run "
+      "`uv sync --all-packages --group dev` to use stratified_regression_split."
+    ),
+  ),
+}
+
+__all__ = [
+  "matbench_dataframe",
+  "matbench_regression",
+  "ndcg_at_k",
+  "dcg",
+  "coverage_at_target",
+  "stratified_regression_split",
+]
+
+
+def __getattr__(name: str) -> Any:
+  if name not in _EXPORTS:
+    msg = f"module {__name__!r} has no attribute {name!r}"
+    raise AttributeError(msg)
+  module_name, attr_name, hint = _EXPORTS[name]
+  try:
+    module = import_module(module_name)
+  except ImportError as e:
+    if hint is None:
+      raise
+    raise ImportError(hint) from e
+  value = getattr(module, attr_name)
+  globals()[name] = value
+  return value
+
+
+def __dir__() -> list[str]:
+  return sorted(set(globals()) | set(__all__))

mattergraph_benchmarks/discovery_metrics.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike
+
+
+def dcg(relevances: ArrayLike) -> float:
+  r = np.asarray(relevances, dtype=float)
+  if r.size == 0:
+    return 0.0
+  g = 2.0**r - 1.0
+  i = np.arange(1, len(r) + 1, dtype=float)
+  return float(np.sum(g / np.log2(i + 1)))
+
+
+def ndcg_at_k(relevances: ArrayLike, k: int = 10) -> float:
+  r = np.asarray(relevances, dtype=float)[:k]
+  if r.size == 0:
+    return 0.0
+  ideal = np.sort(r)[::-1]
+  d = dcg(r)
+  i = dcg(ideal)
+  return d / i if i > 0 else 0.0

mattergraph_benchmarks/matbench_adapter.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+
+def _load_matbench_task(name: str) -> Any:
+  try:
+    from matbench.task import MatbenchTask  # type: ignore[import-not-found, import-untyped]
+  except ImportError as e:  # pragma: no cover
+    msg = "Install the optional `matbench` package to use the Matbench adapter."
+    raise ImportError(msg) from e
+  t = MatbenchTask(name)  # type: ignore[operator]
+  t.load()  # type: ignore[no-untyped-call]
+  return t
+
+
+def matbench_dataframe(task: str = "matbench_v0.1_log_gvrh") -> pd.DataFrame:
+  """
+  Load a Matbench task into a :class:`pandas.DataFrame` with a ``target`` and ``split`` column.
+
+  The ``matbench`` package is an optional install.
+  """
+  t = _load_matbench_task(task)
+  f = t.get_train_and_val()  # type: ignore[no-untyped-call]
+  train = f[0]  # type: ignore[index]
+  test = f[1]  # type: ignore[index]
+  tr_x, tr_y, _ = train  # type: ignore[misc]
+  te_x, te_y, _ = test  # type: ignore[misc]
+
+  def _as_df(x: object, y: object, s: str) -> pd.DataFrame:
+    d = x.to_pandas() if hasattr(x, "to_pandas") else pd.DataFrame(x)  # type: ignore[attr-defined, arg-type, call-overload]  # noqa: E501
+    d = d.copy()
+    d["target"] = np.asarray(y).ravel()
+    d["split"] = s
+    return d
+
+  tr = _as_df(tr_x, tr_y, "train")
+  te = _as_df(te_x, te_y, "test")
+  return pd.concat([tr, te], ignore_index=True)
+
+
+def matbench_regression(task: str = "matbench_v0.1_log_gvrh") -> dict[str, Any]:
+  """Return simple numpy arrays for train/test splits of a Matbench regression task."""
+  df = matbench_dataframe(task=task)
+  train = df[df["split"] == "train"]
+  test = df[df["split"] == "test"]
+  y_tr = train["target"].to_numpy()
+  y_te = test["target"].to_numpy()
+  x_tr = train.drop(columns=["target", "split"], errors="ignore")
+  x_te = test.drop(columns=["target", "split"], errors="ignore")
+  return {
+    "train_X": x_tr,
+    "train_y": y_tr,
+    "test_X": x_te,
+    "test_y": y_te,
+  }

mattergraph_benchmarks/uncertainty.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike
+
+
+def coverage_at_target(
+  y_true: ArrayLike,
+  y_lo: ArrayLike,
+  y_hi: ArrayLike,
+) -> float:
+  """Fraction of points where the interval ``[y_lo, y_hi]`` contains ``y_true``."""
+  yt = np.asarray(y_true, dtype=float).ravel()
+  lo = np.asarray(y_lo, dtype=float).ravel()
+  hi = np.asarray(y_hi, dtype=float).ravel()
+  m = (yt >= lo) & (yt <= hi)
+  return float(np.mean(m)) if m.size else 0.0

mattergraph_benchmarks/validation_split.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from numpy.typing import ArrayLike, NDArray
+
+
+def _load_train_test_split() -> Any:
+  try:
+    from sklearn.model_selection import train_test_split
+  except ImportError as e:
+    msg = "Install the optional `scikit-learn` dependency to use stratified_regression_split."
+    raise ImportError(msg) from e
+  return train_test_split
+
+
+def stratified_regression_split(
+  y: ArrayLike,
+  n_bins: int = 8,
+  test_size: float = 0.2,
+  random_state: int = 0,
+) -> tuple[NDArray[np.int_], NDArray[np.int_]]:
+  """Bin continuous ``y`` and return train/test index arrays (stratified by bin)."""
+  train_test_split = _load_train_test_split()
+  yv = np.asarray(y, dtype=float).ravel()
+  yq, _ = pd.qcut(yv, n_bins, labels=False, retbins=True, duplicates="drop")
+  yq = np.nan_to_num(yq, nan=0.0)
+  i = np.arange(len(yv))
+  i_train, i_test = train_test_split(
+    i, test_size=test_size, random_state=random_state, stratify=yq
+  )
+  return i_train, i_test

mattergraph_benchmarks-0.1.0.dist-info/METADATA

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: mattergraph-benchmarks
+Version: 0.1.0
+Summary: Benchmark adapters and evaluation utilities for MatterGraph.
+Project-URL: Homepage, https://github.com/cyrusmo/MatterGraph
+Project-URL: Repository, https://github.com/cyrusmo/MatterGraph
+Project-URL: Issues, https://github.com/cyrusmo/MatterGraph/issues
+Project-URL: Changelog, https://github.com/cyrusmo/MatterGraph/blob/main/CHANGELOG.md
+Author: MatterGraph contributors
+License-Expression: Apache-2.0
+Keywords: benchmarks,machine-learning,matbench,materials-science,uncertainty
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Chemistry
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Python: >=3.10
+Requires-Dist: httpx
+Requires-Dist: mattergraph-core~=0.1.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: pandas>=2.0
+Requires-Dist: pymatgen>=2024.1.1
+Requires-Dist: scikit-learn>=1.3
+Description-Content-Type: text/markdown
+
+# mattergraph-benchmarks
+
+Benchmark adapters and evaluation utilities for [MatterGraph](https://github.com/cyrusmo/MatterGraph).
+
+## What's in here
+
+- **Discovery metrics** — ranking-quality measures (nDCG and friends) for screening workflows, where what matters is whether good candidates surface near the top.
+- **Uncertainty** — `coverage_at_target` for checking whether predicted intervals are calibrated.
+- **Validation splits** — stratified splitting helpers that respect composition and structure grouping.
+- **Matbench adapter** — optional bridge to [Matbench](https://matbench.materialsproject.org/) tasks. Install `matbench` separately to enable it.
+
+## Install
+
+```bash
+pip install mattergraph-benchmarks
+```
+
+## License
+
+Apache-2.0

mattergraph_benchmarks-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+mattergraph_benchmarks/__init__.py,sha256=b23gJLYVl47s4_wKDF_uziJjwKJnFRoju9XPi6N630A,1942
+mattergraph_benchmarks/discovery_metrics.py,sha256=3iwMf9v-NIblv7BaznX28DUSBop70XQRgmHbfzMCUGw,545
+mattergraph_benchmarks/matbench_adapter.py,sha256=wdpuV_-MSyvCInNw1_HEl_KakcSiLNVLn4n02Z6JmYA,1997
+mattergraph_benchmarks/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mattergraph_benchmarks/uncertainty.py,sha256=ESFT8FVD52nNIXRHu07oAFvuYj-lN54LL_1w_U4vJHQ,484
+mattergraph_benchmarks/validation_split.py,sha256=5Z-Oz74ajXHBboxnOi5J6Zkq9qZXV1WQZr57fSUMZL4,1040
+mattergraph_benchmarks-0.1.0.dist-info/METADATA,sha256=8trmHX8hECh-pjwZZ7KCUyi0riXCh3tJ8q-wxu1Xg7g,2008
+mattergraph_benchmarks-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+mattergraph_benchmarks-0.1.0.dist-info/RECORD,,

mattergraph_benchmarks-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any