benchmark-reliability 0.2.1__tar.gz → 0.3.0__tar.gz

{benchmark_reliability-0.2.1 → benchmark_reliability-0.3.0}/PKG-INFO

@@ -1,11 +1,11 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: benchmark-reliability
-Version: 0.2.1
+Version: 0.3.0
 Summary: Benchmark Reliability Framework (BRF) - dataset-level reliability auditing with built-in benchmark registry
 Author-email: zhanglizhuo <zhanglizhuo@gmail.com>
 License: MIT
-Project-URL: Homepage, https://github.com/zhanglizhuo/BenchmarkReliability
-Project-URL: Repository, https://github.com/zhanglizhuo/BenchmarkReliability
+Project-URL: Homepage, https://github.com/zhanglizhuo/BRFPackage
+Project-URL: Repository, https://github.com/zhanglizhuo/BRFPackage
 Keywords: benchmark reliability,dataset auditing,educational AI,machine learning,registry
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
@@ -65,9 +65,9 @@ print(analyzer.diagnose()["summary"])
 for dim, issue in analyzer.diagnose()["details"].items():
     print(f"  {dim}: {issue}")
 
-# Percentile rank against 25 audited benchmarks
+# Percentile rank against the 51 benchmarks in the BRF Registry (v2.1 reference)
 print(analyzer.rank())
-# {'S_percentile': 16.0, 'E_percentile': 36.0, ...}
+# {'S_percentile': 17.6, 'E_percentile': 27.5, 'reference': 'BRF Registry v2.1 (51 benchmarks)'}
 
 # One-paragraph recommendation
 print(analyzer.recommend())
@@ -77,7 +77,7 @@ print(analyzer.recommend())
 
 ```bash
 $ brf registry list
-BRF Registry -- 16 datasets
+BRF Registry
 
   assistments          ASSISTments 2009-2010           N= 3729  G= 124
   college_scorecard    US College Scorecard            N= 7804  G=  59
@@ -95,7 +95,7 @@ BRF Audit: Teaching Assistant Evaluation (tae)
 ### Download and verify all datasets
 
 ```bash
-$ brf registry sync     # download + SHA-256 verify all 16 datasets
+$ brf registry sync     # download + SHA-256 verify all
 $ brf registry info oulad
   name: oulad
   display_name: Open University Learning Analytics Dataset
@@ -122,7 +122,7 @@ communication shorthand only --- **the signal is in the continuous values**.
 
 Use `analyzer.diagnose()` for per-dimension explanations and actionable
 recommendations, or `analyzer.rank()` to see percentile scores against
-the 25 benchmarks in the BRF Registry.
+the 51 benchmarks in the BRF Registry (v2.1 reference).
 
 ## CLI Reference
 
@@ -152,8 +152,8 @@ To cite the BRF framework and package (JOSS paper forthcoming):
 @software{zhang2026brf,
   author = {Lizhuo Zhang},
   title = {benchmark-reliability: Benchmark Reliability Framework},
-  url = {https://github.com/zhanglizhuo/BenchmarkReliability},
-  version = {0.1.5},
+  url = {https://github.com/zhanglizhuo/BRFPackage},
+  version = {0.3.0},
   year = {2026},
 }
 ```
@@ -162,7 +162,7 @@ The behavior audit protocol is described in:
 
 > Zhang, L. *BehaviorAudit: a four-dimension protocol for auditing
 > benchmark reliability under group-aware evaluation.*
-> Scientific Reports (under review).
+> Scientific Reports (2026). https://doi.org/10.1038/s41598-026-69629-6
 
 ## Related Work
 

{benchmark_reliability-0.2.1 → benchmark_reliability-0.3.0}/README.md

@@ -38,9 +38,9 @@ print(analyzer.diagnose()["summary"])
 for dim, issue in analyzer.diagnose()["details"].items():
     print(f"  {dim}: {issue}")
 
-# Percentile rank against 25 audited benchmarks
+# Percentile rank against the 51 benchmarks in the BRF Registry (v2.1 reference)
 print(analyzer.rank())
-# {'S_percentile': 16.0, 'E_percentile': 36.0, ...}
+# {'S_percentile': 17.6, 'E_percentile': 27.5, 'reference': 'BRF Registry v2.1 (51 benchmarks)'}
 
 # One-paragraph recommendation
 print(analyzer.recommend())
@@ -50,7 +50,7 @@ print(analyzer.recommend())
 
 ```bash
 $ brf registry list
-BRF Registry -- 16 datasets
+BRF Registry
 
   assistments          ASSISTments 2009-2010           N= 3729  G= 124
   college_scorecard    US College Scorecard            N= 7804  G=  59
@@ -68,7 +68,7 @@ BRF Audit: Teaching Assistant Evaluation (tae)
 ### Download and verify all datasets
 
 ```bash
-$ brf registry sync     # download + SHA-256 verify all 16 datasets
+$ brf registry sync     # download + SHA-256 verify all
 $ brf registry info oulad
   name: oulad
   display_name: Open University Learning Analytics Dataset
@@ -95,7 +95,7 @@ communication shorthand only --- **the signal is in the continuous values**.
 
 Use `analyzer.diagnose()` for per-dimension explanations and actionable
 recommendations, or `analyzer.rank()` to see percentile scores against
-the 25 benchmarks in the BRF Registry.
+the 51 benchmarks in the BRF Registry (v2.1 reference).
 
 ## CLI Reference
 
@@ -125,8 +125,8 @@ To cite the BRF framework and package (JOSS paper forthcoming):
 @software{zhang2026brf,
   author = {Lizhuo Zhang},
   title = {benchmark-reliability: Benchmark Reliability Framework},
-  url = {https://github.com/zhanglizhuo/BenchmarkReliability},
-  version = {0.1.5},
+  url = {https://github.com/zhanglizhuo/BRFPackage},
+  version = {0.3.0},
   year = {2026},
 }
 ```
@@ -135,7 +135,7 @@ The behavior audit protocol is described in:
 
 > Zhang, L. *BehaviorAudit: a four-dimension protocol for auditing
 > benchmark reliability under group-aware evaluation.*
-> Scientific Reports (under review).
+> Scientific Reports (2026). https://doi.org/10.1038/s41598-026-69629-6
 
 ## Related Work
 

{benchmark_reliability-0.2.1 → benchmark_reliability-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "benchmark-reliability"
-version = "0.2.1"
+version = "0.3.0"
 description = "Benchmark Reliability Framework (BRF) - dataset-level reliability auditing with built-in benchmark registry"
 readme = "README.md"
 license = { text = "MIT" }
@@ -37,14 +37,18 @@ dependencies = [
 brf = "brf.cli:main"
 
 [project.urls]
-Homepage = "https://github.com/zhanglizhuo/BenchmarkReliability"
-Repository = "https://github.com/zhanglizhuo/BenchmarkReliability"
+Homepage = "https://github.com/zhanglizhuo/BRFPackage"
+Repository = "https://github.com/zhanglizhuo/BRFPackage"
 
 [tool.setuptools]
 license-files = []
 
 [tool.setuptools.packages.find]
 where = ["src"]
+include = ["brf", "brf.metrics", "brf.phase", "brf.registry", "brf.registry.sources", "brf.report"]
 
 [tool.setuptools.package-data]
-"brf.registry" = ["manifest.yaml"]
+"brf.registry" = ["manifest.yaml", "registry_v2.1.json"]
+
+[tool.setuptools.exclude-package-data]
+"brf.registry" = ["cache/*", "cache/**/*"]

{benchmark_reliability-0.2.1 → benchmark_reliability-0.3.0}/setup.py

@@ -2,15 +2,16 @@ from setuptools import setup, find_packages
 
 setup(
     name="benchmark-reliability",
-    version="0.2.1",
+    version="0.3.0",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     package_data={
         "brf.registry": ["manifest.yaml"],
     },
     install_requires=[
-        "numpy>=1.18",
-        "scikit-learn>=0.24",
+        "numpy>=1.21",
+        "scikit-learn>=1.0",
+        "matplotlib>=3.5",
         "pandas>=1.1",
         "scipy>=1.5",
         "openml>=0.12",

{benchmark_reliability-0.2.1 → benchmark_reliability-0.3.0}/src/benchmark_reliability.egg-info/PKG-INFO

@@ -1,11 +1,11 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: benchmark-reliability
-Version: 0.2.1
+Version: 0.3.0
 Summary: Benchmark Reliability Framework (BRF) - dataset-level reliability auditing with built-in benchmark registry
 Author-email: zhanglizhuo <zhanglizhuo@gmail.com>
 License: MIT
-Project-URL: Homepage, https://github.com/zhanglizhuo/BenchmarkReliability
-Project-URL: Repository, https://github.com/zhanglizhuo/BenchmarkReliability
+Project-URL: Homepage, https://github.com/zhanglizhuo/BRFPackage
+Project-URL: Repository, https://github.com/zhanglizhuo/BRFPackage
 Keywords: benchmark reliability,dataset auditing,educational AI,machine learning,registry
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
@@ -65,9 +65,9 @@ print(analyzer.diagnose()["summary"])
 for dim, issue in analyzer.diagnose()["details"].items():
     print(f"  {dim}: {issue}")
 
-# Percentile rank against 25 audited benchmarks
+# Percentile rank against the 51 benchmarks in the BRF Registry (v2.1 reference)
 print(analyzer.rank())
-# {'S_percentile': 16.0, 'E_percentile': 36.0, ...}
+# {'S_percentile': 17.6, 'E_percentile': 27.5, 'reference': 'BRF Registry v2.1 (51 benchmarks)'}
 
 # One-paragraph recommendation
 print(analyzer.recommend())
@@ -77,7 +77,7 @@ print(analyzer.recommend())
 
 ```bash
 $ brf registry list
-BRF Registry -- 16 datasets
+BRF Registry
 
   assistments          ASSISTments 2009-2010           N= 3729  G= 124
   college_scorecard    US College Scorecard            N= 7804  G=  59
@@ -95,7 +95,7 @@ BRF Audit: Teaching Assistant Evaluation (tae)
 ### Download and verify all datasets
 
 ```bash
-$ brf registry sync     # download + SHA-256 verify all 16 datasets
+$ brf registry sync     # download + SHA-256 verify all
 $ brf registry info oulad
   name: oulad
   display_name: Open University Learning Analytics Dataset
@@ -122,7 +122,7 @@ communication shorthand only --- **the signal is in the continuous values**.
 
 Use `analyzer.diagnose()` for per-dimension explanations and actionable
 recommendations, or `analyzer.rank()` to see percentile scores against
-the 25 benchmarks in the BRF Registry.
+the 51 benchmarks in the BRF Registry (v2.1 reference).
 
 ## CLI Reference
 
@@ -152,8 +152,8 @@ To cite the BRF framework and package (JOSS paper forthcoming):
 @software{zhang2026brf,
   author = {Lizhuo Zhang},
   title = {benchmark-reliability: Benchmark Reliability Framework},
-  url = {https://github.com/zhanglizhuo/BenchmarkReliability},
-  version = {0.1.5},
+  url = {https://github.com/zhanglizhuo/BRFPackage},
+  version = {0.3.0},
   year = {2026},
 }
 ```
@@ -162,7 +162,7 @@ The behavior audit protocol is described in:
 
 > Zhang, L. *BehaviorAudit: a four-dimension protocol for auditing
 > benchmark reliability under group-aware evaluation.*
-> Scientific Reports (under review).
+> Scientific Reports (2026). https://doi.org/10.1038/s41598-026-69629-6
 
 ## Related Work
 

{benchmark_reliability-0.2.1 → benchmark_reliability-0.3.0}/src/benchmark_reliability.egg-info/SOURCES.txt

@@ -22,24 +22,54 @@ src/brf/phase/visualization.py
 src/brf/registry/__init__.py
 src/brf/registry/cli.py
 src/brf/registry/manifest.yaml
+src/brf/registry/registry_v2.1.json
 src/brf/registry/verify.py
 src/brf/registry/sources/__init__.py
+src/brf/registry/sources/abalone.py
+src/brf/registry/sources/air_quality_uci.py
 src/brf/registry/sources/assistments.py
+src/brf/registry/sources/auto_mpg.py
+src/brf/registry/sources/boston_housing.py
+src/brf/registry/sources/climate_weather.py
 src/brf/registry/sources/college_scorecard.py
 src/brf/registry/sources/colleges_aaup.py
 src/brf/registry/sources/colleges_usnews.py
+src/brf/registry/sources/cpu_act.py
+src/brf/registry/sources/credit_card.py
+src/brf/registry/sources/cross_domain_batch.py
+src/brf/registry/sources/customer_churn.py
+src/brf/registry/sources/electricity.py
+src/brf/registry/sources/energy_building.py
+src/brf/registry/sources/energy_efficiency.py
 src/brf/registry/sources/entrance_exam.py
+src/brf/registry/sources/external_validation.py
+src/brf/registry/sources/german_credit.py
 src/brf/registry/sources/higher_ed.py
+src/brf/registry/sources/kaggle_students_performance.py
+src/brf/registry/sources/kdd_cup_2010.py
+src/brf/registry/sources/law_school.py
 src/brf/registry/sources/mathe.py
 src/brf/registry/sources/mm_tba.py
+src/brf/registry/sources/nursery.py
 src/brf/registry/sources/oli.py
+src/brf/registry/sources/olympics.py
 src/brf/registry/sources/oulad.py
+src/brf/registry/sources/pisa2015.py
+src/brf/registry/sources/pollution.py
+src/brf/registry/sources/real_estate.py
+src/brf/registry/sources/seoul_bike.py
+src/brf/registry/sources/student_absences.py
 src/brf/registry/sources/student_depression.py
 src/brf/registry/sources/student_dropout.py
+src/brf/registry/sources/student_health.py
+src/brf/registry/sources/students_exam_scores.py
 src/brf/registry/sources/tae.py
 src/brf/registry/sources/turkiye.py
 src/brf/registry/sources/uci_student.py
+src/brf/registry/sources/uci_student_math.py
+src/brf/registry/sources/wine_quality.py
 src/brf/registry/sources/xapi_edu.py
+src/brf/registry/sources/yacht.py
 src/brf/report/__init__.py
 src/brf/report/json_export.py
 src/brf/report/latex_export.py

{benchmark_reliability-0.2.1 → benchmark_reliability-0.3.0}/src/brf/__init__.py

@@ -1,6 +1,8 @@
 from .analyzer import BRFAnalyzer
 
-__all__ = ["BRFAnalyzer"]
+__version__ = "0.3.0"
+
+__all__ = ["BRFAnalyzer", "__version__"]
 
 # Registry is lazily imported to avoid circular dependencies
 # Use: from brf.registry import REGISTRY_SOURCES, list_sources

{benchmark_reliability-0.2.1 → benchmark_reliability-0.3.0}/src/brf/analyzer.py

@@ -27,7 +27,7 @@ class BRFAnalyzer:
             raise ValueError("n_splits must be >= 2")
         self.n_splits = n_splits
         self.n_permutations = n_permutations
-        self.model = model or Ridge(alpha=1.0)
+        self.model = model if model is not None else Ridge(alpha=1.0)
         self.seed = seed
         self.scale = scale
 
@@ -251,10 +251,10 @@ class BRFAnalyzer:
         }
 
     def rank(self) -> Dict[str, float]:
-        """Percentile rank of S and E against the BRF Registry v1.5 benchmarks.
+        """Percentile rank of S and E against the BRF Registry v2.1 benchmarks.
 
         Returns percentiles (0-100) indicating where this dataset's S and E
-        fall relative to the 25 audited benchmarks. Requires the registry
+        fall relative to the audited benchmarks in the bundled Registry reference (v2.1). Requires
         data to be accessible.
         """
         if not self._fitted:
@@ -273,7 +273,7 @@ class BRFAnalyzer:
         return {
             "S_percentile": round(pctile(s_vals, self.S), 1),
             "E_percentile": round(pctile(e_vals, self.E), 1),
-            "reference": f"BRF Registry v1.5 ({len(s_vals)} benchmarks)",
+            "reference": f"BRF Registry v2.1 ({len(s_vals)} benchmarks)",
         }
 
     def recommend(self, n_samples: Optional[int] = None,
@@ -316,12 +316,9 @@ class BRFAnalyzer:
         """Load Registry reference data for percentile ranking."""
         if self._registry_ref is not None:
             return self._registry_ref
-        # Search for registry_v1.5.json in known locations
+        # Bundled registry reference (ships with the package at brf/registry/)
         candidates = [
-            Path(__file__).parent.parent.parent.parent
-            / "BRFRegistry" / "results" / "registry_v1.5.json",
-            Path(__file__).parent.parent.parent
-            / "BRFRegistry" / "results" / "registry_v1.5.json",
+            Path(__file__).parent / "registry" / "registry_v2.1.json",
         ]
         for p in candidates:
             if p.exists():

{benchmark_reliability-0.2.1 → benchmark_reliability-0.3.0}/src/brf/registry/cli.py

@@ -1,38 +1,11 @@
-"""BRF Registry CLI -- download, verify, and prepare datasets.
-
-Usage::
-
-    # List all datasets
-    python -m registry.cli list
-
-    # Download a dataset (with verification)
-    python -m registry.cli download tae
-    python -m registry.cli download --all
-
-    # Verify checksums
-    python -m registry.cli verify tae
-    python -m registry.cli verify --all
-
-    # Sync (download + verify all)
-    python -m registry.cli sync
-
-    # Show metadata
-    python -m registry.cli info tae
-"""
-
-from __future__ import annotations
-
 import argparse
 from pathlib import Path
 
-import sys
-
 from .sources import REGISTRY_SOURCES, list_sources
 from .verify import verify_dataset, verify_all
 
 
 def cmd_list():
-    """List all registered datasets."""
     print(f"BRF Registry -- {len(list_sources())} datasets\n")
     for key in list_sources():
         source = REGISTRY_SOURCES[key]
@@ -40,16 +13,11 @@ def cmd_list():
               f"N={source.n_samples:>5d}  G={source.n_groups:>4d}")
 
 
-def cmd_download(key: str = None, all_: bool = False):
-    """Download dataset(s)."""
-    if all_:
-        keys = list_sources()
-    elif key:
-        keys = [key]
-    else:
+def cmd_download(key=None, all_=False):
+    keys = list_sources() if all_ else ([key] if key else [])
+    if not keys:
         print("Usage: download <key> or download --all")
         return
-
     for k in keys:
         source = REGISTRY_SOURCES.get(k)
         if source is None:
@@ -59,17 +27,14 @@ def cmd_download(key: str = None, all_: bool = False):
         try:
             path = source.download()
             print(f"  -> {path}")
-            # Verify after download
-            if source.sha256:
-                ok = source._check_sha256(path) if isinstance(path, Path) else True
-                if not ok:
+            if source.sha256 and isinstance(path, Path):
+                if not source._check_sha256(path):
                     print(f"  WARNING: SHA-256 mismatch!")
         except Exception as e:
             print(f"  FAILED: {e}")
 
 
-def cmd_verify(key: str = None, all_: bool = False):
-    """Verify dataset checksums."""
+def cmd_verify(key=None, all_=False):
     if all_:
         verify_all()
     elif key:
@@ -79,7 +44,6 @@ def cmd_verify(key: str = None, all_: bool = False):
 
 
 def cmd_sync():
-    """Download + verify all datasets."""
     print("Syncing all datasets...")
     for key in list_sources():
         source = REGISTRY_SOURCES[key]
@@ -87,45 +51,38 @@ def cmd_sync():
         try:
             path = source.download()
             print(f"    Downloaded to {path}")
-            if source.sha256:
-                ok = source._check_sha256(path) if isinstance(path, Path) else True
+            if source.sha256 and isinstance(path, Path):
+                ok = source._check_sha256(path)
                 print(f"    Checksum: {'OK' if ok else 'FAIL'}")
         except Exception as e:
             print(f"    FAILED: {e}")
 
 
-def cmd_info(key: str):
-    """Show dataset metadata."""
+def cmd_info(key):
     source = REGISTRY_SOURCES.get(key)
     if source is None:
         print(f"Unknown dataset: {key}")
         return
-    meta = source.metadata()
-    for k, v in meta.items():
+    for k, v in source.metadata().items():
         print(f"  {k}: {v}")
 
 
 def main():
     parser = argparse.ArgumentParser(description="BRF Registry CLI")
-    sub = parser.add_subparsers(dest="command", help="Command")
-
-    sub.add_parser("list", help="List all datasets")
-
-    dl = sub.add_parser("download", help="Download dataset(s)")
-    dl.add_argument("key", nargs="?", help="Dataset key")
-    dl.add_argument("--all", action="store_true", help="Download all")
-
-    vf = sub.add_parser("verify", help="Verify checksums")
-    vf.add_argument("key", nargs="?", help="Dataset key")
-    vf.add_argument("--all", action="store_true", help="Verify all")
-
-    sub.add_parser("sync", help="Download + verify all datasets")
-
-    info = sub.add_parser("info", help="Show dataset metadata")
-    info.add_argument("key", help="Dataset key")
+    sub = parser.add_subparsers(dest="command")
+
+    sub.add_parser("list")
+    dl = sub.add_parser("download")
+    dl.add_argument("key", nargs="?")
+    dl.add_argument("--all", action="store_true")
+    vf = sub.add_parser("verify")
+    vf.add_argument("key", nargs="?")
+    vf.add_argument("--all", action="store_true")
+    sub.add_parser("sync")
+    info = sub.add_parser("info")
+    info.add_argument("key")
 
     args = parser.parse_args()
-
     if args.command == "list":
         cmd_list()
     elif args.command == "download":

benchmark_reliability-0.3.0/src/brf/registry/manifest.yaml

@@ -0,0 +1,66 @@
+brf_results:
+  fragile: 0
+  reliable: 42
+  void: 9
+datasets:
+- abalone
+- air_quality
+- airfoil
+- assistments
+- auto_mpg
+- bike_sharing
+- boston_housing
+- breast_cancer
+- california_housing
+- ccpp
+- college_scorecard
+- colleges_aaup
+- colleges_usnews
+- concrete
+- cpu_act
+- credit_card
+- customer_churn
+- diabetes
+- energy_building
+- energy_efficiency
+- entrance_exam
+- german_credit
+- higher_ed
+- kaggle_students_performance
+- kdd_cup_2010
+- law_school
+- linnerud
+- mathe
+- mm_tba
+- nursery
+- oli
+- oulad
+- pisa2015
+- pollution
+- real_estate
+- climate_weather
+- seoul_bike
+- olympics
+- student_absences
+- student_depression
+- student_dropout
+- student_health
+- students_exam_scores
+- electricity
+- tae
+- turkiye
+- uci_student
+- uci_student_math
+- wine_quality
+- xapi_edu
+- yacht
+description: 'Curated collection of group-aware benchmark datasets across education,
+  engineering, transportation, energy, healthcare, finance, environmental, business,
+  sports, and fitness domains with standardized metadata, reproducibility scripts,
+  and BRF (Benchmark Reliability Framework) audit results. v2.1 expands from 35 to
+  51 datasets. All source modules are self-contained.
+
+  '
+release_date: '2026-09-10'
+total_datasets: 51
+version: '2.1'