benchmark-reliability 0.3.0__tar.gz → 0.3.2__tar.gz

benchmark_reliability-0.3.2/MANIFEST.in

@@ -0,0 +1,13 @@
+# verification/ holds the archived output of
+#   python scripts/compute_prepare_hashes.py --verify
+# under two interpreters, plus the superseded 3.8 run that motivated the
+# canonical content hash. It is evidence for the prepare-level digests rather
+# than package code, so it lives outside src/brf and is included here rather
+# than via package-data.
+include verification/*.txt
+include verification/*.md
+
+# The alternate reference style stays available for reuse of the class file
+# even though the manuscript selects sn-nature.
+include sn-nature.bst
+include sn-mathphys-num.bst

{benchmark_reliability-0.3.0 → benchmark_reliability-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchmark-reliability
-Version: 0.3.0
+Version: 0.3.2
 Summary: Benchmark Reliability Framework (BRF) - dataset-level reliability auditing with built-in benchmark registry
 Author-email: zhanglizhuo <zhanglizhuo@gmail.com>
 License: MIT
@@ -24,6 +24,10 @@ Requires-Dist: matplotlib>=3.5
 Requires-Dist: pandas>=1.1
 Requires-Dist: scipy>=1.5
 Requires-Dist: openml>=0.12
+Provides-Extra: full
+Requires-Dist: kagglehub>=0.1; extra == "full"
+Requires-Dist: openpyxl>=3.1; extra == "full"
+Requires-Dist: pyreadstat>=1.2; extra == "full"
 
 # benchmark-reliability
 

{benchmark_reliability-0.3.0 → benchmark_reliability-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "benchmark-reliability"
-version = "0.3.0"
+version = "0.3.2"
 description = "Benchmark Reliability Framework (BRF) - dataset-level reliability auditing with built-in benchmark registry"
 readme = "README.md"
 license = { text = "MIT" }
@@ -33,6 +33,21 @@ dependencies = [
     "openml>=0.12",
 ]
 
+# Seven of the 51 datasets need a reader that is not part of the core stack:
+# kagglehub for 5 Kaggle-hosted sources, pyreadstat for the PISA 2015 .sav
+# files, openpyxl for the Taiwan .xlsx files. They are optional rather than
+# required because pyreadstat needs the system-level libreadstat, which would
+# otherwise be imposed on every install for the sake of 44 datasets.
+# pyreadstat also needs libreadstat present on the system, e.g.
+#   Debian/Ubuntu: apt-get install libreadstat-dev
+#   macOS:         brew install readstat
+[project.optional-dependencies]
+full = [
+    "kagglehub>=0.1",
+    "openpyxl>=3.1",
+    "pyreadstat>=1.2",
+]
+
 [project.scripts]
 brf = "brf.cli:main"
 
@@ -48,7 +63,8 @@ where = ["src"]
 include = ["brf", "brf.metrics", "brf.phase", "brf.registry", "brf.registry.sources", "brf.report"]
 
 [tool.setuptools.package-data]
-"brf.registry" = ["manifest.yaml", "registry_v2.1.json"]
+"brf.registry" = ["manifest.yaml", "registry_v2.1.json", "prepare_hashes.json",
+                    "taxonomy.yaml", "version_policy.yaml", "cards/*.yaml"]
 
 [tool.setuptools.exclude-package-data]
 "brf.registry" = ["cache/*", "cache/**/*"]

{benchmark_reliability-0.3.0 → benchmark_reliability-0.3.2}/src/benchmark_reliability.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchmark-reliability
-Version: 0.3.0
+Version: 0.3.2
 Summary: Benchmark Reliability Framework (BRF) - dataset-level reliability auditing with built-in benchmark registry
 Author-email: zhanglizhuo <zhanglizhuo@gmail.com>
 License: MIT
@@ -24,6 +24,10 @@ Requires-Dist: matplotlib>=3.5
 Requires-Dist: pandas>=1.1
 Requires-Dist: scipy>=1.5
 Requires-Dist: openml>=0.12
+Provides-Extra: full
+Requires-Dist: kagglehub>=0.1; extra == "full"
+Requires-Dist: openpyxl>=3.1; extra == "full"
+Requires-Dist: pyreadstat>=1.2; extra == "full"
 
 # benchmark-reliability
 

{benchmark_reliability-0.3.0 → benchmark_reliability-0.3.2}/src/benchmark_reliability.egg-info/SOURCES.txt

@@ -1,3 +1,4 @@
+MANIFEST.in
 README.md
 pyproject.toml
 setup.py
@@ -22,8 +23,62 @@ src/brf/phase/visualization.py
 src/brf/registry/__init__.py
 src/brf/registry/cli.py
 src/brf/registry/manifest.yaml
+src/brf/registry/prepare_hashes.json
 src/brf/registry/registry_v2.1.json
+src/brf/registry/taxonomy.yaml
 src/brf/registry/verify.py
+src/brf/registry/version_policy.yaml
+src/brf/registry/cards/abalone.yaml
+src/brf/registry/cards/air_quality.yaml
+src/brf/registry/cards/airfoil.yaml
+src/brf/registry/cards/assistments.yaml
+src/brf/registry/cards/auto_mpg.yaml
+src/brf/registry/cards/bike_sharing.yaml
+src/brf/registry/cards/boston_housing.yaml
+src/brf/registry/cards/breast_cancer.yaml
+src/brf/registry/cards/california_housing.yaml
+src/brf/registry/cards/ccpp.yaml
+src/brf/registry/cards/climate_weather.yaml
+src/brf/registry/cards/college_scorecard.yaml
+src/brf/registry/cards/colleges_aaup.yaml
+src/brf/registry/cards/colleges_usnews.yaml
+src/brf/registry/cards/concrete.yaml
+src/brf/registry/cards/cpu_act.yaml
+src/brf/registry/cards/credit_card.yaml
+src/brf/registry/cards/customer_churn.yaml
+src/brf/registry/cards/diabetes.yaml
+src/brf/registry/cards/electricity.yaml
+src/brf/registry/cards/energy_building.yaml
+src/brf/registry/cards/energy_efficiency.yaml
+src/brf/registry/cards/entrance_exam.yaml
+src/brf/registry/cards/german_credit.yaml
+src/brf/registry/cards/higher_ed.yaml
+src/brf/registry/cards/kaggle_students_performance.yaml
+src/brf/registry/cards/kdd_cup_2010.yaml
+src/brf/registry/cards/law_school.yaml
+src/brf/registry/cards/linnerud.yaml
+src/brf/registry/cards/mathe.yaml
+src/brf/registry/cards/mm_tba.yaml
+src/brf/registry/cards/nursery.yaml
+src/brf/registry/cards/oli.yaml
+src/brf/registry/cards/olympics.yaml
+src/brf/registry/cards/oulad.yaml
+src/brf/registry/cards/pisa2015.yaml
+src/brf/registry/cards/pollution.yaml
+src/brf/registry/cards/real_estate.yaml
+src/brf/registry/cards/seoul_bike.yaml
+src/brf/registry/cards/student_absences.yaml
+src/brf/registry/cards/student_depression.yaml
+src/brf/registry/cards/student_dropout.yaml
+src/brf/registry/cards/student_health.yaml
+src/brf/registry/cards/students_exam_scores.yaml
+src/brf/registry/cards/tae.yaml
+src/brf/registry/cards/turkiye.yaml
+src/brf/registry/cards/uci_student.yaml
+src/brf/registry/cards/uci_student_math.yaml
+src/brf/registry/cards/wine_quality.yaml
+src/brf/registry/cards/xapi_edu.yaml
+src/brf/registry/cards/yacht.yaml
 src/brf/registry/sources/__init__.py
 src/brf/registry/sources/abalone.py
 src/brf/registry/sources/air_quality_uci.py
@@ -75,4 +130,8 @@ src/brf/report/json_export.py
 src/brf/report/latex_export.py
 tests/test_analyzer.py
 tests/test_metrics.py
-tests/test_phase.py
+tests/test_phase.py
+verification/README.md
+verification/prepare_verify_py313.txt
+verification/prepare_verify_py38.txt
+verification/prepare_verify_py38_legacy.txt

{benchmark_reliability-0.3.0 → benchmark_reliability-0.3.2}/src/benchmark_reliability.egg-info/requires.txt

@@ -4,3 +4,8 @@ matplotlib>=3.5
 pandas>=1.1
 scipy>=1.5
 openml>=0.12
+
+[full]
+kagglehub>=0.1
+openpyxl>=3.1
+pyreadstat>=1.2

{benchmark_reliability-0.3.0 → benchmark_reliability-0.3.2}/src/brf/__init__.py

@@ -1,6 +1,6 @@
 from .analyzer import BRFAnalyzer
 
-__version__ = "0.3.0"
+__version__ = "0.3.2"
 
 __all__ = ["BRFAnalyzer", "__version__"]
 

benchmark_reliability-0.3.2/src/brf/registry/cards/abalone.yaml

@@ -0,0 +1,35 @@
+name: abalone
+key: abalone
+version: '1.0'
+source:
+  url: https://archive.ics.uci.edu/static/public/1/abalone.zip
+  repository: UCI
+  license: CC BY 4.0
+collection:
+  year: ''
+  country: null
+task:
+  type: regression
+  target: continuous
+features:
+  count: 7
+samples:
+  count: 4177
+grouping:
+  variable: Sex (M/F/I, 3 groups)
+  rationale: ''
+  count: 3
+education:
+  level: N/A (Non-Edu)
+  domain: Marine Biology
+reference: Nash et al. (1995); UCI ID 1
+notes: null
+brf_compatibility:
+  S: 0.9292
+  E: 1.4792
+  class: Reliable
+  audited_version: v2.0
+  note: BRF protocol is under review; scores are provisional. BRF class is communication shorthand only. Use continuous S
+    and E values for analysis.
+integrity:
+  sha256: a2daaed9c48ef860360f6438a224a2fde2576f1ebcf4f821ffe23645c229b23e

benchmark_reliability-0.3.2/src/brf/registry/cards/air_quality.yaml

@@ -0,0 +1,35 @@
+name: air_quality
+key: air_quality
+version: '2.0'
+source:
+  url: https://github.com/Gauhar1107/AirQualityUCI/blob/master/AirQualityUCI.csv
+  repository: GitHub
+  license: UCI ML Repository (CC BY 4.0)
+collection:
+  year: ''
+  country: International
+task:
+  type: regression
+  target: continuous
+features:
+  count: 4
+samples:
+  count: 37424
+grouping:
+  variable: "Pollutant (4: CO, NMHC, NOx, NO2)"
+  rationale: ''
+  count: 4
+education:
+  level: N/A (Non-Edu)
+  domain: Environmental Science
+reference: UCI 'Air Quality' (Smola et al. 2011); github.com/Gauhar1107/AirQualityUCI
+notes: "Environmental: cross-pollutant sensor calibration (GT from PT08 + meteorology)."
+brf_compatibility:
+  S: 0.863
+  E: 1.0211
+  class: Reliable
+  audited_version: v2.1
+  note: BRF protocol is under review; scores are provisional. BRF class is communication shorthand only. Use continuous S
+    and E values for analysis.
+integrity:
+  sha256: b0aad84a1f735f2e98359a0ac6f7e6730d0d7dccd27db1fa6ae3e07388159a97

benchmark_reliability-0.3.2/src/brf/registry/cards/airfoil.yaml

@@ -0,0 +1,35 @@
+name: airfoil
+key: airfoil
+version: '1.0'
+source:
+  url: null
+  repository: Public
+  license: CC BY 4.0
+collection:
+  year: ''
+  country: null
+task:
+  type: regression
+  target: continuous
+features:
+  count: 4
+samples:
+  count: 1503
+grouping:
+  variable: Frequency bins (low<2kHz, mid<8kHz, high)
+  rationale: ''
+  count: 3
+education:
+  level: N/A (Non-Edu)
+  domain: Aerospace
+reference: Brooks et al. (1989); UCI ID 291
+notes: null
+brf_compatibility:
+  S: 0.7601
+  E: 0.7802
+  class: Reliable
+  audited_version: v2.0
+  note: BRF protocol is under review; scores are provisional. BRF class is communication shorthand only. Use continuous S
+    and E values for analysis.
+integrity:
+  sha256: fdd4a50b90f2b56d4d69297770a375c1cfa6ee781a3dfb7741af59764baa4191

benchmark_reliability-0.3.2/src/brf/registry/cards/assistments.yaml

@@ -0,0 +1,35 @@
+name: assistments
+key: assistments
+version: '1.0'
+source:
+  url: http://base.ustc.edu.cn/data/ASSISTment/2009_skill_builder_data_corrected.zip
+  repository: USTC Datashop
+  license: TBD (public research data)
+collection:
+  year: 2009-2010
+  country: US
+task:
+  type: ''
+  target: continuous (overall accuracy)
+features:
+  count: 5
+samples:
+  count: 3729
+grouping:
+  variable: Teacher
+  rationale: Teacher determines curriculum and instruction quality
+  count: 124
+education:
+  level: Secondary
+  domain: Intelligent Tutoring
+reference: Feng, Heffernan & Koedinger (2009)
+notes: Skill builder corrected. Student-level aggregates of 328K transactions. Teacher as group.
+brf_compatibility:
+  S: 0.953
+  E: 0.946
+  class: Reliable
+  audited_version: v1.5
+  note: BRF protocol is under review; scores are provisional. BRF class is communication shorthand only. Use continuous S
+    and E values for analysis.
+integrity:
+  sha256: 1aa296e00b6c88c4d6fad4ca2ae4866484d9fe5484f38f5c8c94dfc49f045e08

benchmark_reliability-0.3.2/src/brf/registry/cards/auto_mpg.yaml

@@ -0,0 +1,35 @@
+name: auto_mpg
+key: auto_mpg
+version: '1.0'
+source:
+  url: https://archive.ics.uci.edu/static/public/9/auto+mpg.zip
+  repository: UCI
+  license: CC BY 4.0
+collection:
+  year: ''
+  country: null
+task:
+  type: regression
+  target: continuous
+features:
+  count: 6
+samples:
+  count: 392
+grouping:
+  variable: Origin (1=US, 2=Europe, 3=Japan)
+  rationale: ''
+  count: 3
+education:
+  level: N/A (Non-Edu)
+  domain: Automotive
+reference: Quinlan (1993); UCI ID 9
+notes: null
+brf_compatibility:
+  S: 0.9586
+  E: 1.4366
+  class: Reliable
+  audited_version: v2.0
+  note: BRF protocol is under review; scores are provisional. BRF class is communication shorthand only. Use continuous S
+    and E values for analysis.
+integrity:
+  sha256: 15f00c8a120c8a86ac9faa16d47115c3e103764549a8c3c2fd6d5de31063636a

benchmark_reliability-0.3.2/src/brf/registry/cards/bike_sharing.yaml

@@ -0,0 +1,35 @@
+name: bike_sharing
+key: bike_sharing
+version: '1.0'
+source:
+  url: 
+  repository: Public
+  license: CC BY 4.0
+collection:
+  year: ''
+  country: 
+task:
+  type: regression
+  target: continuous
+features:
+  count: 10
+samples:
+  count: 731
+grouping:
+  variable: "Season (4: spring/summer/fall/winter)"
+  rationale: ''
+  count: 4
+education:
+  level: N/A (Non-Edu)
+  domain: Transportation
+reference: Fanaee-T & Gama (2013); UCI ID 275
+notes: 
+brf_compatibility:
+  S: 0.9618
+  E: 1.7664
+  class: Reliable
+  audited_version: v2.0
+  note: BRF protocol is under review; scores are provisional. Communication shorthand only. Use continuous S and E values for
+    analysis.
+integrity:
+  sha256: 75c2c56f116c9ebbd9bcb06087c2b7fffc7b8f8225dd37d388aa7e1996cf19dc

benchmark_reliability-0.3.2/src/brf/registry/cards/boston_housing.yaml

@@ -0,0 +1,33 @@
+name: boston_housing
+key: boston_housing
+version: '1.0'
+source:
+  url: https://www.openml.org/d/531
+  repository: OpenML
+  license: Public domain
+collection:
+  year: ''
+  country: US
+task:
+  type: regression
+  target: continuous
+features:
+  count: 13
+samples:
+  count: 506
+grouping:
+  variable: Region (4 regions)
+  rationale: ''
+  count: 4
+education:
+  level: N/A (Non-Edu)
+  domain: Real Estate
+reference: OpenML 531
+notes: null
+brf_compatibility:
+  S: 0.9164
+  E: 1.7186
+  class: Reliable
+  audited_version: v2.1
+  note: BRF protocol is under review; scores are provisional. BRF class is communication shorthand only. Use continuous S
+    and E values for analysis.

benchmark_reliability-0.3.2/src/brf/registry/cards/breast_cancer.yaml

@@ -0,0 +1,33 @@
+name: breast_cancer
+key: breast_cancer
+version: '1.0'
+source:
+  url: sklearn.datasets.load_breast_cancer
+  repository: scikit-learn
+  license: sklearn (public domain)
+collection:
+  year: ''
+  country: US
+task:
+  type: regression
+  target: continuous
+features:
+  count: 29
+samples:
+  count: 569
+grouping:
+  variable: "Diagnosis (2: malignant/benign)"
+  rationale: ''
+  count: 2
+education:
+  level: N/A (Non-Edu)
+  domain: Healthcare
+reference: sklearn; UCI Breast Cancer Wisconsin
+notes: "Biomedical: tumor size as regression. Group: diagnosis class."
+brf_compatibility:
+  S: 0.9999
+  E: 1.857
+  class: Reliable
+  audited_version: v2.1
+  note: BRF protocol is under review; scores are provisional. BRF class is communication shorthand only. Use continuous S
+    and E values for analysis.

benchmark_reliability-0.3.2/src/brf/registry/cards/california_housing.yaml

@@ -0,0 +1,33 @@
+name: california_housing
+key: california_housing
+version: '1.0'
+source:
+  url: sklearn.datasets.fetch_california_housing
+  repository: scikit-learn
+  license: sklearn (public domain)
+collection:
+  year: ''
+  country: US
+task:
+  type: regression
+  target: continuous
+features:
+  count: 8
+samples:
+  count: 20640
+grouping:
+  variable: Region (71 lat/lon bins)
+  rationale: ''
+  count: 71
+education:
+  level: N/A (Non-Edu)
+  domain: Real Estate
+reference: sklearn; California Census 1990
+notes: "Real Estate: median house value. Group: geographic region."
+brf_compatibility:
+  S: 0.9794
+  E: 0.9294
+  class: Reliable
+  audited_version: v2.1
+  note: BRF protocol is under review; scores are provisional. BRF class is communication shorthand only. Use continuous S
+    and E values for analysis.

benchmark_reliability-0.3.2/src/brf/registry/cards/ccpp.yaml

@@ -0,0 +1,35 @@
+name: ccpp
+key: ccpp
+version: '1.0'
+source:
+  url: 
+  repository: Public
+  license: CC BY 4.0
+collection:
+  year: ''
+  country: 
+task:
+  type: regression
+  target: continuous
+features:
+  count: 3
+samples:
+  count: 9568
+grouping:
+  variable: Temperature bins (cold<15C, mild<25C, hot)
+  rationale: ''
+  count: 3
+education:
+  level: N/A (Non-Edu)
+  domain: Energy
+reference: Tufekci (2014); UCI ID 294
+notes: 
+brf_compatibility:
+  S: 0.9928
+  E: 1.7385
+  class: Reliable
+  audited_version: v2.0
+  note: BRF protocol is under review; scores are provisional. Communication shorthand only. Use continuous S and E values for
+    analysis.
+integrity:
+  sha256: 51848da3c8b8c29f709f159fad95443eaa8ff2e95c721a00bba582048cd207e4

benchmark_reliability-0.3.2/src/brf/registry/cards/climate_weather.yaml

@@ -0,0 +1,35 @@
+name: climate_weather
+key: climate_weather
+version: '2.0'
+source:
+  url: https://www.openml.org/d/40918
+  repository: OpenML
+  license: OpenML (public domain)
+collection:
+  year: ''
+  country: International
+task:
+  type: regression
+  target: continuous
+features:
+  count: 3
+samples:
+  count: 544811
+grouping:
+  variable: Country (242 countries)
+  rationale: ''
+  count: 242
+education:
+  level: N/A (Non-Edu)
+  domain: Climate Science
+reference: OpenML ID 40918 (Climate); NOAA Global Historical Weather Summary
+notes: "Climate: average temperature. Group: country of origin."
+brf_compatibility:
+  S: 0.9871
+  E: 0.9712
+  class: Reliable
+  audited_version: v2.1
+  note: BRF protocol is under review; scores are provisional. BRF class is communication shorthand only. Use continuous S
+    and E values for analysis.
+integrity:
+  sha256: c83f266cf7b96dc90d8c10e271220078008d9a92b4d2fc73d7bcf77722d5b2e1

benchmark_reliability-0.3.2/src/brf/registry/cards/college_scorecard.yaml

@@ -0,0 +1,35 @@
+name: college_scorecard
+key: college_scorecard
+version: '1.0'
+source:
+  url: https://www.openml.org/search?type=data&id=42121
+  repository: OpenML
+  license: Public Domain (OpenML)
+collection:
+  year: 2015-2016
+  country: US
+task:
+  type: ''
+  target: continuous (admission rate)
+features:
+  count: 30
+samples:
+  count: 2220
+grouping:
+  variable: State
+  rationale: State determines funding, policy, and regulatory environment
+  count: 55
+education:
+  level: Mixed
+  domain: Institutional Outcomes
+reference: US Dept of Education; OpenML ID 42121
+notes: 7804 colleges, filtered to 2220 with admission data.
+brf_compatibility:
+  S: 0.824
+  E: 0.687
+  class: Reliable
+  audited_version: v1.5
+  note: BRF protocol is under review; scores are provisional. Communication shorthand only. Use continuous S and E values for
+    analysis.
+integrity:
+  sha256: d4dca5f3d35fba46a164a1e8a59c35d643a89cbc25a216431a44913dd4cf5e00

benchmark_reliability-0.3.2/src/brf/registry/cards/colleges_aaup.yaml

@@ -0,0 +1,36 @@
+name: colleges_aaup
+key: colleges_aaup
+version: '1.0'
+source:
+  url: https://www.openml.org/search?type=data&id=488
+  repository: OpenML
+  license: Public Domain (OpenML)
+collection:
+  year: '1995'
+  country: US
+task:
+  type: ''
+  target: continuous (avg faculty salary)
+features:
+  count: 9
+samples:
+  count: 1161
+grouping:
+  variable: State
+  rationale: State determines cost of living and funding levels
+  count: 52
+education:
+  level: Mixed
+  domain: Institutional Economics
+reference: AAUP Faculty Salary Survey (OpenML ID 488)
+notes: '1161 US colleges, 52 state groups. Institutional-level. Target: average salary
+  all ranks.'
+brf_compatibility:
+  S: 0.899
+  E: 1.026
+  class: Reliable
+  audited_version: v1.5
+  note: BRF protocol is under review; scores are provisional. Communication shorthand only. Use continuous S and E values for
+    analysis.
+integrity:
+  sha256: 5eafc5406a35b84887cd3bd13721615fbb73e33cf304b77bfbe38b5e16668ace