nirs4all-benchmarks 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. nirs4all_benchmarks-0.1.5/.dockerignore +20 -0
  2. nirs4all_benchmarks-0.1.5/.editorconfig +23 -0
  3. nirs4all_benchmarks-0.1.5/.github/dependabot.yml +21 -0
  4. nirs4all_benchmarks-0.1.5/.github/workflows/ci.yml +79 -0
  5. nirs4all_benchmarks-0.1.5/.github/workflows/pages.yml +55 -0
  6. nirs4all_benchmarks-0.1.5/.github/workflows/publish.yml +51 -0
  7. nirs4all_benchmarks-0.1.5/.github/workflows/version-guard.yml +116 -0
  8. nirs4all_benchmarks-0.1.5/.gitignore +30 -0
  9. nirs4all_benchmarks-0.1.5/.pre-commit-config.yaml +20 -0
  10. nirs4all_benchmarks-0.1.5/.readthedocs.yaml +16 -0
  11. nirs4all_benchmarks-0.1.5/CHANGELOG.md +148 -0
  12. nirs4all_benchmarks-0.1.5/CITATION.cff +24 -0
  13. nirs4all_benchmarks-0.1.5/CODE_OF_CONDUCT.md +113 -0
  14. nirs4all_benchmarks-0.1.5/CONTRIBUTING.md +68 -0
  15. nirs4all_benchmarks-0.1.5/DATA_MANAGEMENT.md +289 -0
  16. nirs4all_benchmarks-0.1.5/DESIGN.md +613 -0
  17. nirs4all_benchmarks-0.1.5/Dockerfile +32 -0
  18. nirs4all_benchmarks-0.1.5/LICENSE +235 -0
  19. nirs4all_benchmarks-0.1.5/LICENSES/AGPL-3.0-or-later.txt +235 -0
  20. nirs4all_benchmarks-0.1.5/LICENSES/CC-BY-4.0.txt +156 -0
  21. nirs4all_benchmarks-0.1.5/LICENSES/COMMERCIAL-LICENSE.md +14 -0
  22. nirs4all_benchmarks-0.1.5/LICENSES/COMMERCIAL-LICENSE_FR.md +15 -0
  23. nirs4all_benchmarks-0.1.5/LICENSES/CeCILL-2.1.txt +518 -0
  24. nirs4all_benchmarks-0.1.5/LICENSING.md +55 -0
  25. nirs4all_benchmarks-0.1.5/LICENSING_FR.md +57 -0
  26. nirs4all_benchmarks-0.1.5/PERSISTENCE_FORMATS.md +453 -0
  27. nirs4all_benchmarks-0.1.5/PKG-INFO +235 -0
  28. nirs4all_benchmarks-0.1.5/README.md +172 -0
  29. nirs4all_benchmarks-0.1.5/SECURITY.md +18 -0
  30. nirs4all_benchmarks-0.1.5/THIRD_PARTY_NOTICES.md +21 -0
  31. nirs4all_benchmarks-0.1.5/VERSION +1 -0
  32. nirs4all_benchmarks-0.1.5/assets/brand/favicon.ico +0 -0
  33. nirs4all_benchmarks-0.1.5/assets/brand/horizontal-dark.svg +37 -0
  34. nirs4all_benchmarks-0.1.5/assets/brand/horizontal.png +0 -0
  35. nirs4all_benchmarks-0.1.5/assets/brand/horizontal.svg +37 -0
  36. nirs4all_benchmarks-0.1.5/assets/brand/icon-180.png +0 -0
  37. nirs4all_benchmarks-0.1.5/assets/brand/icon-256.png +0 -0
  38. nirs4all_benchmarks-0.1.5/assets/brand/icon-32.png +0 -0
  39. nirs4all_benchmarks-0.1.5/assets/brand/icon-512.png +0 -0
  40. nirs4all_benchmarks-0.1.5/assets/brand/icon.svg +33 -0
  41. nirs4all_benchmarks-0.1.5/assets/brand/og.png +0 -0
  42. nirs4all_benchmarks-0.1.5/assets/brand/stacked-dark.svg +36 -0
  43. nirs4all_benchmarks-0.1.5/assets/brand/stacked.png +0 -0
  44. nirs4all_benchmarks-0.1.5/assets/brand/stacked.svg +36 -0
  45. nirs4all_benchmarks-0.1.5/docker-compose.yml +18 -0
  46. nirs4all_benchmarks-0.1.5/docs/ADAPTERS.md +258 -0
  47. nirs4all_benchmarks-0.1.5/docs/API.md +981 -0
  48. nirs4all_benchmarks-0.1.5/docs/ARCHITECTURE.md +133 -0
  49. nirs4all_benchmarks-0.1.5/docs/CLI.md +348 -0
  50. nirs4all_benchmarks-0.1.5/docs/CONTRACT.md +530 -0
  51. nirs4all_benchmarks-0.1.5/docs/DATAVIZ.md +525 -0
  52. nirs4all_benchmarks-0.1.5/docs/DEPLOYMENT.md +368 -0
  53. nirs4all_benchmarks-0.1.5/docs/IDENTITY.md +284 -0
  54. nirs4all_benchmarks-0.1.5/docs/INGESTION.md +590 -0
  55. nirs4all_benchmarks-0.1.5/docs/PERFORMANCE.md +109 -0
  56. nirs4all_benchmarks-0.1.5/docs/ROADMAP.md +50 -0
  57. nirs4all_benchmarks-0.1.5/docs/conf.py +31 -0
  58. nirs4all_benchmarks-0.1.5/docs/index.md +25 -0
  59. nirs4all_benchmarks-0.1.5/docs/maintenance/codex_reviews/03_main_diff_review.md +15 -0
  60. nirs4all_benchmarks-0.1.5/docs/maintenance/codex_reviews/04_final_release_readiness_review.md +13 -0
  61. nirs4all_benchmarks-0.1.5/docs/maintenance/final_report.md +32 -0
  62. nirs4all_benchmarks-0.1.5/docs/maintenance/quality_gates.md +35 -0
  63. nirs4all_benchmarks-0.1.5/docs/maintenance/release_checklist.md +28 -0
  64. nirs4all_benchmarks-0.1.5/docs/maintenance/repository_audit.md +110 -0
  65. nirs4all_benchmarks-0.1.5/docs/requirements.txt +3 -0
  66. nirs4all_benchmarks-0.1.5/pyproject.toml +126 -0
  67. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/__init__.py +31 -0
  68. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/adapters/__init__.py +26 -0
  69. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/adapters/dagml_bundle.py +188 -0
  70. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/adapters/n4a_bundle.py +72 -0
  71. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/adapters/nirs4all_workspace.py +294 -0
  72. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/cli.py +295 -0
  73. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/contract/__init__.py +56 -0
  74. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/contract/arena_run_export.py +200 -0
  75. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/contract/schema/__init__.py +48 -0
  76. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/contract/schema/arena_run_export.schema.json +182 -0
  77. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/datasets/__init__.py +32 -0
  78. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/datasets/catalog.py +160 -0
  79. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/datasets/dataset_card.py +113 -0
  80. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/fixtures/__init__.py +24 -0
  81. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/fixtures/generate.py +382 -0
  82. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/identity/__init__.py +41 -0
  83. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/identity/hashing.py +128 -0
  84. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/identity/pipeline_dag.py +346 -0
  85. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/indexing.py +186 -0
  86. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/ingestion/__init__.py +34 -0
  87. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/ingestion/ingest.py +585 -0
  88. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/ingestion/pseudonymize.py +41 -0
  89. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/ingestion/report.py +45 -0
  90. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/ingestion/repository.py +122 -0
  91. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/ingestion/resolve.py +295 -0
  92. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/ingestion/upload.py +289 -0
  93. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/ingestion/validate.py +91 -0
  94. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/performance_compare.py +728 -0
  95. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/scoring/__init__.py +27 -0
  96. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/scoring/metrics.py +239 -0
  97. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/scoring/score_spec.py +55 -0
  98. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/service/__init__.py +13 -0
  99. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/service/app.py +305 -0
  100. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/site_build.py +130 -0
  101. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/store/__init__.py +20 -0
  102. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/store/arena_store.py +189 -0
  103. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/store/queries.py +777 -0
  104. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/store/residual_store.py +76 -0
  105. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/store/schema.sql +435 -0
  106. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/version.py +3 -0
  107. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/app.js +192 -0
  108. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/favicon.ico +0 -0
  109. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/horizontal-dark.svg +37 -0
  110. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/horizontal.svg +37 -0
  111. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/icon-180.png +0 -0
  112. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/icon-256.png +0 -0
  113. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/icon-32.png +0 -0
  114. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/icon-512.png +0 -0
  115. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/icon.svg +33 -0
  116. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/og.png +0 -0
  117. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/brand/stacked.svg +36 -0
  118. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/config.js +4 -0
  119. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/index.html +66 -0
  120. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/lib/api.js +57 -0
  121. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/lib/dom.js +150 -0
  122. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/lib/graph.js +88 -0
  123. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/lib/plot.js +55 -0
  124. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/lib/static-engine.js +276 -0
  125. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/styles.css +316 -0
  126. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/vendor/cytoscape.min.js +32 -0
  127. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/vendor/plotly.min.js +8 -0
  128. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/compare.js +103 -0
  129. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/composition.js +144 -0
  130. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/datasets.js +50 -0
  131. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/landscape.js +219 -0
  132. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/leaderboard.js +63 -0
  133. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/matrix.js +76 -0
  134. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/network.js +97 -0
  135. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/operator-effect.js +70 -0
  136. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/overview.js +123 -0
  137. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/parallel.js +109 -0
  138. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/param-effect.js +146 -0
  139. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/pipelines.js +77 -0
  140. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/planned.js +70 -0
  141. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/playground.js +0 -0
  142. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/robustness.js +74 -0
  143. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/run-detail.js +173 -0
  144. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/runs.js +74 -0
  145. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/statistics.js +118 -0
  146. nirs4all_benchmarks-0.1.5/src/nirs4all_benchmarks/web/views/upload.js +204 -0
  147. nirs4all_benchmarks-0.1.5/tests/conftest.py +36 -0
  148. nirs4all_benchmarks-0.1.5/tests/test_adapters.py +100 -0
  149. nirs4all_benchmarks-0.1.5/tests/test_contract.py +60 -0
  150. nirs4all_benchmarks-0.1.5/tests/test_fixtures_regression.py +53 -0
  151. nirs4all_benchmarks-0.1.5/tests/test_identity.py +107 -0
  152. nirs4all_benchmarks-0.1.5/tests/test_ingestion.py +75 -0
  153. nirs4all_benchmarks-0.1.5/tests/test_performance_compare.py +343 -0
  154. nirs4all_benchmarks-0.1.5/tests/test_queries.py +132 -0
  155. nirs4all_benchmarks-0.1.5/tests/test_repository_bridge.py +163 -0
  156. nirs4all_benchmarks-0.1.5/tests/test_scoring.py +75 -0
  157. nirs4all_benchmarks-0.1.5/tests/test_service_api.py +178 -0
  158. nirs4all_benchmarks-0.1.5/tests/test_site_build.py +39 -0
  159. nirs4all_benchmarks-0.1.5/tests/test_store.py +54 -0
  160. nirs4all_benchmarks-0.1.5/tests/test_upload_indexing.py +217 -0
  161. nirs4all_benchmarks-0.1.5/tests/test_version_metadata.py +19 -0
@@ -0,0 +1,20 @@
1
+ .git
2
+ .github
3
+ .codegraph
4
+ .venv
5
+ venv
6
+ __pycache__
7
+ *.pyc
8
+ *.egg-info
9
+ .pytest_cache
10
+ .ruff_cache
11
+ .mypy_cache
12
+ .coverage
13
+ htmlcov
14
+ arena-store
15
+ *.sqlite*
16
+ tests
17
+ docs
18
+ *.md
19
+ !README.md
20
+ assets
@@ -0,0 +1,23 @@
1
+ # EditorConfig — https://editorconfig.org
2
+ root = true
3
+
4
+ [*]
5
+ charset = utf-8
6
+ end_of_line = lf
7
+ insert_final_newline = true
8
+ trim_trailing_whitespace = true
9
+ indent_style = space
10
+ indent_size = 4
11
+
12
+ [*.py]
13
+ indent_size = 4
14
+ max_line_length = 120
15
+
16
+ [*.{yml,yaml,json,toml}]
17
+ indent_size = 2
18
+
19
+ [*.md]
20
+ trim_trailing_whitespace = false
21
+
22
+ [Makefile]
23
+ indent_style = tab
@@ -0,0 +1,21 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: "github-actions"
4
+ directory: "/"
5
+ schedule:
6
+ interval: "weekly"
7
+ groups:
8
+ github-actions:
9
+ patterns: ["*"]
10
+ commit-message:
11
+ prefix: "ci"
12
+
13
+ - package-ecosystem: "pip"
14
+ directory: "/"
15
+ schedule:
16
+ interval: "weekly"
17
+ groups:
18
+ python:
19
+ patterns: ["*"]
20
+ commit-message:
21
+ prefix: "chore"
@@ -0,0 +1,79 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main, rc/**]
6
+ pull_request:
7
+ branches: [main, rc/**]
8
+
9
+ jobs:
10
+ green-gate:
11
+ runs-on: ubuntu-latest
12
+ strategy:
13
+ fail-fast: false
14
+ matrix:
15
+ python-version: ["3.10", "3.11", "3.12"]
16
+ steps:
17
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
18
+
19
+ - name: Install uv
20
+ uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
21
+
22
+ - name: Set up Python ${{ matrix.python-version }}
23
+ run: uv python install ${{ matrix.python-version }}
24
+
25
+ - name: Create venv and install
26
+ run: |
27
+ uv venv --python ${{ matrix.python-version }} .venv
28
+ uv pip install --python .venv -e ".[service,dev]" pytest-cov
29
+
30
+ - name: Ruff (lint)
31
+ run: .venv/bin/ruff check src tests
32
+
33
+ - name: Mypy (types)
34
+ run: .venv/bin/mypy --python-version ${{ matrix.python-version }}
35
+
36
+ - name: Pytest (with coverage)
37
+ run: .venv/bin/pytest -q --cov=nirs4all_benchmarks --cov-report=xml
38
+
39
+ - name: Upload coverage
40
+ if: matrix.python-version == '3.11'
41
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
42
+ with:
43
+ name: coverage
44
+ path: coverage.xml
45
+
46
+ frontend-syntax:
47
+ runs-on: ubuntu-latest
48
+ steps:
49
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
50
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
51
+ with:
52
+ node-version: "20"
53
+ - name: node --check every SPA module
54
+ run: |
55
+ set -e
56
+ for f in src/nirs4all_benchmarks/web/app.js \
57
+ src/nirs4all_benchmarks/web/lib/*.js \
58
+ src/nirs4all_benchmarks/web/views/*.js; do
59
+ cp "$f" "${f%.js}.__check.mjs"
60
+ node --check "${f%.js}.__check.mjs"
61
+ rm "${f%.js}.__check.mjs"
62
+ echo "ok: $f"
63
+ done
64
+
65
+ docker:
66
+ runs-on: ubuntu-latest
67
+ needs: green-gate
68
+ steps:
69
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
70
+ - name: Build image
71
+ run: docker build -t nirs4all-benchmarks:ci .
72
+ - name: Smoke test
73
+ run: |
74
+ docker run -d --name arena -p 8000:8000 nirs4all-benchmarks:ci
75
+ for i in $(seq 1 20); do
76
+ if curl -sf http://127.0.0.1:8000/api/healthz; then echo " up"; exit 0; fi
77
+ sleep 1
78
+ done
79
+ docker logs arena; exit 1
@@ -0,0 +1,55 @@
1
+ name: pages
2
+
3
+ # Build the static, client-side Arena (a JSON snapshot of the demo store + the
4
+ # no-build SPA) and deploy it to GitHub Pages at benchmarks.nirs4all.org.
5
+ # The published site needs NO backend — see DESIGN/ROADMAP (static now; a live
6
+ # server managing runs/pipelines/meta-analyses with nirs4all-repository &
7
+ # nirs4all-datasets is the medium-term plan).
8
+
9
+ on:
10
+ push:
11
+ branches: [main]
12
+ paths:
13
+ - ".github/workflows/pages.yml"
14
+ - "pyproject.toml"
15
+ - "VERSION"
16
+ - "src/nirs4all_benchmarks/**"
17
+ workflow_dispatch:
18
+
19
+ permissions:
20
+ contents: read
21
+ pages: write
22
+ id-token: write
23
+
24
+ concurrency:
25
+ group: pages
26
+ cancel-in-progress: true
27
+
28
+ jobs:
29
+ build:
30
+ runs-on: ubuntu-latest
31
+ steps:
32
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
33
+ - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
34
+ - run: uv python install 3.11
35
+ - name: Install
36
+ run: |
37
+ uv venv --python 3.11 .venv
38
+ uv pip install --python .venv -e .
39
+ - name: Seed demo store + build static site
40
+ run: |
41
+ .venv/bin/n4a-benchmarks fixtures --store ./pages-store
42
+ .venv/bin/n4a-benchmarks build-site --store ./pages-store --out ./site --domain benchmarks.nirs4all.org
43
+ - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
44
+ with:
45
+ path: ./site
46
+
47
+ deploy:
48
+ needs: build
49
+ runs-on: ubuntu-latest
50
+ environment:
51
+ name: github-pages
52
+ url: ${{ steps.deployment.outputs.page_url }}
53
+ steps:
54
+ - id: deployment
55
+ uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
@@ -0,0 +1,51 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+ workflow_dispatch:
7
+ inputs:
8
+ dry_run:
9
+ description: "true = build only (no publish)"
10
+ type: string
11
+ default: "true"
12
+
13
+ permissions:
14
+ contents: read
15
+
16
+ jobs:
17
+ build:
18
+ runs-on: ubuntu-latest
19
+ steps:
20
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
21
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
22
+ with:
23
+ python-version: "3.11"
24
+ - name: Build sdist + wheel
25
+ run: |
26
+ python -m pip install --upgrade build twine
27
+ python -m build
28
+ twine check dist/*
29
+ - name: Smoke-check the wheel
30
+ run: |
31
+ python -m pip install dist/*.whl
32
+ n4a-benchmarks --help
33
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
34
+ with:
35
+ name: dist
36
+ path: dist/
37
+
38
+ publish:
39
+ needs: build
40
+ runs-on: ubuntu-latest
41
+ if: github.event_name == 'release' || inputs.dry_run == 'false'
42
+ environment: pypi
43
+ permissions:
44
+ contents: read
45
+ id-token: write
46
+ steps:
47
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
48
+ with:
49
+ name: dist
50
+ path: dist/
51
+ - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
@@ -0,0 +1,116 @@
1
+ name: version-guard
2
+
3
+ # Guardrail: the in-repo version must NEVER be ahead of the latest git tag.
4
+ # Tag ahead of the manifest (a release in flight) is allowed; manifest ahead of
5
+ # the tag (an un-tagged bump) is forbidden — that is the drift this blocks.
6
+ # Part of the nirs4all ecosystem version-sync guardrail.
7
+
8
+ on:
9
+ push:
10
+ branches: [main, rc/**]
11
+ pull_request:
12
+ branches: [main, rc/**]
13
+
14
+ permissions:
15
+ contents: read
16
+
17
+ jobs:
18
+ guard:
19
+ runs-on: ubuntu-latest
20
+ env:
21
+ VG_STRATEGY: "version_file"
22
+ VG_MANIFEST: "VERSION"
23
+ VG_ATTR: ""
24
+ VG_TAG_PREFIX: "v"
25
+ steps:
26
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
27
+ with:
28
+ fetch-depth: 0
29
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
30
+ with:
31
+ python-version: "3.11"
32
+ - run: python -m pip install --quiet packaging
33
+ - name: Manifest must not be ahead of the latest tag
34
+ shell: python
35
+ run: |
36
+ import os, re, subprocess, sys, json
37
+ try:
38
+ import tomllib
39
+ except ModuleNotFoundError:
40
+ import tomli as tomllib
41
+ from packaging.version import Version, InvalidVersion
42
+
43
+ strat = os.environ["VG_STRATEGY"]
44
+ path = os.environ["VG_MANIFEST"]
45
+ attr = os.environ.get("VG_ATTR", "")
46
+ prefix = os.environ.get("VG_TAG_PREFIX", "v")
47
+
48
+
49
+ def read_manifest(strat, path, attr):
50
+ with open(path, "r", encoding="utf-8") as f:
51
+ text = f.read()
52
+ if strat == "python_attr":
53
+ m = re.search(rf"{re.escape(attr or '__version__')}\s*=\s*['\"]([^'\"]+)['\"]", text)
54
+ return m.group(1) if m else None
55
+ if strat == "c_header":
56
+ macro = attr or "N4M_PROJECT_VERSION_STRING"
57
+ m = re.search(rf"#\s*define\s+{re.escape(macro)}\s+\"([^\"]+)\"", text)
58
+ return m.group(1) if m else None
59
+ if strat == "npm_package_json":
60
+ return json.loads(text).get("version")
61
+ if strat == "version_file":
62
+ return text.strip() or None
63
+ if strat in ("python_pyproject", "cargo_workspace", "cargo_package"):
64
+ data = tomllib.loads(text)
65
+ if strat == "python_pyproject":
66
+ return (data.get("project") or {}).get("version")
67
+ if strat == "cargo_workspace":
68
+ return ((data.get("workspace") or {}).get("package") or {}).get("version")
69
+ if strat == "cargo_package":
70
+ return (data.get("package") or {}).get("version")
71
+ return None
72
+
73
+
74
+ def norm(s):
75
+ s = s.strip()
76
+ return s[1:] if s[:1] in ("v", "V") else s
77
+
78
+
79
+ raw = read_manifest(strat, path, attr)
80
+ if not raw:
81
+ print(f"::warning::version-guard: could not read a version ({strat}:{path}); skipping")
82
+ sys.exit(0)
83
+ try:
84
+ manifest = Version(norm(raw))
85
+ except InvalidVersion:
86
+ print(f"::warning::version-guard: manifest version '{raw}' is not PEP 440; skipping")
87
+ sys.exit(0)
88
+
89
+ tags = subprocess.run(["git", "tag"], capture_output=True, text=True).stdout.split()
90
+ cand = []
91
+ for t in tags:
92
+ if prefix:
93
+ if not t.startswith(prefix):
94
+ continue
95
+ base = t[len(prefix):]
96
+ else:
97
+ if t[:1] in ("v", "V"):
98
+ continue
99
+ base = t
100
+ try:
101
+ cand.append((Version(base), t))
102
+ except InvalidVersion:
103
+ continue
104
+ if not cand:
105
+ print("::notice::version-guard: no matching tags yet; nothing to compare against")
106
+ sys.exit(0)
107
+
108
+ latest_v, latest_t = max(cand, key=lambda x: x[0])
109
+ print(f"manifest = {manifest} latest tag = {latest_t} ({latest_v})")
110
+ if manifest > latest_v:
111
+ print(
112
+ f"::error::version-guard: in-repo version {manifest} is AHEAD of the latest tag {latest_t}. "
113
+ f"A version bump must ship as a tag/release (tag {prefix}{manifest}), never merged to main ahead of its tag."
114
+ )
115
+ sys.exit(1)
116
+ print("OK: manifest is not ahead of the latest tag (tag ahead of / equal to the manifest is fine).")
@@ -0,0 +1,30 @@
1
+ # local code-intelligence index
2
+ .codegraph/
3
+
4
+ # python
5
+ __pycache__/
6
+ *.py[cod]
7
+ *.egg-info/
8
+ .eggs/
9
+ build/
10
+ dist/
11
+ .venv/
12
+ venv/
13
+
14
+ # tooling caches
15
+ .pytest_cache/
16
+ .ruff_cache/
17
+ .mypy_cache/
18
+ .coverage
19
+ htmlcov/
20
+
21
+ # arena runtime data (never commit a store)
22
+ arena-store/
23
+ *.sqlite
24
+ *.sqlite-wal
25
+ *.sqlite-shm
26
+ /arrays/
27
+ /exports/
28
+
29
+ # os
30
+ .DS_Store
@@ -0,0 +1,20 @@
1
+ # Local dev hooks mirroring the CI green gate (ruff check + hygiene).
2
+ # Install: uvx pre-commit install Run all: uvx pre-commit run --all-files
3
+ # mypy is run in CI (needs the full dev environment); not duplicated here.
4
+ repos:
5
+ - repo: https://github.com/pre-commit/pre-commit-hooks
6
+ rev: v5.0.0
7
+ hooks:
8
+ - id: trailing-whitespace
9
+ - id: end-of-file-fixer
10
+ - id: check-yaml
11
+ - id: check-toml
12
+ - id: check-merge-conflict
13
+ - id: check-added-large-files
14
+ - id: detect-private-key
15
+
16
+ - repo: https://github.com/astral-sh/ruff-pre-commit
17
+ rev: v0.15.17
18
+ hooks:
19
+ - id: ruff
20
+ name: ruff (lint)
@@ -0,0 +1,16 @@
1
+ version: 2
2
+
3
+ build:
4
+ os: ubuntu-24.04
5
+ tools:
6
+ python: "3.12"
7
+
8
+ sphinx:
9
+ configuration: docs/conf.py
10
+
11
+ python:
12
+ install:
13
+ - requirements: docs/requirements.txt
14
+
15
+ formats:
16
+ - htmlzip
@@ -0,0 +1,148 @@
1
+ # Changelog
2
+
3
+ All notable changes to `nirs4all-benchmarks` (the Arena) are documented here.
4
+ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this project uses
5
+ [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
+
7
+ ## [Unreleased]
8
+
9
+ ## [0.1.5] — 2026-07-07
10
+
11
+ Patch release for the V1 RC benchmark lane.
12
+
13
+ ### Added
14
+
15
+ - Add a benchmark score-agreement gate for performance comparison reports so regressions between
16
+ nirs4all and dag-ml/native results are caught before publication.
17
+
18
+ ## [0.1.4] — 2026-07-06
19
+
20
+ Patch-version release candidate preparation for current HEAD.
21
+
22
+ ### Changed
23
+
24
+ - Bump package and citation release metadata from 0.1.3 to 0.1.4.
25
+ - Add a lightweight metadata consistency check for the release version sources.
26
+
27
+ Second + third iterations — richer ingestion surface, deep faceted dataviz, the
28
+ ecosystem visual identity, and graph/3D visualizations.
29
+
30
+ ### Added (iteration 6 — static GitHub Pages site + roadmap)
31
+
32
+ - **Static, client-side deployment.** `n4a-benchmarks build-site` snapshots a store to JSON and copies
33
+ the SPA; a new client-side engine (`web/lib/static-engine.js`) answers every query in the browser
34
+ from that snapshot (verified at parity with the Python `Queries`), so the whole dataviz runs on
35
+ **GitHub Pages with no backend**. A `pages.yml` workflow builds + deploys to
36
+ **benchmarks.nirs4all.org** (CNAME); `config.js` flips the SPA into static mode.
37
+ - **Prototype notice + roadmap everywhere** — a dismissible prototype banner, and a documented plan
38
+ ([docs/ROADMAP.md](ROADMAP.md), README, deployment docs): *static client-side now → a live server
39
+ managing runs / pipelines / meta-analyses with `nirs4all-repository` & `nirs4all-datasets` next.*
40
+ - **Fixed** a real `operator_effect` bug (it counted the `input` pseudo-operator `X`); now excluded,
41
+ matching `operator_graph`/`composition`.
42
+
43
+ ### Changed (iteration 5 — meta-analysis UX overhaul)
44
+
45
+ - **Single shared lens** — a sticky context bar (Metric · Score level · Dataset) now drives **every**
46
+ analysis view; the per-view metric/scope/dataset selectors were removed (major declutter). Set the
47
+ lens once; the whole app follows.
48
+ - **Meta-analysis IA** — nav reorganized into a narrative (Start · Rankings · Drivers · Structure ·
49
+ Runs · Catalog · Contribute) and every view reframed around the question it answers.
50
+ - **Overview is now a findings dashboard** — headline insights (top pipeline, best model family, best
51
+ preprocessing, augmentation effect, split sensitivity, most robust), each linking to the relevant view.
52
+ - **Professional charts** — centralized Plotly theme (muted grid, readable hover, brand colorway),
53
+ small consistent markers (no more oversized dots), tidy legends/colorbars.
54
+ - **Fixed Composition** — the sunburst/treemap rendered blank because the `branchvalues:"total"` root
55
+ value was 0; the root now totals its children.
56
+
57
+ ### Added (iteration 4 — stats, more 3D, polish, prototype/version)
58
+
59
+ - **Statistics view** + `/api/stats` (`Queries.stats`) — score distribution (histogram), spread per
60
+ dataset (violin), and Pearson correlation of every numeric facet with the score, plus summary tables.
61
+ - **More 3D** — Landscape gains a **Z-axis facet** for a true 3-facet 3D scatter (x × y × z, colored
62
+ by the metric), alongside the 2-facet surface.
63
+ - **Official logos** — nav, favicon, OG image and footer now use the canonical ecosystem brand set
64
+ (`nirs4all-org/assets/brand/nirs4all-benchmarks/`), wired like the other ecosystem pages.
65
+ - **Prototype notice + version tracking** — a "prototype" badge in the nav + footer, a version chip
66
+ fed from `/api/healthz` (VERSION → `version.py` → `__version__`), and a footer with version,
67
+ ecosystem links, changelog, and license. Mirrors the `version-guard` CI guardrail.
68
+ - **Serious polish pass** — refined cards/hover, footer, skeleton shimmer, unobtrusive Plotly modebar,
69
+ gradient hero, and consistent ecosystem theming.
70
+
71
+ ### Added (iteration 3 — identity + graphs)
72
+
73
+ - **Ecosystem restyle** — the SPA now wears the **nirs4all.org** identity: teal `#0d9488`
74
+ primary (+ cyan/green/indigo/amber), warm paper background with grid ruling + aurora glow,
75
+ glassmorphic nav, IBM Plex Sans / Inter / JetBrains Mono, gradient-clip headings, ecosystem nav
76
+ links, and a gradient hero on the overview.
77
+ - **3D + graph dataviz** — three new views: **Landscape (3D)** (Plotly `scatter3d`/`surface` of the
78
+ score over two facets), **Composition** (sunburst/treemap of stage-role → operator usage, colored
79
+ by score), and **Network** — a clustered **mega-graph** (Cytoscape, force-directed) of pipelines
80
+ (edges = shared-operator Jaccard, clusters = model family) and operators (edges = co-occurrence,
81
+ clusters = stage role).
82
+ - **Graph/composition analytics** — `Queries.pipeline_graph/operator_graph/composition` +
83
+ `/api/graph` and `/api/composition`. Plotly upgraded to the full build; Cytoscape.js vendored.
84
+
85
+ ### Added (iteration 2 — ingestion + faceting)
86
+
87
+ - **Unified upload + run/store/display state machine** (`ingestion/upload.py`, `POST /api/upload`,
88
+ CLI `ingest-pipeline`): auto-detects a `.n4a` bundle (with **or** without fitted artifacts —
89
+ weights stripped), a raw nirs4all pipeline as a Python list / JSON / **YAML**, a dag-ml
90
+ `ExecutionBundle`, or an `ArenaRunExport`. Results-bearing inputs are ingested; bare pipelines are
91
+ registered and **planned** against target datasets, with already-run detection per pipeline×dataset.
92
+ - **`planned_runs`** table + `/api/planned` + a Planned view — the not-yet-run state (a runner
93
+ fulfils a plan and ingests the result; the Arena never runs compute).
94
+ - **Role-aware indexing** (`indexing.py`) — every node is classified
95
+ (preprocessing / augmentation / scaler / feature-selection / model / merge) and materialized into a
96
+ long **`run_facets`** table, so the benchmark is groupable by *any* dimension.
97
+ - **Faceted analytics** — `Queries.facets/facet_values/pivot/parallel` + `/api/facets`,
98
+ `/api/pivot`, `/api/parallel`.
99
+ - **New dataviz** — a **Playground** (pivot/group-by any facet: split × aug × pp × model × param;
100
+ bars / line / heatmap), **Parallel coordinates** across dimensions → score, a Planned view, and a
101
+ rewritten **Upload** view driving the unified endpoint. Plotly upgraded to the full build
102
+ (parcoords/violin).
103
+ - **Rich demo fixtures** (`generate_demo_exports`, `n4a-benchmarks fixtures` default) — a ~128-run
104
+ factorial over datasets × seeds × splits × {preprocessing chains, augmentation on/off, models,
105
+ parameter sweeps} so the playground has a real space to explore. The small deterministic set
106
+ (`--basic`) remains the frozen regression contract.
107
+
108
+ ## [0.1.0] — 2026-06-16
109
+
110
+ First implementation — the Arena goes from design docs to a working v1.
111
+
112
+ ### Added
113
+
114
+ - **Identity spine** (`identity/`): canonical-JSON SHA-256 hashing, the six-field
115
+ `run_condition_hash` composition, and a topology-aware, producer-agnostic
116
+ `pipeline_dag_hash` (Merkle hash over a normalized graph; id/order-invariant; order-insensitive
117
+ merge handling; linear-list lifting; records `engine_graph_fingerprint` and nirs4all `get_hash`).
118
+ - **Ingestion contract** (`contract/`): the frozen `ArenaRunExport` v1 — pydantic models + an
119
+ authoritative JSON Schema + the sample-keyed `residuals.parquet` column contract.
120
+ - **Store** (`store/`): `arena.sqlite` (full DESIGN §6/§7 dimension + fact schema, dedup-by-hash
121
+ upserts, forward-version guard) + `ResidualStore` (sample-keyed Zstd Parquet) + a `Queries`
122
+ serving facade (leaderboard, matrix, operator/parameter effect, robustness, run detail,
123
+ residual compare). **No artifacts are ever stored.**
124
+ - **Ingestion pipeline** (`ingestion/`): the verify → dedup → resolve → validate → pseudonymize →
125
+ strip → commit → report state machine; idempotent; leakage-honest (quarantine for releases);
126
+ store-global pseudonymization; score recomputation with supersede.
127
+ - **Dual-compatibility adapters** (`adapters/`): Adapter A (nirs4all workspace), Adapter B
128
+ (dag-ml `ExecutionBundle`), and `.n4a` recipe extraction (weights stripped).
129
+ - **Scoring** (`scoring/`): NumPy-only regression + classification metrics, versioned
130
+ `ScoreComputationSpec`.
131
+ - **Datasets** (`datasets/`): `DatasetCard` / `DatasetFingerprint` models + an optional
132
+ `nirs4all-datasets` catalog loader and a deterministic mock.
133
+ - **Service** (`service/`): a FastAPI app exposing the full query API + `POST /api/ingest`, serving
134
+ the dataviz SPA.
135
+ - **Dataviz web app** (`web/`): a no-build vanilla-JS + Plotly SPA — overview, leaderboard,
136
+ pipeline × dataset matrix, parameter/operator effect, robustness, run explorer, residual
137
+ complementarity compare, dataset/pipeline catalogs, DAG-rendering run detail, and upload.
138
+ - **CLI** (`n4a-benchmarks` / `n4a-arena`): init, ingest-export, ingest-workspace, ingest-bundle,
139
+ inspect-n4a, fixtures, stats, leaderboard, serve.
140
+ - **Fixtures**: a deterministic 2-dataset × {PLS sweep, branch/merge, stacking} grid with synthetic
141
+ OOF residuals, with frozen `run_condition_hash`es as a regression contract.
142
+ - **Docs**: ARCHITECTURE, CONTRACT, IDENTITY, INGESTION, ADAPTERS, API, DATAVIZ, DEPLOYMENT, CLI.
143
+ - **Ops**: Dockerfile, docker-compose, GitHub Actions CI (green gate × Python 3.10–3.12 +
144
+ frontend syntax check + Docker smoke test).
145
+
146
+ [0.1.5]: https://github.com/GBeurier/nirs4all-benchmarks/compare/v0.1.4...v0.1.5
147
+ [0.1.4]: https://github.com/GBeurier/nirs4all-benchmarks/compare/v0.1.3...v0.1.4
148
+ [0.1.0]: https://github.com/GBeurier/nirs4all-benchmarks/releases/tag/v0.1.0
@@ -0,0 +1,24 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use this software, please cite it using the metadata below."
3
+ title: "nirs4all-benchmarks"
4
+ abstract: >-
5
+ Reproducible, scored NIRS pipeline benchmarks on nirs4all-datasets (internal codename the Arena), published as a browsable benchmark resource.
6
+ type: software
7
+ authors:
8
+ - family-names: Beurier
9
+ given-names: Gregory
10
+ email: beurier@cirad.fr
11
+ affiliation: CIRAD
12
+ repository-code: "https://github.com/GBeurier/nirs4all-benchmarks"
13
+ url: "https://benchmarks.nirs4all.org"
14
+ license:
15
+ - CECILL-2.1
16
+ - AGPL-3.0-or-later
17
+ version: "0.1.5"
18
+ keywords:
19
+ - NIRS
20
+ - spectroscopy
21
+ - benchmarks
22
+ - reproducibility
23
+ - chemometrics
24
+ - scoring