endpoints-submission-cli 0.0.0__tar.gz

endpoints_submission_cli-0.0.0/.github/CODEOWNERS

@@ -0,0 +1,9 @@
+# These owners will be the default owners for everything in the repo.
+# Unless a later match takes precedence,they will be requested for review when someone opens a pull request.
+# (uncomment this line when adding an owner) * @username_or_team 
+
+/.github/CODEOWNERS @mlcommons/systems
+
+/.github/workflows/cla.yml @mlcommons/systems
+
+/LICENSE.md @mlcommons/systems

endpoints_submission_cli-0.0.0/.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --extra dev
+
+      - name: Run tests
+        run: uv run pytest
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Install dependencies
+        run: uv sync --extra docs
+
+      - name: Build docs
+        run: uv run sphinx-build -W docs docs/_build/html

endpoints_submission_cli-0.0.0/.github/workflows/cla.yml

@@ -0,0 +1,36 @@
+
+name: "cla-bot"
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened,closed,synchronize]
+
+jobs:
+  cla-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "MLCommons CLA bot check"
+        if: (github.event.comment.body == 'recheck') || github.event_name == 'pull_request_target'
+        # Alpha Release
+        uses: mlcommons/cla-bot@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # the below token should have repo scope and must be manually added by you in the repository's secret
+          PERSONAL_ACCESS_TOKEN : ${{ secrets.MLCOMMONS_BOT_CLA_TOKEN }}
+        with:
+          path-to-signatures: 'cla-bot/v1/cla.json'
+          # branch should not be protected
+          branch: 'main'
+          allowlist: user1,bot*
+          remote-organization-name: mlcommons
+          remote-repository-name: systems
+          
+         #below are the optional inputs - If the optional inputs are not given, then default values will be taken
+          #remote-organization-name: enter the remote organization name where the signatures should be stored (Default is storing the signatures in the same repository)
+          #remote-repository-name:  enter the  remote repository name where the signatures should be stored (Default is storing the signatures in the same repository)
+          #create-file-commit-message: 'For example: Creating file for storing CLA Signatures'
+          #signed-commit-message: 'For example: $contributorName has signed the CLA in #$pullRequestNo'
+          #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign'
+          #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA'
+          #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.'

endpoints_submission_cli-0.0.0/.github/workflows/publish.yml

@@ -0,0 +1,48 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Build sdist and wheel
+        run: uv build
+
+      - name: Upload dist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-pypi:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/endpoints-submission-cli
+    permissions:
+      id-token: write  # required for Trusted Publishing
+    steps:
+      - name: Download dist artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

endpoints_submission_cli-0.0.0/.gitignore

@@ -0,0 +1,31 @@
+.venv/
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.coverage
+.coverage.*
+htmlcov/
+docs/_build/
+.uv/
+
+# Test / lint caches
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+
+# AI tool artifacts
+.claude/
+.specstory/
+.cursor/
+.cursor_artifacts/
+
+# IDE
+.idea/
+.vscode/
+*.iml
+
+# OS
+.DS_Store
+Thumbs.db

endpoints_submission_cli-0.0.0/CONTRIBUTING.md

@@ -0,0 +1,9 @@
+## Contributing
+
+The best way to contribute to the MLCommons is to get involved with one of our many project communities. You can find more information about getting involved with MLCommons [here](https://mlcommons.org/community/). 
+
+Generally we encourage people to become MLCommons members if they wish to contribute to MLCommons projects, but outside pull requests are very welcome too.
+
+Regardless of whether you are a member, your organization (or you as an individual contributor) needs to sign the MLCommons Contributor License Agreement (CLA). Please submit your GitHub username to the [MLCommons Subscription form](https://mlcommons.org/community/subscribe/) to start that process.
+
+MLCommons project work is tracked with issue trackers and pull requests. Modify the project in your own fork and issue a pull request once you want other developers to take a look at what you have done and discuss the proposed changes. Ensure that cla-bot and other checks pass for your pull requests.

endpoints_submission_cli-0.0.0/LICENSE.md

@@ -0,0 +1,177 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

endpoints_submission_cli-0.0.0/PKG-INFO

@@ -0,0 +1,194 @@
+Metadata-Version: 2.4
+Name: endpoints-submission-cli
+Version: 0.0.0
+Summary: CLI tool for managing MLPerf endpoint submissions
+Author-email: MLCommons <arav@mlcommons.org>
+License-Expression: Apache-2.0
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: click>=8.1
+Requires-Dist: httpx>=0.25
+Requires-Dist: pydantic>=2.0
+Requires-Dist: rich>=13.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.0; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.0; extra == "docs"
+Requires-Dist: sphinx-autodoc-typehints>=2.0; extra == "docs"
+Requires-Dist: furo>=2024.1; extra == "docs"
+Dynamic: license-file
+
+# MLCommons Endpoints Submission Tools
+
+This repository contains two tools:
+
+- **`submission-checker`** — validates a submission folder against the §9.1 automated compliance rules (see below).
+- **`endpoints-submission-cli`** — CLI for registering benchmark runs and creating rolling submissions via the PRISM API. See [docs/endpoints-submission-cli.md](docs/endpoints-submission-cli.md) for full usage.
+
+---
+
+# submission-checker
+
+CLI tool for validating MLPerf Endpoints submissions against the §9.1 automated compliance checks.
+
+## Installation
+
+```bash
+uv sync --extra dev
+```
+
+Or with pip:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Usage
+
+### Check a submission
+
+```bash
+submission-checker check /path/to/submission
+```
+
+The tool expects the submission root to contain `systems/` and `pareto/` subdirectories as specified in §8.1.
+
+**Options:**
+
+| Flag | Description |
+|------|-------------|
+| `--strict` | Treat warnings as errors (exit 1 on any warning) |
+| `--quiet` / `-q` | Suppress INFO-level passing checks |
+| `--output FILE` / `-o FILE` | Write full results as JSON to *FILE* |
+
+**Exit codes:** `0` = all checks passed, `1` = one or more errors (or warnings with `--strict`).
+
+### Show region boundaries
+
+```bash
+submission-checker regions --max-concurrency 1024
+```
+
+Prints the concurrency ranges for each region given a declared Maximum Supported Concurrency *M* (§5.5).
+
+## Submission structure
+
+```
+<org>/
+├── systems/
+│   └── <system_desc_id>.json         # §8.2 — hardware + software description
+└── pareto/
+    └── <system_desc_id>/
+        └── <benchmark_model>/
+            ├── points/
+            │   └── point_<N>.yaml    # §8.3 — one config per measurement point
+            ├── results/
+            │   └── point_<N>/
+            │       ├── mlperf_endpoints_log_summary.json
+            │       └── mlperf_endpoints_log_detail.json
+            └── accuracy/
+                ├── accuracy.txt
+                └── accuracy_result.json
+```
+
+## What gets checked
+
+| Rule | Spec | Description |
+|------|------|-------------|
+| `path-exists` | §1 | Submission root directory exists |
+| `required-dir` | §1 | `systems/` and `pareto/` present |
+| `system-description-present` | §1 | At least one `*.json` file found in `systems/` |
+| `system-description-valid` | §1 | `systems/*.json` parses against schema |
+| `src-dir` | §1 | `src/` present for Standardized submissions |
+| `pareto-dir-exists` | §1 | `pareto/<system_id>/` directory exists |
+| `benchmark-model-dir` | §1 | At least one benchmark-model directory in `pareto/<system_id>/` |
+| `pareto-subdir` | §1 | `points/`, `results/`, `accuracy/` present |
+| `measurement-points-present` | §1 | At least one `point_*.yaml` found |
+| `point-config-valid` | §1 | YAML parses against `PointConfig` schema |
+| `point-filename-concurrency` | §1 | Filename concurrency matches declared value |
+| `result-file-present` | §1 | Result summary log exists for each point config |
+| `result-detail-present` | §1 | Result detail log exists for each point config |
+| `result-file-valid` | §1 | Result summary log parses against `PointSummary` schema |
+| `point-count` | §2, §8 | 7–32 measurement points |
+| `point-cap` | §2, §8 | Point count does not exceed 32 |
+| `low-latency-coverage` | §3 | At least one point in Low Latency region |
+| `low-throughput-coverage` | §4 | At least one point in Low Throughput region |
+| `med-throughput-coverage` | §5 | At least one point in Medium Throughput region |
+| `high-throughput-coverage` | §6 | At least one point in High Throughput region |
+| `max-concurrency-declared` | §7 | `max_supported_concurrency` field present |
+| `region-computation` | §7 | *M* > 32 (required for region formula) |
+| `concurrency-in-range` | §9 | Concurrency within region bounds (incl. 10% margin) |
+| `load-pattern` | §10 | `load_pattern` is `concurrency` with a positive concurrency level |
+| `point-duration` | §11 | Point meets per-region minimum duration |
+| `min-query-count` | §12 | `n_samples_completed` meets dataset-specific minimum (§6.4) |
+| `streaming-config` | §13 | `stream_all_chunks` is `True` |
+| `metric-consistency-duration` | §14 | `duration_ns` > 0 |
+| `metric-consistency-accounting` | §14 | `completed + failed == issued` |
+| `metric-consistency-output-tokens` | §14 | `total_output_tokens` ≥ 0 |
+| `metric-consistency-system-tps` | §9.1 | Stored `system_tps` consistent with derived value |
+| `metric-consistency-tps-per-user` | §9.1 | Stored `tps_per_user` consistent with `system_tps / concurrency` |
+| `accuracy-file` | §15 | `accuracy.txt` and `accuracy_result.json` present |
+| `accuracy-valid` | §15 | `accuracy_result.json` parses correctly |
+| `accuracy-consistency` | §15 | `passed` flag consistent with `score >= quality_target` |
+| `accuracy-gate` | §15 | Score ≥ quality target |
+| `config-consistency-dataset` | §16 | All points use the same dataset |
+| `config-consistency-model` | §16 | Directory name matches `benchmark_model` |
+| `region-declared` | §8.3 | Declared `region` field (if present) is valid and matches computed region |
+
+## Programmatic API
+
+```python
+from submission_checker import SubmissionChecker, Report
+
+checker = SubmissionChecker(Path("/submissions/acme_corp"))
+report = checker.run()
+
+if report.passed:
+    print("All checks passed")
+else:
+    for result in report.errors:
+        print(f"[{result.rule}] {result.message}")
+```
+
+The `Report` object also exposes `report.warnings` and serialises cleanly via `report.model_dump_json()`.
+
+## Development
+
+```bash
+uv run pytest                                          # run tests (189 tests, 100% coverage)
+uv run pytest --no-cov -x                             # fast fail on first error
+uv run ruff check src/ tests/                         # lint
+uv run ruff format src/ tests/                        # auto-format
+uv run sphinx-build -W docs docs/_build/html          # build docs
+```
+
+## Architecture
+
+```
+cli.py          Entry point — Click commands, Rich table output
+checker.py      SubmissionChecker — orchestrates loading and validation
+loader.py       File I/O — JSON/YAML loading, returns (model | None, list[CheckResult])
+structure.py    Directory structure validators (§8.1)
+models/
+  results.py         CheckResult, Severity, ok/warn/err helpers
+  regions.py         Region boundary computation (§5.5 reference algorithm)
+  file/              Per-artifact models — each validates a single file
+    system.py          SystemDescription (systems/*.json)
+    point_config.py    PointConfig + RuntimeSettings (points/point_<N>.yaml)
+    point_summary.py   PointSummary + PercentileStats (mlperf_endpoints_log_summary.json)
+    accuracy.py        AccuracyResult (accuracy/accuracy_result.json)
+  aggregate/         Cross-artifact models — validate across multiple files
+    point_result.py    PointResult — pairs one PointConfig with its PointSummary
+    context.py         ModelContext — validates point count, coverage, consistency, accuracy
+```
+
+Validation logic is co-located with the data models: each Pydantic model runs its own
+`@model_validator` methods and accumulates results in a private `_check_results` list.
+`SubmissionChecker.run()` orchestrates loading, instantiates models, and collects results
+into a `Report`. All loaders return `(model | None, list[CheckResult])` — failure surfaces
+every Pydantic validation error, not just the first.